source: josm/trunk/src/org/openstreetmap/josm/data/imagery/TMSCachedTileLoaderJob.java@ 16431

Last change on this file since 16431 was 16431, checked in by simon04, 4 years ago

TMSCachedTileLoaderJob.parseHeaders: use Utils.clamp

Fixes inconsistent TimeUnit.SECONDS.toMillis conversion (in if condition vs. statement).
  • Property svn:eol-style set to native
File size: 13.5 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.imagery;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.ByteArrayInputStream;
7import java.io.IOException;
8import java.net.SocketTimeoutException;
9import java.net.URL;
10import java.nio.charset.StandardCharsets;
11import java.util.HashSet;
12import java.util.List;
13import java.util.Map;
14import java.util.Map.Entry;
15import java.util.Optional;
16import java.util.Set;
17import java.util.concurrent.ConcurrentHashMap;
18import java.util.concurrent.ConcurrentMap;
19import java.util.concurrent.ThreadPoolExecutor;
20import java.util.concurrent.TimeUnit;
21import java.util.regex.Matcher;
22import java.util.regex.Pattern;
23
24import org.apache.commons.jcs3.access.behavior.ICacheAccess;
25import org.openstreetmap.gui.jmapviewer.Tile;
26import org.openstreetmap.gui.jmapviewer.interfaces.TileJob;
27import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener;
28import org.openstreetmap.gui.jmapviewer.interfaces.TileSource;
29import org.openstreetmap.gui.jmapviewer.tilesources.AbstractTMSTileSource;
30import org.openstreetmap.josm.data.cache.BufferedImageCacheEntry;
31import org.openstreetmap.josm.data.cache.CacheEntry;
32import org.openstreetmap.josm.data.cache.CacheEntryAttributes;
33import org.openstreetmap.josm.data.cache.ICachedLoaderListener;
34import org.openstreetmap.josm.data.cache.JCSCachedTileLoaderJob;
35import org.openstreetmap.josm.data.preferences.LongProperty;
36import org.openstreetmap.josm.tools.HttpClient;
37import org.openstreetmap.josm.tools.Logging;
38import org.openstreetmap.josm.tools.Utils;
39
40/**
41 * Class bridging TMS requests to JCS cache requests
42 *
43 * @author Wiktor Niesiobędzki
44 * @since 8168
45 */
46public class TMSCachedTileLoaderJob extends JCSCachedTileLoaderJob<String, BufferedImageCacheEntry> implements TileJob, ICachedLoaderListener {
47 /** General maximum expires for tiles. Might be overridden by imagery settings */
48 public static final LongProperty MAXIMUM_EXPIRES = new LongProperty("imagery.generic.maximum_expires", TimeUnit.DAYS.toMillis(30));
49 /** General minimum expires for tiles. Might be overridden by imagery settings */
50 public static final LongProperty MINIMUM_EXPIRES = new LongProperty("imagery.generic.minimum_expires", TimeUnit.HOURS.toMillis(1));
51 static final Pattern SERVICE_EXCEPTION_PATTERN = Pattern.compile("(?s).+<ServiceException[^>]*>(.+)</ServiceException>.+");
52 static final Pattern CDATA_PATTERN = Pattern.compile("(?s)\\s*<!\\[CDATA\\[(.+)\\]\\]>\\s*");
53 static final Pattern JSON_PATTERN = Pattern.compile("\\{\"message\":\"(.+)\"\\}");
54 protected final Tile tile;
55 private volatile URL url;
56 private final TileJobOptions options;
57
58 // we need another deduplication of Tile Loader listeners, as for each submit, new TMSCachedTileLoaderJob was created
59 // that way, we reduce calls to tileLoadingFinished, and general CPU load due to surplus Map repaints
60 private static final ConcurrentMap<String, Set<TileLoaderListener>> inProgress = new ConcurrentHashMap<>();
61
62 /**
63 * Constructor for creating a job, to get a specific tile from cache
64 * @param listener Tile loader listener
65 * @param tile to be fetched from cache
66 * @param cache object
67 * @param options for job (such as http headers, timeouts etc.)
68 * @param downloadExecutor that will be executing the jobs
69 */
70 public TMSCachedTileLoaderJob(TileLoaderListener listener, Tile tile,
71 ICacheAccess<String, BufferedImageCacheEntry> cache,
72 TileJobOptions options,
73 ThreadPoolExecutor downloadExecutor) {
74 super(cache, options, downloadExecutor);
75 this.tile = tile;
76 this.options = options;
77 if (listener != null) {
78 inProgress.computeIfAbsent(getCacheKey(), k -> new HashSet<>()).add(listener);
79 }
80 }
81
82 @Override
83 public String getCacheKey() {
84 if (tile != null) {
85 TileSource tileSource = tile.getTileSource();
86 return Optional.ofNullable(tileSource.getName()).orElse("").replace(':', '_') + ':'
87 + tileSource.getTileId(tile.getZoom(), tile.getXtile(), tile.getYtile());
88 }
89 return null;
90 }
91
92 /*
93 * this doesn't needs to be synchronized, as it's not that costly to keep only one execution
94 * in parallel, but URL creation and Tile.getUrl() are costly and are not needed when fetching
95 * data from cache, that's why URL creation is postponed until it's needed
96 *
97 * We need to have static url value for TileLoaderJob, as for some TileSources we might get different
98 * URL's each call we made (servers switching), and URL's are used below as a key for duplicate detection
99 *
100 */
101 @Override
102 public URL getUrl() throws IOException {
103 if (url == null) {
104 synchronized (this) {
105 if (url == null) {
106 String sUrl = tile.getUrl();
107 if (!"".equals(sUrl)) {
108 url = new URL(sUrl);
109 }
110 }
111 }
112 }
113 return url;
114 }
115
116 @Override
117 public boolean isObjectLoadable() {
118 if (cacheData != null) {
119 byte[] content = cacheData.getContent();
120 try {
121 return content.length > 0 || cacheData.getImage() != null || isNoTileAtZoom();
122 } catch (IOException e) {
123 Logging.logWithStackTrace(Logging.LEVEL_WARN, e, "JCS TMS - error loading from cache for tile {0}: {1}",
124 tile.getKey(), e.getMessage());
125 }
126 }
127 return false;
128 }
129
130 @Override
131 protected boolean isResponseLoadable(Map<String, List<String>> headers, int statusCode, byte[] content) {
132 attributes.setMetadata(tile.getTileSource().getMetadata(headers));
133 if (tile.getTileSource().isNoTileAtZoom(headers, statusCode, content)) {
134 attributes.setNoTileAtZoom(true);
135 return false; // do no try to load data from no-tile at zoom, cache empty object instead
136 }
137 return super.isResponseLoadable(headers, statusCode, content);
138 }
139
140 @Override
141 protected boolean cacheAsEmpty() {
142 return isNoTileAtZoom() || super.cacheAsEmpty();
143 }
144
145 @Override
146 public void submit(boolean force) {
147 tile.initLoading();
148 try {
149 super.submit(this, force);
150 } catch (IOException | IllegalArgumentException e) {
151 // if we fail to submit the job, mark tile as loaded and set error message
152 Logging.log(Logging.LEVEL_WARN, e);
153 tile.finishLoading();
154 tile.setError(e.getMessage());
155 }
156 }
157
158 @Override
159 public void loadingFinished(CacheEntry object, CacheEntryAttributes attributes, LoadResult result) {
160 this.attributes = attributes; // as we might get notification from other object than our selfs, pass attributes along
161 Set<TileLoaderListener> listeners = inProgress.remove(getCacheKey());
162 boolean status = result == LoadResult.SUCCESS;
163
164 try {
165 tile.finishLoading(); // whatever happened set that loading has finished
166 // set tile metadata
167 if (this.attributes != null) {
168 for (Entry<String, String> e: this.attributes.getMetadata().entrySet()) {
169 tile.putValue(e.getKey(), e.getValue());
170 }
171 }
172
173 switch(result) {
174 case SUCCESS:
175 handleNoTileAtZoom();
176 if (attributes != null) {
177 int httpStatusCode = attributes.getResponseCode();
178 if (httpStatusCode >= 400 && !isNoTileAtZoom()) {
179 status = false;
180 handleError(attributes);
181 }
182 }
183 status &= tryLoadTileImage(object); //try to keep returned image as background
184 break;
185 case FAILURE:
186 handleError(attributes);
187 tryLoadTileImage(object);
188 break;
189 case CANCELED:
190 tile.loadingCanceled();
191 // do nothing
192 }
193
194 // always check, if there is some listener interested in fact, that tile has finished loading
195 if (listeners != null) { // listeners might be null, if some other thread notified already about success
196 for (TileLoaderListener l: listeners) {
197 l.tileLoadingFinished(tile, status);
198 }
199 }
200 } catch (IOException e) {
201 Logging.warn("JCS TMS - error loading object for tile {0}: {1}", tile.getKey(), e.getMessage());
202 tile.setError(e);
203 tile.setLoaded(false);
204 if (listeners != null) { // listeners might be null, if some other thread notified already about success
205 for (TileLoaderListener l: listeners) {
206 l.tileLoadingFinished(tile, false);
207 }
208 }
209 }
210 }
211
212 private void handleError(CacheEntryAttributes attributes) {
213 if (attributes != null) {
214 int httpStatusCode = attributes.getResponseCode();
215 if (attributes.getErrorMessage() == null) {
216 tile.setError(tr("HTTP error {0} when loading tiles", httpStatusCode));
217 } else {
218 tile.setError(tr("Error downloading tiles: {0}", attributes.getErrorMessage()));
219 }
220 if (httpStatusCode >= 500 && httpStatusCode != 599) {
221 // httpStatusCode = 599 is set by JCSCachedTileLoaderJob on IOException
222 tile.setLoaded(false); // treat 500 errors as temporary and try to load it again
223 }
224 // treat SocketTimeoutException as transient error
225 attributes.getException()
226 .filter(x -> x.isAssignableFrom(SocketTimeoutException.class))
227 .ifPresent(x -> tile.setLoaded(false));
228 } else {
229 tile.setError(tr("Problem loading tile"));
230 // treat unknown errors as permanent and do not try to load tile again
231 }
232 }
233
234 /**
235 * For TMS use BaseURL as settings discovery, so for different paths, we will have different settings (useful for developer servers)
236 *
237 * @return base URL of TMS or server url as defined in super class
238 */
239 @Override
240 protected String getServerKey() {
241 TileSource ts = tile.getSource();
242 if (ts instanceof AbstractTMSTileSource) {
243 return ((AbstractTMSTileSource) ts).getBaseUrl();
244 }
245 return super.getServerKey();
246 }
247
248 @Override
249 protected BufferedImageCacheEntry createCacheEntry(byte[] content) {
250 return new BufferedImageCacheEntry(content);
251 }
252
253 @Override
254 public void submit() {
255 submit(false);
256 }
257
258 @Override
259 protected CacheEntryAttributes parseHeaders(HttpClient.Response urlConn) {
260 CacheEntryAttributes ret = super.parseHeaders(urlConn);
261 // keep the expiration time between MINIMUM_EXPIRES and MAXIMUM_EXPIRES, so we will cache the tiles
262 // at least for some short period of time, but not too long
263 ret.setExpirationTime((long) Utils.clamp(
264 ret.getExpirationTime(),
265 now + Math.max(MINIMUM_EXPIRES.get(), TimeUnit.SECONDS.toMillis(options.getMinimumExpiryTime())),
266 now + Math.max(MAXIMUM_EXPIRES.get(), TimeUnit.SECONDS.toMillis(options.getMinimumExpiryTime()))
267 ));
268 return ret;
269 }
270
271 private boolean handleNoTileAtZoom() {
272 if (isNoTileAtZoom()) {
273 Logging.debug("JCS TMS - Tile valid, but no file, as no tiles at this level {0}", tile);
274 tile.setError(tr("No tiles at this zoom level"));
275 tile.putValue("tile-info", "no-tile");
276 return true;
277 }
278 return false;
279 }
280
281 private boolean isNoTileAtZoom() {
282 if (attributes == null) {
283 Logging.warn("Cache attributes are null");
284 }
285 return attributes != null && attributes.isNoTileAtZoom();
286 }
287
288 private boolean tryLoadTileImage(CacheEntry object) throws IOException {
289 if (object != null) {
290 byte[] content = object.getContent();
291 if (content.length > 0) {
292 try (ByteArrayInputStream in = new ByteArrayInputStream(content)) {
293 tile.loadImage(in);
294 if (tile.getImage() == null) {
295 String s = new String(content, StandardCharsets.UTF_8);
296 Matcher m = SERVICE_EXCEPTION_PATTERN.matcher(s);
297 if (m.matches()) {
298 String message = Utils.strip(m.group(1));
299 tile.setError(message);
300 Logging.error(message);
301 Logging.debug(s);
302 } else {
303 tile.setError(tr("Could not load image from tile server"));
304 }
305 return false;
306 }
307 } catch (UnsatisfiedLinkError | SecurityException e) {
308 throw new IOException(e);
309 }
310 }
311 }
312 return true;
313 }
314
315 @Override
316 public String detectErrorMessage(String data) {
317 Matcher xml = SERVICE_EXCEPTION_PATTERN.matcher(data);
318 Matcher json = JSON_PATTERN.matcher(data);
319 return xml.matches() ? removeCdata(Utils.strip(xml.group(1)))
320 : json.matches() ? Utils.strip(json.group(1))
321 : super.detectErrorMessage(data);
322 }
323
324 private static String removeCdata(String msg) {
325 Matcher m = CDATA_PATTERN.matcher(msg);
326 return m.matches() ? Utils.strip(m.group(1)) : msg;
327 }
328}
Note: See TracBrowser for help on using the repository browser.