source: josm/trunk/src/org/openstreetmap/josm/data/imagery/TMSCachedTileLoaderJob.java@ 19050

Last change on this file since 19050 was 19050, checked in by taylor.smock, 20 months ago

Revert most var changes from r19048, fix most new compile warnings and checkstyle issues

Also, document why various ErrorProne checks were originally disabled and fix
generic SonarLint issues.
  • Property svn:eol-style set to native
File size: 15.4 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.imagery;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.ByteArrayInputStream;
7import java.io.IOException;
8import java.net.SocketTimeoutException;
9import java.net.URL;
10import java.nio.charset.StandardCharsets;
11import java.util.HashSet;
12import java.util.List;
13import java.util.Locale;
14import java.util.Map;
15import java.util.Map.Entry;
16import java.util.Optional;
17import java.util.Set;
18import java.util.concurrent.ConcurrentHashMap;
19import java.util.concurrent.ConcurrentMap;
20import java.util.concurrent.ThreadPoolExecutor;
21import java.util.concurrent.TimeUnit;
22import java.util.regex.Matcher;
23import java.util.regex.Pattern;
24
25import org.apache.commons.jcs3.access.behavior.ICacheAccess;
26import org.apache.commons.jcs3.engine.behavior.ICache;
27import org.openstreetmap.gui.jmapviewer.Tile;
28import org.openstreetmap.gui.jmapviewer.interfaces.TileJob;
29import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener;
30import org.openstreetmap.gui.jmapviewer.interfaces.TileSource;
31import org.openstreetmap.gui.jmapviewer.tilesources.AbstractTMSTileSource;
32import org.openstreetmap.josm.data.cache.BufferedImageCacheEntry;
33import org.openstreetmap.josm.data.cache.CacheEntry;
34import org.openstreetmap.josm.data.cache.CacheEntryAttributes;
35import org.openstreetmap.josm.data.cache.ICachedLoaderListener;
36import org.openstreetmap.josm.data.cache.JCSCachedTileLoaderJob;
37import org.openstreetmap.josm.data.imagery.vectortile.VectorTile;
38import org.openstreetmap.josm.data.imagery.vectortile.mapbox.MVTFile;
39import org.openstreetmap.josm.data.preferences.LongProperty;
40import org.openstreetmap.josm.tools.HttpClient;
41import org.openstreetmap.josm.tools.Logging;
42import org.openstreetmap.josm.tools.Utils;
43
44/**
45 * Class bridging TMS requests to JCS cache requests
46 *
47 * @author Wiktor Niesiobędzki
48 * @since 8168
49 */
50public class TMSCachedTileLoaderJob extends JCSCachedTileLoaderJob<String, BufferedImageCacheEntry> implements TileJob, ICachedLoaderListener {
51 /** General maximum expires for tiles. Might be overridden by imagery settings */
52 public static final LongProperty MAXIMUM_EXPIRES = new LongProperty("imagery.generic.maximum_expires", TimeUnit.DAYS.toMillis(30));
53 /** General minimum expires for tiles. Might be overridden by imagery settings */
54 public static final LongProperty MINIMUM_EXPIRES = new LongProperty("imagery.generic.minimum_expires", TimeUnit.HOURS.toMillis(1));
55 static final Pattern SERVICE_EXCEPTION_PATTERN = Pattern.compile("(?s).+<ServiceException[^>]*>(.+)</ServiceException>.+");
56 static final Pattern CDATA_PATTERN = Pattern.compile("(?s)\\s*<!\\[CDATA\\[(.+)\\]\\]>\\s*");
57 static final Pattern JSON_PATTERN = Pattern.compile("\\{\"message\":\"(.+)\"\\}");
58 protected final Tile tile;
59 private volatile URL url;
60 private final TileJobOptions options;
61
62 // we need another deduplication of Tile Loader listeners, as for each submit, new TMSCachedTileLoaderJob was created
63 // that way, we reduce calls to tileLoadingFinished, and general CPU load due to surplus Map repaints
64 private static final ConcurrentMap<String, Set<TileLoaderListener>> inProgress = new ConcurrentHashMap<>();
65
66 /**
67 * Constructor for creating a job, to get a specific tile from cache
68 * @param listener Tile loader listener
69 * @param tile to be fetched from cache
70 * @param cache object
71 * @param options for job (such as http headers, timeouts etc.)
72 * @param downloadExecutor that will be executing the jobs
73 */
74 public TMSCachedTileLoaderJob(TileLoaderListener listener, Tile tile,
75 ICacheAccess<String, BufferedImageCacheEntry> cache,
76 TileJobOptions options,
77 ThreadPoolExecutor downloadExecutor) {
78 super(cache, options, downloadExecutor);
79 this.tile = tile;
80 this.options = options;
81 if (listener != null) {
82 inProgress.computeIfAbsent(getCacheKey(), k -> new HashSet<>()).add(listener);
83 }
84 }
85
86 @Override
87 public String getCacheKey() {
88 if (tile != null) {
89 TileSource tileSource = tile.getTileSource();
90 return Optional.ofNullable(tileSource.getName()).orElse("").replace(ICache.NAME_COMPONENT_DELIMITER, "_")
91 + ICache.NAME_COMPONENT_DELIMITER
92 + tileSource.getTileId(tile.getZoom(), tile.getXtile(), tile.getYtile());
93 }
94 return null;
95 }
96
97 /*
98 * this doesn't needs to be synchronized, as it's not that costly to keep only one execution
99 * in parallel, but URL creation and Tile.getUrl() are costly and are not needed when fetching
100 * data from cache, that's why URL creation is postponed until it's needed
101 *
102 * We need to have static url value for TileLoaderJob, as for some TileSources we might get different
103 * URL's each call we made (servers switching), and URL's are used below as a key for duplicate detection
104 *
105 */
106 @Override
107 public URL getUrl() throws IOException {
108 if (url == null) {
109 synchronized (this) {
110 if (url == null) {
111 String sUrl = tile.getUrl();
112 if (!"".equals(sUrl)) {
113 url = new URL(sUrl);
114 }
115 }
116 }
117 }
118 return url;
119 }
120
121 @Override
122 public boolean isObjectLoadable() {
123 if (cacheData != null) {
124 byte[] content = cacheData.getContent();
125 try {
126 return content.length > 0 || cacheData.getImage() != null || isNoTileAtZoom();
127 } catch (IOException e) {
128 Logging.logWithStackTrace(Logging.LEVEL_WARN, e, "JCS TMS - error loading from cache for tile {0}: {1}",
129 tile.getKey(), e.getMessage());
130 }
131 }
132 return false;
133 }
134
135 @Override
136 protected boolean isResponseLoadable(Map<String, List<String>> headers, int statusCode, byte[] content) {
137 attributes.setMetadata(tile.getTileSource().getMetadata(headers));
138 if (tile.getTileSource().isNoTileAtZoom(headers, statusCode, content)) {
139 attributes.setNoTileAtZoom(true);
140 return false; // do no try to load data from no-tile at zoom, cache empty object instead
141 }
142 if (isNotImage(headers, statusCode)) {
143 String message = detectErrorMessage(new String(content, StandardCharsets.UTF_8));
144 if (!Utils.isEmpty(message)) {
145 tile.setError(message);
146 }
147 return false;
148 }
149 return super.isResponseLoadable(headers, statusCode, content);
150 }
151
152 private boolean isNotImage(Map<String, List<String>> headers, int statusCode) {
153 if (statusCode == 200 && headers.containsKey("Content-Type") && !headers.get("Content-Type").isEmpty()) {
154 String contentType = headers.get("Content-Type").stream().findAny().orElse(null);
155 if (contentType != null && !contentType.startsWith("image") && !MVTFile.MIMETYPE.contains(contentType.toLowerCase(Locale.ROOT))) {
156 Logging.warn("Image not returned for tile: " + url + " content type was: " + contentType);
157 // not an image - do not store response in cache, so next time it will be queried again from the server
158 return true;
159 }
160 }
161 return false;
162 }
163
164 @Override
165 protected boolean cacheAsEmpty(Map<String, List<String>> headerFields, int responseCode) {
166 if (isNotImage(headerFields, responseCode)) {
167 return false;
168 }
169 return isNoTileAtZoom() || super.cacheAsEmpty(headerFields, responseCode);
170 }
171
172 @Override
173 public void submit(boolean force) {
174 tile.initLoading();
175 try {
176 super.submit(this, force);
177 } catch (IOException | IllegalArgumentException e) {
178 // if we fail to submit the job, mark tile as loaded and set error message
179 Logging.log(Logging.LEVEL_WARN, e);
180 tile.finishLoading();
181 tile.setError(e.getMessage());
182 }
183 }
184
185 @Override
186 public void loadingFinished(CacheEntry object, CacheEntryAttributes attributes, LoadResult result) {
187 this.attributes = attributes; // as we might get notification from other object than our selfs, pass attributes along
188 Set<TileLoaderListener> listeners = inProgress.remove(getCacheKey());
189 boolean status = result == LoadResult.SUCCESS;
190
191 try {
192 tile.finishLoading(); // whatever happened set that loading has finished
193 // set tile metadata
194 if (this.attributes != null) {
195 for (Entry<String, String> e: this.attributes.getMetadata().entrySet()) {
196 tile.putValue(e.getKey(), e.getValue());
197 }
198 }
199
200 switch (result) {
201 case SUCCESS:
202 handleNoTileAtZoom();
203 if (attributes != null) {
204 int httpStatusCode = attributes.getResponseCode();
205 if (httpStatusCode >= 400 && !isNoTileAtZoom()) {
206 status = false;
207 handleError(attributes);
208 }
209 }
210 status &= tryLoadTileImage(object); //try to keep returned image as background
211 break;
212 case FAILURE:
213 handleError(attributes);
214 tryLoadTileImage(object);
215 break;
216 case CANCELED:
217 tile.loadingCanceled();
218 // do nothing
219 }
220
221 // always check, if there is some listener interested in fact, that tile has finished loading
222 if (listeners != null) { // listeners might be null, if some other thread notified already about success
223 for (TileLoaderListener l: listeners) {
224 l.tileLoadingFinished(tile, status);
225 }
226 }
227 } catch (IOException e) {
228 Logging.warn("JCS TMS - error loading object for tile {0}: {1}", tile.getKey(), e.getMessage());
229 tile.setError(e);
230 tile.setLoaded(false);
231 if (listeners != null) { // listeners might be null, if some other thread notified already about success
232 for (TileLoaderListener l: listeners) {
233 l.tileLoadingFinished(tile, false);
234 }
235 }
236 }
237 }
238
239 private void handleError(CacheEntryAttributes attributes) {
240 if (tile.hasError() && tile.getErrorMessage() != null) {
241 // tile has already set error message, don't overwrite it
242 return;
243 }
244 if (attributes != null) {
245 int httpStatusCode = attributes.getResponseCode();
246 if (attributes.getErrorMessage() == null) {
247 tile.setError(tr("HTTP error {0} when loading tiles", httpStatusCode));
248 } else {
249 tile.setError(tr("Error downloading tiles: {0}", attributes.getErrorMessage()));
250 }
251 if (httpStatusCode >= 500 && httpStatusCode != 599) {
252 // httpStatusCode = 599 is set by JCSCachedTileLoaderJob on IOException
253 tile.setLoaded(false); // treat 500 errors as temporary and try to load it again
254 }
255 // treat SocketTimeoutException as transient error
256 attributes.getException()
257 .filter(x -> x.isAssignableFrom(SocketTimeoutException.class))
258 .ifPresent(x -> tile.setLoaded(false));
259 } else {
260 tile.setError(tr("Problem loading tile"));
261 // treat unknown errors as permanent and do not try to load tile again
262 }
263 }
264
265 /**
266 * For TMS use BaseURL as settings discovery, so for different paths, we will have different settings (useful for developer servers)
267 *
268 * @return base URL of TMS or server url as defined in super class
269 */
270 @Override
271 protected String getServerKey() {
272 TileSource ts = tile.getSource();
273 if (ts instanceof AbstractTMSTileSource) {
274 return ((AbstractTMSTileSource) ts).getBaseUrl();
275 }
276 return super.getServerKey();
277 }
278
279 @Override
280 protected BufferedImageCacheEntry createCacheEntry(byte[] content) {
281 return new BufferedImageCacheEntry(content);
282 }
283
284 @Override
285 public void submit() {
286 submit(false);
287 }
288
289 @Override
290 protected CacheEntryAttributes parseHeaders(HttpClient.Response urlConn) {
291 CacheEntryAttributes ret = super.parseHeaders(urlConn);
292 // keep the expiration time between MINIMUM_EXPIRES and MAXIMUM_EXPIRES, so we will cache the tiles
293 // at least for some short period of time, but not too long
294 long minimumExpiryTime = TimeUnit.SECONDS.toMillis(options.getMinimumExpiryTime());
295 long nowPlusMin = now + Math.max(MINIMUM_EXPIRES.get(), minimumExpiryTime);
296 if (ret.getExpirationTime() < nowPlusMin) {
297 ret.setExpirationTime(nowPlusMin);
298 }
299 long nowPlusMax = now + Math.max(MAXIMUM_EXPIRES.get(), minimumExpiryTime);
300 if (ret.getExpirationTime() > nowPlusMax) {
301 ret.setExpirationTime(nowPlusMax);
302 }
303 return ret;
304 }
305
306 private boolean handleNoTileAtZoom() {
307 if (isNoTileAtZoom()) {
308 Logging.debug("JCS TMS - Tile valid, but no file, as no tiles at this level {0}", tile);
309 tile.setError(tr("No tiles at this zoom level"));
310 tile.putValue("tile-info", "no-tile");
311 return true;
312 }
313 return false;
314 }
315
316 private boolean isNoTileAtZoom() {
317 if (attributes == null) {
318 Logging.warn("Cache attributes are null");
319 }
320 return attributes != null && attributes.isNoTileAtZoom();
321 }
322
323 private boolean tryLoadTileImage(CacheEntry object) throws IOException {
324 if (object != null) {
325 byte[] content = object.getContent();
326 if (content.length > 0 || tile instanceof VectorTile) {
327 try (ByteArrayInputStream in = new ByteArrayInputStream(content)) {
328 tile.loadImage(in);
329 if ((!(tile instanceof VectorTile) && tile.getImage() == null)
330 || ((tile instanceof VectorTile) && !tile.isLoaded())) {
331 String s = new String(content, StandardCharsets.UTF_8);
332 Matcher m = SERVICE_EXCEPTION_PATTERN.matcher(s);
333 if (m.matches()) {
334 String message = Utils.strip(m.group(1));
335 tile.setError(message);
336 Logging.error(message);
337 Logging.debug(s);
338 } else {
339 tile.setError(tr("Could not load image from tile server"));
340 }
341 return false;
342 }
343 } catch (UnsatisfiedLinkError | SecurityException e) {
344 throw new IOException(e);
345 }
346 }
347 }
348 return true;
349 }
350
351 @Override
352 public String detectErrorMessage(String data) {
353 Matcher xml = SERVICE_EXCEPTION_PATTERN.matcher(data);
354 Matcher json = JSON_PATTERN.matcher(data);
355 return xml.matches() ? removeCdata(Utils.strip(xml.group(1)))
356 : json.matches() ? Utils.strip(json.group(1))
357 : super.detectErrorMessage(data);
358 }
359
360 private static String removeCdata(String msg) {
361 Matcher m = CDATA_PATTERN.matcher(msg);
362 return m.matches() ? Utils.strip(m.group(1)) : msg;
363 }
364}
Note: See TracBrowser for help on using the repository browser.