001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.data.imagery; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.ByteArrayInputStream; 007import java.io.IOException; 008import java.net.SocketTimeoutException; 009import java.net.URL; 010import java.nio.charset.StandardCharsets; 011import java.util.HashSet; 012import java.util.List; 013import java.util.Locale; 014import java.util.Map; 015import java.util.Map.Entry; 016import java.util.Optional; 017import java.util.Set; 018import java.util.concurrent.ConcurrentHashMap; 019import java.util.concurrent.ConcurrentMap; 020import java.util.concurrent.ThreadPoolExecutor; 021import java.util.concurrent.TimeUnit; 022import java.util.regex.Matcher; 023import java.util.regex.Pattern; 024 025import org.apache.commons.jcs3.access.behavior.ICacheAccess; 026import org.apache.commons.jcs3.engine.behavior.ICache; 027import org.openstreetmap.gui.jmapviewer.Tile; 028import org.openstreetmap.gui.jmapviewer.interfaces.TileJob; 029import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener; 030import org.openstreetmap.gui.jmapviewer.interfaces.TileSource; 031import org.openstreetmap.gui.jmapviewer.tilesources.AbstractTMSTileSource; 032import org.openstreetmap.josm.data.cache.BufferedImageCacheEntry; 033import org.openstreetmap.josm.data.cache.CacheEntry; 034import org.openstreetmap.josm.data.cache.CacheEntryAttributes; 035import org.openstreetmap.josm.data.cache.ICachedLoaderListener; 036import org.openstreetmap.josm.data.cache.JCSCachedTileLoaderJob; 037import org.openstreetmap.josm.data.imagery.vectortile.VectorTile; 038import org.openstreetmap.josm.data.imagery.vectortile.mapbox.MVTFile; 039import org.openstreetmap.josm.data.preferences.LongProperty; 040import org.openstreetmap.josm.tools.HttpClient; 041import org.openstreetmap.josm.tools.Logging; 042import org.openstreetmap.josm.tools.Utils; 043 044/** 045 * Class bridging TMS requests to JCS cache requests 046 * 047 * @author Wiktor Niesiobędzki 048 * @since 8168 049 */ 050public class TMSCachedTileLoaderJob extends JCSCachedTileLoaderJob<String, BufferedImageCacheEntry> implements TileJob, ICachedLoaderListener { 051 /** General maximum expires for tiles. Might be overridden by imagery settings */ 052 public static final LongProperty MAXIMUM_EXPIRES = new LongProperty("imagery.generic.maximum_expires", TimeUnit.DAYS.toMillis(30)); 053 /** General minimum expires for tiles. Might be overridden by imagery settings */ 054 public static final LongProperty MINIMUM_EXPIRES = new LongProperty("imagery.generic.minimum_expires", TimeUnit.HOURS.toMillis(1)); 055 static final Pattern SERVICE_EXCEPTION_PATTERN = Pattern.compile("(?s).+<ServiceException[^>]*>(.+)</ServiceException>.+"); 056 static final Pattern CDATA_PATTERN = Pattern.compile("(?s)\\s*<!\\[CDATA\\[(.+)\\]\\]>\\s*"); 057 static final Pattern JSON_PATTERN = Pattern.compile("\\{\"message\":\"(.+)\"\\}"); 058 protected final Tile tile; 059 private volatile URL url; 060 private final TileJobOptions options; 061 062 // we need another deduplication of Tile Loader listeners, as for each submit, new TMSCachedTileLoaderJob was created 063 // that way, we reduce calls to tileLoadingFinished, and general CPU load due to surplus Map repaints 064 private static final ConcurrentMap<String, Set<TileLoaderListener>> inProgress = new ConcurrentHashMap<>(); 065 066 /** 067 * Constructor for creating a job, to get a specific tile from cache 068 * @param listener Tile loader listener 069 * @param tile to be fetched from cache 070 * @param cache object 071 * @param options for job (such as http headers, timeouts etc.) 072 * @param downloadExecutor that will be executing the jobs 073 */ 074 public TMSCachedTileLoaderJob(TileLoaderListener listener, Tile tile, 075 ICacheAccess<String, BufferedImageCacheEntry> cache, 076 TileJobOptions options, 077 ThreadPoolExecutor downloadExecutor) { 078 super(cache, options, downloadExecutor); 079 this.tile = tile; 080 this.options = options; 081 if (listener != null) { 082 inProgress.computeIfAbsent(getCacheKey(), k -> new HashSet<>()).add(listener); 083 } 084 } 085 086 @Override 087 public String getCacheKey() { 088 if (tile != null) { 089 TileSource tileSource = tile.getTileSource(); 090 return Optional.ofNullable(tileSource.getName()).orElse("").replace(ICache.NAME_COMPONENT_DELIMITER, "_") 091 + ICache.NAME_COMPONENT_DELIMITER 092 + tileSource.getTileId(tile.getZoom(), tile.getXtile(), tile.getYtile()); 093 } 094 return null; 095 } 096 097 /* 098 * this doesn't needs to be synchronized, as it's not that costly to keep only one execution 099 * in parallel, but URL creation and Tile.getUrl() are costly and are not needed when fetching 100 * data from cache, that's why URL creation is postponed until it's needed 101 * 102 * We need to have static url value for TileLoaderJob, as for some TileSources we might get different 103 * URL's each call we made (servers switching), and URL's are used below as a key for duplicate detection 104 * 105 */ 106 @Override 107 public URL getUrl() throws IOException { 108 if (url == null) { 109 synchronized (this) { 110 if (url == null) { 111 String sUrl = tile.getUrl(); 112 if (!"".equals(sUrl)) { 113 url = new URL(sUrl); 114 } 115 } 116 } 117 } 118 return url; 119 } 120 121 @Override 122 public boolean isObjectLoadable() { 123 if (cacheData != null) { 124 byte[] content = cacheData.getContent(); 125 try { 126 return content.length > 0 || cacheData.getImage() != null || isNoTileAtZoom(); 127 } catch (IOException e) { 128 Logging.logWithStackTrace(Logging.LEVEL_WARN, e, "JCS TMS - error loading from cache for tile {0}: {1}", 129 tile.getKey(), e.getMessage()); 130 } 131 } 132 return false; 133 } 134 135 @Override 136 protected boolean isResponseLoadable(Map<String, List<String>> headers, int statusCode, byte[] content) { 137 attributes.setMetadata(tile.getTileSource().getMetadata(headers)); 138 if (tile.getTileSource().isNoTileAtZoom(headers, statusCode, content)) { 139 attributes.setNoTileAtZoom(true); 140 return false; // do no try to load data from no-tile at zoom, cache empty object instead 141 } 142 if (isNotImage(headers, statusCode)) { 143 String message = detectErrorMessage(new String(content, StandardCharsets.UTF_8)); 144 if (!Utils.isEmpty(message)) { 145 tile.setError(message); 146 } 147 return false; 148 } 149 return super.isResponseLoadable(headers, statusCode, content); 150 } 151 152 private boolean isNotImage(Map<String, List<String>> headers, int statusCode) { 153 if (statusCode == 200 && headers.containsKey("Content-Type") && !headers.get("Content-Type").isEmpty()) { 154 String contentType = headers.get("Content-Type").stream().findAny().orElse("").toLowerCase(Locale.ROOT); 155 if (!contentType.startsWith("image") 156 && !contentType.startsWith("application/octet-stream") 157 && !MVTFile.MIMETYPE.contains(contentType)) { 158 Logging.warn("Image not returned for tile: " + url + " content type was: " + contentType); 159 // not an image - do not store response in cache, so next time it will be queried again from the server 160 return true; 161 } 162 } 163 return false; 164 } 165 166 @Override 167 protected boolean cacheAsEmpty(Map<String, List<String>> headerFields, int responseCode) { 168 if (isNotImage(headerFields, responseCode)) { 169 return false; 170 } 171 return isNoTileAtZoom() || super.cacheAsEmpty(headerFields, responseCode); 172 } 173 174 @Override 175 public void submit(boolean force) { 176 tile.initLoading(); 177 try { 178 super.submit(this, force); 179 } catch (IOException | IllegalArgumentException e) { 180 // if we fail to submit the job, mark tile as loaded and set error message 181 Logging.log(Logging.LEVEL_WARN, e); 182 tile.finishLoading(); 183 tile.setError(e.getMessage()); 184 } 185 } 186 187 @Override 188 public void loadingFinished(CacheEntry object, CacheEntryAttributes attributes, LoadResult result) { 189 this.attributes = attributes; // as we might get notification from other object than our selfs, pass attributes along 190 Set<TileLoaderListener> listeners = inProgress.remove(getCacheKey()); 191 boolean status = result == LoadResult.SUCCESS; 192 193 try { 194 tile.finishLoading(); // whatever happened set that loading has finished 195 // set tile metadata 196 if (this.attributes != null) { 197 for (Entry<String, String> e: this.attributes.getMetadata().entrySet()) { 198 tile.putValue(e.getKey(), e.getValue()); 199 } 200 } 201 202 switch (result) { 203 case SUCCESS: 204 handleNoTileAtZoom(); 205 if (attributes != null) { 206 int httpStatusCode = attributes.getResponseCode(); 207 if (httpStatusCode >= 400 && !isNoTileAtZoom()) { 208 status = false; 209 handleError(attributes); 210 } 211 } 212 status &= tryLoadTileImage(object); //try to keep returned image as background 213 break; 214 case FAILURE: 215 handleError(attributes); 216 tryLoadTileImage(object); 217 break; 218 case CANCELED: 219 tile.loadingCanceled(); 220 break; 221 default: // This should be removed when we move to Java 17+ 222 // do nothing 223 } 224 225 // always check, if there is some listener interested in fact, that tile has finished loading 226 if (listeners != null) { // listeners might be null, if some other thread notified already about success 227 for (TileLoaderListener l: listeners) { 228 l.tileLoadingFinished(tile, status); 229 } 230 } 231 } catch (IOException e) { 232 Logging.warn("JCS TMS - error loading object for tile {0}: {1}", tile.getKey(), e.getMessage()); 233 tile.setError(e); 234 tile.setLoaded(false); 235 if (listeners != null) { // listeners might be null, if some other thread notified already about success 236 for (TileLoaderListener l: listeners) { 237 l.tileLoadingFinished(tile, false); 238 } 239 } 240 } 241 } 242 243 private void handleError(CacheEntryAttributes attributes) { 244 if (tile.hasError() && tile.getErrorMessage() != null) { 245 // tile has already set error message, don't overwrite it 246 return; 247 } 248 if (attributes != null) { 249 int httpStatusCode = attributes.getResponseCode(); 250 if (attributes.getErrorMessage() == null) { 251 tile.setError(tr("HTTP error {0} when loading tiles", httpStatusCode)); 252 } else { 253 tile.setError(tr("Error downloading tiles: {0}", attributes.getErrorMessage())); 254 } 255 if (httpStatusCode >= 500 && httpStatusCode != 599) { 256 // httpStatusCode = 599 is set by JCSCachedTileLoaderJob on IOException 257 tile.setLoaded(false); // treat 500 errors as temporary and try to load it again 258 } 259 // treat SocketTimeoutException as transient error 260 attributes.getException() 261 .filter(x -> x.isAssignableFrom(SocketTimeoutException.class)) 262 .ifPresent(x -> tile.setLoaded(false)); 263 } else { 264 tile.setError(tr("Problem loading tile")); 265 // treat unknown errors as permanent and do not try to load tile again 266 } 267 } 268 269 /** 270 * For TMS use BaseURL as settings discovery, so for different paths, we will have different settings (useful for developer servers) 271 * 272 * @return base URL of TMS or server url as defined in super class 273 */ 274 @Override 275 protected String getServerKey() { 276 TileSource ts = tile.getSource(); 277 if (ts instanceof AbstractTMSTileSource) { 278 return ((AbstractTMSTileSource) ts).getBaseUrl(); 279 } 280 return super.getServerKey(); 281 } 282 283 @Override 284 protected BufferedImageCacheEntry createCacheEntry(byte[] content) { 285 return new BufferedImageCacheEntry(content); 286 } 287 288 @Override 289 public void submit() { 290 submit(false); 291 } 292 293 @Override 294 protected CacheEntryAttributes parseHeaders(HttpClient.Response urlConn) { 295 CacheEntryAttributes ret = super.parseHeaders(urlConn); 296 // keep the expiration time between MINIMUM_EXPIRES and MAXIMUM_EXPIRES, so we will cache the tiles 297 // at least for some short period of time, but not too long 298 long minimumExpiryTime = TimeUnit.SECONDS.toMillis(options.getMinimumExpiryTime()); 299 long nowPlusMin = now + Math.max(MINIMUM_EXPIRES.get(), minimumExpiryTime); 300 if (ret.getExpirationTime() < nowPlusMin) { 301 ret.setExpirationTime(nowPlusMin); 302 } 303 long nowPlusMax = now + Math.max(MAXIMUM_EXPIRES.get(), minimumExpiryTime); 304 if (ret.getExpirationTime() > nowPlusMax) { 305 ret.setExpirationTime(nowPlusMax); 306 } 307 return ret; 308 } 309 310 private boolean handleNoTileAtZoom() { 311 if (isNoTileAtZoom()) { 312 Logging.debug("JCS TMS - Tile valid, but no file, as no tiles at this level {0}", tile); 313 tile.setError(tr("No tiles at this zoom level")); 314 tile.putValue("tile-info", "no-tile"); 315 return true; 316 } 317 return false; 318 } 319 320 private boolean isNoTileAtZoom() { 321 if (attributes == null) { 322 Logging.warn("Cache attributes are null"); 323 } 324 return attributes != null && attributes.isNoTileAtZoom(); 325 } 326 327 private boolean tryLoadTileImage(CacheEntry object) throws IOException { 328 if (object != null) { 329 byte[] content = object.getContent(); 330 if (content.length > 0 || tile instanceof VectorTile) { 331 try (ByteArrayInputStream in = new ByteArrayInputStream(content)) { 332 tile.loadImage(in); 333 if ((!(tile instanceof VectorTile) && tile.getImage() == null) 334 || ((tile instanceof VectorTile) && !tile.isLoaded())) { 335 String s = new String(content, StandardCharsets.UTF_8); 336 Matcher m = SERVICE_EXCEPTION_PATTERN.matcher(s); 337 if (m.matches()) { 338 String message = Utils.strip(m.group(1)); 339 tile.setError(message); 340 Logging.error(message); 341 Logging.debug(s); 342 } else { 343 tile.setError(tr("Could not load image from tile server")); 344 } 345 return false; 346 } 347 } catch (UnsatisfiedLinkError | SecurityException e) { 348 throw new IOException(e); 349 } 350 } 351 } 352 return true; 353 } 354 355 @Override 356 public String detectErrorMessage(String data) { 357 Matcher xml = SERVICE_EXCEPTION_PATTERN.matcher(data); 358 Matcher json = JSON_PATTERN.matcher(data); 359 return xml.matches() ? removeCdata(Utils.strip(xml.group(1))) 360 : json.matches() ? Utils.strip(json.group(1)) 361 : super.detectErrorMessage(data); 362 } 363 364 private static String removeCdata(String msg) { 365 Matcher m = CDATA_PATTERN.matcher(msg); 366 return m.matches() ? Utils.strip(m.group(1)) : msg; 367 } 368}