001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.data.imagery;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.io.ByteArrayInputStream;
007import java.io.IOException;
008import java.net.SocketTimeoutException;
009import java.net.URL;
010import java.nio.charset.StandardCharsets;
011import java.util.HashSet;
012import java.util.List;
013import java.util.Locale;
014import java.util.Map;
015import java.util.Map.Entry;
016import java.util.Optional;
017import java.util.Set;
018import java.util.concurrent.ConcurrentHashMap;
019import java.util.concurrent.ConcurrentMap;
020import java.util.concurrent.ThreadPoolExecutor;
021import java.util.concurrent.TimeUnit;
022import java.util.regex.Matcher;
023import java.util.regex.Pattern;
024
025import org.apache.commons.jcs3.access.behavior.ICacheAccess;
026import org.apache.commons.jcs3.engine.behavior.ICache;
027import org.openstreetmap.gui.jmapviewer.Tile;
028import org.openstreetmap.gui.jmapviewer.interfaces.TileJob;
029import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener;
030import org.openstreetmap.gui.jmapviewer.interfaces.TileSource;
031import org.openstreetmap.gui.jmapviewer.tilesources.AbstractTMSTileSource;
032import org.openstreetmap.josm.data.cache.BufferedImageCacheEntry;
033import org.openstreetmap.josm.data.cache.CacheEntry;
034import org.openstreetmap.josm.data.cache.CacheEntryAttributes;
035import org.openstreetmap.josm.data.cache.ICachedLoaderListener;
036import org.openstreetmap.josm.data.cache.JCSCachedTileLoaderJob;
037import org.openstreetmap.josm.data.imagery.vectortile.VectorTile;
038import org.openstreetmap.josm.data.imagery.vectortile.mapbox.MVTFile;
039import org.openstreetmap.josm.data.preferences.LongProperty;
040import org.openstreetmap.josm.tools.HttpClient;
041import org.openstreetmap.josm.tools.Logging;
042import org.openstreetmap.josm.tools.Utils;
043
044/**
045 * Class bridging TMS requests to JCS cache requests
046 *
047 * @author Wiktor Niesiobędzki
048 * @since 8168
049 */
050public class TMSCachedTileLoaderJob extends JCSCachedTileLoaderJob<String, BufferedImageCacheEntry> implements TileJob, ICachedLoaderListener {
051    /** General maximum expires for tiles. Might be overridden by imagery settings */
052    public static final LongProperty MAXIMUM_EXPIRES = new LongProperty("imagery.generic.maximum_expires", TimeUnit.DAYS.toMillis(30));
053    /** General minimum expires for tiles. Might be overridden by imagery settings */
054    public static final LongProperty MINIMUM_EXPIRES = new LongProperty("imagery.generic.minimum_expires", TimeUnit.HOURS.toMillis(1));
055    static final Pattern SERVICE_EXCEPTION_PATTERN = Pattern.compile("(?s).+<ServiceException[^>]*>(.+)</ServiceException>.+");
056    static final Pattern CDATA_PATTERN = Pattern.compile("(?s)\\s*<!\\[CDATA\\[(.+)\\]\\]>\\s*");
057    static final Pattern JSON_PATTERN = Pattern.compile("\\{\"message\":\"(.+)\"\\}");
058    protected final Tile tile;
059    private volatile URL url;
060    private final TileJobOptions options;
061
062    // we need another deduplication of Tile Loader listeners, as for each submit, new TMSCachedTileLoaderJob was created
063    // that way, we reduce calls to tileLoadingFinished, and general CPU load due to surplus Map repaints
064    private static final ConcurrentMap<String, Set<TileLoaderListener>> inProgress = new ConcurrentHashMap<>();
065
066    /**
067     * Constructor for creating a job, to get a specific tile from cache
068     * @param listener Tile loader listener
069     * @param tile to be fetched from cache
070     * @param cache object
071     * @param options for job (such as http headers, timeouts etc.)
072     * @param downloadExecutor that will be executing the jobs
073     */
074    public TMSCachedTileLoaderJob(TileLoaderListener listener, Tile tile,
075            ICacheAccess<String, BufferedImageCacheEntry> cache,
076            TileJobOptions options,
077            ThreadPoolExecutor downloadExecutor) {
078        super(cache, options, downloadExecutor);
079        this.tile = tile;
080        this.options = options;
081        if (listener != null) {
082            inProgress.computeIfAbsent(getCacheKey(), k -> new HashSet<>()).add(listener);
083        }
084    }
085
086    @Override
087    public String getCacheKey() {
088        if (tile != null) {
089            TileSource tileSource = tile.getTileSource();
090            return Optional.ofNullable(tileSource.getName()).orElse("").replace(ICache.NAME_COMPONENT_DELIMITER, "_")
091                    + ICache.NAME_COMPONENT_DELIMITER
092                    + tileSource.getTileId(tile.getZoom(), tile.getXtile(), tile.getYtile());
093        }
094        return null;
095    }
096
097    /*
098     *  this doesn't needs to be synchronized, as it's not that costly to keep only one execution
099     *  in parallel, but URL creation and Tile.getUrl() are costly and are not needed when fetching
100     *  data from cache, that's why URL creation is postponed until it's needed
101     *
102     *  We need to have static url value for TileLoaderJob, as for some TileSources we might get different
103     *  URL's each call we made (servers switching), and URL's are used below as a key for duplicate detection
104     *
105     */
106    @Override
107    public URL getUrl() throws IOException {
108        if (url == null) {
109            synchronized (this) {
110                if (url == null) {
111                    String sUrl = tile.getUrl();
112                    if (!"".equals(sUrl)) {
113                        url = new URL(sUrl);
114                    }
115                }
116            }
117        }
118        return url;
119    }
120
121    @Override
122    public boolean isObjectLoadable() {
123        if (cacheData != null) {
124            byte[] content = cacheData.getContent();
125            try {
126                return content.length > 0 || cacheData.getImage() != null || isNoTileAtZoom();
127            } catch (IOException e) {
128                Logging.logWithStackTrace(Logging.LEVEL_WARN, e, "JCS TMS - error loading from cache for tile {0}: {1}",
129                        tile.getKey(), e.getMessage());
130            }
131        }
132        return false;
133    }
134
135    @Override
136    protected boolean isResponseLoadable(Map<String, List<String>> headers, int statusCode, byte[] content) {
137        attributes.setMetadata(tile.getTileSource().getMetadata(headers));
138        if (tile.getTileSource().isNoTileAtZoom(headers, statusCode, content)) {
139            attributes.setNoTileAtZoom(true);
140            return false; // do no try to load data from no-tile at zoom, cache empty object instead
141        }
142        if (isNotImage(headers, statusCode)) {
143            String message = detectErrorMessage(new String(content, StandardCharsets.UTF_8));
144            if (!Utils.isEmpty(message)) {
145                tile.setError(message);
146            }
147            return false;
148        }
149        return super.isResponseLoadable(headers, statusCode, content);
150    }
151
152    private boolean isNotImage(Map<String, List<String>> headers, int statusCode) {
153        if (statusCode == 200 && headers.containsKey("Content-Type") && !headers.get("Content-Type").isEmpty()) {
154            String contentType = headers.get("Content-Type").stream().findAny().orElse("").toLowerCase(Locale.ROOT);
155            if (!contentType.startsWith("image")
156                && !contentType.startsWith("application/octet-stream")
157                && !MVTFile.MIMETYPE.contains(contentType)) {
158                Logging.warn("Image not returned for tile: " + url + " content type was: " + contentType);
159                // not an image - do not store response in cache, so next time it will be queried again from the server
160                return true;
161            }
162        }
163        return false;
164    }
165
166    @Override
167    protected boolean cacheAsEmpty(Map<String, List<String>> headerFields, int responseCode) {
168        if (isNotImage(headerFields, responseCode)) {
169            return false;
170        }
171        return isNoTileAtZoom() || super.cacheAsEmpty(headerFields, responseCode);
172    }
173
174    @Override
175    public void submit(boolean force) {
176        tile.initLoading();
177        try {
178            super.submit(this, force);
179        } catch (IOException | IllegalArgumentException e) {
180            // if we fail to submit the job, mark tile as loaded and set error message
181            Logging.log(Logging.LEVEL_WARN, e);
182            tile.finishLoading();
183            tile.setError(e.getMessage());
184        }
185    }
186
187    @Override
188    public void loadingFinished(CacheEntry object, CacheEntryAttributes attributes, LoadResult result) {
189        this.attributes = attributes; // as we might get notification from other object than our selfs, pass attributes along
190        Set<TileLoaderListener> listeners = inProgress.remove(getCacheKey());
191        boolean status = result == LoadResult.SUCCESS;
192
193        try {
194            tile.finishLoading(); // whatever happened set that loading has finished
195            // set tile metadata
196            if (this.attributes != null) {
197                for (Entry<String, String> e: this.attributes.getMetadata().entrySet()) {
198                    tile.putValue(e.getKey(), e.getValue());
199                }
200            }
201
202            switch (result) {
203            case SUCCESS:
204                handleNoTileAtZoom();
205                if (attributes != null) {
206                    int httpStatusCode = attributes.getResponseCode();
207                    if (httpStatusCode >= 400 && !isNoTileAtZoom()) {
208                        status = false;
209                        handleError(attributes);
210                    }
211                }
212                status &= tryLoadTileImage(object); //try to keep returned image as background
213                break;
214            case FAILURE:
215                handleError(attributes);
216                tryLoadTileImage(object);
217                break;
218            case CANCELED:
219                tile.loadingCanceled();
220                break;
221            default: // This should be removed when we move to Java 17+
222                // do nothing
223            }
224
225            // always check, if there is some listener interested in fact, that tile has finished loading
226            if (listeners != null) { // listeners might be null, if some other thread notified already about success
227                for (TileLoaderListener l: listeners) {
228                    l.tileLoadingFinished(tile, status);
229                }
230            }
231        } catch (IOException e) {
232            Logging.warn("JCS TMS - error loading object for tile {0}: {1}", tile.getKey(), e.getMessage());
233            tile.setError(e);
234            tile.setLoaded(false);
235            if (listeners != null) { // listeners might be null, if some other thread notified already about success
236                for (TileLoaderListener l: listeners) {
237                    l.tileLoadingFinished(tile, false);
238                }
239            }
240        }
241    }
242
243    private void handleError(CacheEntryAttributes attributes) {
244        if (tile.hasError() && tile.getErrorMessage() != null) {
245            // tile has already set error message, don't overwrite it
246            return;
247        }
248        if (attributes != null) {
249            int httpStatusCode = attributes.getResponseCode();
250            if (attributes.getErrorMessage() == null) {
251                tile.setError(tr("HTTP error {0} when loading tiles", httpStatusCode));
252            } else {
253                tile.setError(tr("Error downloading tiles: {0}", attributes.getErrorMessage()));
254            }
255            if (httpStatusCode >= 500 && httpStatusCode != 599) {
256                // httpStatusCode = 599 is set by JCSCachedTileLoaderJob on IOException
257                tile.setLoaded(false); // treat 500 errors as temporary and try to load it again
258            }
259            // treat SocketTimeoutException as transient error
260            attributes.getException()
261            .filter(x -> x.isAssignableFrom(SocketTimeoutException.class))
262            .ifPresent(x -> tile.setLoaded(false));
263        } else {
264            tile.setError(tr("Problem loading tile"));
265            // treat unknown errors as permanent and do not try to load tile again
266        }
267    }
268
269    /**
270     * For TMS use BaseURL as settings discovery, so for different paths, we will have different settings (useful for developer servers)
271     *
272     * @return base URL of TMS or server url as defined in super class
273     */
274    @Override
275    protected String getServerKey() {
276        TileSource ts = tile.getSource();
277        if (ts instanceof AbstractTMSTileSource) {
278            return ((AbstractTMSTileSource) ts).getBaseUrl();
279        }
280        return super.getServerKey();
281    }
282
283    @Override
284    protected BufferedImageCacheEntry createCacheEntry(byte[] content) {
285        return new BufferedImageCacheEntry(content);
286    }
287
288    @Override
289    public void submit() {
290        submit(false);
291    }
292
293    @Override
294    protected CacheEntryAttributes parseHeaders(HttpClient.Response urlConn) {
295        CacheEntryAttributes ret = super.parseHeaders(urlConn);
296        // keep the expiration time between MINIMUM_EXPIRES and MAXIMUM_EXPIRES, so we will cache the tiles
297        // at least for some short period of time, but not too long
298        long minimumExpiryTime = TimeUnit.SECONDS.toMillis(options.getMinimumExpiryTime());
299        long nowPlusMin = now + Math.max(MINIMUM_EXPIRES.get(), minimumExpiryTime);
300        if (ret.getExpirationTime() < nowPlusMin) {
301            ret.setExpirationTime(nowPlusMin);
302        }
303        long nowPlusMax = now + Math.max(MAXIMUM_EXPIRES.get(), minimumExpiryTime);
304        if (ret.getExpirationTime() > nowPlusMax) {
305            ret.setExpirationTime(nowPlusMax);
306        }
307        return ret;
308    }
309
310    private boolean handleNoTileAtZoom() {
311        if (isNoTileAtZoom()) {
312            Logging.debug("JCS TMS - Tile valid, but no file, as no tiles at this level {0}", tile);
313            tile.setError(tr("No tiles at this zoom level"));
314            tile.putValue("tile-info", "no-tile");
315            return true;
316        }
317        return false;
318    }
319
320    private boolean isNoTileAtZoom() {
321        if (attributes == null) {
322            Logging.warn("Cache attributes are null");
323        }
324        return attributes != null && attributes.isNoTileAtZoom();
325    }
326
327    private boolean tryLoadTileImage(CacheEntry object) throws IOException {
328        if (object != null) {
329            byte[] content = object.getContent();
330            if (content.length > 0 || tile instanceof VectorTile) {
331                try (ByteArrayInputStream in = new ByteArrayInputStream(content)) {
332                    tile.loadImage(in);
333                    if ((!(tile instanceof VectorTile) && tile.getImage() == null)
334                        || ((tile instanceof VectorTile) && !tile.isLoaded())) {
335                        String s = new String(content, StandardCharsets.UTF_8);
336                        Matcher m = SERVICE_EXCEPTION_PATTERN.matcher(s);
337                        if (m.matches()) {
338                            String message = Utils.strip(m.group(1));
339                            tile.setError(message);
340                            Logging.error(message);
341                            Logging.debug(s);
342                        } else {
343                            tile.setError(tr("Could not load image from tile server"));
344                        }
345                        return false;
346                    }
347                } catch (UnsatisfiedLinkError | SecurityException e) {
348                    throw new IOException(e);
349                }
350            }
351        }
352        return true;
353    }
354
355    @Override
356    public String detectErrorMessage(String data) {
357        Matcher xml = SERVICE_EXCEPTION_PATTERN.matcher(data);
358        Matcher json = JSON_PATTERN.matcher(data);
359        return xml.matches() ? removeCdata(Utils.strip(xml.group(1)))
360            : json.matches() ? Utils.strip(json.group(1))
361            : super.detectErrorMessage(data);
362    }
363
364    private static String removeCdata(String msg) {
365        Matcher m = CDATA_PATTERN.matcher(msg);
366        return m.matches() ? Utils.strip(m.group(1)) : msg;
367    }
368}