source: osm/applications/viewer/jmapviewer/src/org/openstreetmap/gui/jmapviewer/OsmFileCacheTileLoader.java@ 30849

Last change on this file since 30849 was 30849, checked in by bastik, 10 years ago

see #josm10849 - improve TMS imagery caching
File size: 19.5 KB
Line 
1// License: GPL. For details, see Readme.txt file.
2package org.openstreetmap.gui.jmapviewer;
3
4import java.io.BufferedReader;
5import java.io.ByteArrayInputStream;
6import java.io.ByteArrayOutputStream;
7import java.io.File;
8import java.io.FileInputStream;
9import java.io.FileNotFoundException;
10import java.io.FileOutputStream;
11import java.io.IOException;
12import java.io.InputStream;
13import java.io.InputStreamReader;
14import java.io.OutputStreamWriter;
15import java.io.PrintWriter;
16import java.net.HttpURLConnection;
17import java.net.URL;
18import java.net.URLConnection;
19import java.nio.charset.Charset;
20import java.util.HashMap;
21import java.util.Map;
22import java.util.Map.Entry;
23import java.util.Random;
24import java.util.logging.Level;
25import java.util.logging.Logger;
26
27import org.openstreetmap.gui.jmapviewer.interfaces.CachedTileLoader;
28import org.openstreetmap.gui.jmapviewer.interfaces.TileClearController;
29import org.openstreetmap.gui.jmapviewer.interfaces.TileJob;
30import org.openstreetmap.gui.jmapviewer.interfaces.TileLoader;
31import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener;
32import org.openstreetmap.gui.jmapviewer.interfaces.TileSource;
33import org.openstreetmap.gui.jmapviewer.interfaces.TileSource.TileUpdate;
34
35/**
36 * A {@link TileLoader} implementation that loads tiles from OSM via HTTP and
37 * saves all loaded files in a directory located in the temporary directory.
38 * If a tile is present in this file cache it will not be loaded from OSM again.
39 *
40 * @author Jan Peter Stotz
41 * @author Stefan Zeller
42 */
43public class OsmFileCacheTileLoader extends OsmTileLoader implements CachedTileLoader {
44
45 private static final Logger log = FeatureAdapter.getLogger(OsmFileCacheTileLoader.class.getName());
46
47 private static final String TAGS_FILE_EXT = ".tags";
48
49 private static final Charset TAGS_CHARSET = Charset.forName("UTF-8");
50
51 public static final long FILE_AGE_ONE_DAY = 1000 * 60 * 60 * 24;
52 public static final long FILE_AGE_ONE_WEEK = FILE_AGE_ONE_DAY * 7;
53
54 protected String cacheDirBase;
55
56 protected final Map<TileSource, File> sourceCacheDirMap;
57
58 protected long maxCacheFileAge = Long.MAX_VALUE; // max. age not limited
59 protected long recheckAfter = FILE_AGE_ONE_WEEK;
60
61 public static File getDefaultCacheDir() throws SecurityException {
62 String tempDir = null;
63 String userName = System.getProperty("user.name");
64 try {
65 tempDir = System.getProperty("java.io.tmpdir");
66 } catch (SecurityException e) {
67 log.log(Level.WARNING,
68 "Failed to access system property ''java.io.tmpdir'' for security reasons. Exception was: "
69 + e.toString());
70 throw e; // rethrow
71 }
72 try {
73 if (tempDir == null)
74 throw new IOException("No temp directory set");
75 String subDirName = "JMapViewerTiles";
76 // On Linux/Unix systems we do not have a per user tmp directory.
77 // Therefore we add the user name for getting a unique dir name.
78 if (userName != null && userName.length() > 0) {
79 subDirName += "_" + userName;
80 }
81 File cacheDir = new File(tempDir, subDirName);
82 return cacheDir;
83 } catch (Exception e) {
84 }
85 return null;
86 }
87
88 /**
89 * Create a OSMFileCacheTileLoader with given cache directory.
90 * If cacheDir is not set or invalid, IOException will be thrown.
91 * @param map the listener checking for tile load events (usually the map for display)
92 * @param cacheDir directory to store cached tiles
93 */
94 public OsmFileCacheTileLoader(TileLoaderListener map, File cacheDir) throws IOException {
95 super(map);
96 if (cacheDir == null || (!cacheDir.exists() && !cacheDir.mkdirs()))
97 throw new IOException("Cannot access cache directory");
98
99 log.finest("Tile cache directory: " + cacheDir);
100 cacheDirBase = cacheDir.getAbsolutePath();
101 sourceCacheDirMap = new HashMap<>();
102 }
103
104 /**
105 * Create a OSMFileCacheTileLoader with system property temp dir.
106 * If not set an IOException will be thrown.
107 * @param map the listener checking for tile load events (usually the map for display)
108 */
109 public OsmFileCacheTileLoader(TileLoaderListener map) throws SecurityException, IOException {
110 this(map, getDefaultCacheDir());
111 }
112
113 @Override
114 public TileJob createTileLoaderJob(final Tile tile) {
115 return new FileLoadJob(tile);
116 }
117
118 protected File getSourceCacheDir(TileSource source) {
119 File dir = sourceCacheDirMap.get(source);
120 if (dir == null) {
121 dir = new File(cacheDirBase, source.getName().replaceAll("[\\\\/:*?\"<>|]", "_"));
122 if (!dir.exists()) {
123 dir.mkdirs();
124 }
125 }
126 return dir;
127 }
128
129 protected class FileLoadJob implements TileJob {
130 InputStream input = null;
131
132 Tile tile;
133 File tileCacheDir;
134 File tileFile = null;
135 Long fileAge = null;
136
137 public FileLoadJob(Tile tile) {
138 this.tile = tile;
139 }
140
141 @Override
142 public Tile getTile() {
143 return tile;
144 }
145
146 @Override
147 public void run() {
148 synchronized (tile) {
149 if ((tile.isLoaded() && !tile.hasError()) || tile.isLoading())
150 return;
151 tile.loaded = false;
152 tile.error = false;
153 tile.loading = true;
154 }
155 tileCacheDir = getSourceCacheDir(tile.getSource());
156
157 if (loadTileFromFile(recheckAfter)) {
158 log.log(Level.FINEST, "TMS - found in tile cache: {0}", tile);
159 tile.setLoaded(true);
160 listener.tileLoadingFinished(tile, true);
161 return;
162 }
163 TileJob job = new TileJob() {
164
165 @Override
166 public void run() {
167 if (loadOrUpdateTile()) {
168 tile.setLoaded(true);
169 listener.tileLoadingFinished(tile, true);
170 } else {
171 // failed to download - use old cache file if available
172 if (loadTileFromFile(maxCacheFileAge)) {
173 tile.setLoaded(true);
174 tile.error = false;
175 listener.tileLoadingFinished(tile, true);
176 log.log(Level.FINEST, "TMS - found stale tile in cache: {0}", tile);
177 } else {
178 // failed completely
179 tile.setLoaded(true);
180 listener.tileLoadingFinished(tile, false);
181 }
182 }
183 }
184 @Override
185 public Tile getTile() {
186 return tile;
187 }
188 };
189 JobDispatcher.getInstance().addJob(job);
190 }
191
192 protected boolean loadOrUpdateTile() {
193 try {
194 URLConnection urlConn = loadTileFromOsm(tile);
195 if (fileAge != null) {
196 switch (tile.getSource().getTileUpdate()) {
197 case IfModifiedSince:
198 urlConn.setIfModifiedSince(fileAge);
199 break;
200 case LastModified:
201 if (!isOsmTileNewer(fileAge)) {
202 log.log(Level.FINEST, "TMS - LastModified test: local version is up to date: {0}", tile);
203 tileFile.setLastModified(System.currentTimeMillis());
204 return true;
205 }
206 break;
207 }
208 }
209 if (tile.getSource().getTileUpdate() == TileUpdate.ETag || tile.getSource().getTileUpdate() == TileUpdate.IfNoneMatch) {
210 String fileETag = tile.getValue("etag");
211 if (fileETag != null) {
212 switch (tile.getSource().getTileUpdate()) {
213 case IfNoneMatch:
214 urlConn.addRequestProperty("If-None-Match", fileETag);
215 break;
216 case ETag:
217 if (hasOsmTileETag(fileETag)) {
218 log.log(Level.FINEST, "TMS - ETag test: local version is up to date: {0}", tile);
219 tileFile.setLastModified(System.currentTimeMillis());
220 return true;
221 }
222 }
223 }
224 tile.putValue("etag", urlConn.getHeaderField("ETag"));
225 }
226 if (urlConn instanceof HttpURLConnection && ((HttpURLConnection)urlConn).getResponseCode() == 304) {
227 // If we are isModifiedSince or If-None-Match has been set
228 // and the server answers with a HTTP 304 = "Not Modified"
229 switch (tile.getSource().getTileUpdate()) {
230 case IfModifiedSince:
231 log.log(Level.FINEST, "TMS - IfModifiedSince test: local version is up to date: {0}", tile);
232 break;
233 case IfNoneMatch:
234 log.log(Level.FINEST, "TMS - IfNoneMatch test: local version is up to date: {0}", tile);
235 break;
236 }
237 if (loadTileFromFile(maxCacheFileAge)) {
238 tileFile.setLastModified(System.currentTimeMillis());
239 return true;
240 }
241 }
242
243 loadTileMetadata(tile, urlConn);
244 saveTagsToFile();
245
246 if ("no-tile".equals(tile.getValue("tile-info")))
247 {
248 log.log(Level.FINEST, "TMS - No tile: tile-info=no-tile: {0}", tile);
249 tile.setError("No tile at this zoom level");
250 return true;
251 } else {
252 for (int i = 0; i < 5; ++i) {
253 if (urlConn instanceof HttpURLConnection && ((HttpURLConnection)urlConn).getResponseCode() == 503) {
254 Thread.sleep(5000+(new Random()).nextInt(5000));
255 continue;
256 }
257 byte[] buffer = loadTileInBuffer(urlConn);
258 if (buffer != null) {
259 tile.loadImage(new ByteArrayInputStream(buffer));
260 saveTileToFile(buffer);
261 log.log(Level.FINEST, "TMS - downloaded tile from server: {0}", tile.getUrl());
262 return true;
263 }
264 }
265 }
266 } catch (Exception e) {
267 tile.setError(e.getMessage());
268 if (input == null) {
269 try {
270 log.log(Level.WARNING, "TMS - Failed downloading {0}: {1}", new Object[]{tile.getUrl(), e.getMessage()});
271 return false;
272 } catch(IOException i) {
273 }
274 }
275 }
276 log.log(Level.WARNING, "TMS - Failed downloading tile: {0}", tile);
277 return false;
278 }
279
280 protected boolean loadTileFromFile(long maxAge) {
281 try {
282 tileFile = getTileFile();
283 if (!tileFile.exists())
284 return false;
285 loadTagsFromFile();
286
287 fileAge = tileFile.lastModified();
288 if (System.currentTimeMillis() - fileAge > maxAge)
289 return false;
290
291 if ("no-tile".equals(tile.getValue("tile-info"))) {
292 tile.setError("No tile at this zoom level");
293 if (tileFile.exists()) {
294 tileFile.delete();
295 }
296 tileFile = null;
297 } else {
298 try (FileInputStream fin = new FileInputStream(tileFile)) {
299 if (fin.available() == 0)
300 throw new IOException("File empty");
301 tile.loadImage(fin);
302 }
303 }
304 return true;
305
306 } catch (Exception e) {
307 log.log(Level.WARNING, "TMS - Error while loading image from tile cache: {0}; {1}", new Object[]{e.getMessage(), tile});
308 tileFile.delete();
309 tileFile = null;
310 fileAge = null;
311 }
312 return false;
313 }
314
315 protected byte[] loadTileInBuffer(URLConnection urlConn) throws IOException {
316 input = urlConn.getInputStream();
317 try {
318 ByteArrayOutputStream bout = new ByteArrayOutputStream(input.available());
319 byte[] buffer = new byte[2048];
320 boolean finished = false;
321 do {
322 int read = input.read(buffer);
323 if (read >= 0) {
324 bout.write(buffer, 0, read);
325 } else {
326 finished = true;
327 }
328 } while (!finished);
329 if (bout.size() == 0)
330 return null;
331 return bout.toByteArray();
332 } finally {
333 input.close();
334 input = null;
335 }
336 }
337
338 /**
339 * Performs a <code>HEAD</code> request for retrieving the
340 * <code>LastModified</code> header value.
341 *
342 * Note: This does only work with servers providing the
343 * <code>LastModified</code> header:
344 * <ul>
345 * <li>{@link org.openstreetmap.gui.jmapviewer.tilesources.OsmTileSource.CycleMap} - supported</li>
346 * <li>{@link org.openstreetmap.gui.jmapviewer.tilesources.OsmTileSource.Mapnik} - not supported</li>
347 * </ul>
348 *
349 * @param fileAge time of the
350 * @return <code>true</code> if the tile on the server is newer than the
351 * file
352 * @throws IOException
353 */
354 protected boolean isOsmTileNewer(long fileAge) throws IOException {
355 URL url;
356 url = new URL(tile.getUrl());
357 HttpURLConnection urlConn = (HttpURLConnection) url.openConnection();
358 prepareHttpUrlConnection(urlConn);
359 urlConn.setRequestMethod("HEAD");
360 urlConn.setReadTimeout(30000); // 30 seconds read timeout
361 // System.out.println("Tile age: " + new
362 // Date(urlConn.getLastModified()) + " / "
363 // + new Date(fileAge));
364 long lastModified = urlConn.getLastModified();
365 if (lastModified == 0)
366 return true; // no LastModified time returned
367 return (lastModified > fileAge);
368 }
369
370 protected boolean hasOsmTileETag(String eTag) throws IOException {
371 URL url;
372 url = new URL(tile.getUrl());
373 HttpURLConnection urlConn = (HttpURLConnection) url.openConnection();
374 prepareHttpUrlConnection(urlConn);
375 urlConn.setRequestMethod("HEAD");
376 urlConn.setReadTimeout(30000); // 30 seconds read timeout
377 // System.out.println("Tile age: " + new
378 // Date(urlConn.getLastModified()) + " / "
379 // + new Date(fileAge));
380 String osmETag = urlConn.getHeaderField("ETag");
381 if (osmETag == null)
382 return true;
383 return (osmETag.equals(eTag));
384 }
385
386 protected File getTileFile() {
387 return new File(tileCacheDir + "/" + tile.getZoom() + "_" + tile.getXtile() + "_" + tile.getYtile() + "."
388 + tile.getSource().getTileType());
389 }
390
391 protected File getTagsFile() {
392 return new File(tileCacheDir + "/" + tile.getZoom() + "_" + tile.getXtile() + "_" + tile.getYtile()
393 + TAGS_FILE_EXT);
394 }
395
396 protected void saveTileToFile(byte[] rawData) {
397 try (
398 FileOutputStream f = new FileOutputStream(tileCacheDir + "/" + tile.getZoom() + "_" + tile.getXtile()
399 + "_" + tile.getYtile() + "." + tile.getSource().getTileType())
400 ) {
401 f.write(rawData);
402 } catch (Exception e) {
403 System.err.println("Failed to save tile content: " + e.getLocalizedMessage());
404 }
405 }
406
407 protected void saveTagsToFile() {
408 File tagsFile = getTagsFile();
409 if (tile.getMetadata() == null) {
410 tagsFile.delete();
411 return;
412 }
413 try (PrintWriter f = new PrintWriter(new OutputStreamWriter(new FileOutputStream(tagsFile), TAGS_CHARSET))) {
414 for (Entry<String, String> entry : tile.getMetadata().entrySet()) {
415 f.println(entry.getKey() + "=" + entry.getValue());
416 }
417 } catch (Exception e) {
418 System.err.println("Failed to save tile tags: " + e.getLocalizedMessage());
419 }
420 }
421
422 protected void loadTagsFromFile() {
423 File tagsFile = getTagsFile();
424 try (BufferedReader f = new BufferedReader(new InputStreamReader(new FileInputStream(tagsFile), TAGS_CHARSET))) {
425 for (String line = f.readLine(); line != null; line = f.readLine()) {
426 final int i = line.indexOf('=');
427 if (i == -1 || i == 0) {
428 System.err.println("Malformed tile tag in file '" + tagsFile.getName() + "':" + line);
429 continue;
430 }
431 tile.putValue(line.substring(0,i),line.substring(i+1));
432 }
433 } catch (FileNotFoundException e) {
434 } catch (Exception e) {
435 System.err.println("Failed to load tile tags: " + e.getLocalizedMessage());
436 }
437 }
438 }
439
440 public long getMaxFileAge() {
441 return maxCacheFileAge;
442 }
443
444 /**
445 * Sets the maximum age of the local cached tile in the file system. If a
446 * local tile is older than the specified file age
447 * {@link OsmFileCacheTileLoader} will connect to the tile server and check
448 * if a newer tile is available using the mechanism specified for the
449 * selected tile source/server.
450 *
451 * @param maxFileAge
452 * maximum age in milliseconds
453 * @see #FILE_AGE_ONE_DAY
454 * @see #FILE_AGE_ONE_WEEK
455 * @see TileSource#getTileUpdate()
456 */
457 public void setCacheMaxFileAge(long maxFileAge) {
458 this.maxCacheFileAge = maxFileAge;
459 }
460
461 public String getCacheDirBase() {
462 return cacheDirBase;
463 }
464
465 public void setTileCacheDir(String tileCacheDir) {
466 File dir = new File(tileCacheDir);
467 dir.mkdirs();
468 this.cacheDirBase = dir.getAbsolutePath();
469 }
470
471 @Override
472 public void clearCache(TileSource source) {
473 clearCache(source, null);
474 }
475
476 @Override
477 public void clearCache(TileSource source, TileClearController controller) {
478 File dir = getSourceCacheDir(source);
479 if (dir != null) {
480 if (controller != null) controller.initClearDir(dir);
481 if (dir.isDirectory()) {
482 File[] files = dir.listFiles();
483 if (controller != null) controller.initClearFiles(files);
484 for (File file : files) {
485 if (controller != null && controller.cancel()) return;
486 file.delete();
487 if (controller != null) controller.fileDeleted(file);
488 }
489 }
490 dir.delete();
491 }
492 if (controller != null) controller.clearFinished();
493 }
494}
Note: See TracBrowser for help on using the repository browser.