Ticket #11216: jcs-tms-v07.patch
File jcs-tms-v07.patch, 94.9 KB (added by , 9 years ago) |
---|
-
src/org/apache/commons
-
build.xml
Property changes on: src/org/apache/commons ___________________________________________________________________ Added: svn:externals ## -0,0 +1,2 ## +http://svn.apache.org/repos/asf/commons/proper/jcs/trunk/commons-jcs-core/src/main/java/org/apache/commons/jcs jcs +http://svn.apache.org/repos/asf/commons/proper/logging/trunk/src/main/java/org/apache/commons/logging loggingIndex: build.xml
220 220 destdir="build" target="1.7" source="1.7" debug="on" includeantruntime="false" createMissingPackageInfoClass="false" encoding="iso-8859-1"> 221 221 <!-- get rid of "internal proprietary API" warning --> 222 222 <compilerarg value="-XDignore.symbol.file"/> 223 <exclude name="org/apache/commons/jcs/admin/**"/> 224 <exclude name="org/apache/commons/jcs/auxiliary/disk/jdbc/**"/> 225 <exclude name="org/apache/commons/jcs/auxiliary/remote/**"/> 226 <exclude name="org/apache/commons/jcs/utils/servlet/**"/> 227 <exclude name="org/apache/commons/logging/impl/AvalonLogger.java"/> 228 <exclude name="org/apache/commons/logging/impl/Log4JLogger.java"/> 229 <exclude name="org/apache/commons/logging/impl/LogKitLogger.java"/> 230 <exclude name="org/apache/commons/logging/impl/ServletContextCleaner.java"/> 223 231 </javac> 224 232 <!-- JMapViewer/JOSM --> 225 233 <javac srcdir="${src.dir}" excludes="com/**,oauth/**,org/apache/commons/**,org/glassfish/**,org/openstreetmap/gui/jmapviewer/Demo.java" -
src/org/openstreetmap/josm/Main.java
67 67 import org.openstreetmap.josm.data.ProjectionBounds; 68 68 import org.openstreetmap.josm.data.UndoRedoHandler; 69 69 import org.openstreetmap.josm.data.ViewportData; 70 import org.openstreetmap.josm.data.cache.JCSCacheManager; 70 71 import org.openstreetmap.josm.data.coor.CoordinateFormat; 71 72 import org.openstreetmap.josm.data.coor.LatLon; 72 73 import org.openstreetmap.josm.data.osm.DataSet; … … 1088 1089 * @since 3378 1089 1090 */ 1090 1091 public static boolean exitJosm(boolean exit, int exitCode) { 1092 JCSCacheManager.shutdown(); 1091 1093 if (Main.saveUnsavedModifications()) { 1092 1094 geometry.remember("gui.geometry"); 1093 1095 if (map != null) { -
src/org/openstreetmap/josm/data/cache/JCSCacheManager.java
1 // License: GPL. For details, see LICENSE file. 2 package org.openstreetmap.josm.data.cache; 3 4 import java.io.File; 5 import java.io.IOException; 6 import java.text.MessageFormat; 7 import java.util.Properties; 8 import java.util.logging.Handler; 9 import java.util.logging.Level; 10 import java.util.logging.LogRecord; 11 import java.util.logging.Logger; 12 13 import org.apache.commons.jcs.access.CacheAccess; 14 import org.apache.commons.jcs.auxiliary.AuxiliaryCache; 15 import org.apache.commons.jcs.auxiliary.disk.indexed.IndexedDiskCacheAttributes; 16 import org.apache.commons.jcs.engine.control.CompositeCache; 17 import org.apache.commons.jcs.engine.control.CompositeCacheManager; 18 import org.openstreetmap.gui.jmapviewer.FeatureAdapter; 19 import org.openstreetmap.josm.Main; 20 import org.openstreetmap.josm.data.preferences.IntegerProperty; 21 22 23 /** 24 * @author Wiktor Niesiobędzki 25 * 26 * Wrapper class for JCS Cache. Sets some sane environment and returns instances of cache objects. 27 * Static configuration for now assumes some small LRU cache in memory and larger LRU cache on disk 28 * 29 */ 30 public class JCSCacheManager { 31 private static final Logger log = FeatureAdapter.getLogger(JCSCacheManager.class.getCanonicalName()); 32 33 private static volatile CompositeCacheManager cacheManager = null; 34 private static long maxObjectTTL = Long.MAX_VALUE; 35 private final static String PREFERENCE_PREFIX = "jcs.cache"; 36 37 public static final IntegerProperty DEFAULT_MAX_OBJECTS_IN_MEMORY = new IntegerProperty(PREFERENCE_PREFIX + ".max_objects_in_memory", 1000); 38 public static final IntegerProperty DEFAULT_MAX_OBJECTS_ON_DISK = new IntegerProperty(PREFERENCE_PREFIX + ".max_objects_on_disk", 25000); 39 40 private static void initialize() throws IOException { 41 File cacheDir = new File(Main.pref.getCacheDirectory(), "jcs"); 42 43 if ((!cacheDir.exists() && !cacheDir.mkdirs())) 44 throw new IOException("Cannot access cache directory"); 45 46 // raising logging level gives ~500x performance gain 47 // http://westsworld.dk/blog/2008/01/jcs-and-performance/ 48 Logger jcsLog = Logger.getLogger("org.apache.commons.jcs"); 49 jcsLog.setLevel(Level.INFO); 50 jcsLog.setUseParentHandlers(false); 51 // we need a separate handler from Main's, as we downgrade LEVEL.INFO to DEBUG level 52 jcsLog.addHandler(new Handler() { 53 @Override 54 public void publish(LogRecord record) { 55 String msg = MessageFormat.format(record.getMessage(), record.getParameters()); 56 if (record.getLevel().intValue() >= Level.SEVERE.intValue()) { 57 Main.error(msg); 58 } else if (record.getLevel().intValue() >= Level.WARNING.intValue()) { 59 Main.warn(msg); 60 // downgrade INFO level to debug, as JCS is too verbose at INFO level 61 } else if (record.getLevel().intValue() >= Level.INFO.intValue()) { 62 Main.debug(msg); 63 } else { 64 Main.trace(msg); 65 } 66 } 67 68 @Override 69 public void flush() { 70 } 71 72 @Override 73 public void close() throws SecurityException { 74 } 75 }); 76 77 78 CompositeCacheManager cm = CompositeCacheManager.getUnconfiguredInstance(); 79 // this could be moved to external file 80 Properties props = new Properties(); 81 props.setProperty("jcs.default", "DC"); 82 props.setProperty("jcs.default.cacheattributes", org.apache.commons.jcs.engine.CompositeCacheAttributes.class.getCanonicalName()); 83 props.setProperty("jcs.default.cacheattributes.MaxObjects", DEFAULT_MAX_OBJECTS_IN_MEMORY.get().toString()); 84 props.setProperty("jcs.default.cacheattributes.UseMemoryShrinker", "true"); 85 props.setProperty("jcs.default.cacheattributes.DiskUsagePatternName", "UPDATE"); // store elements on disk on put 86 props.setProperty("jcs.default.elementattributes", CacheEntryAttributes.class.getCanonicalName()); 87 props.setProperty("jcs.default.elementattributes.IsEternal", "false"); 88 props.setProperty("jcs.default.elementattributes.MaxLife", Long.toString(maxObjectTTL)); 89 props.setProperty("jcs.default.elementattributes.IdleTime", Long.toString(maxObjectTTL)); 90 props.setProperty("jcs.default.elementattributes.IsSpool", "true"); 91 props.setProperty("jcs.auxiliary.DC", org.apache.commons.jcs.auxiliary.disk.indexed.IndexedDiskCacheFactory.class.getCanonicalName()); 92 props.setProperty("jcs.auxiliary.DC.attributes", org.apache.commons.jcs.auxiliary.disk.indexed.IndexedDiskCacheAttributes.class.getCanonicalName()); 93 //props.setProperty("jcs.auxiliary.DC", org.apache.commons.jcs.auxiliary.disk.block.BlockDiskCacheFactory.class.getCanonicalName()); 94 //props.setProperty("jcs.auxiliary.DC.attributes", org.apache.commons.jcs.auxiliary.disk.block.BlockDiskCacheAttributes.class.getCanonicalName()); 95 props.setProperty("jcs.auxiliary.DC.attributes.DiskPath", cacheDir.getAbsolutePath()); 96 props.setProperty("jcs.auxiliary.DC.attributes.maxKeySize", DEFAULT_MAX_OBJECTS_ON_DISK.get().toString()); 97 //props.setProperty("jcs.auxiliary.DC.attributes.blockSizeBytes", "1024"); // - for BlockDiskCacheAttributes 98 cm.configure(props); 99 cacheManager = cm; 100 101 } 102 103 /** 104 * Returns configured cache object for named cache region 105 * @param cacheName region name 106 * @return cache access object 107 * @throws IOException if directory is not found 108 */ 109 public static <K,V> CacheAccess<K, V> getCache(String cacheName) throws IOException { 110 return getCache(cacheName, DEFAULT_MAX_OBJECTS_IN_MEMORY.get().intValue(), DEFAULT_MAX_OBJECTS_ON_DISK.get().intValue(), null); 111 } 112 113 /** 114 * Returns configured cache object with defined limits of memory cache and disk cache 115 * @param cacheName region name 116 * @param maxMemoryObjects number of objects to keep in memory 117 * @param maxDiskObjects number of objects to keep on disk 118 * @return cache access object 119 * @throws IOException if directory is not found 120 */ 121 public static <K,V> CacheAccess<K, V> getCache(String cacheName, int maxMemoryObjects, int maxDiskObjects, String cachePath) throws IOException { 122 if (cacheManager != null) 123 return getCacheInner(cacheName, maxMemoryObjects, maxDiskObjects, cachePath); 124 125 synchronized (JCSCacheManager.class) { 126 if (cacheManager == null) 127 initialize(); 128 return getCacheInner(cacheName, maxMemoryObjects, maxDiskObjects, cachePath); 129 } 130 } 131 132 133 private static <K,V> CacheAccess<K, V> getCacheInner(String cacheName, int maxMemoryObjects, int maxDiskObjects, String cachePath) { 134 CompositeCache<K, V> cc = cacheManager.getCache(cacheName); 135 setCacheParams(cc, maxMemoryObjects, maxDiskObjects, cachePath); 136 return new CacheAccess<K, V>(cc); 137 } 138 139 @SuppressWarnings("rawtypes") 140 private static void setCacheParams(CompositeCache cc, int maxMemoryObjects, int maxDiskObjects, String cachePath) { 141 cc.getCacheAttributes().setMaxObjects(maxMemoryObjects); 142 AuxiliaryCache ac[] = cc.getAuxCaches(); 143 if (ac!=null && ac.length > 0) { 144 if (ac[0].getAuxiliaryCacheAttributes() instanceof IndexedDiskCacheAttributes) { 145 IndexedDiskCacheAttributes cacheAttributes = (IndexedDiskCacheAttributes) ac[0].getAuxiliaryCacheAttributes(); 146 147 cacheAttributes.setMaxKeySize(maxDiskObjects); 148 if (cachePath != null) { 149 File path = new File(cachePath); 150 if (!path.exists() && !path.mkdirs()) { 151 log.log(Level.WARNING, "Failed to create cache path: {0}", cachePath); 152 } else { 153 cacheAttributes.setDiskPath(path); 154 } 155 } 156 } 157 } 158 } 159 160 /** 161 * Close all files to ensure, that all indexes and data are properly written 162 */ 163 public static void shutdown() { 164 // use volatile semantics to get consistent object 165 CompositeCacheManager localCacheManager = cacheManager; 166 if (localCacheManager != null) { 167 localCacheManager.shutDown(); 168 } 169 } 170 171 public static void updateSettings() { 172 if (cacheManager != null) { 173 for (String cacheName: cacheManager.getCacheNames()) { 174 setCacheParams(cacheManager.getCache(cacheName), DEFAULT_MAX_OBJECTS_IN_MEMORY.get().intValue(), DEFAULT_MAX_OBJECTS_ON_DISK.get().intValue(), Main.pref.getCacheDirectory().getPath()); 175 } 176 } 177 } 178 } -
src/org/openstreetmap/josm/data/cache/CacheEntry.java
1 // License: GPL. For details, see LICENSE file. 2 package org.openstreetmap.josm.data.cache; 3 4 import java.io.Serializable; 5 6 /** 7 * @author Wiktor Niesiobędzki 8 * 9 * Class that will hold JCS cache entries 10 * 11 */ 12 public class CacheEntry implements Serializable { 13 private static final long serialVersionUID = 1L; //version 14 private byte[] content; 15 16 /** 17 * @param content of the cache entry 18 */ 19 public CacheEntry(byte[] content) { 20 this.content = content; 21 } 22 23 /** 24 * @return cache entry content 25 */ 26 public byte[] getContent() { 27 return content; 28 } 29 } -
src/org/openstreetmap/josm/data/cache/ICachedLoaderListener.java
1 // License: GPL. For details, see LICENSE file. 2 package org.openstreetmap.josm.data.cache; 3 4 public interface ICachedLoaderListener { 5 /** 6 * Will be called when K object was successfully downloaded 7 * 8 * @param object 9 * @param success 10 */ 11 public void loadingFinished(byte[] object, boolean success); 12 13 } -
src/org/openstreetmap/josm/data/cache/JCSCachedTileLoaderJob.java
1 // License: GPL. For details, see LICENSE file. 2 package org.openstreetmap.josm.data.cache; 3 4 import java.io.ByteArrayOutputStream; 5 import java.io.FileNotFoundException; 6 import java.io.IOException; 7 import java.io.InputStream; 8 import java.net.HttpURLConnection; 9 import java.net.MalformedURLException; 10 import java.net.URLConnection; 11 import java.util.HashSet; 12 import java.util.Map; 13 import java.util.Random; 14 import java.util.Set; 15 import java.util.concurrent.ConcurrentHashMap; 16 import java.util.concurrent.ConcurrentMap; 17 import java.util.concurrent.Executor; 18 import java.util.concurrent.Executors; 19 import java.util.concurrent.LinkedBlockingDeque; 20 import java.util.concurrent.RejectedExecutionException; 21 import java.util.concurrent.ThreadPoolExecutor; 22 import java.util.concurrent.TimeUnit; 23 import java.util.logging.Level; 24 import java.util.logging.Logger; 25 26 import org.apache.commons.jcs.access.behavior.ICacheAccess; 27 import org.apache.commons.jcs.engine.behavior.ICacheElement; 28 import org.openstreetmap.gui.jmapviewer.FeatureAdapter; 29 import org.openstreetmap.josm.data.preferences.IntegerProperty; 30 31 /** 32 * @author Wiktor Niesiobędzki 33 * 34 * @param <K> cache entry key type 35 * 36 * Generic loader for HTTP based tiles. Uses custom attribute, to check, if entry has expired 37 * according to HTTP headers sent with tile. If so, it tries to verify using Etags 38 * or If-Modified-Since / Last-Modified. 39 * 40 * If the tile is not valid, it will try to download it from remote service and put it 41 * to cache. If remote server will fail it will try to use stale entry. 42 * 43 * This class will keep only one Job running for specified tile. All others will just finish, but 44 * listeners will be gathered and notified, once download job will be finished 45 */ 46 public abstract class JCSCachedTileLoaderJob<K> implements ICachedLoaderJob<K>, Runnable { 47 private static final Logger log = FeatureAdapter.getLogger(JCSCachedTileLoaderJob.class.getCanonicalName()); 48 protected static final long DEFAULT_EXPIRE_TIME = 1000L * 60 * 60 * 24 * 7; // 7 days 49 // Limit for the max-age value send by the server. 50 protected static final long EXPIRE_TIME_SERVER_LIMIT = 1000L * 60 * 60 * 24 * 28; // 4 weeks 51 // Absolute expire time limit. Cached tiles that are older will not be used, 52 // even if the refresh from the server fails. 53 protected static final long ABSOLUTE_EXPIRE_TIME_LIMIT = Long.MAX_VALUE; // unlimited 54 55 /** 56 * maximum download threads that will be started 57 */ 58 public static IntegerProperty THREAD_LIMIT = new IntegerProperty("cache.jcs.max_threads", 10); 59 private static Executor DOWNLOAD_JOB_DISPATCHER = new ThreadPoolExecutor( 60 2, // we have a small queue, so threads will be quickly started (threads are started only, when queue is full) 61 THREAD_LIMIT.get().intValue(), // do not this number of threads 62 30, // keepalive for thread 63 TimeUnit.SECONDS, 64 // make queue of LIFO type - so recently requested tiles will be loaded first (assuming that these are which user is waiting to see) 65 new LinkedBlockingDeque<Runnable>(5) { 66 /* keep the queue size fairly small, we do not want to 67 download a lot of tiles, that user is not seeing anyway */ 68 @Override 69 public boolean offer(Runnable t) { 70 return super.offerFirst(t); 71 } 72 73 @Override 74 public Runnable remove() { 75 return super.removeFirst(); 76 } 77 } 78 ); 79 80 /* 81 * Executor that handles cache access. As the cache might be on (slow) disk to avoid 82 * blocking EDT, new thread is spawned, when cache is accessed. 83 * 84 * This is cached threaded executor, as the task should be easy and fast, but 85 * Executors.newSingleThreadExecutor() was too slow, to keep the download threads busy. 86 * 87 * This is separate thread pool from download pool, to do not delay cache accesses, with 88 * download tasks 89 */ 90 91 private static Executor CACHE_ACCESS_EXECUTOR = Executors.newCachedThreadPool(); 92 93 private ICacheAccess<K, CacheEntry> cache; 94 private long now; 95 private ICacheElement<K,CacheEntry> cacheElement; 96 private int connectTimeout; 97 private int readTimeout; 98 private Map<String, String> headers; 99 private static ConcurrentMap<String,Set<ICachedLoaderListener>> inProgress = new ConcurrentHashMap<>(); 100 101 protected CacheEntryAttributes attributes = null; 102 protected byte[] data = null; 103 private static ConcurrentMap<String, Boolean> useHead = new ConcurrentHashMap<>(); 104 105 106 107 /** 108 * @param cache cache instance that we will work on 109 * @param headers 110 * @param readTimeout 111 * @param connectTimeout 112 */ 113 public JCSCachedTileLoaderJob(ICacheAccess<K,CacheEntry> cache, 114 int connectTimeout, int readTimeout, 115 Map<String, String> headers) { 116 117 this.cache = cache; 118 this.now = System.currentTimeMillis(); 119 this.connectTimeout = connectTimeout; 120 this.readTimeout = readTimeout; 121 this.headers = headers; 122 } 123 124 private void ensureCacheElement() { 125 if (cacheElement == null && getCacheKey() != null) { 126 cacheElement = cache.getCacheElement(getCacheKey()); 127 if (cacheElement != null) { 128 attributes = (CacheEntryAttributes) cacheElement.getElementAttributes(); 129 data = cacheElement.getVal().getContent(); 130 } 131 } 132 } 133 134 public byte[] get() { 135 ensureCacheElement(); 136 return data; 137 } 138 139 @Override 140 public void submit(ICachedLoaderListener listener) { 141 boolean first = false; 142 String url = getUrl().toString(); 143 if (url == null) { 144 log.log(Level.WARNING, "No url returned for: {0}, skipping", getCacheKey()); 145 return; 146 } 147 synchronized (inProgress) { 148 Set<ICachedLoaderListener> newListeners = inProgress.get(url); 149 if (newListeners == null) { 150 newListeners = new HashSet<>(); 151 inProgress.put(url, newListeners); 152 first = true; 153 } 154 newListeners.add(listener); 155 } 156 157 if (first) { 158 // we need to get executor to local namespace, so if subclass overrides the Executor, it will be used 159 final Executor downloadExecutor = getDownloadExecutor(); 160 CACHE_ACCESS_EXECUTOR.execute(new Runnable() { 161 public void run() { 162 ensureCacheElement(); 163 if (cacheElement != null && isCacheElementValid() && (isObjectLoadable())) { 164 // we got something in cache, and it's valid, so lets return it 165 log.log(Level.FINE, "JCS - Returning object from cache: {0}", getCacheKey()); 166 finishLoading(true); 167 return; 168 } 169 // object not in cache, so submit work to separate thread 170 try { 171 downloadExecutor.execute(JCSCachedTileLoaderJob.this); 172 } catch (RejectedExecutionException e) { 173 // queue was full, try again later 174 log.log(Level.FINE, "JCS - rejected job for: {0}", getCacheKey()); 175 finishLoading(false); 176 } 177 } 178 }); 179 } 180 } 181 182 /** 183 * 184 * @return checks if object from cache has sufficient data to be returned 185 */ 186 protected boolean isObjectLoadable() { 187 return data != null && data.length > 0; 188 } 189 190 /** 191 * 192 * @return cache object as empty, regardless of what remote resource has returned (ex. based on headers) 193 */ 194 protected boolean cacheAsEmpty() { 195 return false; 196 } 197 198 /** 199 * @return key under which discovered server settings will be kept 200 */ 201 protected String getServerKey() { 202 return getUrl().getHost(); 203 } 204 205 /** 206 * this needs to be non-static, so it can be overridden by subclasses 207 */ 208 protected Executor getDownloadExecutor() { 209 return DOWNLOAD_JOB_DISPATCHER; 210 } 211 212 213 public void run() { 214 final Thread currentThread = Thread.currentThread(); 215 final String oldName = currentThread.getName(); 216 currentThread.setName("JCS Downloading: " + getUrl()); 217 try { 218 // try to load object from remote resource 219 if (loadObject()) { 220 finishLoading(true); 221 } else { 222 // if loading failed - check if we can return stale entry 223 if (isObjectLoadable()) { 224 // try to get stale entry in cache 225 finishLoading(true); 226 log.log(Level.FINE, "JCS - found stale object in cache: {0}", getUrl()); 227 } else { 228 // failed completely 229 finishLoading(false); 230 } 231 } 232 } finally { 233 currentThread.setName(oldName); 234 } 235 } 236 237 238 private void finishLoading(boolean success) { 239 Set<ICachedLoaderListener> listeners = null; 240 synchronized (inProgress) { 241 listeners = inProgress.remove(getUrl().toString()); 242 } 243 if (listeners == null) { 244 log.log(Level.WARNING, "Listener not found for URL: {0}. Listener not notified!", getUrl()); 245 return; 246 } 247 try { 248 for (ICachedLoaderListener l: listeners) { 249 l.loadingFinished(data, success); 250 } 251 } catch (Exception e) { 252 log.log(Level.WARNING, "JCS - Error while loading object from cache: {0}; {1}", new Object[]{e.getMessage(), getUrl()}); 253 log.log(Level.FINE, "Stacktrace", e); 254 for (ICachedLoaderListener l: listeners) { 255 l.loadingFinished(data, false); 256 } 257 258 } 259 260 } 261 262 private boolean isCacheElementValid() { 263 long expires = attributes.getExpirationTime(); 264 265 // check by expire date set by server 266 if (expires != 0L) { 267 // put a limit to the expire time (some servers send a value 268 // that is too large) 269 expires = Math.min(expires, attributes.getCreateTime() + EXPIRE_TIME_SERVER_LIMIT); 270 if (now > expires) { 271 log.log(Level.FINE, "JCS - Object {0} has expired -> valid to {1}, now is: {2}", new Object[]{getUrl(), Long.toString(expires), Long.toString(now)}); 272 return false; 273 } 274 } else { 275 // check by file modification date 276 if (now - attributes.getLastModification() > DEFAULT_EXPIRE_TIME) { 277 log.log(Level.FINE, "JCS - Object has expired, maximum file age reached {0}", getUrl()); 278 return false; 279 } 280 } 281 return true; 282 } 283 284 private boolean loadObject() { 285 try { 286 // if we have object in cache, and host doesn't support If-Modified-Since nor If-None-Match 287 // then just use HEAD request and check returned values 288 if (isObjectLoadable() && 289 Boolean.TRUE.equals(useHead.get(getServerKey())) && 290 isCacheValidUsingHead()) { 291 log.log(Level.FINE, "JCS - cache entry verified using HEAD request: {0}", getUrl()); 292 return true; 293 } 294 URLConnection urlConn = getURLConnection(); 295 296 if (isObjectLoadable() && 297 (now - attributes.getLastModification()) <= ABSOLUTE_EXPIRE_TIME_LIMIT) { 298 urlConn.setIfModifiedSince(attributes.getLastModification()); 299 } 300 if (isObjectLoadable() && attributes.getEtag() != null) { 301 urlConn.addRequestProperty("If-None-Match", attributes.getEtag()); 302 } 303 if (urlConn instanceof HttpURLConnection && ((HttpURLConnection)urlConn).getResponseCode() == 304) { 304 // If isModifiedSince or If-None-Match has been set 305 // and the server answers with a HTTP 304 = "Not Modified" 306 log.log(Level.FINE, "JCS - IfModifiedSince/Etag test: local version is up to date: {0}", getUrl()); 307 return true; 308 } else if (isObjectLoadable()) { 309 // we have an object in cache, but we haven't received 304 resposne code 310 // check if we should use HEAD request to verify 311 if((attributes.getEtag() != null && attributes.getEtag().equals(urlConn.getRequestProperty("ETag"))) || 312 attributes.getLastModification() == urlConn.getLastModified()) { 313 // we sent ETag or If-Modified-Since, but didn't get 304 response code 314 // for further requests - use HEAD 315 String serverKey = getServerKey(); 316 log.log(Level.INFO, "JCS - Host: {0} found not to return 304 codes for If-Modifed-Since or If-None-Match headers", serverKey); 317 useHead.put(serverKey, Boolean.TRUE); 318 } 319 } 320 321 attributes = parseHeaders(urlConn); 322 323 for (int i = 0; i < 5; ++i) { 324 if (urlConn instanceof HttpURLConnection && ((HttpURLConnection)urlConn).getResponseCode() == 503) { 325 Thread.sleep(5000+(new Random()).nextInt(5000)); 326 continue; 327 } 328 data = read(urlConn); 329 synchronized (cache) { // FIXME: does it protect against corrupted cache files? 330 if (!cacheAsEmpty() && data != null && data.length > 0) { 331 cache.put(getCacheKey(), new CacheEntry(data), attributes); 332 log.log(Level.FINE, "JCS - downloaded key: {0}, length: {1}, url: {2}", 333 new Object[] {getCacheKey(), data.length, getUrl()}); 334 return true; 335 } else { 336 log.log(Level.FINE, "JCS - Caching empty object {0}", getUrl()); 337 cache.put(getCacheKey(), new CacheEntry(new byte[]{}), attributes); 338 return true; 339 } 340 } 341 } 342 } catch (FileNotFoundException e) { 343 log.log(Level.FINE, "JCS - Caching empty object as server returned 404 for: {0}", getUrl()); 344 cache.put(getCacheKey(), new CacheEntry(new byte[]{}), attributes); 345 handleNotFound(); 346 return true; 347 } catch (Exception e) { 348 log.log(Level.WARNING, "JCS - Exception during download " + getUrl(), e); 349 } 350 log.log(Level.WARNING, "JCS - Silent failure during download: {0}", getUrl()); 351 return false; 352 353 } 354 355 protected abstract void handleNotFound(); 356 357 private CacheEntryAttributes parseHeaders(URLConnection urlConn) { 358 CacheEntryAttributes ret = new CacheEntryAttributes(); 359 ret.setNoTileAtZoom("no-tile".equals(urlConn.getHeaderField("X-VE-Tile-Info"))); 360 361 Long lng = urlConn.getExpiration(); 362 if (lng.equals(0L)) { 363 try { 364 String str = urlConn.getHeaderField("Cache-Control"); 365 if (str != null) { 366 for (String token: str.split(",")) { 367 if (token.startsWith("max-age=")) { 368 lng = Long.parseLong(token.substring(8)) * 1000 + 369 System.currentTimeMillis(); 370 } 371 } 372 } 373 } catch (NumberFormatException e) {} //ignore malformed Cache-Control headers 374 } 375 376 ret.setExpirationTime(lng); 377 ret.setLastModification(now); 378 ret.setEtag(urlConn.getHeaderField("ETag")); 379 return ret; 380 } 381 382 private HttpURLConnection getURLConnection() throws IOException, MalformedURLException { 383 HttpURLConnection urlConn = (HttpURLConnection) getUrl().openConnection(); 384 urlConn.setRequestProperty("Accept", "text/html, image/png, image/jpeg, image/gif, */*"); 385 urlConn.setReadTimeout(readTimeout); // 30 seconds read timeout 386 urlConn.setConnectTimeout(connectTimeout); 387 for(Map.Entry<String, String> e: headers.entrySet()) { 388 urlConn.setRequestProperty(e.getKey(), e.getValue()); 389 } 390 return urlConn; 391 } 392 393 private boolean isCacheValidUsingHead() throws IOException { 394 HttpURLConnection urlConn = (HttpURLConnection) getUrl().openConnection(); 395 urlConn.setRequestMethod("HEAD"); 396 long lastModified = urlConn.getLastModified(); 397 return ( 398 (attributes.getEtag() != null && attributes.getEtag().equals(urlConn.getRequestProperty("ETag"))) || 399 (lastModified != 0 && lastModified <= attributes.getLastModification()) 400 ); 401 } 402 403 private static byte[] read(URLConnection urlConn) throws IOException { 404 InputStream input = urlConn.getInputStream(); 405 try { 406 ByteArrayOutputStream bout = new ByteArrayOutputStream(input.available()); 407 byte[] buffer = new byte[2048]; 408 boolean finished = false; 409 do { 410 int read = input.read(buffer); 411 if (read >= 0) { 412 bout.write(buffer, 0, read); 413 } else { 414 finished = true; 415 } 416 } while (!finished); 417 if (bout.size() == 0) 418 return null; 419 return bout.toByteArray(); 420 } finally { 421 input.close(); 422 } 423 } 424 } -
src/org/openstreetmap/josm/data/cache/CacheEntryAttributes.java
1 // License: GPL. For details, see LICENSE file. 2 package org.openstreetmap.josm.data.cache; 3 4 import java.util.HashMap; 5 import java.util.Map; 6 7 import org.apache.commons.jcs.engine.ElementAttributes; 8 9 public class CacheEntryAttributes extends ElementAttributes { 10 private static final long serialVersionUID = 1L; //version 11 private Map<String, String> attrs = new HashMap<String, String>(); 12 private final static String NO_TILE_AT_ZOOM = "noTileAtZoom"; 13 private final static String ETAG = "Etag"; 14 private final static String LAST_MODIFICATION = "lastModification"; 15 private final static String EXPIRATION_TIME = "expirationTime"; 16 //private boolean noTileAtZoom = false; 17 private String Etag = null; 18 private long lastModification = 0; 19 private long expirationTime = 0; 20 21 22 public CacheEntryAttributes() { 23 super(); 24 attrs.put(NO_TILE_AT_ZOOM, "false"); 25 attrs.put(ETAG, null); 26 attrs.put(LAST_MODIFICATION, "0"); 27 attrs.put(EXPIRATION_TIME, "0"); 28 } 29 public boolean isNoTileAtZoom() { 30 return Boolean.toString(true).equals(attrs.get(NO_TILE_AT_ZOOM)); 31 } 32 public void setNoTileAtZoom(boolean noTileAtZoom) { 33 attrs.put(NO_TILE_AT_ZOOM, Boolean.toString(noTileAtZoom)); 34 } 35 public String getEtag() { 36 return attrs.get(ETAG); 37 } 38 public void setEtag(String etag) { 39 attrs.put(ETAG, etag); 40 } 41 42 private long getLongAttr(String key) { 43 try { 44 return Long.parseLong(attrs.get(key)); 45 } catch (NumberFormatException e) { 46 attrs.put(key, "0"); 47 return 0; 48 } 49 } 50 51 public long getLastModification() { 52 return getLongAttr(LAST_MODIFICATION); 53 } 54 public void setLastModification(long lastModification) { 55 attrs.put(LAST_MODIFICATION, Long.toString(lastModification)); 56 } 57 public long getExpirationTime() { 58 return getLongAttr(EXPIRATION_TIME); 59 } 60 public void setExpirationTime(long expirationTime) { 61 attrs.put(EXPIRATION_TIME, Long.toString(expirationTime)); 62 } 63 64 } -
src/org/openstreetmap/josm/data/cache/ICachedLoaderJob.java
1 // License: GPL. For details, see LICENSE file. 2 package org.openstreetmap.josm.data.cache; 3 4 import java.net.URL; 5 6 7 /** 8 * 9 * @author Wiktor Niesiobędzki 10 * 11 * @param <K> cache key type 12 * @param <V> value that is returned from cache 13 */ 14 public interface ICachedLoaderJob<K> { 15 /** 16 * returns cache entry key 17 * 18 * @param tile 19 * @return cache key for tile 20 */ 21 public K getCacheKey(); 22 23 /** 24 * method to get download URL for Job 25 * @return URL that should be fetched 26 * 27 */ 28 public URL getUrl(); 29 /** 30 * implements the main algorithm for fetching 31 */ 32 public void run(); 33 34 /** 35 * fetches object from cache, or returns null when object is not found 36 * 37 * @return filled tile with data or null when no cache entry found 38 */ 39 public byte[] get(); 40 41 /** 42 * Submit job for background fetch, and listener will be 43 * fed with value object 44 * 45 * @param listener 46 */ 47 public void submit(ICachedLoaderListener listener); 48 } -
src/org/openstreetmap/josm/data/imagery/TMSCachedTileLoader.java
1 // License: GPL. For details, see LICENSE file. 2 package org.openstreetmap.josm.data.imagery; 3 4 import java.io.IOException; 5 import java.util.Map; 6 7 import org.apache.commons.jcs.access.behavior.ICacheAccess; 8 import org.openstreetmap.gui.jmapviewer.Tile; 9 import org.openstreetmap.gui.jmapviewer.interfaces.CachedTileLoader; 10 import org.openstreetmap.gui.jmapviewer.interfaces.TileJob; 11 import org.openstreetmap.gui.jmapviewer.interfaces.TileLoader; 12 import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener; 13 import org.openstreetmap.josm.data.cache.CacheEntry; 14 import org.openstreetmap.josm.data.cache.JCSCacheManager; 15 import org.openstreetmap.josm.data.preferences.IntegerProperty; 16 17 /** 18 * @author Wiktor Niesiobędzki 19 * 20 * Wrapper class that bridges between JCS cache and Tile Loaders 21 * 22 */ 23 public class TMSCachedTileLoader implements TileLoader, CachedTileLoader { 24 25 private ICacheAccess<String, CacheEntry> cache; 26 private int connectTimeout; 27 private int readTimeout; 28 private Map<String, String> headers; 29 private TileLoaderListener listener; 30 public static final String PREFERENCE_PREFIX = "imagery.tms.cache."; 31 // average tile size is about 20kb 32 public static IntegerProperty MAX_OBJECTS_ON_DISK = new IntegerProperty(PREFERENCE_PREFIX + "max_objects_disk", 25000); // 25000 is around 500MB under this assumptions 33 34 35 /** 36 * Constructor 37 * @param listener called when tile loading has finished 38 * @param name of the cache 39 * @param connectTimeout to remote resource 40 * @param readTimeout to remote resource 41 * @param headers to be sent along with request 42 * @param cacheDir where cache file shall reside 43 * @throws IOException when cache initialization fails 44 */ 45 public TMSCachedTileLoader(TileLoaderListener listener, String name, int connectTimeout, int readTimeout, Map<String, String> headers, String cacheDir) throws IOException { 46 this.cache = JCSCacheManager.getCache(name, 47 0, // don't use memory cache - we have already cache with BufferedImage 48 MAX_OBJECTS_ON_DISK.get(), 49 cacheDir); 50 this.connectTimeout = connectTimeout; 51 this.readTimeout = readTimeout; 52 this.headers = headers; 53 this.listener = listener; 54 } 55 56 @Override 57 public TileJob createTileLoaderJob(Tile tile) { 58 return new TMSCachedTileLoaderJob(listener, tile, cache, connectTimeout, readTimeout, headers); 59 } 60 61 @Override 62 public void clearCache() { 63 this.cache.clear(); 64 } 65 } -
src/org/openstreetmap/josm/data/imagery/TMSCachedTileLoaderJob.java
1 // License: GPL. For details, see LICENSE file. 2 package org.openstreetmap.josm.data.imagery; 3 4 import java.io.ByteArrayInputStream; 5 import java.io.IOException; 6 import java.net.URL; 7 import java.util.Map; 8 import java.util.concurrent.Executor; 9 import java.util.concurrent.LinkedBlockingDeque; 10 import java.util.concurrent.ThreadPoolExecutor; 11 import java.util.concurrent.TimeUnit; 12 import java.util.logging.Level; 13 import java.util.logging.Logger; 14 15 import org.apache.commons.jcs.access.behavior.ICacheAccess; 16 import org.openstreetmap.gui.jmapviewer.FeatureAdapter; 17 import org.openstreetmap.gui.jmapviewer.Tile; 18 import org.openstreetmap.gui.jmapviewer.interfaces.TileJob; 19 import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener; 20 import org.openstreetmap.gui.jmapviewer.interfaces.TileSource; 21 import org.openstreetmap.gui.jmapviewer.tilesources.AbstractTMSTileSource; 22 import org.openstreetmap.josm.data.cache.CacheEntry; 23 import org.openstreetmap.josm.data.cache.ICachedLoaderListener; 24 import org.openstreetmap.josm.data.cache.JCSCachedTileLoaderJob; 25 import org.openstreetmap.josm.data.preferences.IntegerProperty; 26 27 /** 28 * @author Wiktor Niesiobędzki 29 * 30 * Class bridging TMS requests to JCS cache requests 31 * 32 */ 33 public class TMSCachedTileLoaderJob extends JCSCachedTileLoaderJob<String> implements TileJob, ICachedLoaderListener { 34 private static final Logger log = FeatureAdapter.getLogger(TMSCachedTileLoaderJob.class.getCanonicalName()); 35 private Tile tile; 36 private TileLoaderListener listener; 37 private URL url; 38 39 /** 40 * overrides the THREAD_LIMIT in superclass, as we want to have separate limit and pool for TMS 41 */ 42 public static IntegerProperty THREAD_LIMIT = new IntegerProperty("imagery.tms.tmsloader.maxjobs", 25); 43 /** 44 * separate from JCS thread pool for TMS loader 45 */ 46 private static ThreadPoolExecutor DOWNLOAD_JOB_DISPATCHER = new ThreadPoolExecutor( 47 THREAD_LIMIT.get().intValue(), // keep the thread number constant 48 THREAD_LIMIT.get().intValue(), // do not this number of threads 49 30, // keepalive for thread 50 TimeUnit.SECONDS, 51 // make queue of LIFO type - so recently requested tiles will be loaded first (assuming that these are which user is waiting to see) 52 new LinkedBlockingDeque<Runnable>(5) { 53 /* keep the queue size fairly small, we do not want to 54 download a lot of tiles, that user is not seeing anyway */ 55 @Override 56 public boolean offer(Runnable t) { 57 return super.offerFirst(t); 58 } 59 60 @Override 61 public Runnable remove() { 62 return super.removeFirst(); 63 } 64 } 65 ); 66 67 /** 68 * Constructor for creating a job, to get a specific tile from cache 69 * @param listener 70 * @param tile to be fetched from cache 71 * @param cache object 72 * @param connectTimeout when connecting to remote resource 73 * @param readTimeout when connecting to remote resource 74 * @param headers to be sent together with request 75 */ 76 public TMSCachedTileLoaderJob(TileLoaderListener listener, Tile tile, ICacheAccess<String, CacheEntry> cache, int connectTimeout, int readTimeout, 77 Map<String, String> headers) { 78 super(cache, connectTimeout, readTimeout, headers); 79 this.tile = tile; 80 this.listener = listener; 81 // URLs tend to change for some tile providers. Make a static reference here, so the tile URL might be used as a key 82 // for request deduplication 83 try { 84 this.url = new URL(tile.getUrl()); 85 } catch (IOException e) { 86 log.log(Level.WARNING, "JCS TMS Cache - error creating URL for tile {0}: {1}", new Object[] {tile.getKey(), e.getMessage()}); 87 } 88 89 } 90 91 @Override 92 public Tile getTile() { 93 return tile; 94 } 95 96 @Override 97 public String getCacheKey() { 98 if (tile != null) 99 return tile.getKey(); 100 return null; 101 } 102 103 @Override 104 public URL getUrl() { 105 return url; 106 } 107 108 @Override 109 public boolean isObjectLoadable() { 110 return (data != null && data.length > 0) || cacheAsEmpty(); 111 } 112 113 @Override 114 protected boolean cacheAsEmpty() { 115 if (attributes != null && attributes.isNoTileAtZoom()) { 116 // do not remove file - keep the information, that there is no tile, for further requests 117 // the code above will check, if this information is still valid 118 log.log(Level.FINE, "JCS TMS - Tile valid, but no file, as no tiles at this level {0}", tile); 119 tile.setError("No tile at this zoom level"); 120 tile.putValue("tile-info", "no-tile"); 121 return true; 122 } 123 return false; 124 } 125 126 @Override 127 protected Executor getDownloadExecutor() { 128 return DOWNLOAD_JOB_DISPATCHER; 129 } 130 131 public void submit() { 132 tile.initLoading(); 133 super.submit(this); 134 } 135 136 @Override 137 public void loadingFinished(byte[] object, boolean success) { 138 try { 139 tile.finishLoading(); 140 if (object != null && object.length > 0) { 141 tile.loadImage(new ByteArrayInputStream(object)); 142 } 143 if (listener != null) { 144 listener.tileLoadingFinished(tile, success); 145 } 146 } catch (IOException e) { 147 log.log(Level.WARNING, "JCS TMS - error loading object for tile {0}: {1}", new Object[] {tile.getKey(), e.getMessage()}); 148 tile.setError(e.getMessage()); 149 tile.setLoaded(false); 150 if (listener != null) { 151 listener.tileLoadingFinished(tile, false); 152 } 153 } 154 } 155 156 /** 157 * Method for getting the tile from cache only, without trying to reach remote resource 158 * @return tile or null, if nothing (useful) was found in cache 159 */ 160 public Tile getCachedTile() { 161 byte[] data = super.get(); 162 if (isObjectLoadable()) { 163 loadingFinished(data, true); 164 return tile; 165 } else { 166 return null; 167 } 168 } 169 170 @Override 171 protected void handleNotFound() { 172 tile.setError("No tile at this zoom level"); 173 tile.putValue("tile-info", "no-tile"); 174 } 175 176 @Override 177 protected String getServerKey() { 178 TileSource ts = tile.getSource(); 179 if (ts instanceof AbstractTMSTileSource) { 180 return ((AbstractTMSTileSource) ts).getBaseUrl(); 181 } 182 return super.getServerKey(); 183 } 184 } -
src/org/openstreetmap/josm/gui/preferences/imagery/TMSSettingsPanel.java
11 11 import javax.swing.JSpinner; 12 12 import javax.swing.SpinnerNumberModel; 13 13 14 import org.openstreetmap.josm.data.imagery.TMSCachedTileLoader; 15 import org.openstreetmap.josm.data.imagery.TMSCachedTileLoaderJob; 14 16 import org.openstreetmap.josm.gui.layer.TMSLayer; 15 import org.openstreetmap.josm.tools.GBC;16 17 import org.openstreetmap.josm.gui.widgets.JosmTextField; 18 import org.openstreetmap.josm.tools.GBC; 17 19 18 20 /** 19 21 * {@code JPanel} giving access to TMS settings. … … 28 30 private final JSpinner maxZoomLvl; 29 31 private final JCheckBox addToSlippyMapChosser = new JCheckBox(); 30 32 private final JosmTextField tilecacheDir = new JosmTextField(); 33 private final JSpinner maxElementsOnDisk; 34 private final JSpinner maxConcurrentDownloads; 35 31 36 32 37 /** 33 38 * Constructs a new {@code TMSSettingsPanel}. … … 36 41 super(new GridBagLayout()); 37 42 minZoomLvl = new JSpinner(new SpinnerNumberModel(TMSLayer.DEFAULT_MIN_ZOOM, TMSLayer.MIN_ZOOM, TMSLayer.MAX_ZOOM, 1)); 38 43 maxZoomLvl = new JSpinner(new SpinnerNumberModel(TMSLayer.DEFAULT_MAX_ZOOM, TMSLayer.MIN_ZOOM, TMSLayer.MAX_ZOOM, 1)); 44 maxElementsOnDisk = new JSpinner(new SpinnerNumberModel(TMSCachedTileLoader.MAX_OBJECTS_ON_DISK.get().intValue(), 0, Integer.MAX_VALUE, 1)); 45 maxConcurrentDownloads = new JSpinner(new SpinnerNumberModel(TMSCachedTileLoaderJob.THREAD_LIMIT.get().intValue(), 0, Integer.MAX_VALUE, 1)); 39 46 40 47 add(new JLabel(tr("Auto zoom by default: ")), GBC.std()); 41 48 add(GBC.glue(5, 0), GBC.std()); … … 60 67 add(new JLabel(tr("Tile cache directory: ")), GBC.std()); 61 68 add(GBC.glue(5, 0), GBC.std()); 62 69 add(tilecacheDir, GBC.eol().fill(GBC.HORIZONTAL)); 70 71 add(new JLabel(tr("Maximum concurrent downloads: ")), GBC.std()); 72 add(GBC.glue(5, 0), GBC.std()); 73 add(maxConcurrentDownloads, GBC.eol()); 74 75 add(new JLabel(tr("Maximum elements in disk cache: ")), GBC.std()); 76 add(GBC.glue(5, 0), GBC.std()); 77 add(this.maxElementsOnDisk, GBC.eol()); 78 63 79 } 64 80 65 81 /** 66 82 * Loads the TMS settings. 67 83 */ … … 72 88 this.maxZoomLvl.setValue(TMSLayer.getMaxZoomLvl(null)); 73 89 this.minZoomLvl.setValue(TMSLayer.getMinZoomLvl(null)); 74 90 this.tilecacheDir.setText(TMSLayer.PROP_TILECACHE_DIR.get()); 91 this.maxElementsOnDisk.setValue(TMSCachedTileLoader.MAX_OBJECTS_ON_DISK.get()); 92 this.maxConcurrentDownloads.setValue(TMSCachedTileLoaderJob.THREAD_LIMIT.get()); 75 93 } 76 94 77 95 /** 78 96 * Saves the TMS settings. 79 97 * @return true when restart is required 80 98 */ 81 99 public boolean saveSettings() { 82 100 boolean restartRequired = false; 83 101 84 102 if (TMSLayer.PROP_ADD_TO_SLIPPYMAP_CHOOSER.get() != this.addToSlippyMapChosser.isSelected()) { 85 103 restartRequired = true; 86 104 } … … 89 107 TMSLayer.PROP_DEFAULT_AUTOLOAD.put(this.autoloadTiles.isSelected()); 90 108 TMSLayer.setMaxZoomLvl((Integer)this.maxZoomLvl.getValue()); 91 109 TMSLayer.setMinZoomLvl((Integer)this.minZoomLvl.getValue()); 92 TMSLayer.PROP_TILECACHE_DIR.put(this.tilecacheDir.getText()); 93 110 111 TMSCachedTileLoader.MAX_OBJECTS_ON_DISK.put((Integer) this.maxElementsOnDisk.getValue()); 112 113 if (!TMSCachedTileLoaderJob.THREAD_LIMIT.get().equals(this.maxConcurrentDownloads.getValue())) { 114 restartRequired = true; 115 TMSCachedTileLoaderJob.THREAD_LIMIT.put((Integer) this.maxConcurrentDownloads.getValue()); 116 } 117 118 if (!TMSLayer.PROP_TILECACHE_DIR.get().equals(this.tilecacheDir.getText())) { 119 restartRequired = true; 120 TMSLayer.PROP_TILECACHE_DIR.put(this.tilecacheDir.getText()); 121 } 122 94 123 return restartRequired; 95 124 } 96 125 } -
src/org/openstreetmap/josm/gui/layer/TMSLayer.java
21 21 import java.net.URL; 22 22 import java.util.ArrayList; 23 23 import java.util.Collections; 24 import java.util.Hash Set;24 import java.util.HashMap; 25 25 import java.util.LinkedList; 26 26 import java.util.List; 27 27 import java.util.Map; 28 import java.util.Map.Entry;29 28 import java.util.Scanner; 30 import java.util.Set;31 29 import java.util.concurrent.Callable; 32 30 import java.util.regex.Matcher; 33 31 import java.util.regex.Pattern; … … 43 41 import org.openstreetmap.gui.jmapviewer.Coordinate; 44 42 import org.openstreetmap.gui.jmapviewer.JobDispatcher; 45 43 import org.openstreetmap.gui.jmapviewer.MemoryTileCache; 46 import org.openstreetmap.gui.jmapviewer.OsmFileCacheTileLoader;47 44 import org.openstreetmap.gui.jmapviewer.OsmTileLoader; 48 import org.openstreetmap.gui.jmapviewer.TMSFileCacheTileLoader;49 45 import org.openstreetmap.gui.jmapviewer.Tile; 50 46 import org.openstreetmap.gui.jmapviewer.interfaces.CachedTileLoader; 51 import org.openstreetmap.gui.jmapviewer.interfaces.Tile ClearController;47 import org.openstreetmap.gui.jmapviewer.interfaces.TileLoader; 52 48 import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener; 53 49 import org.openstreetmap.gui.jmapviewer.interfaces.TileSource; 54 50 import org.openstreetmap.gui.jmapviewer.tilesources.BingAerialTileSource; … … 63 59 import org.openstreetmap.josm.data.coor.LatLon; 64 60 import org.openstreetmap.josm.data.imagery.ImageryInfo; 65 61 import org.openstreetmap.josm.data.imagery.ImageryInfo.ImageryType; 62 import org.openstreetmap.josm.data.imagery.TMSCachedTileLoader; 66 63 import org.openstreetmap.josm.data.osm.visitor.BoundingXYVisitor; 67 64 import org.openstreetmap.josm.data.preferences.BooleanProperty; 68 65 import org.openstreetmap.josm.data.preferences.IntegerProperty; … … 75 72 import org.openstreetmap.josm.gui.dialogs.LayerListDialog; 76 73 import org.openstreetmap.josm.gui.dialogs.LayerListPopup; 77 74 import org.openstreetmap.josm.gui.progress.ProgressMonitor; 78 import org.openstreetmap.josm.gui.progress.ProgressMonitor.CancelListener;79 75 import org.openstreetmap.josm.io.CacheCustomContent; 80 76 import org.openstreetmap.josm.io.OsmTransferException; 81 77 import org.openstreetmap.josm.io.UTFInputStreamReader; … … 108 104 public static final IntegerProperty PROP_MAX_ZOOM_LVL = new IntegerProperty(PREFERENCE_PREFIX + ".max_zoom_lvl", DEFAULT_MAX_ZOOM); 109 105 //public static final BooleanProperty PROP_DRAW_DEBUG = new BooleanProperty(PREFERENCE_PREFIX + ".draw_debug", false); 110 106 public static final BooleanProperty PROP_ADD_TO_SLIPPYMAP_CHOOSER = new BooleanProperty(PREFERENCE_PREFIX + ".add_to_slippymap_chooser", true); 111 public static final IntegerProperty PROP_TMS_JOBS = new IntegerProperty("tmsloader.maxjobs", 25);107 //public static final IntegerProperty PROP_TMS_JOBS = new IntegerProperty("tmsloader.maxjobs", 25); 112 108 public static final StringProperty PROP_TILECACHE_DIR; 113 109 114 110 static { … … 122 118 } 123 119 124 120 public interface TileLoaderFactory { 125 OsmTileLoader makeTileLoader(TileLoaderListener listener); 121 TileLoader makeTileLoader(TileLoaderListener listener); 122 TileLoader makeTileLoader(TileLoaderListener listener, Map<String, String> headers); 126 123 } 127 124 125 // MemoryTileCache caches rendered tiles, to reduce latency during map panning 126 // ImageIO.read() takes a lot of time, so we can't use JCS cache 128 127 protected MemoryTileCache tileCache; 129 128 protected TileSource tileSource; 130 protected OsmTileLoader tileLoader; 129 protected TileLoader tileLoader; 130 131 131 132 132 public static TileLoaderFactory loaderFactory = new TileLoaderFactory() { 133 133 @Override 134 public OsmTileLoader makeTileLoader(TileLoaderListener listener) { 135 String cachePath = TMSLayer.PROP_TILECACHE_DIR.get(); 136 if (cachePath != null && !cachePath.isEmpty()) { 137 try { 138 OsmFileCacheTileLoader loader; 139 loader = new TMSFileCacheTileLoader(listener, new File(cachePath)); 140 loader.headers.put("User-Agent", Version.getInstance().getFullAgentString()); 141 return loader; 142 } catch (IOException e) { 143 Main.warn(e); 144 } 134 public TileLoader makeTileLoader(TileLoaderListener listener, Map<String, String> inputHeaders) { 135 Map<String, String> headers = new HashMap<>(); 136 headers.put("User-Agent", Version.getInstance().getFullAgentString()); 137 headers.put("Accept", "text/html, image/png, image/jpeg, image/gif, */*"); 138 if (inputHeaders != null) 139 headers.putAll(inputHeaders); 140 141 try { 142 return new TMSCachedTileLoader(listener, "TMS", 143 Main.pref.getInteger("socket.timeout.connect",15) * 1000, 144 Main.pref.getInteger("socket.timeout.read", 30) * 1000, 145 headers, 146 PROP_TILECACHE_DIR.get()); 147 } catch (IOException e) { 148 Main.warn(e); 145 149 } 146 150 return null; 147 151 } 152 153 @Override 154 public TileLoader makeTileLoader(TileLoaderListener listener) { 155 return makeTileLoader(listener, null); 156 } 148 157 }; 149 158 150 159 /** 151 160 * Plugins that wish to set custom tile loader should call this method 152 161 */ 162 153 163 public static void setCustomTileLoaderFactory(TileLoaderFactory loaderFactory) { 154 164 TMSLayer.loaderFactory = loaderFactory; 155 165 } 156 166 157 private Set<Tile> tileRequestsOutstanding = new HashSet<>();158 159 167 @Override 160 168 public synchronized void tileLoadingFinished(Tile tile, boolean success) { 161 169 if (tile.hasError()) { … … 163 171 tile.setImage(null); 164 172 } 165 173 if (sharpenLevel != 0 && success) { 166 tile.setImage(sharpenImage(tile.getImage()));174 // tile.setImage(sharpenImage(tile.getImage())); 167 175 } 168 tile.setLoaded( true);176 tile.setLoaded(success); 169 177 needRedraw = true; 170 178 if (Main.map != null) { 171 179 Main.map.repaint(100); 172 180 } 173 tileRequestsOutstanding.remove(tile);174 181 if (Main.isDebugEnabled()) { 175 182 Main.debug("tileLoadingFinished() tile: " + tile + " success: " + success); 176 183 } 177 184 } 178 185 179 private static class TmsTileClearController implements TileClearController, CancelListener {180 181 private final ProgressMonitor monitor;182 private boolean cancel = false;183 184 public TmsTileClearController(ProgressMonitor monitor) {185 this.monitor = monitor;186 this.monitor.addCancelListener(this);187 }188 189 @Override190 public void initClearDir(File dir) {191 }192 193 @Override194 public void initClearFiles(File[] files) {195 monitor.setTicksCount(files.length);196 monitor.setTicks(0);197 }198 199 @Override200 public boolean cancel() {201 return cancel;202 }203 204 @Override205 public void fileDeleted(File file) {206 monitor.setTicks(monitor.getTicks()+1);207 }208 209 @Override210 public void clearFinished() {211 monitor.finishTask();212 }213 214 @Override215 public void operationCanceled() {216 cancel = true;217 }218 }219 220 186 /** 221 187 * Clears the tile cache. 222 188 * … … 231 197 void clearTileCache(ProgressMonitor monitor) { 232 198 tileCache.clear(); 233 199 if (tileLoader instanceof CachedTileLoader) { 234 ((CachedTileLoader)tileLoader).clearCache( tileSource, new TmsTileClearController(monitor));200 ((CachedTileLoader)tileLoader).clearCache(); 235 201 } 236 202 } 237 203 … … 415 381 416 382 currentZoomLevel = getBestZoom(); 417 383 418 tileCache = new MemoryTileCache(); 419 420 tileLoader = loaderFactory.makeTileLoader(this); 421 if (tileLoader == null) { 422 tileLoader = new OsmTileLoader(this); 423 } 424 tileLoader.timeoutConnect = Main.pref.getInteger("socket.timeout.connect",15) * 1000; 425 tileLoader.timeoutRead = Main.pref.getInteger("socket.timeout.read", 30) * 1000; 384 Map<String, String> headers = null; 426 385 if (tileSource instanceof TemplatedTMSTileSource) { 427 for(Entry<String, String> e : ((TemplatedTMSTileSource)tileSource).getHeaders().entrySet()) { 428 tileLoader.headers.put(e.getKey(), e.getValue()); 429 } 386 headers = (((TemplatedTMSTileSource)tileSource).getHeaders()); 430 387 } 431 tileLoader.headers.put("User-Agent", Version.getInstance().getFullAgentString()); 388 389 tileCache = new MemoryTileCache(); 390 tileLoader = loaderFactory.makeTileLoader(this, headers); 391 if (tileLoader == null) 392 tileLoader = new OsmTileLoader(this); 432 393 } 433 394 434 395 @Override … … 475 436 * Function to set the maximum number of workers for tile loading to the value defined 476 437 * in preferences. 477 438 */ 478 public static void setMaxWorkers() {479 JobDispatcher.setMaxWorkers(PROP_TMS_JOBS.get());480 JobDispatcher.getInstance().setLIFO(true);481 }482 439 483 440 @SuppressWarnings("serial") 484 441 public TMSLayer(ImageryInfo info) { 485 442 super(info); 486 443 487 setMaxWorkers();488 444 if(!isProjectionSupported(Main.getProjection())) { 489 445 JOptionPane.showMessageDialog(Main.parent, 490 tr("TMS layers do not support the projection {0}.\n{1}\n"491 + "Change the projection or remove the layer.",492 Main.getProjection().toCode(), nameSupportedProjections()),493 tr("Warning"),494 JOptionPane.WARNING_MESSAGE);446 tr("TMS layers do not support the projection {0}.\n{1}\n" 447 + "Change the projection or remove the layer.", 448 Main.getProjection().toCode(), nameSupportedProjections()), 449 tr("Warning"), 450 JOptionPane.WARNING_MESSAGE); 495 451 } 496 452 497 453 setBackgroundLayer(true); … … 629 585 new PleaseWaitRunnable(tr("Flush Tile Cache")) { 630 586 @Override 631 587 protected void realRun() throws SAXException, IOException, 632 588 OsmTransferException { 633 589 clearTileCache(getProgressMonitor()); 634 590 } 635 591 … … 685 641 } 686 642 needRedraw = true; 687 643 JobDispatcher.getInstance().cancelOutstandingJobs(); 688 tileRequestsOutstanding.clear();689 644 } 690 645 691 646 int getMaxZoomLvl() { … … 770 725 * are temporary only and intentionally not inserted 771 726 * into the tileCache. 772 727 */ 773 synchronizedTile tempCornerTile(Tile t) {728 Tile tempCornerTile(Tile t) { 774 729 int x = t.getXtile() + 1; 775 730 int y = t.getYtile() + 1; 776 731 int zoom = t.getZoom(); … … 780 735 return new Tile(tileSource, x, y, zoom); 781 736 } 782 737 783 synchronizedTile getOrCreateTile(int x, int y, int zoom) {738 Tile getOrCreateTile(int x, int y, int zoom) { 784 739 Tile tile = getTile(x, y, zoom); 785 740 if (tile == null) { 786 741 tile = new Tile(tileSource, x, y, zoom); … … 794 749 * This can and will return null for tiles that are not 795 750 * already in the cache. 796 751 */ 797 synchronizedTile getTile(int x, int y, int zoom) {752 Tile getTile(int x, int y, int zoom) { 798 753 int max = (1 << zoom); 799 754 if (x < 0 || x >= max || y < 0 || y >= max) 800 755 return null; 801 756 return tileCache.getTile(tileSource, x, y, zoom); 802 757 } 803 758 804 synchronizedboolean loadTile(Tile tile, boolean force) {759 boolean loadTile(Tile tile, boolean force) { 805 760 if (tile == null) 806 761 return false; 807 if (!force && (tile. hasError() || tile.isLoaded()))762 if (!force && (tile.isLoaded() || tile.hasError())) 808 763 return false; 809 764 if (tile.isLoading()) 810 765 return false; 811 if (tileRequestsOutstanding.contains(tile)) 812 return false; 813 tileRequestsOutstanding.add(tile); 814 JobDispatcher.getInstance().addJob(tileLoader.createTileLoaderJob(tile)); 766 tileLoader.createTileLoaderJob(tile).submit(); 815 767 return true; 816 768 } 817 769 … … 1268 1220 public TileSet getTileSet(int zoom) { 1269 1221 if (zoom < minZoom) 1270 1222 return nullTileSet; 1271 TileSet ts = tileSets[zoom-minZoom]; 1272 if (ts == null) { 1273 ts = new TileSet(topLeft, botRight, zoom); 1274 tileSets[zoom-minZoom] = ts; 1223 synchronized (tileSets) { 1224 TileSet ts = tileSets[zoom-minZoom]; 1225 if (ts == null) { 1226 ts = new TileSet(topLeft, botRight, zoom); 1227 tileSets[zoom-minZoom] = ts; 1228 } 1229 return ts; 1275 1230 } 1276 return ts;1277 1231 } 1232 1278 1233 public TileSetInfo getTileSetInfo(int zoom) { 1279 1234 if (zoom < minZoom) 1280 1235 return new TileSetInfo(); 1281 TileSetInfo tsi = tileSetInfos[zoom-minZoom]; 1282 if (tsi == null) { 1283 tsi = TMSLayer.getTileSetInfo(getTileSet(zoom)); 1284 tileSetInfos[zoom-minZoom] = tsi; 1236 synchronized (tileSetInfos) { 1237 TileSetInfo tsi = tileSetInfos[zoom-minZoom]; 1238 if (tsi == null) { 1239 tsi = TMSLayer.getTileSetInfo(getTileSet(zoom)); 1240 tileSetInfos[zoom-minZoom] = tsi; 1241 } 1242 return tsi; 1285 1243 } 1286 return tsi;1287 1244 } 1288 1245 } 1289 1246 1290 1247 @Override 1291 1248 public void paint(Graphics2D g, MapView mv, Bounds bounds) { 1292 //long start = System.currentTimeMillis();1293 1249 EastNorth topLeft = mv.getEastNorth(0, 0); 1294 1250 EastNorth botRight = mv.getEastNorth(mv.getWidth(), mv.getHeight()); 1295 1251 -
src/org/openstreetmap/josm/gui/bbox/SlippyMapBBoxChooser.java
11 11 import java.util.ArrayList; 12 12 import java.util.Arrays; 13 13 import java.util.Collections; 14 import java.util.HashMap; 14 15 import java.util.HashSet; 15 16 import java.util.List; 17 import java.util.Map; 16 18 import java.util.Set; 17 19 import java.util.concurrent.CopyOnWriteArrayList; 18 20 … … 25 27 import org.openstreetmap.gui.jmapviewer.MemoryTileCache; 26 28 import org.openstreetmap.gui.jmapviewer.OsmTileLoader; 27 29 import org.openstreetmap.gui.jmapviewer.interfaces.MapMarker; 30 import org.openstreetmap.gui.jmapviewer.interfaces.TileLoader; 28 31 import org.openstreetmap.gui.jmapviewer.interfaces.TileSource; 29 32 import org.openstreetmap.gui.jmapviewer.tilesources.MapQuestOpenAerialTileSource; 30 33 import org.openstreetmap.gui.jmapviewer.tilesources.MapQuestOsmTileSource; … … 113 116 private static final StringProperty PROP_MAPSTYLE = new StringProperty("slippy_map_chooser.mapstyle", "Mapnik"); 114 117 public static final String RESIZE_PROP = SlippyMapBBoxChooser.class.getName() + ".resize"; 115 118 116 private OsmTileLoader cachedLoader;119 private TileLoader cachedLoader; 117 120 private OsmTileLoader uncachedLoader; 118 121 119 122 private final SizeButton iSizeButton; … … 131 134 debug = Main.isDebugEnabled(); 132 135 SpringLayout springLayout = new SpringLayout(); 133 136 setLayout(springLayout); 134 TMSLayer.setMaxWorkers(); 135 cachedLoader = TMSLayer.loaderFactory.makeTileLoader(this); 137 138 Map<String, String> headers = new HashMap<>(); 139 headers.put("User-Agent", Version.getInstance().getFullAgentString()); 140 141 cachedLoader = TMSLayer.loaderFactory.makeTileLoader(this, headers); 136 142 137 143 uncachedLoader = new OsmTileLoader(this); 138 uncachedLoader.headers.put ("User-Agent", Version.getInstance().getFullAgentString());144 uncachedLoader.headers.putAll(headers); 139 145 setZoomContolsVisible(Main.pref.getBoolean("slippy_map_chooser.zoomcontrols",false)); 140 146 setMapMarkerVisible(false); 141 147 setMinimumSize(new Dimension(350, 350 / 2)); -
src/org/openstreetmap/gui/jmapviewer/TMSFileCacheTileLoader.java
1 // License: GPL. For details, see Readme.txt file.2 package org.openstreetmap.gui.jmapviewer;3 4 import java.io.File;5 import java.io.IOException;6 import org.openstreetmap.gui.jmapviewer.interfaces.TileJob;7 import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener;8 import org.openstreetmap.gui.jmapviewer.interfaces.TileSource;9 10 /**11 * Reworked version of the OsmFileCacheTileLoader.12 *13 * When class OsmFileCacheTileLoader is no longer needed, it can be integrated14 * here and removed.15 */16 public class TMSFileCacheTileLoader extends OsmFileCacheTileLoader {17 18 public TMSFileCacheTileLoader(TileLoaderListener map, File cacheDir) throws IOException {19 super(map, cacheDir);20 }21 22 @Override23 public TileJob createTileLoaderJob(final Tile tile) {24 return new TMSFileLoadJob(tile);25 }26 27 protected class TMSFileLoadJob extends FileLoadJob {28 29 public TMSFileLoadJob(Tile tile) {30 super(tile);31 }32 33 @Override34 protected File getTileFile() {35 return getDataFile(tile.getSource().getTileType());36 }37 38 @Override39 protected File getTagsFile() {40 return getDataFile(TAGS_FILE_EXT);41 }42 43 protected File getDataFile(String ext) {44 int nDigits = (int) Math.ceil(Math.log10(1 << tile.getZoom()));45 String x = String.format("%0" + nDigits + "d", tile.getXtile());46 String y = String.format("%0" + nDigits + "d", tile.getYtile());47 File path = new File(tileCacheDir, "z" + tile.getZoom());48 for (int i=0; i<nDigits; i++) {49 String component = "x" + x.substring(i, i+1) + "y" + y.substring(i, i+1);50 if (i == nDigits -1 ) {51 component += "." + ext;52 }53 path = new File(path, component);54 }55 return path;56 }57 }58 59 @Override60 protected File getSourceCacheDir(TileSource source) {61 File dir = sourceCacheDirMap.get(source);62 if (dir == null) {63 String id = source.getId();64 if (id != null) {65 dir = new File(cacheDirBase, id);66 } else {67 dir = new File(cacheDirBase, source.getName().replaceAll("[\\\\/:*?\"<>|]", "_"));68 }69 if (!dir.exists()) {70 dir.mkdirs();71 }72 }73 return dir;74 }75 76 } -
src/org/openstreetmap/gui/jmapviewer/OsmFileCacheTileLoader.java
1 // License: GPL. For details, see Readme.txt file.2 package org.openstreetmap.gui.jmapviewer;3 4 import java.io.BufferedReader;5 import java.io.ByteArrayInputStream;6 import java.io.ByteArrayOutputStream;7 import java.io.File;8 import java.io.FileInputStream;9 import java.io.FileNotFoundException;10 import java.io.FileOutputStream;11 import java.io.IOException;12 import java.io.InputStream;13 import java.io.InputStreamReader;14 import java.io.OutputStreamWriter;15 import java.io.PrintWriter;16 import java.net.HttpURLConnection;17 import java.net.URL;18 import java.net.URLConnection;19 import java.nio.charset.Charset;20 import java.util.HashMap;21 import java.util.Map;22 import java.util.Map.Entry;23 import java.util.Random;24 import java.util.logging.Level;25 import java.util.logging.Logger;26 27 import org.openstreetmap.gui.jmapviewer.interfaces.CachedTileLoader;28 import org.openstreetmap.gui.jmapviewer.interfaces.TileClearController;29 import org.openstreetmap.gui.jmapviewer.interfaces.TileJob;30 import org.openstreetmap.gui.jmapviewer.interfaces.TileLoader;31 import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener;32 import org.openstreetmap.gui.jmapviewer.interfaces.TileSource;33 import org.openstreetmap.gui.jmapviewer.interfaces.TileSource.TileUpdate;34 35 /**36 * A {@link TileLoader} implementation that loads tiles from OSM via HTTP and37 * saves all loaded files in a directory located in the temporary directory.38 * If a tile is present in this file cache it will not be loaded from OSM again.39 *40 * @author Jan Peter Stotz41 * @author Stefan Zeller42 */43 public class OsmFileCacheTileLoader extends OsmTileLoader implements CachedTileLoader {44 45 private static final Logger log = FeatureAdapter.getLogger(OsmFileCacheTileLoader.class.getName());46 47 protected static final String TAGS_FILE_EXT = "tags";48 49 private static final Charset TAGS_CHARSET = Charset.forName("UTF-8");50 51 // Default expire time (i.e. maximum age of cached tile before refresh).52 // Used when the server does not send an expires or max-age value in the http header.53 protected static final long DEFAULT_EXPIRE_TIME = 1000L * 60 * 60 * 24 * 7; // 7 days54 // Limit for the max-age value send by the server.55 protected static final long EXPIRE_TIME_SERVER_LIMIT = 1000L * 60 * 60 * 24 * 28; // 4 weeks56 // Absolute expire time limit. Cached tiles that are older will not be used,57 // even if the refresh from the server fails.58 protected static final long ABSOLUTE_EXPIRE_TIME_LIMIT = Long.MAX_VALUE; // unlimited59 60 protected String cacheDirBase;61 62 protected final Map<TileSource, File> sourceCacheDirMap;63 64 65 public static File getDefaultCacheDir() throws SecurityException {66 String tempDir = null;67 String userName = System.getProperty("user.name");68 try {69 tempDir = System.getProperty("java.io.tmpdir");70 } catch (SecurityException e) {71 log.log(Level.WARNING,72 "Failed to access system property ''java.io.tmpdir'' for security reasons. Exception was: "73 + e.toString());74 throw e; // rethrow75 }76 try {77 if (tempDir == null)78 throw new IOException("No temp directory set");79 String subDirName = "JMapViewerTiles";80 // On Linux/Unix systems we do not have a per user tmp directory.81 // Therefore we add the user name for getting a unique dir name.82 if (userName != null && userName.length() > 0) {83 subDirName += "_" + userName;84 }85 File cacheDir = new File(tempDir, subDirName);86 return cacheDir;87 } catch (Exception e) {88 }89 return null;90 }91 92 /**93 * Create a OSMFileCacheTileLoader with given cache directory.94 * If cacheDir is not set or invalid, IOException will be thrown.95 * @param map the listener checking for tile load events (usually the map for display)96 * @param cacheDir directory to store cached tiles97 */98 public OsmFileCacheTileLoader(TileLoaderListener map, File cacheDir) throws IOException {99 super(map);100 if (cacheDir == null || (!cacheDir.exists() && !cacheDir.mkdirs()))101 throw new IOException("Cannot access cache directory");102 103 log.finest("Tile cache directory: " + cacheDir);104 cacheDirBase = cacheDir.getAbsolutePath();105 sourceCacheDirMap = new HashMap<>();106 }107 108 /**109 * Create a OSMFileCacheTileLoader with system property temp dir.110 * If not set an IOException will be thrown.111 * @param map the listener checking for tile load events (usually the map for display)112 */113 public OsmFileCacheTileLoader(TileLoaderListener map) throws SecurityException, IOException {114 this(map, getDefaultCacheDir());115 }116 117 @Override118 public TileJob createTileLoaderJob(final Tile tile) {119 return new FileLoadJob(tile);120 }121 122 protected File getSourceCacheDir(TileSource source) {123 File dir = sourceCacheDirMap.get(source);124 if (dir == null) {125 dir = new File(cacheDirBase, source.getName().replaceAll("[\\\\/:*?\"<>|]", "_"));126 if (!dir.exists()) {127 dir.mkdirs();128 }129 }130 return dir;131 }132 133 protected class FileLoadJob implements TileJob {134 InputStream input = null;135 136 Tile tile;137 File tileCacheDir;138 File tileFile = null;139 File tagsFile = null;140 Long fileMtime = null;141 Long now = null; // current time in milliseconds (keep consistent value for the whole run)142 143 public FileLoadJob(Tile tile) {144 this.tile = tile;145 }146 147 @Override148 public Tile getTile() {149 return tile;150 }151 152 @Override153 public void run() {154 synchronized (tile) {155 if ((tile.isLoaded() && !tile.hasError()) || tile.isLoading())156 return;157 tile.loaded = false;158 tile.error = false;159 tile.loading = true;160 }161 now = System.currentTimeMillis();162 tileCacheDir = getSourceCacheDir(tile.getSource());163 tileFile = getTileFile();164 tagsFile = getTagsFile();165 166 loadTagsFromFile();167 168 if (isCacheValid() && (isNoTileAtZoom() || loadTileFromFile())) {169 log.log(Level.FINE, "TMS - found in tile cache: {0}", tile);170 tile.setLoaded(true);171 listener.tileLoadingFinished(tile, true);172 return;173 }174 175 TileJob job = new TileJob() {176 177 @Override178 public void run() {179 if (loadOrUpdateTile()) {180 tile.setLoaded(true);181 listener.tileLoadingFinished(tile, true);182 } else {183 // failed to download - use old cache file if available184 if (isNoTileAtZoom() || loadTileFromFile()) {185 tile.setLoaded(true);186 tile.error = false;187 listener.tileLoadingFinished(tile, true);188 log.log(Level.FINE, "TMS - found stale tile in cache: {0}", tile);189 } else {190 // failed completely191 tile.setLoaded(true);192 listener.tileLoadingFinished(tile, false);193 }194 }195 }196 @Override197 public Tile getTile() {198 return tile;199 }200 };201 JobDispatcher.getInstance().addJob(job);202 }203 204 protected boolean loadOrUpdateTile() {205 try {206 URLConnection urlConn = loadTileFromOsm(tile);207 if (fileMtime != null && now - fileMtime <= ABSOLUTE_EXPIRE_TIME_LIMIT) {208 switch (tile.getSource().getTileUpdate()) {209 case IfModifiedSince:210 urlConn.setIfModifiedSince(fileMtime);211 break;212 case LastModified:213 if (!isOsmTileNewer(fileMtime)) {214 log.log(Level.FINE, "TMS - LastModified test: local version is up to date: {0}", tile);215 tileFile.setLastModified(now);216 return true;217 }218 break;219 default:220 break;221 }222 }223 if (tile.getSource().getTileUpdate() == TileUpdate.ETag || tile.getSource().getTileUpdate() == TileUpdate.IfNoneMatch) {224 String fileETag = tile.getValue("etag");225 if (fileETag != null) {226 switch (tile.getSource().getTileUpdate()) {227 case IfNoneMatch:228 urlConn.addRequestProperty("If-None-Match", fileETag);229 break;230 case ETag:231 if (hasOsmTileETag(fileETag)) {232 log.log(Level.FINE, "TMS - ETag test: local version is up to date: {0}", tile);233 tileFile.setLastModified(now);234 return true;235 }236 default:237 break;238 }239 }240 tile.putValue("etag", urlConn.getHeaderField("ETag"));241 }242 if (urlConn instanceof HttpURLConnection && ((HttpURLConnection)urlConn).getResponseCode() == 304) {243 // If isModifiedSince or If-None-Match has been set244 // and the server answers with a HTTP 304 = "Not Modified"245 switch (tile.getSource().getTileUpdate()) {246 case IfModifiedSince:247 log.log(Level.FINE, "TMS - IfModifiedSince test: local version is up to date: {0}", tile);248 break;249 case IfNoneMatch:250 log.log(Level.FINE, "TMS - IfNoneMatch test: local version is up to date: {0}", tile);251 break;252 default:253 break;254 }255 loadTileFromFile();256 tileFile.setLastModified(now);257 return true;258 }259 260 loadTileMetadata(tile, urlConn);261 saveTagsToFile();262 263 if ("no-tile".equals(tile.getValue("tile-info")))264 {265 log.log(Level.FINE, "TMS - No tile: tile-info=no-tile: {0}", tile);266 tile.setError("No tile at this zoom level");267 return true;268 } else {269 for (int i = 0; i < 5; ++i) {270 if (urlConn instanceof HttpURLConnection && ((HttpURLConnection)urlConn).getResponseCode() == 503) {271 Thread.sleep(5000+(new Random()).nextInt(5000));272 continue;273 }274 byte[] buffer = loadTileInBuffer(urlConn);275 if (buffer != null) {276 tile.loadImage(new ByteArrayInputStream(buffer));277 saveTileToFile(buffer);278 log.log(Level.FINE, "TMS - downloaded tile from server: {0}", tile.getUrl());279 return true;280 }281 }282 }283 } catch (Exception e) {284 tile.setError(e.getMessage());285 if (input == null) {286 try {287 log.log(Level.WARNING, "TMS - Failed downloading {0}: {1}", new Object[]{tile.getUrl(), e.getMessage()});288 return false;289 } catch(IOException i) {290 }291 }292 }293 log.log(Level.WARNING, "TMS - Failed downloading tile: {0}", tile);294 return false;295 }296 297 protected boolean isCacheValid() {298 Long expires = null;299 if (tileFile.exists()) {300 fileMtime = tileFile.lastModified();301 } else if (tagsFile.exists()) {302 fileMtime = tagsFile.lastModified();303 } else304 return false;305 306 try {307 expires = Long.parseLong(tile.getValue("expires"));308 } catch (NumberFormatException e) {}309 310 // check by expire date set by server311 if (expires != null && !expires.equals(0L)) {312 // put a limit to the expire time (some servers send a value313 // that is too large)314 expires = Math.min(expires, fileMtime + EXPIRE_TIME_SERVER_LIMIT);315 if (now > expires) {316 log.log(Level.FINE, "TMS - Tile has expired -> not valid {0}", tile);317 return false;318 }319 } else {320 // check by file modification date321 if (now - fileMtime > DEFAULT_EXPIRE_TIME) {322 log.log(Level.FINE, "TMS - Tile has expired, maximum file age reached {0}", tile);323 return false;324 }325 }326 return true;327 }328 329 protected boolean isNoTileAtZoom() {330 if ("no-tile".equals(tile.getValue("tile-info"))) {331 // do not remove file - keep the information, that there is no tile, for further requests332 // the code above will check, if this information is still valid333 log.log(Level.FINE, "TMS - Tile valid, but no file, as no tiles at this level {0}", tile);334 tile.setError("No tile at this zoom level");335 return true;336 }337 return false;338 }339 340 protected boolean loadTileFromFile() {341 if (!tileFile.exists())342 return false;343 344 try (FileInputStream fin = new FileInputStream(tileFile)) {345 if (fin.available() == 0)346 throw new IOException("File empty");347 tile.loadImage(fin);348 return true;349 } catch (Exception e) {350 log.log(Level.WARNING, "TMS - Error while loading image from tile cache: {0}; {1}", new Object[]{e.getMessage(), tile});351 tileFile.delete();352 if (tagsFile.exists()) {353 tagsFile.delete();354 }355 tileFile = null;356 fileMtime = null;357 }358 return false;359 }360 361 protected byte[] loadTileInBuffer(URLConnection urlConn) throws IOException {362 input = urlConn.getInputStream();363 try {364 ByteArrayOutputStream bout = new ByteArrayOutputStream(input.available());365 byte[] buffer = new byte[2048];366 boolean finished = false;367 do {368 int read = input.read(buffer);369 if (read >= 0) {370 bout.write(buffer, 0, read);371 } else {372 finished = true;373 }374 } while (!finished);375 if (bout.size() == 0)376 return null;377 return bout.toByteArray();378 } finally {379 input.close();380 input = null;381 }382 }383 384 /**385 * Performs a <code>HEAD</code> request for retrieving the386 * <code>LastModified</code> header value.387 *388 * Note: This does only work with servers providing the389 * <code>LastModified</code> header:390 * <ul>391 * <li>{@link org.openstreetmap.gui.jmapviewer.tilesources.OsmTileSource.CycleMap} - supported</li>392 * <li>{@link org.openstreetmap.gui.jmapviewer.tilesources.OsmTileSource.Mapnik} - not supported</li>393 * </ul>394 *395 * @param fileAge time of the396 * @return <code>true</code> if the tile on the server is newer than the397 * file398 * @throws IOException399 */400 protected boolean isOsmTileNewer(long fileAge) throws IOException {401 URL url;402 url = new URL(tile.getUrl());403 HttpURLConnection urlConn = (HttpURLConnection) url.openConnection();404 prepareHttpUrlConnection(urlConn);405 urlConn.setRequestMethod("HEAD");406 urlConn.setReadTimeout(30000); // 30 seconds read timeout407 // System.out.println("Tile age: " + new408 // Date(urlConn.getLastModified()) + " / "409 // + new Date(fileMtime));410 long lastModified = urlConn.getLastModified();411 if (lastModified == 0)412 return true; // no LastModified time returned413 return (lastModified > fileAge);414 }415 416 protected boolean hasOsmTileETag(String eTag) throws IOException {417 URL url;418 url = new URL(tile.getUrl());419 HttpURLConnection urlConn = (HttpURLConnection) url.openConnection();420 prepareHttpUrlConnection(urlConn);421 urlConn.setRequestMethod("HEAD");422 urlConn.setReadTimeout(30000); // 30 seconds read timeout423 // System.out.println("Tile age: " + new424 // Date(urlConn.getLastModified()) + " / "425 // + new Date(fileMtime));426 String osmETag = urlConn.getHeaderField("ETag");427 if (osmETag == null)428 return true;429 return (osmETag.equals(eTag));430 }431 432 protected File getTileFile() {433 return new File(tileCacheDir + "/" + tile.getZoom() + "_" + tile.getXtile() + "_" + tile.getYtile() + "."434 + tile.getSource().getTileType());435 }436 437 protected File getTagsFile() {438 return new File(tileCacheDir + "/" + tile.getZoom() + "_" + tile.getXtile() + "_" + tile.getYtile() + "."439 + TAGS_FILE_EXT);440 }441 442 protected void saveTileToFile(byte[] rawData) {443 File file = getTileFile();444 file.getParentFile().mkdirs();445 try (FileOutputStream f = new FileOutputStream(file)) {446 f.write(rawData);447 } catch (Exception e) {448 log.log(Level.SEVERE, "Failed to save tile content: {0}", e.getLocalizedMessage());449 }450 }451 452 protected void saveTagsToFile() {453 File tagsFile = getTagsFile();454 tagsFile.getParentFile().mkdirs();455 if (tile.getMetadata() == null) {456 tagsFile.delete();457 return;458 }459 try (PrintWriter f = new PrintWriter(new OutputStreamWriter(new FileOutputStream(tagsFile), TAGS_CHARSET))) {460 for (Entry<String, String> entry : tile.getMetadata().entrySet()) {461 f.println(entry.getKey() + "=" + entry.getValue());462 }463 } catch (Exception e) {464 System.err.println("Failed to save tile tags: " + e.getLocalizedMessage());465 }466 }467 468 protected boolean loadTagsFromFile() {469 File tagsFile = getTagsFile();470 try (BufferedReader f = new BufferedReader(new InputStreamReader(new FileInputStream(tagsFile), TAGS_CHARSET))) {471 for (String line = f.readLine(); line != null; line = f.readLine()) {472 final int i = line.indexOf('=');473 if (i == -1 || i == 0) {474 System.err.println("Malformed tile tag in file '" + tagsFile.getName() + "':" + line);475 continue;476 }477 tile.putValue(line.substring(0,i),line.substring(i+1));478 }479 } catch (FileNotFoundException e) {480 } catch (Exception e) {481 System.err.println("Failed to load tile tags: " + e.getLocalizedMessage());482 }483 484 return true;485 }486 }487 488 public String getCacheDirBase() {489 return cacheDirBase;490 }491 492 public void setTileCacheDir(String tileCacheDir) {493 File dir = new File(tileCacheDir);494 dir.mkdirs();495 this.cacheDirBase = dir.getAbsolutePath();496 }497 498 @Override499 public void clearCache(TileSource source) {500 clearCache(source, null);501 }502 503 @Override504 public void clearCache(TileSource source, TileClearController controller) {505 File dir = getSourceCacheDir(source);506 if (dir != null) {507 if (controller != null) controller.initClearDir(dir);508 if (dir.isDirectory()) {509 File[] files = dir.listFiles();510 if (controller != null) controller.initClearFiles(files);511 for (File file : files) {512 if (controller != null && controller.cancel()) return;513 file.delete();514 if (controller != null) controller.fileDeleted(file);515 }516 }517 dir.delete();518 }519 if (controller != null) controller.clearFinished();520 }521 } -
src/org/openstreetmap/gui/jmapviewer/interfaces/TileJob.java
17 17 * @return {@link Tile} to be handled 18 18 */ 19 19 public Tile getTile(); 20 21 /** 22 * submits download job to backend. 23 */ 24 void submit(); 20 25 } -
src/org/openstreetmap/gui/jmapviewer/interfaces/CachedTileLoader.java
5 5 * Interface that allow cleaning the tile cache without specifying exact type of loader 6 6 */ 7 7 public interface CachedTileLoader { 8 public void clearCache(TileSource source); 9 public void clearCache(TileSource source, TileClearController controller); 8 public void clearCache(); 10 9 } -
src/org/openstreetmap/gui/jmapviewer/Demo.java
10 10 import java.awt.event.ItemListener; 11 11 import java.awt.event.MouseAdapter; 12 12 import java.awt.event.MouseEvent; 13 import java.io.IOException;14 13 15 14 import javax.swing.JButton; 16 15 import javax.swing.JCheckBox; … … 101 100 } 102 101 }); 103 102 JComboBox<TileLoader> tileLoaderSelector; 104 try { 105 tileLoaderSelector = new JComboBox<>(new TileLoader[] { new OsmFileCacheTileLoader(map()), new OsmTileLoader(map()) }); 106 } catch (IOException e) { 107 tileLoaderSelector = new JComboBox<>(new TileLoader[] { new OsmTileLoader(map()) }); 108 } 103 tileLoaderSelector = new JComboBox<>(new TileLoader[] { new OsmTileLoader(map()) }); 109 104 tileLoaderSelector.addItemListener(new ItemListener() { 110 105 public void itemStateChanged(ItemEvent e) { 111 106 map().setTileLoader((TileLoader) e.getItem()); -
src/org/openstreetmap/gui/jmapviewer/TileController.java
45 45 tile.loadPlaceholderFromCache(tileCache); 46 46 } 47 47 if (!tile.isLoaded()) { 48 jobDispatcher.addJob(tileLoader.createTileLoaderJob(tile));48 tileLoader.createTileLoaderJob(tile).submit(); 49 49 } 50 50 return tile; 51 51 } -
src/org/openstreetmap/gui/jmapviewer/JMapViewer.java
985 985 } 986 986 987 987 public void tileLoadingFinished(Tile tile, boolean success) { 988 tile.setLoaded(success); 988 989 repaint(); 989 990 } 990 991 -
src/org/openstreetmap/gui/jmapviewer/OsmTileLoader.java
85 85 public Tile getTile() { 86 86 return tile; 87 87 } 88 89 @Override 90 public void submit() { 91 run(); 92 93 } 88 94 }; 89 95 } 90 96