source: josm/trunk/src/org/openstreetmap/josm/data/cache/JCSCachedTileLoaderJob.java@ 9228

Last change on this file since 9228 was 9228, checked in by wiktorn, 8 years ago

Do not use cache when downloading.

This avoids getting objects from Java Web Start Cache, though it will
also skip any tile cache in between, which may result in bigger load of
tile servers.

See #12235
  • Property svn:eol-style set to native
File size: 22.0 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.cache;
3
4import java.io.FileNotFoundException;
5import java.io.IOException;
6import java.net.HttpURLConnection;
7import java.net.URL;
8import java.net.URLConnection;
9import java.util.HashSet;
10import java.util.List;
11import java.util.Map;
12import java.util.Map.Entry;
13import java.util.Random;
14import java.util.Set;
15import java.util.concurrent.ConcurrentHashMap;
16import java.util.concurrent.ConcurrentMap;
17import java.util.concurrent.LinkedBlockingDeque;
18import java.util.concurrent.ThreadPoolExecutor;
19import java.util.concurrent.TimeUnit;
20import java.util.logging.Level;
21import java.util.logging.Logger;
22
23import org.apache.commons.jcs.access.behavior.ICacheAccess;
24import org.apache.commons.jcs.engine.behavior.ICacheElement;
25import org.openstreetmap.gui.jmapviewer.FeatureAdapter;
26import org.openstreetmap.josm.Main;
27import org.openstreetmap.josm.data.cache.ICachedLoaderListener.LoadResult;
28import org.openstreetmap.josm.data.preferences.IntegerProperty;
29import org.openstreetmap.josm.tools.Utils;
30
31/**
32 * @author Wiktor Niesiobędzki
33 *
34 * @param <K> cache entry key type
35 * @param <V> cache value type
36 *
37 * Generic loader for HTTP based tiles. Uses custom attribute, to check, if entry has expired
38 * according to HTTP headers sent with tile. If so, it tries to verify using Etags
39 * or If-Modified-Since / Last-Modified.
40 *
41 * If the tile is not valid, it will try to download it from remote service and put it
42 * to cache. If remote server will fail it will try to use stale entry.
43 *
44 * This class will keep only one Job running for specified tile. All others will just finish, but
45 * listeners will be gathered and notified, once download job will be finished
46 *
47 * @since 8168
48 */
49public abstract class JCSCachedTileLoaderJob<K, V extends CacheEntry> implements ICachedLoaderJob<K>, Runnable {
50 private static final Logger log = FeatureAdapter.getLogger(JCSCachedTileLoaderJob.class.getCanonicalName());
51 protected static final long DEFAULT_EXPIRE_TIME = 1000L * 60 * 60 * 24 * 7; // 7 days
52 // Limit for the max-age value send by the server.
53 protected static final long EXPIRE_TIME_SERVER_LIMIT = 1000L * 60 * 60 * 24 * 28; // 4 weeks
54 // Absolute expire time limit. Cached tiles that are older will not be used,
55 // even if the refresh from the server fails.
56 protected static final long ABSOLUTE_EXPIRE_TIME_LIMIT = 1000L * 60 * 60 * 24 * 365; // 1 year
57
58 /**
59 * maximum download threads that will be started
60 */
61 public static final IntegerProperty THREAD_LIMIT = new IntegerProperty("cache.jcs.max_threads", 10);
62
63 /*
64 * ThreadPoolExecutor starts new threads, until THREAD_LIMIT is reached. Then it puts tasks into LinkedBlockingDeque.
65 *
66 * The queue works FIFO, so one needs to take care about ordering of the entries submitted
67 *
68 * There is no point in canceling tasks, that are already taken by worker threads (if we made so much effort, we can at least cache
69 * the response, so later it could be used). We could actually cancel what is in LIFOQueue, but this is a tradeoff between simplicity
70 * and performance (we do want to have something to offer to worker threads before tasks will be resubmitted by class consumer)
71 */
72
73 private static ThreadPoolExecutor DEFAULT_DOWNLOAD_JOB_DISPATCHER = new ThreadPoolExecutor(
74 2, // we have a small queue, so threads will be quickly started (threads are started only, when queue is full)
75 THREAD_LIMIT.get().intValue(), // do not this number of threads
76 30, // keepalive for thread
77 TimeUnit.SECONDS,
78 // make queue of LIFO type - so recently requested tiles will be loaded first (assuming that these are which user is waiting to see)
79 new LinkedBlockingDeque<Runnable>(),
80 Utils.newThreadFactory("JCS-downloader-%d", Thread.NORM_PRIORITY)
81 );
82
83
84
85 private static ConcurrentMap<String, Set<ICachedLoaderListener>> inProgress = new ConcurrentHashMap<>();
86 private static ConcurrentMap<String, Boolean> useHead = new ConcurrentHashMap<>();
87
88 protected long now; // when the job started
89
90 private ICacheAccess<K, V> cache;
91 private ICacheElement<K, V> cacheElement;
92 protected V cacheData;
93 protected CacheEntryAttributes attributes;
94
95 // HTTP connection parameters
96 private int connectTimeout;
97 private int readTimeout;
98 private Map<String, String> headers;
99 private ThreadPoolExecutor downloadJobExecutor;
100 private Runnable finishTask;
101 private boolean force;
102
103 /**
104 * @param cache cache instance that we will work on
105 * @param headers HTTP headers to be sent together with request
106 * @param readTimeout when connecting to remote resource
107 * @param connectTimeout when connecting to remote resource
108 * @param downloadJobExecutor that will be executing the jobs
109 */
110 public JCSCachedTileLoaderJob(ICacheAccess<K, V> cache,
111 int connectTimeout, int readTimeout,
112 Map<String, String> headers,
113 ThreadPoolExecutor downloadJobExecutor) {
114
115 this.cache = cache;
116 this.now = System.currentTimeMillis();
117 this.connectTimeout = connectTimeout;
118 this.readTimeout = readTimeout;
119 this.headers = headers;
120 this.downloadJobExecutor = downloadJobExecutor;
121 }
122
123 /**
124 * @param cache cache instance that we will work on
125 * @param headers HTTP headers to be sent together with request
126 * @param readTimeout when connecting to remote resource
127 * @param connectTimeout when connecting to remote resource
128 */
129 public JCSCachedTileLoaderJob(ICacheAccess<K, V> cache,
130 int connectTimeout, int readTimeout,
131 Map<String, String> headers) {
132 this(cache, connectTimeout, readTimeout,
133 headers, DEFAULT_DOWNLOAD_JOB_DISPATCHER);
134 }
135
136 private void ensureCacheElement() {
137 if (cacheElement == null && getCacheKey() != null) {
138 cacheElement = cache.getCacheElement(getCacheKey());
139 if (cacheElement != null) {
140 attributes = (CacheEntryAttributes) cacheElement.getElementAttributes();
141 cacheData = cacheElement.getVal();
142 }
143 }
144 }
145
146 public V get() {
147 ensureCacheElement();
148 return cacheData;
149 }
150
151 @Override
152 public void submit(ICachedLoaderListener listener, boolean force) throws IOException {
153 this.force = force;
154 boolean first = false;
155 URL url = getUrl();
156 String deduplicationKey = null;
157 if (url != null) {
158 // url might be null, for example when Bing Attribution is not loaded yet
159 deduplicationKey = url.toString();
160 }
161 if (deduplicationKey == null) {
162 log.log(Level.WARNING, "No url returned for: {0}, skipping", getCacheKey());
163 throw new IllegalArgumentException("No url returned");
164 }
165 synchronized (inProgress) {
166 Set<ICachedLoaderListener> newListeners = inProgress.get(deduplicationKey);
167 if (newListeners == null) {
168 newListeners = new HashSet<>();
169 inProgress.put(deduplicationKey, newListeners);
170 first = true;
171 }
172 newListeners.add(listener);
173 }
174
175 if (first || force) {
176 // submit all jobs to separate thread, so calling thread is not blocked with IO when loading from disk
177 log.log(Level.FINE, "JCS - Submitting job for execution for url: {0}", getUrlNoException());
178 downloadJobExecutor.execute(this);
179 }
180 }
181
182 /**
183 * This method is run when job has finished
184 */
185 protected void executionFinished() {
186 if (finishTask != null) {
187 finishTask.run();
188 }
189 }
190
191 /**
192 *
193 * @return checks if object from cache has sufficient data to be returned
194 */
195 protected boolean isObjectLoadable() {
196 if (cacheData == null) {
197 return false;
198 }
199 byte[] content = cacheData.getContent();
200 return content != null && content.length > 0;
201 }
202
203 /**
204 * Simple implementation. All errors should be cached as empty. Though some JDK (JDK8 on Windows for example)
205 * doesn't return 4xx error codes, instead they do throw an FileNotFoundException or IOException
206 *
207 * @return true if we should put empty object into cache, regardless of what remote resource has returned
208 */
209 protected boolean cacheAsEmpty() {
210 return attributes.getResponseCode() < 500;
211 }
212
213 /**
214 * @return key under which discovered server settings will be kept
215 */
216 protected String getServerKey() {
217 return getUrlNoException().getHost();
218 }
219
220 @Override
221 public void run() {
222 final Thread currentThread = Thread.currentThread();
223 final String oldName = currentThread.getName();
224 currentThread.setName("JCS Downloading: " + getUrlNoException());
225 log.log(Level.FINE, "JCS - starting fetch of url: {0} ", getUrlNoException());
226 ensureCacheElement();
227 try {
228 // try to fetch from cache
229 if (!force && cacheElement != null && isCacheElementValid() && isObjectLoadable()) {
230 // we got something in cache, and it's valid, so lets return it
231 log.log(Level.FINE, "JCS - Returning object from cache: {0}", getCacheKey());
232 finishLoading(LoadResult.SUCCESS);
233 return;
234 }
235
236 // try to load object from remote resource
237 if (loadObject()) {
238 finishLoading(LoadResult.SUCCESS);
239 } else {
240 // if loading failed - check if we can return stale entry
241 if (isObjectLoadable()) {
242 // try to get stale entry in cache
243 finishLoading(LoadResult.SUCCESS);
244 log.log(Level.FINE, "JCS - found stale object in cache: {0}", getUrlNoException());
245 } else {
246 // failed completely
247 finishLoading(LoadResult.FAILURE);
248 }
249 }
250 } finally {
251 executionFinished();
252 currentThread.setName(oldName);
253 }
254 }
255
256 private void finishLoading(LoadResult result) {
257 Set<ICachedLoaderListener> listeners = null;
258 synchronized (inProgress) {
259 listeners = inProgress.remove(getUrlNoException().toString());
260 }
261 if (listeners == null) {
262 log.log(Level.WARNING, "Listener not found for URL: {0}. Listener not notified!", getUrlNoException());
263 return;
264 }
265 for (ICachedLoaderListener l: listeners) {
266 l.loadingFinished(cacheData, attributes, result);
267 }
268 }
269
270 protected boolean isCacheElementValid() {
271 long expires = attributes.getExpirationTime();
272
273 // check by expire date set by server
274 if (expires != 0L) {
275 // put a limit to the expire time (some servers send a value
276 // that is too large)
277 expires = Math.min(expires, attributes.getCreateTime() + EXPIRE_TIME_SERVER_LIMIT);
278 if (now > expires) {
279 log.log(Level.FINE, "JCS - Object {0} has expired -> valid to {1}, now is: {2}",
280 new Object[]{getUrlNoException(), Long.toString(expires), Long.toString(now)});
281 return false;
282 }
283 } else if (attributes.getLastModification() > 0 &&
284 now - attributes.getLastModification() > DEFAULT_EXPIRE_TIME) {
285 // check by file modification date
286 log.log(Level.FINE, "JCS - Object has expired, maximum file age reached {0}", getUrlNoException());
287 return false;
288 } else if (now - attributes.getCreateTime() > DEFAULT_EXPIRE_TIME) {
289 log.log(Level.FINE, "JCS - Object has expired, maximum time since object creation reached {0}", getUrlNoException());
290 return false;
291 }
292 return true;
293 }
294
295 /**
296 * @return true if object was successfully downloaded, false, if there was a loading failure
297 */
298
299 private boolean loadObject() {
300 if (attributes == null) {
301 attributes = new CacheEntryAttributes();
302 }
303 try {
304 // if we have object in cache, and host doesn't support If-Modified-Since nor If-None-Match
305 // then just use HEAD request and check returned values
306 if (isObjectLoadable() &&
307 Boolean.TRUE.equals(useHead.get(getServerKey())) &&
308 isCacheValidUsingHead()) {
309 log.log(Level.FINE, "JCS - cache entry verified using HEAD request: {0}", getUrl());
310 return true;
311 }
312
313 HttpURLConnection urlConn = getURLConnection(getUrl(), true);
314
315 if (isObjectLoadable() &&
316 (now - attributes.getLastModification()) <= ABSOLUTE_EXPIRE_TIME_LIMIT) {
317 urlConn.setIfModifiedSince(attributes.getLastModification());
318 }
319 if (isObjectLoadable() && attributes.getEtag() != null) {
320 urlConn.addRequestProperty("If-None-Match", attributes.getEtag());
321 }
322
323 log.log(Level.INFO, "GET {0} -> {1}", new Object[]{getUrl(), urlConn.getResponseCode()});
324
325 // follow redirects
326 for (int i = 0; i < 5; i++) {
327 if (urlConn.getResponseCode() == 302) {
328 urlConn = getURLConnection(new URL(urlConn.getHeaderField("Location")), true);
329 } else {
330 break;
331 }
332 }
333 if (urlConn.getResponseCode() == 304) {
334 // If isModifiedSince or If-None-Match has been set
335 // and the server answers with a HTTP 304 = "Not Modified"
336 log.log(Level.FINE, "JCS - IfModifiedSince/Etag test: local version is up to date: {0}", getUrl());
337 return true;
338 } else if (isObjectLoadable() // we have an object in cache, but we haven't received 304 resposne code
339 && (
340 (attributes.getEtag() != null && attributes.getEtag().equals(urlConn.getRequestProperty("ETag"))) ||
341 attributes.getLastModification() == urlConn.getLastModified())
342 ) {
343 // we sent ETag or If-Modified-Since, but didn't get 304 response code
344 // for further requests - use HEAD
345 String serverKey = getServerKey();
346 log.log(Level.INFO, "JCS - Host: {0} found not to return 304 codes for If-Modifed-Since or If-None-Match headers",
347 serverKey);
348 useHead.put(serverKey, Boolean.TRUE);
349 }
350
351
352 attributes = parseHeaders(urlConn);
353
354 for (int i = 0; i < 5; ++i) {
355 if (urlConn.getResponseCode() == 503) {
356 Thread.sleep(5000+(new Random()).nextInt(5000));
357 continue;
358 }
359
360 attributes.setResponseCode(urlConn.getResponseCode());
361 byte[] raw;
362 if (urlConn.getResponseCode() == 200) {
363 raw = Utils.readBytesFromStream(urlConn.getInputStream());
364 } else {
365 raw = new byte[]{};
366 }
367
368 if (isResponseLoadable(urlConn.getHeaderFields(), urlConn.getResponseCode(), raw)) {
369 // we need to check cacheEmpty, so for cases, when data is returned, but we want to store
370 // as empty (eg. empty tile images) to save some space
371 cacheData = createCacheEntry(raw);
372 cache.put(getCacheKey(), cacheData, attributes);
373 log.log(Level.FINE, "JCS - downloaded key: {0}, length: {1}, url: {2}",
374 new Object[] {getCacheKey(), raw.length, getUrl()});
375 return true;
376 } else if (cacheAsEmpty()) {
377 cacheData = createCacheEntry(new byte[]{});
378 cache.put(getCacheKey(), cacheData, attributes);
379 log.log(Level.FINE, "JCS - Caching empty object {0}", getUrl());
380 return true;
381 } else {
382 log.log(Level.FINE, "JCS - failure during load - reponse is not loadable nor cached as empty");
383 return false;
384 }
385 }
386 } catch (FileNotFoundException e) {
387 log.log(Level.FINE, "JCS - Caching empty object as server returned 404 for: {0}", getUrlNoException());
388 attributes.setResponseCode(404);
389 attributes.setErrorMessage(e.toString());
390 boolean doCache = isResponseLoadable(null, 404, null) || cacheAsEmpty();
391 if (doCache) {
392 cacheData = createCacheEntry(new byte[]{});
393 cache.put(getCacheKey(), cacheData, attributes);
394 }
395 return doCache;
396 } catch (IOException e) {
397 log.log(Level.FINE, "JCS - IOExecption during communication with server for: {0}", getUrlNoException());
398 attributes.setErrorMessage(e.toString());
399 attributes.setResponseCode(499); // set dummy error code
400 boolean doCache = isResponseLoadable(null, 499, null) || cacheAsEmpty(); //generic 499 error code returned
401 if (doCache) {
402 cacheData = createCacheEntry(new byte[]{});
403 cache.put(getCacheKey(), createCacheEntry(new byte[]{}), attributes);
404 }
405 return doCache;
406 } catch (Exception e) {
407 attributes.setErrorMessage(e.toString());
408 log.log(Level.WARNING, "JCS - Exception during download {0}", getUrlNoException());
409 Main.warn(e);
410 }
411 log.log(Level.WARNING, "JCS - Silent failure during download: {0}", getUrlNoException());
412 return false;
413
414 }
415
416 /**
417 * Check if the object is loadable. This means, if the data will be parsed, and if this response
418 * will finish as successful retrieve.
419 *
420 * This simple implementation doesn't load empty response, nor client (4xx) and server (5xx) errors
421 *
422 * @param headerFields headers sent by server
423 * @param responseCode http status code
424 * @param raw data read from server
425 * @return true if object should be cached and returned to listener
426 */
427 protected boolean isResponseLoadable(Map<String, List<String>> headerFields, int responseCode, byte[] raw) {
428 if (raw == null || raw.length == 0 || responseCode >= 400) {
429 return false;
430 }
431 return true;
432 }
433
434 protected abstract V createCacheEntry(byte[] content);
435
436 protected CacheEntryAttributes parseHeaders(URLConnection urlConn) {
437 CacheEntryAttributes ret = new CacheEntryAttributes();
438
439 Long lng = urlConn.getExpiration();
440 if (lng.equals(0L)) {
441 try {
442 String str = urlConn.getHeaderField("Cache-Control");
443 if (str != null) {
444 for (String token: str.split(",")) {
445 if (token.startsWith("max-age=")) {
446 lng = Long.parseLong(token.substring(8)) * 1000 +
447 System.currentTimeMillis();
448 }
449 }
450 }
451 } catch (NumberFormatException e) {
452 // ignore malformed Cache-Control headers
453 if (Main.isTraceEnabled()) {
454 Main.trace(e.getMessage());
455 }
456 }
457 }
458
459 ret.setExpirationTime(lng);
460 ret.setLastModification(now);
461 ret.setEtag(urlConn.getHeaderField("ETag"));
462
463 if (Main.isDebugEnabled()) {
464 for (Entry<String, List<String>> header: urlConn.getHeaderFields().entrySet()) {
465 log.log(Level.FINE, "Response header - {0}: {1}", new Object[]{header.getKey(), header.getValue()});
466 }
467 }
468
469 return ret;
470 }
471
472 private HttpURLConnection getURLConnection(URL url, boolean noCache) throws IOException {
473 HttpURLConnection urlConn = (HttpURLConnection) url.openConnection();
474 urlConn.setRequestProperty("Accept", "text/html, image/png, image/jpeg, image/gif, */*");
475 urlConn.setReadTimeout(readTimeout); // 30 seconds read timeout
476 urlConn.setConnectTimeout(connectTimeout);
477 if (headers != null) {
478 for (Map.Entry<String, String> e: headers.entrySet()) {
479 urlConn.setRequestProperty(e.getKey(), e.getValue());
480 }
481 }
482
483 if (force || noCache) {
484 urlConn.setUseCaches(false);
485 }
486 return urlConn;
487 }
488
489 private boolean isCacheValidUsingHead() throws IOException {
490 HttpURLConnection urlConn = getURLConnection(getUrl(), false);
491 urlConn.setRequestMethod("HEAD");
492 for (int i = 0; i < 5; i++) {
493 if (urlConn.getResponseCode() == 302) {
494 urlConn = getURLConnection(new URL(urlConn.getHeaderField("Location")), false);
495 } else {
496 break;
497 }
498 }
499 long lastModified = urlConn.getLastModified();
500 return (attributes.getEtag() != null && attributes.getEtag().equals(urlConn.getRequestProperty("ETag"))) ||
501 (lastModified != 0 && lastModified <= attributes.getLastModification());
502 }
503
504 /**
505 * TODO: move to JobFactory
506 * cancels all outstanding tasks in the queue.
507 */
508 public void cancelOutstandingTasks() {
509 for (Runnable r: downloadJobExecutor.getQueue()) {
510 if (downloadJobExecutor.remove(r) && r instanceof JCSCachedTileLoaderJob) {
511 ((JCSCachedTileLoaderJob<?, ?>) r).handleJobCancellation();
512 }
513 }
514 }
515
516 /**
517 * Sets a job, that will be run, when job will finish execution
518 * @param runnable that will be executed
519 */
520 public void setFinishedTask(Runnable runnable) {
521 this.finishTask = runnable;
522
523 }
524
525 /**
526 * Marks this job as canceled
527 */
528 public void handleJobCancellation() {
529 finishLoading(LoadResult.CANCELED);
530 }
531
532 private URL getUrlNoException() {
533 try {
534 return getUrl();
535 } catch (IOException e) {
536 return null;
537 }
538 }
539}
Note: See TracBrowser for help on using the repository browser.