source: josm/trunk/src/org/openstreetmap/josm/data/imagery/TMSCachedTileLoaderJob.java@ 8389

Last change on this file since 8389 was 8389, checked in by wiktorn, 9 years ago

addresses #11437 - properly pass information about errors during load from cache to upper layers
File size: 13.6 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.imagery;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.ByteArrayInputStream;
7import java.io.IOException;
8import java.net.URL;
9import java.util.HashSet;
10import java.util.List;
11import java.util.Map;
12import java.util.Set;
13import java.util.concurrent.ConcurrentHashMap;
14import java.util.concurrent.ConcurrentMap;
15import java.util.concurrent.Executor;
16import java.util.concurrent.Semaphore;
17import java.util.concurrent.ThreadPoolExecutor;
18import java.util.concurrent.TimeUnit;
19import java.util.logging.Level;
20import java.util.logging.Logger;
21
22import org.apache.commons.jcs.access.behavior.ICacheAccess;
23import org.openstreetmap.gui.jmapviewer.FeatureAdapter;
24import org.openstreetmap.gui.jmapviewer.Tile;
25import org.openstreetmap.gui.jmapviewer.interfaces.TileJob;
26import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener;
27import org.openstreetmap.gui.jmapviewer.interfaces.TileSource;
28import org.openstreetmap.gui.jmapviewer.tilesources.AbstractTMSTileSource;
29import org.openstreetmap.josm.Main;
30import org.openstreetmap.josm.data.cache.BufferedImageCacheEntry;
31import org.openstreetmap.josm.data.cache.CacheEntry;
32import org.openstreetmap.josm.data.cache.CacheEntryAttributes;
33import org.openstreetmap.josm.data.cache.ICachedLoaderListener;
34import org.openstreetmap.josm.data.cache.JCSCachedTileLoaderJob;
35import org.openstreetmap.josm.data.preferences.IntegerProperty;
36
37/**
38 * @author Wiktor Niesiobędzki
39 *
40 * Class bridging TMS requests to JCS cache requests
41 * @since 8168
42 */
43public class TMSCachedTileLoaderJob extends JCSCachedTileLoaderJob<String, BufferedImageCacheEntry> implements TileJob, ICachedLoaderListener {
44 private static final Logger log = FeatureAdapter.getLogger(TMSCachedTileLoaderJob.class.getCanonicalName());
45 private Tile tile;
46 private volatile URL url;
47
48 // we need another deduplication of Tile Loader listeners, as for each submit, new TMSCachedTileLoaderJob was created
49 // that way, we reduce calls to tileLoadingFinished, and general CPU load due to surplus Map repaints
50 private static final ConcurrentMap<String,Set<TileLoaderListener>> inProgress = new ConcurrentHashMap<>();
51
52 /**
53 * Limit definition for per host concurrent connections
54 */
55 public static final IntegerProperty HOST_LIMIT = new IntegerProperty("imagery.tms.tmsloader.maxjobsperhost", 6);
56
57 /*
58 * Host limit guards the area - between submission to the queue up to loading is finished. It uses executionGuard method
59 * from JCSCachedTileLoaderJob to acquire the semaphore, and releases it - when loadingFinished is called (but not when
60 * LoadResult.GUARD_REJECTED is set)
61 *
62 */
63
64 private Semaphore getSemaphore() {
65 String host = getUrl().getHost();
66 Semaphore limit = HOST_LIMITS.get(host);
67 if (limit == null) {
68 synchronized(HOST_LIMITS) {
69 limit = HOST_LIMITS.get(host);
70 if (limit == null) {
71 limit = new Semaphore(HOST_LIMIT.get().intValue());
72 HOST_LIMITS.put(host, limit);
73 }
74 }
75 }
76 return limit;
77 }
78
79 private boolean acquireSemaphore() {
80 boolean ret = true;
81 Semaphore limit = getSemaphore();
82 if (limit != null) {
83 ret = limit.tryAcquire();
84 if (!ret) {
85 Main.debug("rejecting job because of per host limit");
86 }
87 }
88 return ret;
89 }
90
91 private void releaseSemaphore() {
92 Semaphore limit = getSemaphore();
93 if (limit != null) {
94 limit.release();
95 }
96 }
97
98 private static Map<String, Semaphore> HOST_LIMITS = new ConcurrentHashMap<>();
99
100 /**
101 * overrides the THREAD_LIMIT in superclass, as we want to have separate limit and pool for TMS
102 */
103 public static final IntegerProperty THREAD_LIMIT = new IntegerProperty("imagery.tms.tmsloader.maxjobs", 25);
104
105 /**
106 * separate from JCS thread pool for TMS loader, so we can have different thread pools for default JCS
107 * and for TMS imagery
108 */
109 private static ThreadPoolExecutor DOWNLOAD_JOB_DISPATCHER = getThreadPoolExecutor();
110
111 private static ThreadPoolExecutor getThreadPoolExecutor() {
112 return new ThreadPoolExecutor(
113 THREAD_LIMIT.get().intValue(), // keep the thread number constant
114 THREAD_LIMIT.get().intValue(), // do not this number of threads
115 30, // keepalive for thread
116 TimeUnit.SECONDS,
117 // make queue of LIFO type - so recently requested tiles will be loaded first (assuming that these are which user is waiting to see)
118 new LIFOQueue(5)
119 /* keep the queue size fairly small, we do not want to
120 download a lot of tiles, that user is not seeing anyway */
121 );
122 }
123
124 /**
125 * Reconfigures download dispatcher using current values of THREAD_LIMIT and HOST_LIMIT
126 */
127 public static final void reconfigureDownloadDispatcher() {
128 HOST_LIMITS = new ConcurrentHashMap<>();
129 DOWNLOAD_JOB_DISPATCHER = getThreadPoolExecutor();
130 }
131
132 /**
133 * Constructor for creating a job, to get a specific tile from cache
134 * @param listener
135 * @param tile to be fetched from cache
136 * @param cache object
137 * @param connectTimeout when connecting to remote resource
138 * @param readTimeout when connecting to remote resource
139 * @param headers to be sent together with request
140 */
141 public TMSCachedTileLoaderJob(TileLoaderListener listener, Tile tile, ICacheAccess<String, BufferedImageCacheEntry> cache, int connectTimeout, int readTimeout,
142 Map<String, String> headers) {
143 super(cache, connectTimeout, readTimeout, headers);
144 this.tile = tile;
145 if (listener != null) {
146 String deduplicationKey = getCacheKey();
147 synchronized (inProgress) {
148 Set<TileLoaderListener> newListeners = inProgress.get(deduplicationKey);
149 if (newListeners == null) {
150 newListeners = new HashSet<>();
151 inProgress.put(deduplicationKey, newListeners);
152 }
153 newListeners.add(listener);
154 }
155 }
156 }
157
158 @Override
159 public Tile getTile() {
160 return getCachedTile();
161 }
162
163 @Override
164 public String getCacheKey() {
165 if (tile != null)
166 return tile.getKey();
167 return null;
168 }
169
170 /*
171 * this doesn't needs to be synchronized, as it's not that costly to keep only one execution
172 * in parallel, but URL creation and Tile.getUrl() are costly and are not needed when fetching
173 * data from cache, that's why URL creation is postponed until it's needed
174 *
175 * We need to have static url value for TileLoaderJob, as for some TileSources we might get different
176 * URL's each call we made (servers switching), and URL's are used below as a key for duplicate detection
177 *
178 */
179 @Override
180 public URL getUrl() {
181 if (url == null) {
182 try {
183 synchronized (this) {
184 if (url == null)
185 url = new URL(tile.getUrl());
186 }
187 } catch (IOException e) {
188 log.log(Level.WARNING, "JCS TMS Cache - error creating URL for tile {0}: {1}", new Object[] {tile.getKey(), e.getMessage()});
189 log.log(Level.INFO, "Exception: ", e);
190 }
191 }
192 return url;
193 }
194
195 @Override
196 public boolean isObjectLoadable() {
197 if (cacheData != null) {
198 byte[] content = cacheData.getContent();
199 try {
200 return content != null || cacheData.getImage() != null || isNoTileAtZoom();
201 } catch (IOException e) {
202 log.log(Level.WARNING, "JCS TMS - error loading from cache for tile {0}: {1}", new Object[] {tile.getKey(), e.getMessage()});
203 }
204 }
205 return false;
206 }
207
208 private boolean isNoTileAtZoom() {
209 if (attributes == null) {
210 log.warning("Cache attributes are null");
211 }
212 return attributes != null && attributes.isNoTileAtZoom();
213 }
214
215 @Override
216 protected boolean cacheAsEmpty(Map<String, List<String>> headers, int statusCode, byte[] content) {
217 if (tile.getTileSource().isNoTileAtZoom(headers, statusCode, content)) {
218 attributes.setNoTileAtZoom(true);
219 return true;
220 }
221 return false;
222 }
223
224 private boolean handleNoTileAtZoom() {
225 if (isNoTileAtZoom()) {
226 log.log(Level.FINE, "JCS TMS - Tile valid, but no file, as no tiles at this level {0}", tile);
227 tile.setError("No tile at this zoom level");
228 tile.putValue("tile-info", "no-tile");
229 return true;
230 }
231 return false;
232 }
233
234 @Override
235 protected Executor getDownloadExecutor() {
236 return DOWNLOAD_JOB_DISPATCHER;
237 }
238
239 @Override
240 protected boolean executionGuard() {
241 return acquireSemaphore();
242 }
243
244 @Override
245 protected void executionFinished() {
246 releaseSemaphore();
247 }
248
249 public void submit() {
250 tile.initLoading();
251 super.submit(this);
252 }
253
254 @Override
255 public void loadingFinished(CacheEntry object, CacheEntryAttributes attributes, LoadResult result) {
256 this.attributes = attributes; // as we might get notification from other object than our selfs, pass attributes along
257 Set<TileLoaderListener> listeners;
258 synchronized (inProgress) {
259 listeners = inProgress.remove(getCacheKey());
260 }
261
262 try {
263 if(!tile.isLoaded()) { //if someone else already loaded tile, skip all the handling
264 tile.finishLoading(); // whatever happened set that loading has finished
265 switch(result){
266 case FAILURE:
267 tile.setError("Problem loading tile");
268 // no break intentional here
269 case SUCCESS:
270 handleNoTileAtZoom();
271 if (object != null) {
272 byte[] content = object.getContent();
273 if (content != null && content.length > 0) {
274 tile.loadImage(new ByteArrayInputStream(content));
275 }
276 }
277 int httpStatusCode = attributes.getResponseCode();
278 if (!isNoTileAtZoom() && httpStatusCode >= 400) {
279 tile.setError(tr("HTTP error {0} when loading tiles", httpStatusCode));
280 }
281 // no break intentional here
282 case REJECTED:
283 // do nothing
284 }
285 }
286
287 // always check, if there is some listener interested in fact, that tile has finished loading
288 if (listeners != null) { // listeners might be null, if some other thread notified already about success
289 for(TileLoaderListener l: listeners) {
290 l.tileLoadingFinished(tile, result.equals(LoadResult.SUCCESS));
291 }
292 }
293 } catch (IOException e) {
294 log.log(Level.WARNING, "JCS TMS - error loading object for tile {0}: {1}", new Object[] {tile.getKey(), e.getMessage()});
295 tile.setError(e.getMessage());
296 tile.setLoaded(false);
297 if (listeners != null) { // listeners might be null, if some other thread notified already about success
298 for(TileLoaderListener l: listeners) {
299 l.tileLoadingFinished(tile, false);
300 }
301 }
302 }
303 }
304
305 /**
306 * Method for getting the tile from cache only, without trying to reach remote resource
307 * @return tile or null, if nothing (useful) was found in cache
308 */
309 public Tile getCachedTile() {
310 BufferedImageCacheEntry data = get();
311 if (isObjectLoadable()) {
312 try {
313 if (data != null && data.getImage() != null) {
314 tile.setImage(data.getImage());
315 tile.finishLoading();
316 }
317 if (isNoTileAtZoom()) {
318 handleNoTileAtZoom();
319 tile.finishLoading();
320 }
321 if (attributes.getResponseCode() >= 400) {
322 tile.setError(tr("HTTP error {0} when loading tiles", attributes.getResponseCode()));
323 }
324 return tile;
325 } catch (IOException e) {
326 log.log(Level.WARNING, "JCS TMS - error loading object for tile {0}: {1}", new Object[] {tile.getKey(), e.getMessage()});
327 return null;
328 }
329
330 } else {
331 return tile;
332 }
333 }
334
335 @Override
336 protected boolean handleNotFound() {
337 if (tile.getSource().isNoTileAtZoom(null, 404, null)) {
338 tile.setError("No tile at this zoom level");
339 tile.putValue("tile-info", "no-tile");
340 return true;
341 }
342 return false;
343 }
344
345 /**
346 * For TMS use BaseURL as settings discovery, so for different paths, we will have different settings (useful for developer servers)
347 *
348 * @return base URL of TMS or server url as defined in super class
349 */
350 @Override
351 protected String getServerKey() {
352 TileSource ts = tile.getSource();
353 if (ts instanceof AbstractTMSTileSource) {
354 return ((AbstractTMSTileSource) ts).getBaseUrl();
355 }
356 return super.getServerKey();
357 }
358
359 @Override
360 protected BufferedImageCacheEntry createCacheEntry(byte[] content) {
361 return new BufferedImageCacheEntry(content);
362 }
363}
Note: See TracBrowser for help on using the repository browser.