source: josm/trunk/src/org/openstreetmap/josm/data/imagery/TMSCachedTileLoaderJob.java@ 8326

Last change on this file since 8326 was 8326, checked in by Don-vip, 9 years ago

fix #11404 - High CPU load during tile loading in TMS layer and download box (patch by wiktorn)
File size: 12.4 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.imagery;
3
4import java.io.ByteArrayInputStream;
5import java.io.IOException;
6import java.net.URL;
7import java.util.HashSet;
8import java.util.Map;
9import java.util.Set;
10import java.util.concurrent.ConcurrentHashMap;
11import java.util.concurrent.ConcurrentMap;
12import java.util.concurrent.Executor;
13import java.util.concurrent.Semaphore;
14import java.util.concurrent.ThreadPoolExecutor;
15import java.util.concurrent.TimeUnit;
16import java.util.logging.Level;
17import java.util.logging.Logger;
18
19import org.apache.commons.jcs.access.behavior.ICacheAccess;
20import org.openstreetmap.gui.jmapviewer.FeatureAdapter;
21import org.openstreetmap.gui.jmapviewer.Tile;
22import org.openstreetmap.gui.jmapviewer.interfaces.TileJob;
23import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener;
24import org.openstreetmap.gui.jmapviewer.interfaces.TileSource;
25import org.openstreetmap.gui.jmapviewer.tilesources.AbstractTMSTileSource;
26import org.openstreetmap.josm.Main;
27import org.openstreetmap.josm.data.cache.BufferedImageCacheEntry;
28import org.openstreetmap.josm.data.cache.CacheEntry;
29import org.openstreetmap.josm.data.cache.ICachedLoaderListener;
30import org.openstreetmap.josm.data.cache.JCSCachedTileLoaderJob;
31import org.openstreetmap.josm.data.preferences.IntegerProperty;
32
33/**
34 * @author Wiktor Niesiobędzki
35 *
36 * Class bridging TMS requests to JCS cache requests
37 * @since 8168
38 */
39public class TMSCachedTileLoaderJob extends JCSCachedTileLoaderJob<String, BufferedImageCacheEntry> implements TileJob, ICachedLoaderListener {
40 private static final Logger log = FeatureAdapter.getLogger(TMSCachedTileLoaderJob.class.getCanonicalName());
41 private Tile tile;
42 private volatile URL url;
43
44 // we need another deduplication of Tile Loader listeners, as for each submit, new TMSCachedTileLoaderJob was created
45 // that way, we reduce calls to tileLoadingFinished, and general CPU load due to surplus Map repaints
46 private static final ConcurrentMap<String,Set<TileLoaderListener>> inProgress = new ConcurrentHashMap<>();
47
48 /**
49 * Limit definition for per host concurrent connections
50 */
51 public static final IntegerProperty HOST_LIMIT = new IntegerProperty("imagery.tms.tmsloader.maxjobsperhost", 6);
52
53 /*
54 * Host limit guards the area - between submission to the queue up to loading is finished. It uses executionGuard method
55 * from JCSCachedTileLoaderJob to acquire the semaphore, and releases it - when loadingFinished is called (but not when
56 * LoadResult.GUARD_REJECTED is set)
57 *
58 */
59
60 private Semaphore getSemaphore() {
61 String host = getUrl().getHost();
62 Semaphore limit = HOST_LIMITS.get(host);
63 if (limit == null) {
64 synchronized(HOST_LIMITS) {
65 limit = HOST_LIMITS.get(host);
66 if (limit == null) {
67 limit = new Semaphore(HOST_LIMIT.get().intValue());
68 HOST_LIMITS.put(host, limit);
69 }
70 }
71 }
72 return limit;
73 }
74
75 private boolean acquireSemaphore() {
76 boolean ret = true;
77 Semaphore limit = getSemaphore();
78 if (limit != null) {
79 ret = limit.tryAcquire();
80 if (!ret) {
81 Main.debug("rejecting job because of per host limit");
82 }
83 }
84 return ret;
85 }
86
87 private void releaseSemaphore() {
88 Semaphore limit = getSemaphore();
89 if (limit != null) {
90 limit.release();
91 }
92 }
93
94 private static Map<String, Semaphore> HOST_LIMITS = new ConcurrentHashMap<>();
95
96 /**
97 * overrides the THREAD_LIMIT in superclass, as we want to have separate limit and pool for TMS
98 */
99 public static final IntegerProperty THREAD_LIMIT = new IntegerProperty("imagery.tms.tmsloader.maxjobs", 25);
100
101 /**
102 * separate from JCS thread pool for TMS loader, so we can have different thread pools for default JCS
103 * and for TMS imagery
104 */
105 private static ThreadPoolExecutor DOWNLOAD_JOB_DISPATCHER = getThreadPoolExecutor();
106
107 private static ThreadPoolExecutor getThreadPoolExecutor() {
108 return new ThreadPoolExecutor(
109 THREAD_LIMIT.get().intValue(), // keep the thread number constant
110 THREAD_LIMIT.get().intValue(), // do not this number of threads
111 30, // keepalive for thread
112 TimeUnit.SECONDS,
113 // make queue of LIFO type - so recently requested tiles will be loaded first (assuming that these are which user is waiting to see)
114 new LIFOQueue(5)
115 /* keep the queue size fairly small, we do not want to
116 download a lot of tiles, that user is not seeing anyway */
117 );
118 }
119
120 /**
121 * Reconfigures download dispatcher using current values of THREAD_LIMIT and HOST_LIMIT
122 */
123 public static final void reconfigureDownloadDispatcher() {
124 HOST_LIMITS = new ConcurrentHashMap<>();
125 DOWNLOAD_JOB_DISPATCHER = getThreadPoolExecutor();
126 }
127
128 /**
129 * Constructor for creating a job, to get a specific tile from cache
130 * @param listener
131 * @param tile to be fetched from cache
132 * @param cache object
133 * @param connectTimeout when connecting to remote resource
134 * @param readTimeout when connecting to remote resource
135 * @param headers to be sent together with request
136 */
137 public TMSCachedTileLoaderJob(TileLoaderListener listener, Tile tile, ICacheAccess<String, BufferedImageCacheEntry> cache, int connectTimeout, int readTimeout,
138 Map<String, String> headers) {
139 super(cache, connectTimeout, readTimeout, headers);
140 this.tile = tile;
141 if (listener != null) {
142 String deduplicationKey = getCacheKey();
143 synchronized (inProgress) {
144 Set<TileLoaderListener> newListeners = inProgress.get(deduplicationKey);
145 if (newListeners == null) {
146 newListeners = new HashSet<>();
147 inProgress.put(deduplicationKey, newListeners);
148 }
149 newListeners.add(listener);
150 }
151 }
152 }
153
154 @Override
155 public Tile getTile() {
156 return getCachedTile();
157 }
158
159 @Override
160 public String getCacheKey() {
161 if (tile != null)
162 return tile.getKey();
163 return null;
164 }
165
166 /*
167 * this doesn't needs to be synchronized, as it's not that costly to keep only one execution
168 * in parallel, but URL creation and Tile.getUrl() are costly and are not needed when fetching
169 * data from cache, that's why URL creation is postponed until it's needed
170 *
171 * We need to have static url value for TileLoaderJob, as for some TileSources we might get different
172 * URL's each call we made (servers switching), and URL's are used below as a key for duplicate detection
173 *
174 */
175 @Override
176 public URL getUrl() {
177 if (url == null) {
178 try {
179 synchronized (this) {
180 if (url == null)
181 url = new URL(tile.getUrl());
182 }
183 } catch (IOException e) {
184 log.log(Level.WARNING, "JCS TMS Cache - error creating URL for tile {0}: {1}", new Object[] {tile.getKey(), e.getMessage()});
185 log.log(Level.INFO, "Exception: ", e);
186 }
187 }
188 return url;
189 }
190
191 @Override
192 public boolean isObjectLoadable() {
193 if (cacheData != null) {
194 byte[] content = cacheData.getContent();
195 try {
196 return content != null || cacheData.getImage() != null || cacheAsEmpty();
197 } catch (IOException e) {
198 log.log(Level.WARNING, "JCS TMS - error loading from cache for tile {0}: {1}", new Object[] {tile.getKey(), e.getMessage()});
199 }
200 }
201 return false;
202 }
203
204 private boolean isNoTileAtZoom() {
205 return attributes != null && attributes.isNoTileAtZoom();
206 }
207
208 @Override
209 protected boolean cacheAsEmpty() {
210 return isNoTileAtZoom();
211 }
212
213 private boolean handleNoTileAtZoom() {
214 if (isNoTileAtZoom()) {
215 log.log(Level.FINE, "JCS TMS - Tile valid, but no file, as no tiles at this level {0}", tile);
216 tile.setError("No tile at this zoom level");
217 tile.putValue("tile-info", "no-tile");
218 return true;
219 }
220 return false;
221 }
222
223 @Override
224 protected Executor getDownloadExecutor() {
225 return DOWNLOAD_JOB_DISPATCHER;
226 }
227
228 @Override
229 protected boolean executionGuard() {
230 return acquireSemaphore();
231 }
232
233 @Override
234 protected void executionFinished() {
235 releaseSemaphore();
236 }
237
238 public void submit() {
239 tile.initLoading();
240 super.submit(this);
241 }
242
243 @Override
244 public void loadingFinished(CacheEntry object, LoadResult result) {
245 Set<TileLoaderListener> listeners;
246 synchronized (inProgress) {
247 listeners = inProgress.remove(getCacheKey());
248 }
249
250 try {
251 if(!tile.isLoaded()) { //if someone else already loaded tile, skip all the handling
252 tile.finishLoading(); // whatever happened set that loading has finished
253 switch(result){
254 case FAILURE:
255 tile.setError("Problem loading tile");
256 // no break intentional here
257 case SUCCESS:
258 handleNoTileAtZoom();
259 if (object != null) {
260 byte[] content = object.getContent();
261 if (content != null && content.length > 0) {
262 tile.loadImage(new ByteArrayInputStream(content));
263 }
264 }
265 // no break intentional here
266 case REJECTED:
267 // do nothing
268 }
269 }
270
271 // always check, if there is some listener interested in fact, that tile has finished loading
272 if (listeners != null) { // listeners might be null, if some other thread notified already about success
273 for(TileLoaderListener l: listeners) {
274 l.tileLoadingFinished(tile, result.equals(LoadResult.SUCCESS));
275 }
276 }
277 } catch (IOException e) {
278 log.log(Level.WARNING, "JCS TMS - error loading object for tile {0}: {1}", new Object[] {tile.getKey(), e.getMessage()});
279 tile.setError(e.getMessage());
280 tile.setLoaded(false);
281 if (listeners != null) { // listeners might be null, if some other thread notified already about success
282 for(TileLoaderListener l: listeners) {
283 l.tileLoadingFinished(tile, false);
284 }
285 }
286 }
287 }
288
289 /**
290 * Method for getting the tile from cache only, without trying to reach remote resource
291 * @return tile or null, if nothing (useful) was found in cache
292 */
293 public Tile getCachedTile() {
294 BufferedImageCacheEntry data = get();
295 if (isObjectLoadable()) {
296 try {
297 if (data != null && data.getImage() != null) {
298 tile.setImage(data.getImage());
299 tile.finishLoading();
300 }
301 if (isNoTileAtZoom()) {
302 handleNoTileAtZoom();
303 tile.finishLoading();
304 }
305 return tile;
306 } catch (IOException e) {
307 log.log(Level.WARNING, "JCS TMS - error loading object for tile {0}: {1}", new Object[] {tile.getKey(), e.getMessage()});
308 return null;
309 }
310
311 } else {
312 return tile;
313 }
314 }
315
316 @Override
317 protected boolean handleNotFound() {
318 tile.setError("No tile at this zoom level");
319 tile.putValue("tile-info", "no-tile");
320 return true;
321 }
322
323 /**
324 * For TMS use BaseURL as settings discovery, so for different paths, we will have different settings (useful for developer servers)
325 *
326 * @return base URL of TMS or server url as defined in super class
327 */
328 @Override
329 protected String getServerKey() {
330 TileSource ts = tile.getSource();
331 if (ts instanceof AbstractTMSTileSource) {
332 return ((AbstractTMSTileSource) ts).getBaseUrl();
333 }
334 return super.getServerKey();
335 }
336
337 @Override
338 protected BufferedImageCacheEntry createCacheEntry(byte[] content) {
339 return new BufferedImageCacheEntry(content);
340 }
341}
Note: See TracBrowser for help on using the repository browser.