source: josm/trunk/src/org/openstreetmap/josm/data/imagery/WmsCache.java@ 8308

Last change on this file since 8308 was 8308, checked in by Don-vip, 9 years ago

fix potential NPEs and Sonar issues related to serialization
  • Property svn:eol-style set to native
File size: 23.3 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.imagery;
3
4import java.awt.Graphics2D;
5import java.awt.image.BufferedImage;
6import java.io.BufferedOutputStream;
7import java.io.File;
8import java.io.FileInputStream;
9import java.io.FileNotFoundException;
10import java.io.FileOutputStream;
11import java.io.IOException;
12import java.io.InputStream;
13import java.io.OutputStream;
14import java.lang.ref.SoftReference;
15import java.net.URLConnection;
16import java.util.ArrayList;
17import java.util.Calendar;
18import java.util.Collections;
19import java.util.Comparator;
20import java.util.HashMap;
21import java.util.HashSet;
22import java.util.Iterator;
23import java.util.List;
24import java.util.Map;
25import java.util.Properties;
26import java.util.Set;
27
28import javax.imageio.ImageIO;
29import javax.xml.bind.JAXBContext;
30import javax.xml.bind.Marshaller;
31import javax.xml.bind.Unmarshaller;
32
33import org.openstreetmap.josm.Main;
34import org.openstreetmap.josm.data.ProjectionBounds;
35import org.openstreetmap.josm.data.SystemOfMeasurement;
36import org.openstreetmap.josm.data.coor.EastNorth;
37import org.openstreetmap.josm.data.coor.LatLon;
38import org.openstreetmap.josm.data.imagery.types.EntryType;
39import org.openstreetmap.josm.data.imagery.types.ProjectionType;
40import org.openstreetmap.josm.data.imagery.types.WmsCacheType;
41import org.openstreetmap.josm.data.preferences.StringProperty;
42import org.openstreetmap.josm.data.projection.Projection;
43import org.openstreetmap.josm.gui.layer.WMSLayer;
44import org.openstreetmap.josm.tools.ImageProvider;
45import org.openstreetmap.josm.tools.Utils;
46import org.openstreetmap.josm.tools.date.DateUtils;
47
48public class WmsCache {
49 //TODO Property for maximum cache size
50 //TODO Property for maximum age of tile, automatically remove old tiles
51 //TODO Measure time for partially loading from cache, compare with time to download tile. If slower, disable partial cache
52 //TODO Do loading from partial cache and downloading at the same time, don't wait for partial cache to load
53
54 private static final StringProperty PROP_CACHE_PATH = new StringProperty("imagery.wms-cache.path", "wms");
55 private static final String INDEX_FILENAME = "index.xml";
56 private static final String LAYERS_INDEX_FILENAME = "layers.properties";
57
58 private static class CacheEntry {
59 private final double pixelPerDegree;
60 private final double east;
61 private final double north;
62 private final ProjectionBounds bounds;
63 private final String filename;
64
65 private long lastUsed;
66 private long lastModified;
67
68 CacheEntry(double pixelPerDegree, double east, double north, int tileSize, String filename) {
69 this.pixelPerDegree = pixelPerDegree;
70 this.east = east;
71 this.north = north;
72 this.bounds = new ProjectionBounds(east, north, east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree);
73 this.filename = filename;
74 }
75
76 @Override
77 public String toString() {
78 return "CacheEntry [pixelPerDegree=" + pixelPerDegree + ", east=" + east + ", north=" + north + ", bounds="
79 + bounds + ", filename=" + filename + ", lastUsed=" + lastUsed + ", lastModified=" + lastModified
80 + "]";
81 }
82 }
83
84 private static class ProjectionEntries {
85 private final String projection;
86 private final String cacheDirectory;
87 private final List<CacheEntry> entries = new ArrayList<>();
88
89 ProjectionEntries(String projection, String cacheDirectory) {
90 this.projection = projection;
91 this.cacheDirectory = cacheDirectory;
92 }
93 }
94
95 private final Map<String, ProjectionEntries> entries = new HashMap<>();
96 private final File cacheDir;
97 private final int tileSize; // Should be always 500
98 private int totalFileSize;
99 private boolean totalFileSizeDirty; // Some file was missing - size needs to be recalculated
100 // No need for hashCode/equals on CacheEntry, object identity is enough. Comparing by values can lead to error - CacheEntry for wrong projection could be found
101 private Map<CacheEntry, SoftReference<BufferedImage>> memoryCache = new HashMap<>();
102 private Set<ProjectionBounds> areaToCache;
103
104 protected String cacheDirPath() {
105 String cPath = PROP_CACHE_PATH.get();
106 if (!(new File(cPath).isAbsolute())) {
107 cPath = Main.pref.getCacheDirectory() + File.separator + cPath;
108 }
109 return cPath;
110 }
111
112 public WmsCache(String url, int tileSize) {
113 File globalCacheDir = new File(cacheDirPath());
114 globalCacheDir.mkdirs();
115 cacheDir = new File(globalCacheDir, getCacheDirectory(url));
116 cacheDir.mkdirs();
117 this.tileSize = tileSize;
118 }
119
120 private String getCacheDirectory(String url) {
121 String cacheDirName = null;
122 Properties layersIndex = new Properties();
123 File layerIndexFile = new File(cacheDirPath(), LAYERS_INDEX_FILENAME);
124 try (InputStream fis = new FileInputStream(layerIndexFile)) {
125 layersIndex.load(fis);
126 } catch (FileNotFoundException e) {
127 Main.error("Unable to load layers index for wms cache (file " + layerIndexFile + " not found)");
128 } catch (IOException e) {
129 Main.error("Unable to load layers index for wms cache");
130 Main.error(e);
131 }
132
133 for (Object propKey: layersIndex.keySet()) {
134 String s = (String)propKey;
135 if (url.equals(layersIndex.getProperty(s))) {
136 cacheDirName = s;
137 break;
138 }
139 }
140
141 if (cacheDirName == null) {
142 int counter = 0;
143 while (true) {
144 counter++;
145 if (!layersIndex.keySet().contains(String.valueOf(counter))) {
146 break;
147 }
148 }
149 cacheDirName = String.valueOf(counter);
150 layersIndex.setProperty(cacheDirName, url);
151 try (OutputStream fos = new FileOutputStream(layerIndexFile)) {
152 layersIndex.store(fos, "");
153 } catch (IOException e) {
154 Main.error("Unable to save layer index for wms cache");
155 Main.error(e);
156 }
157 }
158
159 return cacheDirName;
160 }
161
162 private ProjectionEntries getProjectionEntries(Projection projection) {
163 return getProjectionEntries(projection.toCode(), projection.getCacheDirectoryName());
164 }
165
166 private ProjectionEntries getProjectionEntries(String projection, String cacheDirectory) {
167 ProjectionEntries result = entries.get(projection);
168 if (result == null) {
169 result = new ProjectionEntries(projection, cacheDirectory);
170 entries.put(projection, result);
171 }
172
173 return result;
174 }
175
176 public synchronized void loadIndex() {
177 File indexFile = new File(cacheDir, INDEX_FILENAME);
178 try {
179 JAXBContext context = JAXBContext.newInstance(
180 WmsCacheType.class.getPackage().getName(),
181 WmsCacheType.class.getClassLoader());
182 Unmarshaller unmarshaller = context.createUnmarshaller();
183 WmsCacheType cacheEntries;
184 try (InputStream is = new FileInputStream(indexFile)) {
185 cacheEntries = (WmsCacheType)unmarshaller.unmarshal(is);
186 }
187 totalFileSize = cacheEntries.getTotalFileSize();
188 if (cacheEntries.getTileSize() != tileSize) {
189 Main.info("Cache created with different tileSize, cache will be discarded");
190 return;
191 }
192 for (ProjectionType projectionType: cacheEntries.getProjection()) {
193 ProjectionEntries projection = getProjectionEntries(projectionType.getName(), projectionType.getCacheDirectory());
194 for (EntryType entry: projectionType.getEntry()) {
195 CacheEntry ce = new CacheEntry(entry.getPixelPerDegree(), entry.getEast(), entry.getNorth(), tileSize, entry.getFilename());
196 ce.lastUsed = entry.getLastUsed().getTimeInMillis();
197 ce.lastModified = entry.getLastModified().getTimeInMillis();
198 projection.entries.add(ce);
199 }
200 }
201 } catch (Exception e) {
202 if (indexFile.exists()) {
203 Main.error(e);
204 Main.info("Unable to load index for wms-cache, new file will be created");
205 } else {
206 Main.info("Index for wms-cache doesn't exist, new file will be created");
207 }
208 }
209
210 removeNonReferencedFiles();
211 }
212
213 private void removeNonReferencedFiles() {
214
215 Set<String> usedProjections = new HashSet<>();
216
217 for (ProjectionEntries projectionEntries: entries.values()) {
218
219 usedProjections.add(projectionEntries.cacheDirectory);
220
221 File projectionDir = new File(cacheDir, projectionEntries.cacheDirectory);
222 if (projectionDir.exists()) {
223 Set<String> referencedFiles = new HashSet<>();
224
225 for (CacheEntry ce: projectionEntries.entries) {
226 referencedFiles.add(ce.filename);
227 }
228
229 File[] files = projectionDir.listFiles();
230 if (files != null) {
231 for (File file: files) {
232 if (!referencedFiles.contains(file.getName())) {
233 file.delete();
234 }
235 }
236 }
237 }
238 }
239
240 File[] files = cacheDir.listFiles();
241 if (files != null) {
242 for (File projectionDir: files) {
243 if (projectionDir.isDirectory() && !usedProjections.contains(projectionDir.getName())) {
244 Utils.deleteDirectory(projectionDir);
245 }
246 }
247 }
248 }
249
250 private int calculateTotalFileSize() {
251 int result = 0;
252 for (ProjectionEntries projectionEntries: entries.values()) {
253 Iterator<CacheEntry> it = projectionEntries.entries.iterator();
254 while (it.hasNext()) {
255 CacheEntry entry = it.next();
256 File imageFile = getImageFile(projectionEntries, entry);
257 if (!imageFile.exists()) {
258 it.remove();
259 } else {
260 result += imageFile.length();
261 }
262 }
263 }
264 return result;
265 }
266
267 public synchronized void saveIndex() {
268 WmsCacheType index = new WmsCacheType();
269
270 if (totalFileSizeDirty) {
271 totalFileSize = calculateTotalFileSize();
272 }
273
274 index.setTileSize(tileSize);
275 index.setTotalFileSize(totalFileSize);
276 for (ProjectionEntries projectionEntries: entries.values()) {
277 if (!projectionEntries.entries.isEmpty()) {
278 ProjectionType projectionType = new ProjectionType();
279 projectionType.setName(projectionEntries.projection);
280 projectionType.setCacheDirectory(projectionEntries.cacheDirectory);
281 index.getProjection().add(projectionType);
282 for (CacheEntry ce: projectionEntries.entries) {
283 EntryType entry = new EntryType();
284 entry.setPixelPerDegree(ce.pixelPerDegree);
285 entry.setEast(ce.east);
286 entry.setNorth(ce.north);
287 Calendar c = Calendar.getInstance();
288 c.setTimeInMillis(ce.lastUsed);
289 entry.setLastUsed(c);
290 c = Calendar.getInstance();
291 c.setTimeInMillis(ce.lastModified);
292 entry.setLastModified(c);
293 entry.setFilename(ce.filename);
294 projectionType.getEntry().add(entry);
295 }
296 }
297 }
298 try {
299 JAXBContext context = JAXBContext.newInstance(
300 WmsCacheType.class.getPackage().getName(),
301 WmsCacheType.class.getClassLoader());
302 Marshaller marshaller = context.createMarshaller();
303 try (OutputStream fos = new FileOutputStream(new File(cacheDir, INDEX_FILENAME))) {
304 marshaller.marshal(index, fos);
305 }
306 } catch (Exception e) {
307 Main.error("Failed to save wms-cache file");
308 Main.error(e);
309 }
310 }
311
312 private File getImageFile(ProjectionEntries projection, CacheEntry entry) {
313 return new File(cacheDir, projection.cacheDirectory + "/" + entry.filename);
314 }
315
316 private BufferedImage loadImage(ProjectionEntries projectionEntries, CacheEntry entry, boolean enforceTransparency) throws IOException {
317 synchronized (this) {
318 entry.lastUsed = System.currentTimeMillis();
319
320 SoftReference<BufferedImage> memCache = memoryCache.get(entry);
321 if (memCache != null) {
322 BufferedImage result = memCache.get();
323 if (result != null) {
324 if (enforceTransparency == ImageProvider.isTransparencyForced(result)) {
325 return result;
326 } else if (Main.isDebugEnabled()) {
327 Main.debug("Skipping "+entry+" from memory cache (transparency enforcement)");
328 }
329 }
330 }
331 }
332
333 try {
334 // Reading can't be in synchronized section, it's too slow
335 BufferedImage result = ImageProvider.read(getImageFile(projectionEntries, entry), true, enforceTransparency);
336 synchronized (this) {
337 if (result == null) {
338 projectionEntries.entries.remove(entry);
339 totalFileSizeDirty = true;
340 }
341 return result;
342 }
343 } catch (IOException e) {
344 synchronized (this) {
345 projectionEntries.entries.remove(entry);
346 totalFileSizeDirty = true;
347 throw e;
348 }
349 }
350 }
351
352 private CacheEntry findEntry(ProjectionEntries projectionEntries, double pixelPerDegree, double east, double north) {
353 for (CacheEntry entry: projectionEntries.entries) {
354 if (entry.pixelPerDegree == pixelPerDegree && entry.east == east && entry.north == north)
355 return entry;
356 }
357 return null;
358 }
359
360 public synchronized boolean hasExactMatch(Projection projection, double pixelPerDegree, double east, double north) {
361 ProjectionEntries projectionEntries = getProjectionEntries(projection);
362 CacheEntry entry = findEntry(projectionEntries, pixelPerDegree, east, north);
363 return (entry != null);
364 }
365
366 public BufferedImage getExactMatch(Projection projection, double pixelPerDegree, double east, double north) {
367 CacheEntry entry = null;
368 ProjectionEntries projectionEntries = null;
369 synchronized (this) {
370 projectionEntries = getProjectionEntries(projection);
371 entry = findEntry(projectionEntries, pixelPerDegree, east, north);
372 }
373 if (entry != null) {
374 try {
375 return loadImage(projectionEntries, entry, WMSLayer.PROP_ALPHA_CHANNEL.get());
376 } catch (IOException e) {
377 Main.error("Unable to load file from wms cache");
378 Main.error(e);
379 return null;
380 }
381 }
382 return null;
383 }
384
385 public BufferedImage getPartialMatch(Projection projection, double pixelPerDegree, double east, double north) {
386 ProjectionEntries projectionEntries;
387 List<CacheEntry> matches;
388 synchronized (this) {
389 matches = new ArrayList<>();
390
391 double minPPD = pixelPerDegree / 5;
392 double maxPPD = pixelPerDegree * 5;
393 projectionEntries = getProjectionEntries(projection);
394
395 double size2 = tileSize / pixelPerDegree;
396 double border = tileSize * 0.01; // Make sure not to load neighboring tiles that intersects this tile only slightly
397 ProjectionBounds bounds = new ProjectionBounds(east + border, north + border,
398 east + size2 - border, north + size2 - border);
399
400 //TODO Do not load tile if it is completely overlapped by other tile with better ppd
401 for (CacheEntry entry: projectionEntries.entries) {
402 if (entry.pixelPerDegree >= minPPD && entry.pixelPerDegree <= maxPPD && entry.bounds.intersects(bounds)) {
403 entry.lastUsed = System.currentTimeMillis();
404 matches.add(entry);
405 }
406 }
407
408 if (matches.isEmpty())
409 return null;
410
411 Collections.sort(matches, new Comparator<CacheEntry>() {
412 @Override
413 public int compare(CacheEntry o1, CacheEntry o2) {
414 return Double.compare(o2.pixelPerDegree, o1.pixelPerDegree);
415 }
416 });
417 }
418
419 // Use alpha layer only when enabled on wms layer
420 boolean alpha = WMSLayer.PROP_ALPHA_CHANNEL.get();
421 BufferedImage result = new BufferedImage(tileSize, tileSize,
422 alpha ? BufferedImage.TYPE_4BYTE_ABGR : BufferedImage.TYPE_3BYTE_BGR);
423 Graphics2D g = result.createGraphics();
424
425 boolean drawAtLeastOnce = false;
426 Map<CacheEntry, SoftReference<BufferedImage>> localCache = new HashMap<>();
427 for (CacheEntry ce: matches) {
428 BufferedImage img;
429 try {
430 // Enforce transparency only when alpha enabled on wms layer too
431 img = loadImage(projectionEntries, ce, alpha);
432 localCache.put(ce, new SoftReference<>(img));
433 } catch (IOException e) {
434 continue;
435 }
436
437 drawAtLeastOnce = true;
438
439 int xDiff = (int)((ce.east - east) * pixelPerDegree);
440 int yDiff = (int)((ce.north - north) * pixelPerDegree);
441 int size = (int)(pixelPerDegree / ce.pixelPerDegree * tileSize);
442
443 int x = xDiff;
444 int y = -size + tileSize - yDiff;
445
446 g.drawImage(img, x, y, size, size, null);
447 }
448
449 if (drawAtLeastOnce) {
450 synchronized (this) {
451 memoryCache.putAll(localCache);
452 }
453 return result;
454 } else
455 return null;
456 }
457
458 private String generateFileName(ProjectionEntries projectionEntries, double pixelPerDegree, Projection projection, double east, double north, String mimeType) {
459 LatLon ll1 = projection.eastNorth2latlon(new EastNorth(east, north));
460 LatLon ll2 = projection.eastNorth2latlon(new EastNorth(east + 100 / pixelPerDegree, north));
461 LatLon ll3 = projection.eastNorth2latlon(new EastNorth(east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree));
462
463 double deltaLat = Math.abs(ll3.lat() - ll1.lat());
464 double deltaLon = Math.abs(ll3.lon() - ll1.lon());
465 int precisionLat = Math.max(0, -(int)Math.ceil(Math.log10(deltaLat)) + 1);
466 int precisionLon = Math.max(0, -(int)Math.ceil(Math.log10(deltaLon)) + 1);
467
468 String zoom = SystemOfMeasurement.METRIC.getDistText(ll1.greatCircleDistance(ll2));
469 String extension = "dat";
470 if (mimeType != null) {
471 switch(mimeType) {
472 case "image/jpeg":
473 case "image/jpg":
474 extension = "jpg";
475 break;
476 case "image/png":
477 extension = "png";
478 break;
479 case "image/gif":
480 extension = "gif";
481 break;
482 default:
483 Main.warn("Unrecognized MIME type: "+mimeType);
484 }
485 }
486
487 int counter = 0;
488 FILENAME_LOOP:
489 while (true) {
490 String result = String.format("%s_%." + precisionLat + "f_%." + precisionLon +"f%s.%s", zoom, ll1.lat(), ll1.lon(), counter==0?"":"_" + counter, extension);
491 for (CacheEntry entry: projectionEntries.entries) {
492 if (entry.filename.equals(result)) {
493 counter++;
494 continue FILENAME_LOOP;
495 }
496 }
497 return result;
498 }
499 }
500
501 /**
502 *
503 * @param img Used only when overlapping is used, when not used, used raw from imageData
504 * @param imageData
505 * @param projection
506 * @param pixelPerDegree
507 * @param east
508 * @param north
509 * @throws IOException
510 */
511 public synchronized void saveToCache(BufferedImage img, InputStream imageData, Projection projection, double pixelPerDegree, double east, double north) throws IOException {
512 ProjectionEntries projectionEntries = getProjectionEntries(projection);
513 CacheEntry entry = findEntry(projectionEntries, pixelPerDegree, east, north);
514 File imageFile;
515 if (entry == null) {
516
517 String mimeType;
518 if (img != null) {
519 mimeType = "image/png";
520 } else {
521 mimeType = URLConnection.guessContentTypeFromStream(imageData);
522 }
523 entry = new CacheEntry(pixelPerDegree, east, north, tileSize,generateFileName(projectionEntries, pixelPerDegree, projection, east, north, mimeType));
524 entry.lastUsed = System.currentTimeMillis();
525 entry.lastModified = entry.lastUsed;
526 projectionEntries.entries.add(entry);
527 imageFile = getImageFile(projectionEntries, entry);
528 } else {
529 imageFile = getImageFile(projectionEntries, entry);
530 totalFileSize -= imageFile.length();
531 }
532
533 imageFile.getParentFile().mkdirs();
534
535 if (img != null) {
536 BufferedImage copy = new BufferedImage(tileSize, tileSize, img.getType());
537 copy.createGraphics().drawImage(img, 0, 0, tileSize, tileSize, 0, img.getHeight() - tileSize, tileSize, img.getHeight(), null);
538 ImageIO.write(copy, "png", imageFile);
539 totalFileSize += imageFile.length();
540 } else {
541 try (OutputStream os = new BufferedOutputStream(new FileOutputStream(imageFile))) {
542 totalFileSize += Utils.copyStream(imageData, os);
543 }
544 }
545 }
546
547 public synchronized void cleanSmallFiles(int size) {
548 for (ProjectionEntries projectionEntries: entries.values()) {
549 Iterator<CacheEntry> it = projectionEntries.entries.iterator();
550 while (it.hasNext()) {
551 File file = getImageFile(projectionEntries, it.next());
552 long length = file.length();
553 if (length <= size) {
554 if (length == 0) {
555 totalFileSizeDirty = true; // File probably doesn't exist
556 }
557 totalFileSize -= size;
558 file.delete();
559 it.remove();
560 }
561 }
562 }
563 }
564
565 public static String printDate(Calendar c) {
566 return DateUtils.newIsoDateFormat().format(c.getTime());
567 }
568
569 private boolean isInsideAreaToCache(CacheEntry cacheEntry) {
570 for (ProjectionBounds b: areaToCache) {
571 if (cacheEntry.bounds.intersects(b))
572 return true;
573 }
574 return false;
575 }
576
577 public synchronized void setAreaToCache(Set<ProjectionBounds> areaToCache) {
578 this.areaToCache = areaToCache;
579 Iterator<CacheEntry> it = memoryCache.keySet().iterator();
580 while (it.hasNext()) {
581 if (!isInsideAreaToCache(it.next())) {
582 it.remove();
583 }
584 }
585 }
586}
Note: See TracBrowser for help on using the repository browser.