source: josm/trunk/src/org/openstreetmap/josm/data/imagery/WmsCache.java@ 8509

Last change on this file since 8509 was 8509, checked in by Don-vip, 9 years ago

fix many checkstyle violations
  • Property svn:eol-style set to native
File size: 23.9 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.imagery;
3
4import java.awt.Graphics2D;
5import java.awt.image.BufferedImage;
6import java.io.BufferedOutputStream;
7import java.io.File;
8import java.io.FileInputStream;
9import java.io.FileNotFoundException;
10import java.io.FileOutputStream;
11import java.io.IOException;
12import java.io.InputStream;
13import java.io.OutputStream;
14import java.lang.ref.SoftReference;
15import java.net.URLConnection;
16import java.util.ArrayList;
17import java.util.Calendar;
18import java.util.Collections;
19import java.util.Comparator;
20import java.util.HashMap;
21import java.util.HashSet;
22import java.util.Iterator;
23import java.util.List;
24import java.util.Map;
25import java.util.Properties;
26import java.util.Set;
27
28import javax.imageio.ImageIO;
29import javax.xml.bind.JAXBContext;
30import javax.xml.bind.Marshaller;
31import javax.xml.bind.Unmarshaller;
32
33import org.openstreetmap.josm.Main;
34import org.openstreetmap.josm.data.ProjectionBounds;
35import org.openstreetmap.josm.data.SystemOfMeasurement;
36import org.openstreetmap.josm.data.coor.EastNorth;
37import org.openstreetmap.josm.data.coor.LatLon;
38import org.openstreetmap.josm.data.imagery.types.EntryType;
39import org.openstreetmap.josm.data.imagery.types.ProjectionType;
40import org.openstreetmap.josm.data.imagery.types.WmsCacheType;
41import org.openstreetmap.josm.data.preferences.StringProperty;
42import org.openstreetmap.josm.data.projection.Projection;
43import org.openstreetmap.josm.gui.layer.WMSLayer;
44import org.openstreetmap.josm.tools.ImageProvider;
45import org.openstreetmap.josm.tools.Utils;
46import org.openstreetmap.josm.tools.date.DateUtils;
47
48public class WmsCache {
49 //TODO Property for maximum cache size
50 //TODO Property for maximum age of tile, automatically remove old tiles
51 //TODO Measure time for partially loading from cache, compare with time to download tile. If slower, disable partial cache
52 //TODO Do loading from partial cache and downloading at the same time, don't wait for partial cache to load
53
54 private static final StringProperty PROP_CACHE_PATH = new StringProperty("imagery.wms-cache.path", "wms");
55 private static final String INDEX_FILENAME = "index.xml";
56 private static final String LAYERS_INDEX_FILENAME = "layers.properties";
57
58 private static class CacheEntry {
59 private final double pixelPerDegree;
60 private final double east;
61 private final double north;
62 private final ProjectionBounds bounds;
63 private final String filename;
64
65 private long lastUsed;
66 private long lastModified;
67
68 CacheEntry(double pixelPerDegree, double east, double north, int tileSize, String filename) {
69 this.pixelPerDegree = pixelPerDegree;
70 this.east = east;
71 this.north = north;
72 this.bounds = new ProjectionBounds(east, north, east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree);
73 this.filename = filename;
74 }
75
76 @Override
77 public String toString() {
78 return "CacheEntry [pixelPerDegree=" + pixelPerDegree + ", east=" + east + ", north=" + north + ", bounds="
79 + bounds + ", filename=" + filename + ", lastUsed=" + lastUsed + ", lastModified=" + lastModified
80 + "]";
81 }
82 }
83
84 private static class ProjectionEntries {
85 private final String projection;
86 private final String cacheDirectory;
87 private final List<CacheEntry> entries = new ArrayList<>();
88
89 ProjectionEntries(String projection, String cacheDirectory) {
90 this.projection = projection;
91 this.cacheDirectory = cacheDirectory;
92 }
93 }
94
95 private final Map<String, ProjectionEntries> entries = new HashMap<>();
96 private final File cacheDir;
97 private final int tileSize; // Should be always 500
98 private int totalFileSize;
99 private boolean totalFileSizeDirty; // Some file was missing - size needs to be recalculated
100 // No need for hashCode/equals on CacheEntry, object identity is enough.
101 // Comparing by values can lead to error - CacheEntry for wrong projection could be found
102 private Map<CacheEntry, SoftReference<BufferedImage>> memoryCache = new HashMap<>();
103 private Set<ProjectionBounds> areaToCache;
104
105 protected String cacheDirPath() {
106 String cPath = PROP_CACHE_PATH.get();
107 if (!(new File(cPath).isAbsolute())) {
108 cPath = Main.pref.getCacheDirectory() + File.separator + cPath;
109 }
110 return cPath;
111 }
112
113 public WmsCache(String url, int tileSize) {
114 File globalCacheDir = new File(cacheDirPath());
115 if (!globalCacheDir.mkdirs()) {
116 Main.warn("Unable to create global cache directory: "+globalCacheDir.getAbsolutePath());
117 }
118 cacheDir = new File(globalCacheDir, getCacheDirectory(url));
119 cacheDir.mkdirs();
120 this.tileSize = tileSize;
121 }
122
123 private String getCacheDirectory(String url) {
124 String cacheDirName = null;
125 Properties layersIndex = new Properties();
126 File layerIndexFile = new File(cacheDirPath(), LAYERS_INDEX_FILENAME);
127 try (InputStream fis = new FileInputStream(layerIndexFile)) {
128 layersIndex.load(fis);
129 } catch (FileNotFoundException e) {
130 Main.error("Unable to load layers index for wms cache (file " + layerIndexFile + " not found)");
131 } catch (IOException e) {
132 Main.error("Unable to load layers index for wms cache");
133 Main.error(e);
134 }
135
136 for (Object propKey: layersIndex.keySet()) {
137 String s = (String)propKey;
138 if (url.equals(layersIndex.getProperty(s))) {
139 cacheDirName = s;
140 break;
141 }
142 }
143
144 if (cacheDirName == null) {
145 int counter = 0;
146 while (true) {
147 counter++;
148 if (!layersIndex.keySet().contains(String.valueOf(counter))) {
149 break;
150 }
151 }
152 cacheDirName = String.valueOf(counter);
153 layersIndex.setProperty(cacheDirName, url);
154 try (OutputStream fos = new FileOutputStream(layerIndexFile)) {
155 layersIndex.store(fos, "");
156 } catch (IOException e) {
157 Main.error("Unable to save layer index for wms cache");
158 Main.error(e);
159 }
160 }
161
162 return cacheDirName;
163 }
164
165 private ProjectionEntries getProjectionEntries(Projection projection) {
166 return getProjectionEntries(projection.toCode(), projection.getCacheDirectoryName());
167 }
168
169 private ProjectionEntries getProjectionEntries(String projection, String cacheDirectory) {
170 ProjectionEntries result = entries.get(projection);
171 if (result == null) {
172 result = new ProjectionEntries(projection, cacheDirectory);
173 entries.put(projection, result);
174 }
175
176 return result;
177 }
178
179 public synchronized void loadIndex() {
180 File indexFile = new File(cacheDir, INDEX_FILENAME);
181 try {
182 JAXBContext context = JAXBContext.newInstance(
183 WmsCacheType.class.getPackage().getName(),
184 WmsCacheType.class.getClassLoader());
185 Unmarshaller unmarshaller = context.createUnmarshaller();
186 WmsCacheType cacheEntries;
187 try (InputStream is = new FileInputStream(indexFile)) {
188 cacheEntries = (WmsCacheType)unmarshaller.unmarshal(is);
189 }
190 totalFileSize = cacheEntries.getTotalFileSize();
191 if (cacheEntries.getTileSize() != tileSize) {
192 Main.info("Cache created with different tileSize, cache will be discarded");
193 return;
194 }
195 for (ProjectionType projectionType: cacheEntries.getProjection()) {
196 ProjectionEntries projection = getProjectionEntries(projectionType.getName(), projectionType.getCacheDirectory());
197 for (EntryType entry: projectionType.getEntry()) {
198 CacheEntry ce = new CacheEntry(entry.getPixelPerDegree(), entry.getEast(), entry.getNorth(), tileSize, entry.getFilename());
199 ce.lastUsed = entry.getLastUsed().getTimeInMillis();
200 ce.lastModified = entry.getLastModified().getTimeInMillis();
201 projection.entries.add(ce);
202 }
203 }
204 } catch (Exception e) {
205 if (indexFile.exists()) {
206 Main.error(e);
207 Main.info("Unable to load index for wms-cache, new file will be created");
208 } else {
209 Main.info("Index for wms-cache doesn't exist, new file will be created");
210 }
211 }
212
213 removeNonReferencedFiles();
214 }
215
216 private void removeNonReferencedFiles() {
217
218 Set<String> usedProjections = new HashSet<>();
219
220 for (ProjectionEntries projectionEntries: entries.values()) {
221
222 usedProjections.add(projectionEntries.cacheDirectory);
223
224 File projectionDir = new File(cacheDir, projectionEntries.cacheDirectory);
225 if (projectionDir.exists()) {
226 Set<String> referencedFiles = new HashSet<>();
227
228 for (CacheEntry ce: projectionEntries.entries) {
229 referencedFiles.add(ce.filename);
230 }
231
232 File[] files = projectionDir.listFiles();
233 if (files != null) {
234 for (File file: files) {
235 if (!referencedFiles.contains(file.getName()) && !file.delete()) {
236 Main.warn("Unable to delete file: "+file.getAbsolutePath());
237 }
238 }
239 }
240 }
241 }
242
243 File[] files = cacheDir.listFiles();
244 if (files != null) {
245 for (File projectionDir: files) {
246 if (projectionDir.isDirectory() && !usedProjections.contains(projectionDir.getName())) {
247 Utils.deleteDirectory(projectionDir);
248 }
249 }
250 }
251 }
252
253 private int calculateTotalFileSize() {
254 int result = 0;
255 for (ProjectionEntries projectionEntries: entries.values()) {
256 Iterator<CacheEntry> it = projectionEntries.entries.iterator();
257 while (it.hasNext()) {
258 CacheEntry entry = it.next();
259 File imageFile = getImageFile(projectionEntries, entry);
260 if (!imageFile.exists()) {
261 it.remove();
262 } else {
263 result += imageFile.length();
264 }
265 }
266 }
267 return result;
268 }
269
270 public synchronized void saveIndex() {
271 WmsCacheType index = new WmsCacheType();
272
273 if (totalFileSizeDirty) {
274 totalFileSize = calculateTotalFileSize();
275 }
276
277 index.setTileSize(tileSize);
278 index.setTotalFileSize(totalFileSize);
279 for (ProjectionEntries projectionEntries: entries.values()) {
280 if (!projectionEntries.entries.isEmpty()) {
281 ProjectionType projectionType = new ProjectionType();
282 projectionType.setName(projectionEntries.projection);
283 projectionType.setCacheDirectory(projectionEntries.cacheDirectory);
284 index.getProjection().add(projectionType);
285 for (CacheEntry ce: projectionEntries.entries) {
286 EntryType entry = new EntryType();
287 entry.setPixelPerDegree(ce.pixelPerDegree);
288 entry.setEast(ce.east);
289 entry.setNorth(ce.north);
290 Calendar c = Calendar.getInstance();
291 c.setTimeInMillis(ce.lastUsed);
292 entry.setLastUsed(c);
293 c = Calendar.getInstance();
294 c.setTimeInMillis(ce.lastModified);
295 entry.setLastModified(c);
296 entry.setFilename(ce.filename);
297 projectionType.getEntry().add(entry);
298 }
299 }
300 }
301 try {
302 JAXBContext context = JAXBContext.newInstance(
303 WmsCacheType.class.getPackage().getName(),
304 WmsCacheType.class.getClassLoader());
305 Marshaller marshaller = context.createMarshaller();
306 try (OutputStream fos = new FileOutputStream(new File(cacheDir, INDEX_FILENAME))) {
307 marshaller.marshal(index, fos);
308 }
309 } catch (Exception e) {
310 Main.error("Failed to save wms-cache file");
311 Main.error(e);
312 }
313 }
314
315 private File getImageFile(ProjectionEntries projection, CacheEntry entry) {
316 return new File(cacheDir, projection.cacheDirectory + "/" + entry.filename);
317 }
318
319 private BufferedImage loadImage(ProjectionEntries projectionEntries, CacheEntry entry, boolean enforceTransparency) throws IOException {
320 synchronized (this) {
321 entry.lastUsed = System.currentTimeMillis();
322
323 SoftReference<BufferedImage> memCache = memoryCache.get(entry);
324 if (memCache != null) {
325 BufferedImage result = memCache.get();
326 if (result != null) {
327 if (enforceTransparency == ImageProvider.isTransparencyForced(result)) {
328 return result;
329 } else if (Main.isDebugEnabled()) {
330 Main.debug("Skipping "+entry+" from memory cache (transparency enforcement)");
331 }
332 }
333 }
334 }
335
336 try {
337 // Reading can't be in synchronized section, it's too slow
338 BufferedImage result = ImageProvider.read(getImageFile(projectionEntries, entry), true, enforceTransparency);
339 synchronized (this) {
340 if (result == null) {
341 projectionEntries.entries.remove(entry);
342 totalFileSizeDirty = true;
343 }
344 return result;
345 }
346 } catch (IOException e) {
347 synchronized (this) {
348 projectionEntries.entries.remove(entry);
349 totalFileSizeDirty = true;
350 throw e;
351 }
352 }
353 }
354
355 private CacheEntry findEntry(ProjectionEntries projectionEntries, double pixelPerDegree, double east, double north) {
356 for (CacheEntry entry: projectionEntries.entries) {
357 if (Utils.equalsEpsilon(entry.pixelPerDegree, pixelPerDegree)
358 && Utils.equalsEpsilon(entry.east, east) && Utils.equalsEpsilon(entry.north, north))
359 return entry;
360 }
361 return null;
362 }
363
364 public synchronized boolean hasExactMatch(Projection projection, double pixelPerDegree, double east, double north) {
365 ProjectionEntries projectionEntries = getProjectionEntries(projection);
366 return findEntry(projectionEntries, pixelPerDegree, east, north) != null;
367 }
368
369 public BufferedImage getExactMatch(Projection projection, double pixelPerDegree, double east, double north) {
370 CacheEntry entry = null;
371 ProjectionEntries projectionEntries = null;
372 synchronized (this) {
373 projectionEntries = getProjectionEntries(projection);
374 entry = findEntry(projectionEntries, pixelPerDegree, east, north);
375 }
376 if (entry != null) {
377 try {
378 return loadImage(projectionEntries, entry, WMSLayer.PROP_ALPHA_CHANNEL.get());
379 } catch (IOException e) {
380 Main.error("Unable to load file from wms cache");
381 Main.error(e);
382 return null;
383 }
384 }
385 return null;
386 }
387
388 public BufferedImage getPartialMatch(Projection projection, double pixelPerDegree, double east, double north) {
389 ProjectionEntries projectionEntries;
390 List<CacheEntry> matches;
391 synchronized (this) {
392 matches = new ArrayList<>();
393
394 double minPPD = pixelPerDegree / 5;
395 double maxPPD = pixelPerDegree * 5;
396 projectionEntries = getProjectionEntries(projection);
397
398 double size2 = tileSize / pixelPerDegree;
399 double border = tileSize * 0.01; // Make sure not to load neighboring tiles that intersects this tile only slightly
400 ProjectionBounds bounds = new ProjectionBounds(east + border, north + border,
401 east + size2 - border, north + size2 - border);
402
403 //TODO Do not load tile if it is completely overlapped by other tile with better ppd
404 for (CacheEntry entry: projectionEntries.entries) {
405 if (entry.pixelPerDegree >= minPPD && entry.pixelPerDegree <= maxPPD && entry.bounds.intersects(bounds)) {
406 entry.lastUsed = System.currentTimeMillis();
407 matches.add(entry);
408 }
409 }
410
411 if (matches.isEmpty())
412 return null;
413
414 Collections.sort(matches, new Comparator<CacheEntry>() {
415 @Override
416 public int compare(CacheEntry o1, CacheEntry o2) {
417 return Double.compare(o2.pixelPerDegree, o1.pixelPerDegree);
418 }
419 });
420 }
421
422 // Use alpha layer only when enabled on wms layer
423 boolean alpha = WMSLayer.PROP_ALPHA_CHANNEL.get();
424 BufferedImage result = new BufferedImage(tileSize, tileSize,
425 alpha ? BufferedImage.TYPE_4BYTE_ABGR : BufferedImage.TYPE_3BYTE_BGR);
426 Graphics2D g = result.createGraphics();
427
428 boolean drawAtLeastOnce = false;
429 Map<CacheEntry, SoftReference<BufferedImage>> localCache = new HashMap<>();
430 for (CacheEntry ce: matches) {
431 BufferedImage img;
432 try {
433 // Enforce transparency only when alpha enabled on wms layer too
434 img = loadImage(projectionEntries, ce, alpha);
435 localCache.put(ce, new SoftReference<>(img));
436 } catch (IOException e) {
437 continue;
438 }
439
440 drawAtLeastOnce = true;
441
442 int xDiff = (int)((ce.east - east) * pixelPerDegree);
443 int yDiff = (int)((ce.north - north) * pixelPerDegree);
444 int size = (int)(pixelPerDegree / ce.pixelPerDegree * tileSize);
445
446 int x = xDiff;
447 int y = -size + tileSize - yDiff;
448
449 g.drawImage(img, x, y, size, size, null);
450 }
451
452 if (drawAtLeastOnce) {
453 synchronized (this) {
454 memoryCache.putAll(localCache);
455 }
456 return result;
457 } else
458 return null;
459 }
460
461 private String generateFileName(ProjectionEntries projectionEntries, double pixelPerDegree, Projection projection,
462 double east, double north, String mimeType) {
463 LatLon ll1 = projection.eastNorth2latlon(new EastNorth(east, north));
464 LatLon ll2 = projection.eastNorth2latlon(new EastNorth(east + 100 / pixelPerDegree, north));
465 LatLon ll3 = projection.eastNorth2latlon(new EastNorth(east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree));
466
467 double deltaLat = Math.abs(ll3.lat() - ll1.lat());
468 double deltaLon = Math.abs(ll3.lon() - ll1.lon());
469 int precisionLat = Math.max(0, -(int)Math.ceil(Math.log10(deltaLat)) + 1);
470 int precisionLon = Math.max(0, -(int)Math.ceil(Math.log10(deltaLon)) + 1);
471
472 String zoom = SystemOfMeasurement.METRIC.getDistText(ll1.greatCircleDistance(ll2));
473 String extension = "dat";
474 if (mimeType != null) {
475 switch(mimeType) {
476 case "image/jpeg":
477 case "image/jpg":
478 extension = "jpg";
479 break;
480 case "image/png":
481 extension = "png";
482 break;
483 case "image/gif":
484 extension = "gif";
485 break;
486 default:
487 Main.warn("Unrecognized MIME type: "+mimeType);
488 }
489 }
490
491 int counter = 0;
492 FILENAME_LOOP:
493 while (true) {
494 String result = String.format("%s_%." + precisionLat + "f_%." + precisionLon +"f%s.%s",
495 zoom, ll1.lat(), ll1.lon(), counter==0?"":"_" + counter, extension);
496 for (CacheEntry entry: projectionEntries.entries) {
497 if (entry.filename.equals(result)) {
498 counter++;
499 continue FILENAME_LOOP;
500 }
501 }
502 return result;
503 }
504 }
505
506 /**
507 *
508 * @param img Used only when overlapping is used, when not used, used raw from imageData
509 * @param imageData input stream to raw image data
510 * @param projection current projection
511 * @param pixelPerDegree number of pixels per degree
512 * @param east easting
513 * @param north northing
514 * @throws IOException if any I/O error occurs
515 */
516 public synchronized void saveToCache(BufferedImage img, InputStream imageData, Projection projection, double pixelPerDegree,
517 double east, double north) throws IOException {
518 ProjectionEntries projectionEntries = getProjectionEntries(projection);
519 CacheEntry entry = findEntry(projectionEntries, pixelPerDegree, east, north);
520 File imageFile;
521 if (entry == null) {
522
523 String mimeType;
524 if (img != null) {
525 mimeType = "image/png";
526 } else {
527 mimeType = URLConnection.guessContentTypeFromStream(imageData);
528 }
529 entry = new CacheEntry(pixelPerDegree, east, north,
530 tileSize,generateFileName(projectionEntries, pixelPerDegree, projection, east, north, mimeType));
531 entry.lastUsed = System.currentTimeMillis();
532 entry.lastModified = entry.lastUsed;
533 projectionEntries.entries.add(entry);
534 imageFile = getImageFile(projectionEntries, entry);
535 } else {
536 imageFile = getImageFile(projectionEntries, entry);
537 totalFileSize -= imageFile.length();
538 }
539
540 if (!imageFile.getParentFile().mkdirs()) {
541 Main.warn("Unable to create parent directory: "+imageFile.getParentFile().getAbsolutePath());
542 }
543
544 if (img != null) {
545 BufferedImage copy = new BufferedImage(tileSize, tileSize, img.getType());
546 copy.createGraphics().drawImage(img, 0, 0, tileSize, tileSize, 0, img.getHeight() - tileSize, tileSize, img.getHeight(), null);
547 ImageIO.write(copy, "png", imageFile);
548 totalFileSize += imageFile.length();
549 } else {
550 try (OutputStream os = new BufferedOutputStream(new FileOutputStream(imageFile))) {
551 totalFileSize += Utils.copyStream(imageData, os);
552 }
553 }
554 }
555
556 public synchronized void cleanSmallFiles(int size) {
557 for (ProjectionEntries projectionEntries: entries.values()) {
558 Iterator<CacheEntry> it = projectionEntries.entries.iterator();
559 while (it.hasNext()) {
560 File file = getImageFile(projectionEntries, it.next());
561 long length = file.length();
562 if (length <= size) {
563 if (length == 0) {
564 totalFileSizeDirty = true; // File probably doesn't exist
565 }
566 totalFileSize -= size;
567 if (!file.delete()) {
568 Main.warn("Unable to delete file: "+file.getAbsolutePath());
569 }
570 it.remove();
571 }
572 }
573 }
574 }
575
576 public static String printDate(Calendar c) {
577 return DateUtils.newIsoDateFormat().format(c.getTime());
578 }
579
580 private boolean isInsideAreaToCache(CacheEntry cacheEntry) {
581 for (ProjectionBounds b: areaToCache) {
582 if (cacheEntry.bounds.intersects(b))
583 return true;
584 }
585 return false;
586 }
587
588 public synchronized void setAreaToCache(Set<ProjectionBounds> areaToCache) {
589 this.areaToCache = areaToCache;
590 Iterator<CacheEntry> it = memoryCache.keySet().iterator();
591 while (it.hasNext()) {
592 if (!isInsideAreaToCache(it.next())) {
593 it.remove();
594 }
595 }
596 }
597}
Note: See TracBrowser for help on using the repository browser.