source: josm/trunk/src/org/openstreetmap/josm/data/imagery/WmsCache.java@ 7132

Last change on this file since 7132 was 7132, checked in by Don-vip, 10 years ago

fix #9984 - Add support for WMS tiles defining a transparent color in RGB space (tRNS PNG chunk for example), instead of a proper alpha channel. Surprisingly, Java does not support that out of the box, ImageIO.read always returns opaque images. Allows to switch between this mode and standard mode using WMS layer contextual entry "Use Alpha Channel", for consistency with images defining an alpha channel. Does not impact other images than WMS tiles.
File size: 23.0 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.imagery;
3
4import java.awt.Graphics2D;
5import java.awt.image.BufferedImage;
6import java.io.BufferedOutputStream;
7import java.io.File;
8import java.io.FileInputStream;
9import java.io.FileNotFoundException;
10import java.io.FileOutputStream;
11import java.io.IOException;
12import java.io.InputStream;
13import java.io.OutputStream;
14import java.lang.ref.SoftReference;
15import java.net.URLConnection;
16import java.text.SimpleDateFormat;
17import java.util.ArrayList;
18import java.util.Calendar;
19import java.util.Collections;
20import java.util.Comparator;
21import java.util.HashMap;
22import java.util.HashSet;
23import java.util.Iterator;
24import java.util.List;
25import java.util.Map;
26import java.util.Properties;
27import java.util.Set;
28
29import javax.imageio.ImageIO;
30import javax.xml.bind.JAXBContext;
31import javax.xml.bind.Marshaller;
32import javax.xml.bind.Unmarshaller;
33
34import org.openstreetmap.josm.Main;
35import org.openstreetmap.josm.data.ProjectionBounds;
36import org.openstreetmap.josm.data.SystemOfMeasurement;
37import org.openstreetmap.josm.data.coor.EastNorth;
38import org.openstreetmap.josm.data.coor.LatLon;
39import org.openstreetmap.josm.data.imagery.types.EntryType;
40import org.openstreetmap.josm.data.imagery.types.ProjectionType;
41import org.openstreetmap.josm.data.imagery.types.WmsCacheType;
42import org.openstreetmap.josm.data.preferences.StringProperty;
43import org.openstreetmap.josm.data.projection.Projection;
44import org.openstreetmap.josm.gui.layer.WMSLayer;
45import org.openstreetmap.josm.tools.ImageProvider;
46import org.openstreetmap.josm.tools.Utils;
47
48public class WmsCache {
49 //TODO Property for maximum cache size
50 //TODO Property for maximum age of tile, automatically remove old tiles
51 //TODO Measure time for partially loading from cache, compare with time to download tile. If slower, disable partial cache
52 //TODO Do loading from partial cache and downloading at the same time, don't wait for partial cache to load
53
54 private static final StringProperty PROP_CACHE_PATH = new StringProperty("imagery.wms-cache.path", "wms");
55 private static final String INDEX_FILENAME = "index.xml";
56 private static final String LAYERS_INDEX_FILENAME = "layers.properties";
57
58 private static class CacheEntry {
59 final double pixelPerDegree;
60 final double east;
61 final double north;
62 final ProjectionBounds bounds;
63 final String filename;
64
65 long lastUsed;
66 long lastModified;
67
68 CacheEntry(double pixelPerDegree, double east, double north, int tileSize, String filename) {
69 this.pixelPerDegree = pixelPerDegree;
70 this.east = east;
71 this.north = north;
72 this.bounds = new ProjectionBounds(east, north, east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree);
73 this.filename = filename;
74 }
75
76 @Override
77 public String toString() {
78 return "CacheEntry [pixelPerDegree=" + pixelPerDegree + ", east=" + east + ", north=" + north + ", bounds="
79 + bounds + ", filename=" + filename + ", lastUsed=" + lastUsed + ", lastModified=" + lastModified
80 + "]";
81 }
82 }
83
84 private static class ProjectionEntries {
85 final String projection;
86 final String cacheDirectory;
87 final List<CacheEntry> entries = new ArrayList<>();
88
89 ProjectionEntries(String projection, String cacheDirectory) {
90 this.projection = projection;
91 this.cacheDirectory = cacheDirectory;
92 }
93 }
94
95 private final Map<String, ProjectionEntries> entries = new HashMap<>();
96 private final File cacheDir;
97 private final int tileSize; // Should be always 500
98 private int totalFileSize;
99 private boolean totalFileSizeDirty; // Some file was missing - size needs to be recalculated
100 // No need for hashCode/equals on CacheEntry, object identity is enough. Comparing by values can lead to error - CacheEntry for wrong projection could be found
101 private Map<CacheEntry, SoftReference<BufferedImage>> memoryCache = new HashMap<>();
102 private Set<ProjectionBounds> areaToCache;
103
104 protected String cacheDirPath() {
105 String cPath = PROP_CACHE_PATH.get();
106 if (!(new File(cPath).isAbsolute())) {
107 cPath = Main.pref.getCacheDirectory() + File.separator + cPath;
108 }
109 return cPath;
110 }
111
112 public WmsCache(String url, int tileSize) {
113 File globalCacheDir = new File(cacheDirPath());
114 globalCacheDir.mkdirs();
115 cacheDir = new File(globalCacheDir, getCacheDirectory(url));
116 cacheDir.mkdirs();
117 this.tileSize = tileSize;
118 }
119
120 private String getCacheDirectory(String url) {
121 String cacheDirName = null;
122 Properties layersIndex = new Properties();
123 File layerIndexFile = new File(cacheDirPath(), LAYERS_INDEX_FILENAME);
124 try (InputStream fis = new FileInputStream(layerIndexFile)) {
125 layersIndex.load(fis);
126 } catch (FileNotFoundException e) {
127 Main.error("Unable to load layers index for wms cache (file " + layerIndexFile + " not found)");
128 } catch (IOException e) {
129 Main.error("Unable to load layers index for wms cache");
130 Main.error(e);
131 }
132
133 for (Object propKey: layersIndex.keySet()) {
134 String s = (String)propKey;
135 if (url.equals(layersIndex.getProperty(s))) {
136 cacheDirName = s;
137 break;
138 }
139 }
140
141 if (cacheDirName == null) {
142 int counter = 0;
143 while (true) {
144 counter++;
145 if (!layersIndex.keySet().contains(String.valueOf(counter))) {
146 break;
147 }
148 }
149 cacheDirName = String.valueOf(counter);
150 layersIndex.setProperty(cacheDirName, url);
151 try (OutputStream fos = new FileOutputStream(layerIndexFile)) {
152 layersIndex.store(fos, "");
153 } catch (IOException e) {
154 Main.error("Unable to save layer index for wms cache");
155 Main.error(e);
156 }
157 }
158
159 return cacheDirName;
160 }
161
162 private ProjectionEntries getProjectionEntries(Projection projection) {
163 return getProjectionEntries(projection.toCode(), projection.getCacheDirectoryName());
164 }
165
166 private ProjectionEntries getProjectionEntries(String projection, String cacheDirectory) {
167 ProjectionEntries result = entries.get(projection);
168 if (result == null) {
169 result = new ProjectionEntries(projection, cacheDirectory);
170 entries.put(projection, result);
171 }
172
173 return result;
174 }
175
176 public synchronized void loadIndex() {
177 File indexFile = new File(cacheDir, INDEX_FILENAME);
178 try {
179 JAXBContext context = JAXBContext.newInstance(
180 WmsCacheType.class.getPackage().getName(),
181 WmsCacheType.class.getClassLoader());
182 Unmarshaller unmarshaller = context.createUnmarshaller();
183 WmsCacheType cacheEntries;
184 try (InputStream is = new FileInputStream(indexFile)) {
185 cacheEntries = (WmsCacheType)unmarshaller.unmarshal(is);
186 }
187 totalFileSize = cacheEntries.getTotalFileSize();
188 if (cacheEntries.getTileSize() != tileSize) {
189 Main.info("Cache created with different tileSize, cache will be discarded");
190 return;
191 }
192 for (ProjectionType projectionType: cacheEntries.getProjection()) {
193 ProjectionEntries projection = getProjectionEntries(projectionType.getName(), projectionType.getCacheDirectory());
194 for (EntryType entry: projectionType.getEntry()) {
195 CacheEntry ce = new CacheEntry(entry.getPixelPerDegree(), entry.getEast(), entry.getNorth(), tileSize, entry.getFilename());
196 ce.lastUsed = entry.getLastUsed().getTimeInMillis();
197 ce.lastModified = entry.getLastModified().getTimeInMillis();
198 projection.entries.add(ce);
199 }
200 }
201 } catch (Exception e) {
202 if (indexFile.exists()) {
203 Main.error(e);
204 Main.info("Unable to load index for wms-cache, new file will be created");
205 } else {
206 Main.info("Index for wms-cache doesn't exist, new file will be created");
207 }
208 }
209
210 removeNonReferencedFiles();
211 }
212
213 private void removeNonReferencedFiles() {
214
215 Set<String> usedProjections = new HashSet<>();
216
217 for (ProjectionEntries projectionEntries: entries.values()) {
218
219 usedProjections.add(projectionEntries.cacheDirectory);
220
221 File projectionDir = new File(cacheDir, projectionEntries.cacheDirectory);
222 if (projectionDir.exists()) {
223 Set<String> referencedFiles = new HashSet<>();
224
225 for (CacheEntry ce: projectionEntries.entries) {
226 referencedFiles.add(ce.filename);
227 }
228
229 for (File file: projectionDir.listFiles()) {
230 if (!referencedFiles.contains(file.getName())) {
231 file.delete();
232 }
233 }
234 }
235 }
236
237 for (File projectionDir: cacheDir.listFiles()) {
238 if (projectionDir.isDirectory() && !usedProjections.contains(projectionDir.getName())) {
239 Utils.deleteDirectory(projectionDir);
240 }
241 }
242 }
243
244 private int calculateTotalFileSize() {
245 int result = 0;
246 for (ProjectionEntries projectionEntries: entries.values()) {
247 Iterator<CacheEntry> it = projectionEntries.entries.iterator();
248 while (it.hasNext()) {
249 CacheEntry entry = it.next();
250 File imageFile = getImageFile(projectionEntries, entry);
251 if (!imageFile.exists()) {
252 it.remove();
253 } else {
254 result += imageFile.length();
255 }
256 }
257 }
258 return result;
259 }
260
261 public synchronized void saveIndex() {
262 WmsCacheType index = new WmsCacheType();
263
264 if (totalFileSizeDirty) {
265 totalFileSize = calculateTotalFileSize();
266 }
267
268 index.setTileSize(tileSize);
269 index.setTotalFileSize(totalFileSize);
270 for (ProjectionEntries projectionEntries: entries.values()) {
271 if (!projectionEntries.entries.isEmpty()) {
272 ProjectionType projectionType = new ProjectionType();
273 projectionType.setName(projectionEntries.projection);
274 projectionType.setCacheDirectory(projectionEntries.cacheDirectory);
275 index.getProjection().add(projectionType);
276 for (CacheEntry ce: projectionEntries.entries) {
277 EntryType entry = new EntryType();
278 entry.setPixelPerDegree(ce.pixelPerDegree);
279 entry.setEast(ce.east);
280 entry.setNorth(ce.north);
281 Calendar c = Calendar.getInstance();
282 c.setTimeInMillis(ce.lastUsed);
283 entry.setLastUsed(c);
284 c = Calendar.getInstance();
285 c.setTimeInMillis(ce.lastModified);
286 entry.setLastModified(c);
287 entry.setFilename(ce.filename);
288 projectionType.getEntry().add(entry);
289 }
290 }
291 }
292 try {
293 JAXBContext context = JAXBContext.newInstance(
294 WmsCacheType.class.getPackage().getName(),
295 WmsCacheType.class.getClassLoader());
296 Marshaller marshaller = context.createMarshaller();
297 try (OutputStream fos = new FileOutputStream(new File(cacheDir, INDEX_FILENAME))) {
298 marshaller.marshal(index, fos);
299 }
300 } catch (Exception e) {
301 Main.error("Failed to save wms-cache file");
302 Main.error(e);
303 }
304 }
305
306 private File getImageFile(ProjectionEntries projection, CacheEntry entry) {
307 return new File(cacheDir, projection.cacheDirectory + "/" + entry.filename);
308 }
309
310 private BufferedImage loadImage(ProjectionEntries projectionEntries, CacheEntry entry, boolean enforceTransparency) throws IOException {
311 synchronized (this) {
312 entry.lastUsed = System.currentTimeMillis();
313
314 SoftReference<BufferedImage> memCache = memoryCache.get(entry);
315 if (memCache != null) {
316 BufferedImage result = memCache.get();
317 if (result != null) {
318 if (enforceTransparency == ImageProvider.isTransparencyForced(result)) {
319 return result;
320 } else if (Main.isDebugEnabled()) {
321 Main.debug("Skipping "+entry+" from memory cache (transparency enforcement)");
322 }
323 }
324 }
325 }
326
327 try {
328 // Reading can't be in synchronized section, it's too slow
329 BufferedImage result = ImageProvider.read(getImageFile(projectionEntries, entry), true, enforceTransparency);
330 synchronized (this) {
331 if (result == null) {
332 projectionEntries.entries.remove(entry);
333 totalFileSizeDirty = true;
334 }
335 return result;
336 }
337 } catch (IOException e) {
338 synchronized (this) {
339 projectionEntries.entries.remove(entry);
340 totalFileSizeDirty = true;
341 throw e;
342 }
343 }
344 }
345
346 private CacheEntry findEntry(ProjectionEntries projectionEntries, double pixelPerDegree, double east, double north) {
347 for (CacheEntry entry: projectionEntries.entries) {
348 if (entry.pixelPerDegree == pixelPerDegree && entry.east == east && entry.north == north)
349 return entry;
350 }
351 return null;
352 }
353
354 public synchronized boolean hasExactMatch(Projection projection, double pixelPerDegree, double east, double north) {
355 ProjectionEntries projectionEntries = getProjectionEntries(projection);
356 CacheEntry entry = findEntry(projectionEntries, pixelPerDegree, east, north);
357 return (entry != null);
358 }
359
360 public BufferedImage getExactMatch(Projection projection, double pixelPerDegree, double east, double north) {
361 CacheEntry entry = null;
362 ProjectionEntries projectionEntries = null;
363 synchronized (this) {
364 projectionEntries = getProjectionEntries(projection);
365 entry = findEntry(projectionEntries, pixelPerDegree, east, north);
366 }
367 if (entry != null) {
368 try {
369 return loadImage(projectionEntries, entry, WMSLayer.PROP_ALPHA_CHANNEL.get());
370 } catch (IOException e) {
371 Main.error("Unable to load file from wms cache");
372 Main.error(e);
373 return null;
374 }
375 }
376 return null;
377 }
378
379 public BufferedImage getPartialMatch(Projection projection, double pixelPerDegree, double east, double north) {
380 ProjectionEntries projectionEntries;
381 List<CacheEntry> matches;
382 synchronized (this) {
383 matches = new ArrayList<>();
384
385 double minPPD = pixelPerDegree / 5;
386 double maxPPD = pixelPerDegree * 5;
387 projectionEntries = getProjectionEntries(projection);
388
389 double size2 = tileSize / pixelPerDegree;
390 double border = tileSize * 0.01; // Make sure not to load neighboring tiles that intersects this tile only slightly
391 ProjectionBounds bounds = new ProjectionBounds(east + border, north + border,
392 east + size2 - border, north + size2 - border);
393
394 //TODO Do not load tile if it is completely overlapped by other tile with better ppd
395 for (CacheEntry entry: projectionEntries.entries) {
396 if (entry.pixelPerDegree >= minPPD && entry.pixelPerDegree <= maxPPD && entry.bounds.intersects(bounds)) {
397 entry.lastUsed = System.currentTimeMillis();
398 matches.add(entry);
399 }
400 }
401
402 if (matches.isEmpty())
403 return null;
404
405 Collections.sort(matches, new Comparator<CacheEntry>() {
406 @Override
407 public int compare(CacheEntry o1, CacheEntry o2) {
408 return Double.compare(o2.pixelPerDegree, o1.pixelPerDegree);
409 }
410 });
411 }
412
413 // Use alpha layer only when enabled on wms layer
414 boolean alpha = WMSLayer.PROP_ALPHA_CHANNEL.get();
415 BufferedImage result = new BufferedImage(tileSize, tileSize,
416 alpha ? BufferedImage.TYPE_4BYTE_ABGR : BufferedImage.TYPE_3BYTE_BGR);
417 Graphics2D g = result.createGraphics();
418
419 boolean drawAtLeastOnce = false;
420 Map<CacheEntry, SoftReference<BufferedImage>> localCache = new HashMap<>();
421 for (CacheEntry ce: matches) {
422 BufferedImage img;
423 try {
424 // Enforce transparency only when alpha enabled on wms layer too
425 img = loadImage(projectionEntries, ce, alpha);
426 localCache.put(ce, new SoftReference<>(img));
427 } catch (IOException e) {
428 continue;
429 }
430
431 drawAtLeastOnce = true;
432
433 int xDiff = (int)((ce.east - east) * pixelPerDegree);
434 int yDiff = (int)((ce.north - north) * pixelPerDegree);
435 int size = (int)(pixelPerDegree / ce.pixelPerDegree * tileSize);
436
437 int x = xDiff;
438 int y = -size + tileSize - yDiff;
439
440 g.drawImage(img, x, y, size, size, null);
441 }
442
443 if (drawAtLeastOnce) {
444 synchronized (this) {
445 memoryCache.putAll(localCache);
446 }
447 return result;
448 } else
449 return null;
450 }
451
452 private String generateFileName(ProjectionEntries projectionEntries, double pixelPerDegree, Projection projection, double east, double north, String mimeType) {
453 LatLon ll1 = projection.eastNorth2latlon(new EastNorth(east, north));
454 LatLon ll2 = projection.eastNorth2latlon(new EastNorth(east + 100 / pixelPerDegree, north));
455 LatLon ll3 = projection.eastNorth2latlon(new EastNorth(east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree));
456
457 double deltaLat = Math.abs(ll3.lat() - ll1.lat());
458 double deltaLon = Math.abs(ll3.lon() - ll1.lon());
459 int precisionLat = Math.max(0, -(int)Math.ceil(Math.log10(deltaLat)) + 1);
460 int precisionLon = Math.max(0, -(int)Math.ceil(Math.log10(deltaLon)) + 1);
461
462 String zoom = SystemOfMeasurement.METRIC.getDistText(ll1.greatCircleDistance(ll2));
463 String extension = "dat";
464 if (mimeType != null) {
465 switch(mimeType) {
466 case "image/jpeg":
467 case "image/jpg":
468 extension = "jpg";
469 break;
470 case "image/png":
471 extension = "png";
472 break;
473 case "image/gif":
474 extension = "gif";
475 break;
476 default:
477 Main.warn("Unrecognized MIME type: "+mimeType);
478 }
479 }
480
481 int counter = 0;
482 FILENAME_LOOP:
483 while (true) {
484 String result = String.format("%s_%." + precisionLat + "f_%." + precisionLon +"f%s.%s", zoom, ll1.lat(), ll1.lon(), counter==0?"":"_" + counter, extension);
485 for (CacheEntry entry: projectionEntries.entries) {
486 if (entry.filename.equals(result)) {
487 counter++;
488 continue FILENAME_LOOP;
489 }
490 }
491 return result;
492 }
493 }
494
495 /**
496 *
497 * @param img Used only when overlapping is used, when not used, used raw from imageData
498 * @param imageData
499 * @param projection
500 * @param pixelPerDegree
501 * @param east
502 * @param north
503 * @throws IOException
504 */
505 public synchronized void saveToCache(BufferedImage img, InputStream imageData, Projection projection, double pixelPerDegree, double east, double north) throws IOException {
506 ProjectionEntries projectionEntries = getProjectionEntries(projection);
507 CacheEntry entry = findEntry(projectionEntries, pixelPerDegree, east, north);
508 File imageFile;
509 if (entry == null) {
510
511 String mimeType;
512 if (img != null) {
513 mimeType = "image/png";
514 } else {
515 mimeType = URLConnection.guessContentTypeFromStream(imageData);
516 }
517 entry = new CacheEntry(pixelPerDegree, east, north, tileSize,generateFileName(projectionEntries, pixelPerDegree, projection, east, north, mimeType));
518 entry.lastUsed = System.currentTimeMillis();
519 entry.lastModified = entry.lastUsed;
520 projectionEntries.entries.add(entry);
521 imageFile = getImageFile(projectionEntries, entry);
522 } else {
523 imageFile = getImageFile(projectionEntries, entry);
524 totalFileSize -= imageFile.length();
525 }
526
527 imageFile.getParentFile().mkdirs();
528
529 if (img != null) {
530 BufferedImage copy = new BufferedImage(tileSize, tileSize, img.getType());
531 copy.createGraphics().drawImage(img, 0, 0, tileSize, tileSize, 0, img.getHeight() - tileSize, tileSize, img.getHeight(), null);
532 ImageIO.write(copy, "png", imageFile);
533 totalFileSize += imageFile.length();
534 } else {
535 try (OutputStream os = new BufferedOutputStream(new FileOutputStream(imageFile))) {
536 totalFileSize += Utils.copyStream(imageData, os);
537 }
538 }
539 }
540
541 public synchronized void cleanSmallFiles(int size) {
542 for (ProjectionEntries projectionEntries: entries.values()) {
543 Iterator<CacheEntry> it = projectionEntries.entries.iterator();
544 while (it.hasNext()) {
545 File file = getImageFile(projectionEntries, it.next());
546 long length = file.length();
547 if (length <= size) {
548 if (length == 0) {
549 totalFileSizeDirty = true; // File probably doesn't exist
550 }
551 totalFileSize -= size;
552 file.delete();
553 it.remove();
554 }
555 }
556 }
557 }
558
559 public static String printDate(Calendar c) {
560 return (new SimpleDateFormat("yyyy-MM-dd")).format(c.getTime());
561 }
562
563 private boolean isInsideAreaToCache(CacheEntry cacheEntry) {
564 for (ProjectionBounds b: areaToCache) {
565 if (cacheEntry.bounds.intersects(b))
566 return true;
567 }
568 return false;
569 }
570
571 public synchronized void setAreaToCache(Set<ProjectionBounds> areaToCache) {
572 this.areaToCache = areaToCache;
573 Iterator<CacheEntry> it = memoryCache.keySet().iterator();
574 while (it.hasNext()) {
575 if (!isInsideAreaToCache(it.next())) {
576 it.remove();
577 }
578 }
579 }
580}
Note: See TracBrowser for help on using the repository browser.