source: josm/trunk/src/org/openstreetmap/josm/data/imagery/WmsCache.java@ 7937

Last change on this file since 7937 was 7937, checked in by bastiK, 9 years ago

add subversion property svn:eol=native
  • Property svn:eol-style set to native
File size: 23.0 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.imagery;
3
4import java.awt.Graphics2D;
5import java.awt.image.BufferedImage;
6import java.io.BufferedOutputStream;
7import java.io.File;
8import java.io.FileInputStream;
9import java.io.FileNotFoundException;
10import java.io.FileOutputStream;
11import java.io.IOException;
12import java.io.InputStream;
13import java.io.OutputStream;
14import java.lang.ref.SoftReference;
15import java.net.URLConnection;
16import java.util.ArrayList;
17import java.util.Calendar;
18import java.util.Collections;
19import java.util.Comparator;
20import java.util.HashMap;
21import java.util.HashSet;
22import java.util.Iterator;
23import java.util.List;
24import java.util.Map;
25import java.util.Properties;
26import java.util.Set;
27
28import javax.imageio.ImageIO;
29import javax.xml.bind.JAXBContext;
30import javax.xml.bind.Marshaller;
31import javax.xml.bind.Unmarshaller;
32
33import org.openstreetmap.josm.Main;
34import org.openstreetmap.josm.data.ProjectionBounds;
35import org.openstreetmap.josm.data.SystemOfMeasurement;
36import org.openstreetmap.josm.data.coor.EastNorth;
37import org.openstreetmap.josm.data.coor.LatLon;
38import org.openstreetmap.josm.data.imagery.types.EntryType;
39import org.openstreetmap.josm.data.imagery.types.ProjectionType;
40import org.openstreetmap.josm.data.imagery.types.WmsCacheType;
41import org.openstreetmap.josm.data.preferences.StringProperty;
42import org.openstreetmap.josm.data.projection.Projection;
43import org.openstreetmap.josm.gui.layer.WMSLayer;
44import org.openstreetmap.josm.tools.ImageProvider;
45import org.openstreetmap.josm.tools.Utils;
46import org.openstreetmap.josm.tools.date.DateUtils;
47
48public class WmsCache {
49 //TODO Property for maximum cache size
50 //TODO Property for maximum age of tile, automatically remove old tiles
51 //TODO Measure time for partially loading from cache, compare with time to download tile. If slower, disable partial cache
52 //TODO Do loading from partial cache and downloading at the same time, don't wait for partial cache to load
53
54 private static final StringProperty PROP_CACHE_PATH = new StringProperty("imagery.wms-cache.path", "wms");
55 private static final String INDEX_FILENAME = "index.xml";
56 private static final String LAYERS_INDEX_FILENAME = "layers.properties";
57
58 private static class CacheEntry {
59 final double pixelPerDegree;
60 final double east;
61 final double north;
62 final ProjectionBounds bounds;
63 final String filename;
64
65 long lastUsed;
66 long lastModified;
67
68 CacheEntry(double pixelPerDegree, double east, double north, int tileSize, String filename) {
69 this.pixelPerDegree = pixelPerDegree;
70 this.east = east;
71 this.north = north;
72 this.bounds = new ProjectionBounds(east, north, east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree);
73 this.filename = filename;
74 }
75
76 @Override
77 public String toString() {
78 return "CacheEntry [pixelPerDegree=" + pixelPerDegree + ", east=" + east + ", north=" + north + ", bounds="
79 + bounds + ", filename=" + filename + ", lastUsed=" + lastUsed + ", lastModified=" + lastModified
80 + "]";
81 }
82 }
83
84 private static class ProjectionEntries {
85 final String projection;
86 final String cacheDirectory;
87 final List<CacheEntry> entries = new ArrayList<>();
88
89 ProjectionEntries(String projection, String cacheDirectory) {
90 this.projection = projection;
91 this.cacheDirectory = cacheDirectory;
92 }
93 }
94
95 private final Map<String, ProjectionEntries> entries = new HashMap<>();
96 private final File cacheDir;
97 private final int tileSize; // Should be always 500
98 private int totalFileSize;
99 private boolean totalFileSizeDirty; // Some file was missing - size needs to be recalculated
100 // No need for hashCode/equals on CacheEntry, object identity is enough. Comparing by values can lead to error - CacheEntry for wrong projection could be found
101 private Map<CacheEntry, SoftReference<BufferedImage>> memoryCache = new HashMap<>();
102 private Set<ProjectionBounds> areaToCache;
103
104 protected String cacheDirPath() {
105 String cPath = PROP_CACHE_PATH.get();
106 if (!(new File(cPath).isAbsolute())) {
107 cPath = Main.pref.getCacheDirectory() + File.separator + cPath;
108 }
109 return cPath;
110 }
111
112 public WmsCache(String url, int tileSize) {
113 File globalCacheDir = new File(cacheDirPath());
114 globalCacheDir.mkdirs();
115 cacheDir = new File(globalCacheDir, getCacheDirectory(url));
116 cacheDir.mkdirs();
117 this.tileSize = tileSize;
118 }
119
120 private String getCacheDirectory(String url) {
121 String cacheDirName = null;
122 Properties layersIndex = new Properties();
123 File layerIndexFile = new File(cacheDirPath(), LAYERS_INDEX_FILENAME);
124 try (InputStream fis = new FileInputStream(layerIndexFile)) {
125 layersIndex.load(fis);
126 } catch (FileNotFoundException e) {
127 Main.error("Unable to load layers index for wms cache (file " + layerIndexFile + " not found)");
128 } catch (IOException e) {
129 Main.error("Unable to load layers index for wms cache");
130 Main.error(e);
131 }
132
133 for (Object propKey: layersIndex.keySet()) {
134 String s = (String)propKey;
135 if (url.equals(layersIndex.getProperty(s))) {
136 cacheDirName = s;
137 break;
138 }
139 }
140
141 if (cacheDirName == null) {
142 int counter = 0;
143 while (true) {
144 counter++;
145 if (!layersIndex.keySet().contains(String.valueOf(counter))) {
146 break;
147 }
148 }
149 cacheDirName = String.valueOf(counter);
150 layersIndex.setProperty(cacheDirName, url);
151 try (OutputStream fos = new FileOutputStream(layerIndexFile)) {
152 layersIndex.store(fos, "");
153 } catch (IOException e) {
154 Main.error("Unable to save layer index for wms cache");
155 Main.error(e);
156 }
157 }
158
159 return cacheDirName;
160 }
161
162 private ProjectionEntries getProjectionEntries(Projection projection) {
163 return getProjectionEntries(projection.toCode(), projection.getCacheDirectoryName());
164 }
165
166 private ProjectionEntries getProjectionEntries(String projection, String cacheDirectory) {
167 ProjectionEntries result = entries.get(projection);
168 if (result == null) {
169 result = new ProjectionEntries(projection, cacheDirectory);
170 entries.put(projection, result);
171 }
172
173 return result;
174 }
175
176 public synchronized void loadIndex() {
177 File indexFile = new File(cacheDir, INDEX_FILENAME);
178 try {
179 JAXBContext context = JAXBContext.newInstance(
180 WmsCacheType.class.getPackage().getName(),
181 WmsCacheType.class.getClassLoader());
182 Unmarshaller unmarshaller = context.createUnmarshaller();
183 WmsCacheType cacheEntries;
184 try (InputStream is = new FileInputStream(indexFile)) {
185 cacheEntries = (WmsCacheType)unmarshaller.unmarshal(is);
186 }
187 totalFileSize = cacheEntries.getTotalFileSize();
188 if (cacheEntries.getTileSize() != tileSize) {
189 Main.info("Cache created with different tileSize, cache will be discarded");
190 return;
191 }
192 for (ProjectionType projectionType: cacheEntries.getProjection()) {
193 ProjectionEntries projection = getProjectionEntries(projectionType.getName(), projectionType.getCacheDirectory());
194 for (EntryType entry: projectionType.getEntry()) {
195 CacheEntry ce = new CacheEntry(entry.getPixelPerDegree(), entry.getEast(), entry.getNorth(), tileSize, entry.getFilename());
196 ce.lastUsed = entry.getLastUsed().getTimeInMillis();
197 ce.lastModified = entry.getLastModified().getTimeInMillis();
198 projection.entries.add(ce);
199 }
200 }
201 } catch (Exception e) {
202 if (indexFile.exists()) {
203 Main.error(e);
204 Main.info("Unable to load index for wms-cache, new file will be created");
205 } else {
206 Main.info("Index for wms-cache doesn't exist, new file will be created");
207 }
208 }
209
210 removeNonReferencedFiles();
211 }
212
213 private void removeNonReferencedFiles() {
214
215 Set<String> usedProjections = new HashSet<>();
216
217 for (ProjectionEntries projectionEntries: entries.values()) {
218
219 usedProjections.add(projectionEntries.cacheDirectory);
220
221 File projectionDir = new File(cacheDir, projectionEntries.cacheDirectory);
222 if (projectionDir.exists()) {
223 Set<String> referencedFiles = new HashSet<>();
224
225 for (CacheEntry ce: projectionEntries.entries) {
226 referencedFiles.add(ce.filename);
227 }
228
229 for (File file: projectionDir.listFiles()) {
230 if (!referencedFiles.contains(file.getName())) {
231 file.delete();
232 }
233 }
234 }
235 }
236
237 for (File projectionDir: cacheDir.listFiles()) {
238 if (projectionDir.isDirectory() && !usedProjections.contains(projectionDir.getName())) {
239 Utils.deleteDirectory(projectionDir);
240 }
241 }
242 }
243
244 private int calculateTotalFileSize() {
245 int result = 0;
246 for (ProjectionEntries projectionEntries: entries.values()) {
247 Iterator<CacheEntry> it = projectionEntries.entries.iterator();
248 while (it.hasNext()) {
249 CacheEntry entry = it.next();
250 File imageFile = getImageFile(projectionEntries, entry);
251 if (!imageFile.exists()) {
252 it.remove();
253 } else {
254 result += imageFile.length();
255 }
256 }
257 }
258 return result;
259 }
260
261 public synchronized void saveIndex() {
262 WmsCacheType index = new WmsCacheType();
263
264 if (totalFileSizeDirty) {
265 totalFileSize = calculateTotalFileSize();
266 }
267
268 index.setTileSize(tileSize);
269 index.setTotalFileSize(totalFileSize);
270 for (ProjectionEntries projectionEntries: entries.values()) {
271 if (!projectionEntries.entries.isEmpty()) {
272 ProjectionType projectionType = new ProjectionType();
273 projectionType.setName(projectionEntries.projection);
274 projectionType.setCacheDirectory(projectionEntries.cacheDirectory);
275 index.getProjection().add(projectionType);
276 for (CacheEntry ce: projectionEntries.entries) {
277 EntryType entry = new EntryType();
278 entry.setPixelPerDegree(ce.pixelPerDegree);
279 entry.setEast(ce.east);
280 entry.setNorth(ce.north);
281 Calendar c = Calendar.getInstance();
282 c.setTimeInMillis(ce.lastUsed);
283 entry.setLastUsed(c);
284 c = Calendar.getInstance();
285 c.setTimeInMillis(ce.lastModified);
286 entry.setLastModified(c);
287 entry.setFilename(ce.filename);
288 projectionType.getEntry().add(entry);
289 }
290 }
291 }
292 try {
293 JAXBContext context = JAXBContext.newInstance(
294 WmsCacheType.class.getPackage().getName(),
295 WmsCacheType.class.getClassLoader());
296 Marshaller marshaller = context.createMarshaller();
297 try (OutputStream fos = new FileOutputStream(new File(cacheDir, INDEX_FILENAME))) {
298 marshaller.marshal(index, fos);
299 }
300 } catch (Exception e) {
301 Main.error("Failed to save wms-cache file");
302 Main.error(e);
303 }
304 }
305
306 private File getImageFile(ProjectionEntries projection, CacheEntry entry) {
307 return new File(cacheDir, projection.cacheDirectory + "/" + entry.filename);
308 }
309
310 private BufferedImage loadImage(ProjectionEntries projectionEntries, CacheEntry entry, boolean enforceTransparency) throws IOException {
311 synchronized (this) {
312 entry.lastUsed = System.currentTimeMillis();
313
314 SoftReference<BufferedImage> memCache = memoryCache.get(entry);
315 if (memCache != null) {
316 BufferedImage result = memCache.get();
317 if (result != null) {
318 if (enforceTransparency == ImageProvider.isTransparencyForced(result)) {
319 return result;
320 } else if (Main.isDebugEnabled()) {
321 Main.debug("Skipping "+entry+" from memory cache (transparency enforcement)");
322 }
323 }
324 }
325 }
326
327 try {
328 // Reading can't be in synchronized section, it's too slow
329 BufferedImage result = ImageProvider.read(getImageFile(projectionEntries, entry), true, enforceTransparency);
330 synchronized (this) {
331 if (result == null) {
332 projectionEntries.entries.remove(entry);
333 totalFileSizeDirty = true;
334 }
335 return result;
336 }
337 } catch (IOException e) {
338 synchronized (this) {
339 projectionEntries.entries.remove(entry);
340 totalFileSizeDirty = true;
341 throw e;
342 }
343 }
344 }
345
346 private CacheEntry findEntry(ProjectionEntries projectionEntries, double pixelPerDegree, double east, double north) {
347 for (CacheEntry entry: projectionEntries.entries) {
348 if (entry.pixelPerDegree == pixelPerDegree && entry.east == east && entry.north == north)
349 return entry;
350 }
351 return null;
352 }
353
354 public synchronized boolean hasExactMatch(Projection projection, double pixelPerDegree, double east, double north) {
355 ProjectionEntries projectionEntries = getProjectionEntries(projection);
356 CacheEntry entry = findEntry(projectionEntries, pixelPerDegree, east, north);
357 return (entry != null);
358 }
359
360 public BufferedImage getExactMatch(Projection projection, double pixelPerDegree, double east, double north) {
361 CacheEntry entry = null;
362 ProjectionEntries projectionEntries = null;
363 synchronized (this) {
364 projectionEntries = getProjectionEntries(projection);
365 entry = findEntry(projectionEntries, pixelPerDegree, east, north);
366 }
367 if (entry != null) {
368 try {
369 return loadImage(projectionEntries, entry, WMSLayer.PROP_ALPHA_CHANNEL.get());
370 } catch (IOException e) {
371 Main.error("Unable to load file from wms cache");
372 Main.error(e);
373 return null;
374 }
375 }
376 return null;
377 }
378
379 public BufferedImage getPartialMatch(Projection projection, double pixelPerDegree, double east, double north) {
380 ProjectionEntries projectionEntries;
381 List<CacheEntry> matches;
382 synchronized (this) {
383 matches = new ArrayList<>();
384
385 double minPPD = pixelPerDegree / 5;
386 double maxPPD = pixelPerDegree * 5;
387 projectionEntries = getProjectionEntries(projection);
388
389 double size2 = tileSize / pixelPerDegree;
390 double border = tileSize * 0.01; // Make sure not to load neighboring tiles that intersects this tile only slightly
391 ProjectionBounds bounds = new ProjectionBounds(east + border, north + border,
392 east + size2 - border, north + size2 - border);
393
394 //TODO Do not load tile if it is completely overlapped by other tile with better ppd
395 for (CacheEntry entry: projectionEntries.entries) {
396 if (entry.pixelPerDegree >= minPPD && entry.pixelPerDegree <= maxPPD && entry.bounds.intersects(bounds)) {
397 entry.lastUsed = System.currentTimeMillis();
398 matches.add(entry);
399 }
400 }
401
402 if (matches.isEmpty())
403 return null;
404
405 Collections.sort(matches, new Comparator<CacheEntry>() {
406 @Override
407 public int compare(CacheEntry o1, CacheEntry o2) {
408 return Double.compare(o2.pixelPerDegree, o1.pixelPerDegree);
409 }
410 });
411 }
412
413 // Use alpha layer only when enabled on wms layer
414 boolean alpha = WMSLayer.PROP_ALPHA_CHANNEL.get();
415 BufferedImage result = new BufferedImage(tileSize, tileSize,
416 alpha ? BufferedImage.TYPE_4BYTE_ABGR : BufferedImage.TYPE_3BYTE_BGR);
417 Graphics2D g = result.createGraphics();
418
419 boolean drawAtLeastOnce = false;
420 Map<CacheEntry, SoftReference<BufferedImage>> localCache = new HashMap<>();
421 for (CacheEntry ce: matches) {
422 BufferedImage img;
423 try {
424 // Enforce transparency only when alpha enabled on wms layer too
425 img = loadImage(projectionEntries, ce, alpha);
426 localCache.put(ce, new SoftReference<>(img));
427 } catch (IOException e) {
428 continue;
429 }
430
431 drawAtLeastOnce = true;
432
433 int xDiff = (int)((ce.east - east) * pixelPerDegree);
434 int yDiff = (int)((ce.north - north) * pixelPerDegree);
435 int size = (int)(pixelPerDegree / ce.pixelPerDegree * tileSize);
436
437 int x = xDiff;
438 int y = -size + tileSize - yDiff;
439
440 g.drawImage(img, x, y, size, size, null);
441 }
442
443 if (drawAtLeastOnce) {
444 synchronized (this) {
445 memoryCache.putAll(localCache);
446 }
447 return result;
448 } else
449 return null;
450 }
451
452 private String generateFileName(ProjectionEntries projectionEntries, double pixelPerDegree, Projection projection, double east, double north, String mimeType) {
453 LatLon ll1 = projection.eastNorth2latlon(new EastNorth(east, north));
454 LatLon ll2 = projection.eastNorth2latlon(new EastNorth(east + 100 / pixelPerDegree, north));
455 LatLon ll3 = projection.eastNorth2latlon(new EastNorth(east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree));
456
457 double deltaLat = Math.abs(ll3.lat() - ll1.lat());
458 double deltaLon = Math.abs(ll3.lon() - ll1.lon());
459 int precisionLat = Math.max(0, -(int)Math.ceil(Math.log10(deltaLat)) + 1);
460 int precisionLon = Math.max(0, -(int)Math.ceil(Math.log10(deltaLon)) + 1);
461
462 String zoom = SystemOfMeasurement.METRIC.getDistText(ll1.greatCircleDistance(ll2));
463 String extension = "dat";
464 if (mimeType != null) {
465 switch(mimeType) {
466 case "image/jpeg":
467 case "image/jpg":
468 extension = "jpg";
469 break;
470 case "image/png":
471 extension = "png";
472 break;
473 case "image/gif":
474 extension = "gif";
475 break;
476 default:
477 Main.warn("Unrecognized MIME type: "+mimeType);
478 }
479 }
480
481 int counter = 0;
482 FILENAME_LOOP:
483 while (true) {
484 String result = String.format("%s_%." + precisionLat + "f_%." + precisionLon +"f%s.%s", zoom, ll1.lat(), ll1.lon(), counter==0?"":"_" + counter, extension);
485 for (CacheEntry entry: projectionEntries.entries) {
486 if (entry.filename.equals(result)) {
487 counter++;
488 continue FILENAME_LOOP;
489 }
490 }
491 return result;
492 }
493 }
494
495 /**
496 *
497 * @param img Used only when overlapping is used, when not used, used raw from imageData
498 * @param imageData
499 * @param projection
500 * @param pixelPerDegree
501 * @param east
502 * @param north
503 * @throws IOException
504 */
505 public synchronized void saveToCache(BufferedImage img, InputStream imageData, Projection projection, double pixelPerDegree, double east, double north) throws IOException {
506 ProjectionEntries projectionEntries = getProjectionEntries(projection);
507 CacheEntry entry = findEntry(projectionEntries, pixelPerDegree, east, north);
508 File imageFile;
509 if (entry == null) {
510
511 String mimeType;
512 if (img != null) {
513 mimeType = "image/png";
514 } else {
515 mimeType = URLConnection.guessContentTypeFromStream(imageData);
516 }
517 entry = new CacheEntry(pixelPerDegree, east, north, tileSize,generateFileName(projectionEntries, pixelPerDegree, projection, east, north, mimeType));
518 entry.lastUsed = System.currentTimeMillis();
519 entry.lastModified = entry.lastUsed;
520 projectionEntries.entries.add(entry);
521 imageFile = getImageFile(projectionEntries, entry);
522 } else {
523 imageFile = getImageFile(projectionEntries, entry);
524 totalFileSize -= imageFile.length();
525 }
526
527 imageFile.getParentFile().mkdirs();
528
529 if (img != null) {
530 BufferedImage copy = new BufferedImage(tileSize, tileSize, img.getType());
531 copy.createGraphics().drawImage(img, 0, 0, tileSize, tileSize, 0, img.getHeight() - tileSize, tileSize, img.getHeight(), null);
532 ImageIO.write(copy, "png", imageFile);
533 totalFileSize += imageFile.length();
534 } else {
535 try (OutputStream os = new BufferedOutputStream(new FileOutputStream(imageFile))) {
536 totalFileSize += Utils.copyStream(imageData, os);
537 }
538 }
539 }
540
541 public synchronized void cleanSmallFiles(int size) {
542 for (ProjectionEntries projectionEntries: entries.values()) {
543 Iterator<CacheEntry> it = projectionEntries.entries.iterator();
544 while (it.hasNext()) {
545 File file = getImageFile(projectionEntries, it.next());
546 long length = file.length();
547 if (length <= size) {
548 if (length == 0) {
549 totalFileSizeDirty = true; // File probably doesn't exist
550 }
551 totalFileSize -= size;
552 file.delete();
553 it.remove();
554 }
555 }
556 }
557 }
558
559 public static String printDate(Calendar c) {
560 return DateUtils.newIsoDateFormat().format(c.getTime());
561 }
562
563 private boolean isInsideAreaToCache(CacheEntry cacheEntry) {
564 for (ProjectionBounds b: areaToCache) {
565 if (cacheEntry.bounds.intersects(b))
566 return true;
567 }
568 return false;
569 }
570
571 public synchronized void setAreaToCache(Set<ProjectionBounds> areaToCache) {
572 this.areaToCache = areaToCache;
573 Iterator<CacheEntry> it = memoryCache.keySet().iterator();
574 while (it.hasNext()) {
575 if (!isInsideAreaToCache(it.next())) {
576 it.remove();
577 }
578 }
579 }
580}
Note: See TracBrowser for help on using the repository browser.