source: josm/trunk/src/org/openstreetmap/josm/data/imagery/WmsCache.java@ 6093

Last change on this file since 6093 was 6093, checked in by akks, 11 years ago

see #8902 - collection size ==/!= 0 -> isEmpty()/!isEmpty() (patch by shinigami)
File size: 22.3 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.imagery;
3
4
5import java.awt.Graphics2D;
6import java.awt.image.BufferedImage;
7import java.io.BufferedOutputStream;
8import java.io.File;
9import java.io.FileInputStream;
10import java.io.FileNotFoundException;
11import java.io.FileOutputStream;
12import java.io.IOException;
13import java.io.InputStream;
14import java.io.OutputStream;
15import java.lang.ref.SoftReference;
16import java.net.URLConnection;
17import java.text.SimpleDateFormat;
18import java.util.ArrayList;
19import java.util.Calendar;
20import java.util.Collections;
21import java.util.Comparator;
22import java.util.HashMap;
23import java.util.HashSet;
24import java.util.Iterator;
25import java.util.List;
26import java.util.Map;
27import java.util.Properties;
28import java.util.Set;
29
30import javax.imageio.ImageIO;
31import javax.xml.bind.JAXBContext;
32import javax.xml.bind.Marshaller;
33import javax.xml.bind.Unmarshaller;
34
35import org.openstreetmap.josm.Main;
36import org.openstreetmap.josm.data.ProjectionBounds;
37import org.openstreetmap.josm.data.coor.EastNorth;
38import org.openstreetmap.josm.data.coor.LatLon;
39import org.openstreetmap.josm.data.imagery.types.EntryType;
40import org.openstreetmap.josm.data.imagery.types.ProjectionType;
41import org.openstreetmap.josm.data.imagery.types.WmsCacheType;
42import org.openstreetmap.josm.data.preferences.StringProperty;
43import org.openstreetmap.josm.data.projection.Projection;
44import org.openstreetmap.josm.gui.NavigatableComponent;
45import org.openstreetmap.josm.tools.Utils;
46
47
48
49public class WmsCache {
50 //TODO Property for maximum cache size
51 //TODO Property for maximum age of tile, automatically remove old tiles
52 //TODO Measure time for partially loading from cache, compare with time to download tile. If slower, disable partial cache
53 //TODO Do loading from partial cache and downloading at the same time, don't wait for partical cache to load
54
55 private static final StringProperty PROP_CACHE_PATH = new StringProperty("imagery.wms-cache.path", "wms");
56 private static final String INDEX_FILENAME = "index.xml";
57 private static final String LAYERS_INDEX_FILENAME = "layers.properties";
58
59 private static class CacheEntry {
60 final double pixelPerDegree;
61 final double east;
62 final double north;
63 final ProjectionBounds bounds;
64 final String filename;
65
66 long lastUsed;
67 long lastModified;
68
69 CacheEntry(double pixelPerDegree, double east, double north, int tileSize, String filename) {
70 this.pixelPerDegree = pixelPerDegree;
71 this.east = east;
72 this.north = north;
73 this.bounds = new ProjectionBounds(east, north, east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree);
74 this.filename = filename;
75 }
76 }
77
78 private static class ProjectionEntries {
79 final String projection;
80 final String cacheDirectory;
81 final List<CacheEntry> entries = new ArrayList<WmsCache.CacheEntry>();
82
83 ProjectionEntries(String projection, String cacheDirectory) {
84 this.projection = projection;
85 this.cacheDirectory = cacheDirectory;
86 }
87 }
88
89 private final Map<String, ProjectionEntries> entries = new HashMap<String, ProjectionEntries>();
90 private final File cacheDir;
91 private final int tileSize; // Should be always 500
92 private int totalFileSize;
93 private boolean totalFileSizeDirty; // Some file was missing - size needs to be recalculated
94 // No need for hashCode/equals on CacheEntry, object identity is enough. Comparing by values can lead to error - CacheEntry for wrong projection could be found
95 private Map<CacheEntry, SoftReference<BufferedImage>> memoryCache = new HashMap<WmsCache.CacheEntry, SoftReference<BufferedImage>>();
96 private Set<ProjectionBounds> areaToCache;
97
98 protected String cacheDirPath() {
99 String cPath = PROP_CACHE_PATH.get();
100 if (!(new File(cPath).isAbsolute())) {
101 cPath = Main.pref.getCacheDirectory() + File.separator + cPath;
102 }
103 return cPath;
104 }
105
106 public WmsCache(String url, int tileSize) {
107 File globalCacheDir = new File(cacheDirPath());
108 globalCacheDir.mkdirs();
109 cacheDir = new File(globalCacheDir, getCacheDirectory(url));
110 cacheDir.mkdirs();
111 this.tileSize = tileSize;
112 }
113
114 private String getCacheDirectory(String url) {
115 String cacheDirName = null;
116 InputStream fis = null;
117 OutputStream fos = null;
118 try {
119 Properties layersIndex = new Properties();
120 File layerIndexFile = new File(cacheDirPath(), LAYERS_INDEX_FILENAME);
121 try {
122 fis = new FileInputStream(layerIndexFile);
123 layersIndex.load(fis);
124 } catch (FileNotFoundException e) {
125 System.out.println("Unable to load layers index for wms cache (file " + layerIndexFile + " not found)");
126 } catch (IOException e) {
127 System.err.println("Unable to load layers index for wms cache");
128 e.printStackTrace();
129 }
130
131 for (Object propKey: layersIndex.keySet()) {
132 String s = (String)propKey;
133 if (url.equals(layersIndex.getProperty(s))) {
134 cacheDirName = s;
135 break;
136 }
137 }
138
139 if (cacheDirName == null) {
140 int counter = 0;
141 while (true) {
142 counter++;
143 if (!layersIndex.keySet().contains(String.valueOf(counter))) {
144 break;
145 }
146 }
147 cacheDirName = String.valueOf(counter);
148 layersIndex.setProperty(cacheDirName, url);
149 try {
150 fos = new FileOutputStream(layerIndexFile);
151 layersIndex.store(fos, "");
152 } catch (IOException e) {
153 System.err.println("Unable to save layer index for wms cache");
154 e.printStackTrace();
155 }
156 }
157 } finally {
158 Utils.close(fos);
159 Utils.close(fis);
160 }
161
162 return cacheDirName;
163 }
164
165 private ProjectionEntries getProjectionEntries(Projection projection) {
166 return getProjectionEntries(projection.toCode(), projection.getCacheDirectoryName());
167 }
168
169 private ProjectionEntries getProjectionEntries(String projection, String cacheDirectory) {
170 ProjectionEntries result = entries.get(projection);
171 if (result == null) {
172 result = new ProjectionEntries(projection, cacheDirectory);
173 entries.put(projection, result);
174 }
175
176 return result;
177 }
178
179 public synchronized void loadIndex() {
180 File indexFile = new File(cacheDir, INDEX_FILENAME);
181 try {
182 JAXBContext context = JAXBContext.newInstance(
183 WmsCacheType.class.getPackage().getName(),
184 WmsCacheType.class.getClassLoader());
185 Unmarshaller unmarshaller = context.createUnmarshaller();
186 WmsCacheType cacheEntries = (WmsCacheType)unmarshaller.unmarshal(new FileInputStream(indexFile));
187 totalFileSize = cacheEntries.getTotalFileSize();
188 if (cacheEntries.getTileSize() != tileSize) {
189 System.out.println("Cache created with different tileSize, cache will be discarded");
190 return;
191 }
192 for (ProjectionType projectionType: cacheEntries.getProjection()) {
193 ProjectionEntries projection = getProjectionEntries(projectionType.getName(), projectionType.getCacheDirectory());
194 for (EntryType entry: projectionType.getEntry()) {
195 CacheEntry ce = new CacheEntry(entry.getPixelPerDegree(), entry.getEast(), entry.getNorth(), tileSize, entry.getFilename());
196 ce.lastUsed = entry.getLastUsed().getTimeInMillis();
197 ce.lastModified = entry.getLastModified().getTimeInMillis();
198 projection.entries.add(ce);
199 }
200 }
201 } catch (Exception e) {
202 if (indexFile.exists()) {
203 e.printStackTrace();
204 System.out.println("Unable to load index for wms-cache, new file will be created");
205 } else {
206 System.out.println("Index for wms-cache doesn't exist, new file will be created");
207 }
208 }
209
210 removeNonReferencedFiles();
211 }
212
213 private void removeNonReferencedFiles() {
214
215 Set<String> usedProjections = new HashSet<String>();
216
217 for (ProjectionEntries projectionEntries: entries.values()) {
218
219 usedProjections.add(projectionEntries.cacheDirectory);
220
221 File projectionDir = new File(cacheDir, projectionEntries.cacheDirectory);
222 if (projectionDir.exists()) {
223 Set<String> referencedFiles = new HashSet<String>();
224
225 for (CacheEntry ce: projectionEntries.entries) {
226 referencedFiles.add(ce.filename);
227 }
228
229 for (File file: projectionDir.listFiles()) {
230 if (!referencedFiles.contains(file.getName())) {
231 file.delete();
232 }
233 }
234 }
235 }
236
237 for (File projectionDir: cacheDir.listFiles()) {
238 if (projectionDir.isDirectory() && !usedProjections.contains(projectionDir.getName())) {
239 Utils.deleteDirectory(projectionDir);
240 }
241 }
242 }
243
244 private int calculateTotalFileSize() {
245 int result = 0;
246 for (ProjectionEntries projectionEntries: entries.values()) {
247 Iterator<CacheEntry> it = projectionEntries.entries.iterator();
248 while (it.hasNext()) {
249 CacheEntry entry = it.next();
250 File imageFile = getImageFile(projectionEntries, entry);
251 if (!imageFile.exists()) {
252 it.remove();
253 } else {
254 result += imageFile.length();
255 }
256 }
257 }
258 return result;
259 }
260
261 public synchronized void saveIndex() {
262 WmsCacheType index = new WmsCacheType();
263
264 if (totalFileSizeDirty) {
265 totalFileSize = calculateTotalFileSize();
266 }
267
268 index.setTileSize(tileSize);
269 index.setTotalFileSize(totalFileSize);
270 for (ProjectionEntries projectionEntries: entries.values()) {
271 if (!projectionEntries.entries.isEmpty()) {
272 ProjectionType projectionType = new ProjectionType();
273 projectionType.setName(projectionEntries.projection);
274 projectionType.setCacheDirectory(projectionEntries.cacheDirectory);
275 index.getProjection().add(projectionType);
276 for (CacheEntry ce: projectionEntries.entries) {
277 EntryType entry = new EntryType();
278 entry.setPixelPerDegree(ce.pixelPerDegree);
279 entry.setEast(ce.east);
280 entry.setNorth(ce.north);
281 Calendar c = Calendar.getInstance();
282 c.setTimeInMillis(ce.lastUsed);
283 entry.setLastUsed(c);
284 c = Calendar.getInstance();
285 c.setTimeInMillis(ce.lastModified);
286 entry.setLastModified(c);
287 entry.setFilename(ce.filename);
288 projectionType.getEntry().add(entry);
289 }
290 }
291 }
292 try {
293 JAXBContext context = JAXBContext.newInstance(
294 WmsCacheType.class.getPackage().getName(),
295 WmsCacheType.class.getClassLoader());
296 Marshaller marshaller = context.createMarshaller();
297 marshaller.marshal(index, new FileOutputStream(new File(cacheDir, INDEX_FILENAME)));
298 } catch (Exception e) {
299 System.err.println("Failed to save wms-cache file");
300 e.printStackTrace();
301 }
302 }
303
304 private File getImageFile(ProjectionEntries projection, CacheEntry entry) {
305 return new File(cacheDir, projection.cacheDirectory + "/" + entry.filename);
306 }
307
308
309 private BufferedImage loadImage(ProjectionEntries projectionEntries, CacheEntry entry) throws IOException {
310
311 synchronized (this) {
312 entry.lastUsed = System.currentTimeMillis();
313
314 SoftReference<BufferedImage> memCache = memoryCache.get(entry);
315 if (memCache != null) {
316 BufferedImage result = memCache.get();
317 if (result != null)
318 return result;
319 }
320 }
321
322 try {
323 // Reading can't be in synchronized section, it's too slow
324 BufferedImage result = ImageIO.read(getImageFile(projectionEntries, entry));
325 synchronized (this) {
326 if (result == null) {
327 projectionEntries.entries.remove(entry);
328 totalFileSizeDirty = true;
329 }
330 return result;
331 }
332 } catch (IOException e) {
333 synchronized (this) {
334 projectionEntries.entries.remove(entry);
335 totalFileSizeDirty = true;
336 throw e;
337 }
338 }
339 }
340
341 private CacheEntry findEntry(ProjectionEntries projectionEntries, double pixelPerDegree, double east, double north) {
342 for (CacheEntry entry: projectionEntries.entries) {
343 if (entry.pixelPerDegree == pixelPerDegree && entry.east == east && entry.north == north)
344 return entry;
345 }
346 return null;
347 }
348
349 public synchronized boolean hasExactMatch(Projection projection, double pixelPerDegree, double east, double north) {
350 ProjectionEntries projectionEntries = getProjectionEntries(projection);
351 CacheEntry entry = findEntry(projectionEntries, pixelPerDegree, east, north);
352 return (entry != null);
353 }
354
355 public BufferedImage getExactMatch(Projection projection, double pixelPerDegree, double east, double north) {
356 CacheEntry entry = null;
357 ProjectionEntries projectionEntries = null;
358 synchronized (this) {
359 projectionEntries = getProjectionEntries(projection);
360 entry = findEntry(projectionEntries, pixelPerDegree, east, north);
361 }
362 if (entry != null) {
363 try {
364 return loadImage(projectionEntries, entry);
365 } catch (IOException e) {
366 System.err.println("Unable to load file from wms cache");
367 e.printStackTrace();
368 return null;
369 }
370 }
371 return null;
372 }
373
374 public BufferedImage getPartialMatch(Projection projection, double pixelPerDegree, double east, double north) {
375 ProjectionEntries projectionEntries;
376 List<CacheEntry> matches;
377 synchronized (this) {
378 matches = new ArrayList<WmsCache.CacheEntry>();
379
380 double minPPD = pixelPerDegree / 5;
381 double maxPPD = pixelPerDegree * 5;
382 projectionEntries = getProjectionEntries(projection);
383
384 double size2 = tileSize / pixelPerDegree;
385 double border = tileSize * 0.01; // Make sure not to load neighboring tiles that intersects this tile only slightly
386 ProjectionBounds bounds = new ProjectionBounds(east + border, north + border,
387 east + size2 - border, north + size2 - border);
388
389 //TODO Do not load tile if it is completely overlapped by other tile with better ppd
390 for (CacheEntry entry: projectionEntries.entries) {
391 if (entry.pixelPerDegree >= minPPD && entry.pixelPerDegree <= maxPPD && entry.bounds.intersects(bounds)) {
392 entry.lastUsed = System.currentTimeMillis();
393 matches.add(entry);
394 }
395 }
396
397 if (matches.isEmpty())
398 return null;
399
400
401 Collections.sort(matches, new Comparator<CacheEntry>() {
402 @Override
403 public int compare(CacheEntry o1, CacheEntry o2) {
404 return Double.compare(o2.pixelPerDegree, o1.pixelPerDegree);
405 }
406 });
407 }
408
409 //TODO Use alpha layer only when enabled on wms layer
410 BufferedImage result = new BufferedImage(tileSize, tileSize, BufferedImage.TYPE_4BYTE_ABGR);
411 Graphics2D g = result.createGraphics();
412
413
414 boolean drawAtLeastOnce = false;
415 Map<CacheEntry, SoftReference<BufferedImage>> localCache = new HashMap<WmsCache.CacheEntry, SoftReference<BufferedImage>>();
416 for (CacheEntry ce: matches) {
417 BufferedImage img;
418 try {
419 img = loadImage(projectionEntries, ce);
420 localCache.put(ce, new SoftReference<BufferedImage>(img));
421 } catch (IOException e) {
422 continue;
423 }
424
425 drawAtLeastOnce = true;
426
427 int xDiff = (int)((ce.east - east) * pixelPerDegree);
428 int yDiff = (int)((ce.north - north) * pixelPerDegree);
429 int size = (int)(pixelPerDegree / ce.pixelPerDegree * tileSize);
430
431 int x = xDiff;
432 int y = -size + tileSize - yDiff;
433
434 g.drawImage(img, x, y, size, size, null);
435 }
436
437 if (drawAtLeastOnce) {
438 synchronized (this) {
439 memoryCache.putAll(localCache);
440 }
441 return result;
442 } else
443 return null;
444 }
445
446 private String generateFileName(ProjectionEntries projectionEntries, double pixelPerDegree, Projection projection, double east, double north, String mimeType) {
447 LatLon ll1 = projection.eastNorth2latlon(new EastNorth(east, north));
448 LatLon ll2 = projection.eastNorth2latlon(new EastNorth(east + 100 / pixelPerDegree, north));
449 LatLon ll3 = projection.eastNorth2latlon(new EastNorth(east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree));
450
451 double deltaLat = Math.abs(ll3.lat() - ll1.lat());
452 double deltaLon = Math.abs(ll3.lon() - ll1.lon());
453 int precisionLat = Math.max(0, -(int)Math.ceil(Math.log10(deltaLat)) + 1);
454 int precisionLon = Math.max(0, -(int)Math.ceil(Math.log10(deltaLon)) + 1);
455
456 String zoom = NavigatableComponent.METRIC_SOM.getDistText(ll1.greatCircleDistance(ll2));
457 String extension;
458 if ("image/jpeg".equals(mimeType) || "image/jpg".equals(mimeType)) {
459 extension = "jpg";
460 } else if ("image/png".equals(mimeType)) {
461 extension = "png";
462 } else if ("image/gif".equals(mimeType)) {
463 extension = "gif";
464 } else {
465 extension = "dat";
466 }
467
468 int counter = 0;
469 FILENAME_LOOP:
470 while (true) {
471 String result = String.format("%s_%." + precisionLat + "f_%." + precisionLon +"f%s.%s", zoom, ll1.lat(), ll1.lon(), counter==0?"":"_" + counter, extension);
472 for (CacheEntry entry: projectionEntries.entries) {
473 if (entry.filename.equals(result)) {
474 counter++;
475 continue FILENAME_LOOP;
476 }
477 }
478 return result;
479 }
480 }
481
482 /**
483 *
484 * @param img Used only when overlapping is used, when not used, used raw from imageData
485 * @param imageData
486 * @param projection
487 * @param pixelPerDegree
488 * @param east
489 * @param north
490 * @throws IOException
491 */
492 public synchronized void saveToCache(BufferedImage img, InputStream imageData, Projection projection, double pixelPerDegree, double east, double north) throws IOException {
493 ProjectionEntries projectionEntries = getProjectionEntries(projection);
494 CacheEntry entry = findEntry(projectionEntries, pixelPerDegree, east, north);
495 File imageFile;
496 if (entry == null) {
497
498 String mimeType;
499 if (img != null) {
500 mimeType = "image/png";
501 } else {
502 mimeType = URLConnection.guessContentTypeFromStream(imageData);
503 }
504 entry = new CacheEntry(pixelPerDegree, east, north, tileSize,generateFileName(projectionEntries, pixelPerDegree, projection, east, north, mimeType));
505 entry.lastUsed = System.currentTimeMillis();
506 entry.lastModified = entry.lastUsed;
507 projectionEntries.entries.add(entry);
508 imageFile = getImageFile(projectionEntries, entry);
509 } else {
510 imageFile = getImageFile(projectionEntries, entry);
511 totalFileSize -= imageFile.length();
512 }
513
514 imageFile.getParentFile().mkdirs();
515
516 if (img != null) {
517 BufferedImage copy = new BufferedImage(tileSize, tileSize, img.getType());
518 copy.createGraphics().drawImage(img, 0, 0, tileSize, tileSize, 0, img.getHeight() - tileSize, tileSize, img.getHeight(), null);
519 ImageIO.write(copy, "png", imageFile);
520 totalFileSize += imageFile.length();
521 } else {
522 OutputStream os = new BufferedOutputStream(new FileOutputStream(imageFile));
523 try {
524 totalFileSize += Utils.copyStream(imageData, os);
525 } finally {
526 Utils.close(os);
527 }
528 }
529 }
530
531 public synchronized void cleanSmallFiles(int size) {
532 for (ProjectionEntries projectionEntries: entries.values()) {
533 Iterator<CacheEntry> it = projectionEntries.entries.iterator();
534 while (it.hasNext()) {
535 File file = getImageFile(projectionEntries, it.next());
536 long length = file.length();
537 if (length <= size) {
538 if (length == 0) {
539 totalFileSizeDirty = true; // File probably doesn't exist
540 }
541 totalFileSize -= size;
542 file.delete();
543 it.remove();
544 }
545 }
546 }
547 }
548
549 public static String printDate(Calendar c) {
550 return (new SimpleDateFormat("yyyy-MM-dd")).format(c.getTime());
551 }
552
553 private boolean isInsideAreaToCache(CacheEntry cacheEntry) {
554 for (ProjectionBounds b: areaToCache) {
555 if (cacheEntry.bounds.intersects(b))
556 return true;
557 }
558 return false;
559 }
560
561 public synchronized void setAreaToCache(Set<ProjectionBounds> areaToCache) {
562 this.areaToCache = areaToCache;
563 Iterator<CacheEntry> it = memoryCache.keySet().iterator();
564 while (it.hasNext()) {
565 if (!isInsideAreaToCache(it.next())) {
566 it.remove();
567 }
568 }
569 }
570}
Note: See TracBrowser for help on using the repository browser.