source: josm/trunk/src/org/openstreetmap/josm/data/imagery/WmsCache.java@ 4745

Last change on this file since 4745 was 4745, checked in by jttt, 12 years ago

Add precache wms tiles action to gpx layers (it will download wms tiles along track to cache for faster work afterwards)
  • Property svn:mime-type set to text/plain
File size: 22.6 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.imagery;
3
4
5import java.awt.Graphics2D;
6import java.awt.image.BufferedImage;
7import java.io.BufferedOutputStream;
8import java.io.File;
9import java.io.FileInputStream;
10import java.io.FileNotFoundException;
11import java.io.FileOutputStream;
12import java.io.IOException;
13import java.io.InputStream;
14import java.io.OutputStream;
15import java.lang.ref.SoftReference;
16import java.net.URLConnection;
17import java.text.SimpleDateFormat;
18import java.util.ArrayList;
19import java.util.Calendar;
20import java.util.Collections;
21import java.util.Comparator;
22import java.util.HashMap;
23import java.util.HashSet;
24import java.util.Iterator;
25import java.util.List;
26import java.util.Map;
27import java.util.Properties;
28import java.util.Set;
29
30import javax.imageio.ImageIO;
31import javax.xml.bind.JAXBContext;
32import javax.xml.bind.Marshaller;
33import javax.xml.bind.Unmarshaller;
34
35import org.openstreetmap.josm.Main;
36import org.openstreetmap.josm.data.ProjectionBounds;
37import org.openstreetmap.josm.data.coor.EastNorth;
38import org.openstreetmap.josm.data.coor.LatLon;
39import org.openstreetmap.josm.data.imagery.types.EntryType;
40import org.openstreetmap.josm.data.imagery.types.ProjectionType;
41import org.openstreetmap.josm.data.imagery.types.WmsCacheType;
42import org.openstreetmap.josm.data.preferences.StringProperty;
43import org.openstreetmap.josm.data.projection.Projection;
44import org.openstreetmap.josm.gui.NavigatableComponent;
45import org.openstreetmap.josm.tools.Utils;
46
47
48
49public class WmsCache {
50 //TODO Property for maximum cache size
51 //TODO Property for maximum age of tile, automatically remove old tiles
52 //TODO Measure time for partially loading from cache, compare with time to download tile. If slower, disable partial cache
53 //TODO Do loading from partial cache and downloading at the same time, don't wait for partical cache to load
54
55 private static final StringProperty PROP_CACHE_PATH = new StringProperty("imagery.wms-cache.path", "wms-cache");
56 private static final String INDEX_FILENAME = "index.xml";
57 private static final String LAYERS_INDEX_FILENAME = "layers.properties";
58
59 private static class CacheEntry {
60 final double pixelPerDegree;
61 final double east;
62 final double north;
63 final ProjectionBounds bounds;
64 final String filename;
65
66 long lastUsed;
67 long lastModified;
68
69 CacheEntry(double pixelPerDegree, double east, double north, int tileSize, String filename) {
70 this.pixelPerDegree = pixelPerDegree;
71 this.east = east;
72 this.north = north;
73 this.bounds = new ProjectionBounds(east, north, east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree);
74 this.filename = filename;
75 }
76 }
77
78 private static class ProjectionEntries {
79 final String projection;
80 final String cacheDirectory;
81 final List<CacheEntry> entries = new ArrayList<WmsCache.CacheEntry>();
82
83 ProjectionEntries(String projection, String cacheDirectory) {
84 this.projection = projection;
85 this.cacheDirectory = cacheDirectory;
86 }
87 }
88
89 private final Map<String, ProjectionEntries> entries = new HashMap<String, ProjectionEntries>();
90 private final File cacheDir;
91 private final int tileSize; // Should be always 500
92 private int totalFileSize;
93 private boolean totalFileSizeDirty; // Some file was missing - size needs to be recalculated
94 // No need for hashCode/equals on CacheEntry, object identity is enough. Comparing by values can lead to error - CacheEntry for wrong projection could be found
95 private Map<CacheEntry, SoftReference<BufferedImage>> memoryCache = new HashMap<WmsCache.CacheEntry, SoftReference<BufferedImage>>();
96 private Set<ProjectionBounds> areaToCache;
97
98 protected String cacheDirPath() {
99 String cPath = PROP_CACHE_PATH.get();
100 if (!(cPath.startsWith("/") || cPath.startsWith(":/",1))) {
101 //Not an absolute path
102 cPath = Main.pref.getPreferencesDir() + cPath;
103 }
104 return cPath;
105 }
106
107 public WmsCache(String url, int tileSize) {
108 File globalCacheDir = new File(cacheDirPath());
109 globalCacheDir.mkdirs();
110 cacheDir = new File(globalCacheDir, getCacheDirectory(url));
111 cacheDir.mkdirs();
112 this.tileSize = tileSize;
113 }
114
115 private String getCacheDirectory(String url) {
116 String cacheDirName = null;
117 InputStream fis = null;
118 OutputStream fos = null;
119 try {
120 Properties layersIndex = new Properties();
121 File layerIndexFile = new File(cacheDirPath(), LAYERS_INDEX_FILENAME);
122 try {
123 fis = new FileInputStream(layerIndexFile);
124 layersIndex.load(fis);
125 } catch (FileNotFoundException e) {
126 System.out.println("Unable to load layers index for wms cache (file " + layerIndexFile + " not found)");
127 } catch (IOException e) {
128 System.err.println("Unable to load layers index for wms cache");
129 e.printStackTrace();
130 }
131
132 for (Object propKey: layersIndex.keySet()) {
133 String s = (String)propKey;
134 if (url.equals(layersIndex.getProperty(s))) {
135 cacheDirName = s;
136 break;
137 }
138 }
139
140 if (cacheDirName == null) {
141 int counter = 0;
142 while (true) {
143 counter++;
144 if (!layersIndex.keySet().contains(String.valueOf(counter))) {
145 break;
146 }
147 }
148 cacheDirName = String.valueOf(counter);
149 layersIndex.setProperty(cacheDirName, url);
150 try {
151 fos = new FileOutputStream(layerIndexFile);
152 layersIndex.store(fos, "");
153 } catch (IOException e) {
154 System.err.println("Unable to save layer index for wms cache");
155 e.printStackTrace();
156 }
157 }
158 } finally {
159 try {
160 if (fis != null) {
161 fis.close();
162 }
163 if (fos != null) {
164 fos.close();
165 }
166 } catch (IOException e) {
167 e.printStackTrace();
168 }
169 }
170
171 return cacheDirName;
172 }
173
174 private ProjectionEntries getProjectionEntries(Projection projection) {
175 return getProjectionEntries(projection.toCode(), projection.getCacheDirectoryName());
176 }
177
178 private ProjectionEntries getProjectionEntries(String projection, String cacheDirectory) {
179 ProjectionEntries result = entries.get(projection);
180 if (result == null) {
181 result = new ProjectionEntries(projection, cacheDirectory);
182 entries.put(projection, result);
183 }
184
185 return result;
186 }
187
188 public synchronized void loadIndex() {
189 File indexFile = new File(cacheDir, INDEX_FILENAME);
190 try {
191 JAXBContext context = JAXBContext.newInstance(
192 WmsCacheType.class.getPackage().getName(),
193 WmsCacheType.class.getClassLoader());
194 Unmarshaller unmarshaller = context.createUnmarshaller();
195 WmsCacheType cacheEntries = (WmsCacheType)unmarshaller.unmarshal(new FileInputStream(indexFile));
196 totalFileSize = cacheEntries.getTotalFileSize();
197 if (cacheEntries.getTileSize() != tileSize) {
198 System.out.println("Cache created with different tileSize, cache will be discarded");
199 return;
200 }
201 for (ProjectionType projectionType: cacheEntries.getProjection()) {
202 ProjectionEntries projection = getProjectionEntries(projectionType.getName(), projectionType.getCacheDirectory());
203 for (EntryType entry: projectionType.getEntry()) {
204 CacheEntry ce = new CacheEntry(entry.getPixelPerDegree(), entry.getEast(), entry.getNorth(), tileSize, entry.getFilename());
205 ce.lastUsed = entry.getLastUsed().getTimeInMillis();
206 ce.lastModified = entry.getLastModified().getTimeInMillis();
207 projection.entries.add(ce);
208 }
209 }
210 } catch (Exception e) {
211 if (indexFile.exists()) {
212 e.printStackTrace();
213 System.out.println("Unable to load index for wms-cache, new file will be created");
214 } else {
215 System.out.println("Index for wms-cache doesn't exist, new file will be created");
216 }
217 }
218
219 removeNonReferencedFiles();
220 }
221
222 private void removeNonReferencedFiles() {
223
224 Set<String> usedProjections = new HashSet<String>();
225
226 for (ProjectionEntries projectionEntries: entries.values()) {
227
228 usedProjections.add(projectionEntries.cacheDirectory);
229
230 File projectionDir = new File(cacheDir, projectionEntries.cacheDirectory);
231 if (projectionDir.exists()) {
232 Set<String> referencedFiles = new HashSet<String>();
233
234 for (CacheEntry ce: projectionEntries.entries) {
235 referencedFiles.add(ce.filename);
236 }
237
238 for (File file: projectionDir.listFiles()) {
239 if (!referencedFiles.contains(file.getName())) {
240 file.delete();
241 }
242 }
243 }
244 }
245
246 for (File projectionDir: cacheDir.listFiles()) {
247 if (projectionDir.isDirectory() && !usedProjections.contains(projectionDir.getName())) {
248 Utils.deleteDirectory(projectionDir);
249 }
250 }
251 }
252
253 private int calculateTotalFileSize() {
254 int result = 0;
255 for (ProjectionEntries projectionEntries: entries.values()) {
256 Iterator<CacheEntry> it = projectionEntries.entries.iterator();
257 while (it.hasNext()) {
258 CacheEntry entry = it.next();
259 File imageFile = getImageFile(projectionEntries, entry);
260 if (!imageFile.exists()) {
261 it.remove();
262 } else {
263 result += imageFile.length();
264 }
265 }
266 }
267 return result;
268 }
269
270 public synchronized void saveIndex() {
271 WmsCacheType index = new WmsCacheType();
272
273 if (totalFileSizeDirty) {
274 totalFileSize = calculateTotalFileSize();
275 }
276
277 index.setTileSize(tileSize);
278 index.setTotalFileSize(totalFileSize);
279 for (ProjectionEntries projectionEntries: entries.values()) {
280 if (projectionEntries.entries.size() > 0) {
281 ProjectionType projectionType = new ProjectionType();
282 projectionType.setName(projectionEntries.projection);
283 projectionType.setCacheDirectory(projectionEntries.cacheDirectory);
284 index.getProjection().add(projectionType);
285 for (CacheEntry ce: projectionEntries.entries) {
286 EntryType entry = new EntryType();
287 entry.setPixelPerDegree(ce.pixelPerDegree);
288 entry.setEast(ce.east);
289 entry.setNorth(ce.north);
290 Calendar c = Calendar.getInstance();
291 c.setTimeInMillis(ce.lastUsed);
292 entry.setLastUsed(c);
293 c = Calendar.getInstance();
294 c.setTimeInMillis(ce.lastModified);
295 entry.setLastModified(c);
296 entry.setFilename(ce.filename);
297 projectionType.getEntry().add(entry);
298 }
299 }
300 }
301 try {
302 JAXBContext context = JAXBContext.newInstance(
303 WmsCacheType.class.getPackage().getName(),
304 WmsCacheType.class.getClassLoader());
305 Marshaller marshaller = context.createMarshaller();
306 marshaller.marshal(index, new FileOutputStream(new File(cacheDir, INDEX_FILENAME)));
307 } catch (Exception e) {
308 System.err.println("Failed to save wms-cache file");
309 e.printStackTrace();
310 }
311 }
312
313 private File getImageFile(ProjectionEntries projection, CacheEntry entry) {
314 return new File(cacheDir, projection.cacheDirectory + "/" + entry.filename);
315 }
316
317
318 private BufferedImage loadImage(ProjectionEntries projectionEntries, CacheEntry entry) throws IOException {
319
320 synchronized (this) {
321 entry.lastUsed = System.currentTimeMillis();
322
323 SoftReference<BufferedImage> memCache = memoryCache.get(entry);
324 if (memCache != null) {
325 BufferedImage result = memCache.get();
326 if (result != null)
327 return result;
328 }
329 }
330
331 try {
332 // Reading can't be in synchronized section, it's too slow
333 BufferedImage result = ImageIO.read(getImageFile(projectionEntries, entry));
334 synchronized (this) {
335 if (result == null) {
336 projectionEntries.entries.remove(entry);
337 totalFileSizeDirty = true;
338 }
339 return result;
340 }
341 } catch (IOException e) {
342 synchronized (this) {
343 projectionEntries.entries.remove(entry);
344 totalFileSizeDirty = true;
345 throw e;
346 }
347 }
348 }
349
350 private CacheEntry findEntry(ProjectionEntries projectionEntries, double pixelPerDegree, double east, double north) {
351 for (CacheEntry entry: projectionEntries.entries) {
352 if (entry.pixelPerDegree == pixelPerDegree && entry.east == east && entry.north == north)
353 return entry;
354 }
355 return null;
356 }
357
358 public synchronized boolean hasExactMatch(Projection projection, double pixelPerDegree, double east, double north) {
359 ProjectionEntries projectionEntries = getProjectionEntries(projection);
360 CacheEntry entry = findEntry(projectionEntries, pixelPerDegree, east, north);
361 return (entry != null);
362 }
363
364 public BufferedImage getExactMatch(Projection projection, double pixelPerDegree, double east, double north) {
365 CacheEntry entry = null;
366 ProjectionEntries projectionEntries = null;
367 synchronized (this) {
368 projectionEntries = getProjectionEntries(projection);
369 entry = findEntry(projectionEntries, pixelPerDegree, east, north);
370 }
371 if (entry != null) {
372 try {
373 return loadImage(projectionEntries, entry);
374 } catch (IOException e) {
375 System.err.println("Unable to load file from wms cache");
376 e.printStackTrace();
377 return null;
378 }
379 }
380 return null;
381 }
382
383 public BufferedImage getPartialMatch(Projection projection, double pixelPerDegree, double east, double north) {
384 ProjectionEntries projectionEntries;
385 List<CacheEntry> matches;
386 synchronized (this) {
387 matches = new ArrayList<WmsCache.CacheEntry>();
388
389 double minPPD = pixelPerDegree / 5;
390 double maxPPD = pixelPerDegree * 5;
391 projectionEntries = getProjectionEntries(projection);
392
393 double size2 = tileSize / pixelPerDegree;
394 double border = tileSize * 0.01; // Make sure not to load neighboring tiles that intersects this tile only slightly
395 ProjectionBounds bounds = new ProjectionBounds(east + border, north + border,
396 east + size2 - border, north + size2 - border);
397
398 //TODO Do not load tile if it is completely overlapped by other tile with better ppd
399 for (CacheEntry entry: projectionEntries.entries) {
400 if (entry.pixelPerDegree >= minPPD && entry.pixelPerDegree <= maxPPD && entry.bounds.intersects(bounds)) {
401 entry.lastUsed = System.currentTimeMillis();
402 matches.add(entry);
403 }
404 }
405
406 if (matches.isEmpty())
407 return null;
408
409
410 Collections.sort(matches, new Comparator<CacheEntry>() {
411 @Override
412 public int compare(CacheEntry o1, CacheEntry o2) {
413 return Double.compare(o2.pixelPerDegree, o1.pixelPerDegree);
414 }
415 });
416 }
417
418 //TODO Use alpha layer only when enabled on wms layer
419 BufferedImage result = new BufferedImage(tileSize, tileSize, BufferedImage.TYPE_4BYTE_ABGR);
420 Graphics2D g = result.createGraphics();
421
422
423 boolean drawAtLeastOnce = false;
424 Map<CacheEntry, SoftReference<BufferedImage>> localCache = new HashMap<WmsCache.CacheEntry, SoftReference<BufferedImage>>();
425 for (CacheEntry ce: matches) {
426 BufferedImage img;
427 try {
428 img = loadImage(projectionEntries, ce);
429 localCache.put(ce, new SoftReference<BufferedImage>(img));
430 } catch (IOException e) {
431 continue;
432 }
433
434 drawAtLeastOnce = true;
435
436 int xDiff = (int)((ce.east - east) * pixelPerDegree);
437 int yDiff = (int)((ce.north - north) * pixelPerDegree);
438 int size = (int)(pixelPerDegree / ce.pixelPerDegree * tileSize);
439
440 int x = xDiff;
441 int y = -size + tileSize - yDiff;
442
443 g.drawImage(img, x, y, size, size, null);
444 }
445
446 if (drawAtLeastOnce) {
447 synchronized (this) {
448 memoryCache.putAll(localCache);
449 }
450 return result;
451 } else
452 return null;
453 }
454
455 private String generateFileName(ProjectionEntries projectionEntries, double pixelPerDegree, Projection projection, double east, double north, String mimeType) {
456 LatLon ll1 = projection.eastNorth2latlon(new EastNorth(east, north));
457 LatLon ll2 = projection.eastNorth2latlon(new EastNorth(east + 100 / pixelPerDegree, north));
458 LatLon ll3 = projection.eastNorth2latlon(new EastNorth(east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree));
459
460 double deltaLat = Math.abs(ll3.lat() - ll1.lat());
461 double deltaLon = Math.abs(ll3.lon() - ll1.lon());
462 int precisionLat = Math.max(0, -(int)Math.ceil(Math.log10(deltaLat)) + 1);
463 int precisionLon = Math.max(0, -(int)Math.ceil(Math.log10(deltaLon)) + 1);
464
465 String zoom = NavigatableComponent.METRIC_SOM.getDistText(ll1.greatCircleDistance(ll2));
466 String extension;
467 if ("image/jpeg".equals(mimeType) || "image/jpg".equals(mimeType)) {
468 extension = "jpg";
469 } else if ("image/png".equals(mimeType)) {
470 extension = "png";
471 } else if ("image/gif".equals(mimeType)) {
472 extension = "gif";
473 } else {
474 extension = "dat";
475 }
476
477 int counter = 0;
478 FILENAME_LOOP:
479 while (true) {
480 String result = String.format("%s_%." + precisionLat + "f_%." + precisionLon +"f%s.%s", zoom, ll1.lat(), ll1.lon(), counter==0?"":"_" + counter, extension);
481 for (CacheEntry entry: projectionEntries.entries) {
482 if (entry.filename.equals(result)) {
483 counter++;
484 continue FILENAME_LOOP;
485 }
486 }
487 return result;
488 }
489 }
490
491 /**
492 *
493 * @param img Used only when overlapping is used, when not used, used raw from imageData
494 * @param imageData
495 * @param projection
496 * @param pixelPerDegree
497 * @param east
498 * @param north
499 * @throws IOException
500 */
501 public synchronized void saveToCache(BufferedImage img, InputStream imageData, Projection projection, double pixelPerDegree, double east, double north) throws IOException {
502 ProjectionEntries projectionEntries = getProjectionEntries(projection);
503 CacheEntry entry = findEntry(projectionEntries, pixelPerDegree, east, north);
504 File imageFile;
505 if (entry == null) {
506
507 String mimeType;
508 if (img != null) {
509 mimeType = "image/png";
510 } else {
511 mimeType = URLConnection.guessContentTypeFromStream(imageData);
512 }
513 entry = new CacheEntry(pixelPerDegree, east, north, tileSize,generateFileName(projectionEntries, pixelPerDegree, projection, east, north, mimeType));
514 entry.lastUsed = System.currentTimeMillis();
515 entry.lastModified = entry.lastUsed;
516 projectionEntries.entries.add(entry);
517 imageFile = getImageFile(projectionEntries, entry);
518 } else {
519 imageFile = getImageFile(projectionEntries, entry);
520 totalFileSize -= imageFile.length();
521 }
522
523 imageFile.getParentFile().mkdirs();
524
525 if (img != null) {
526 BufferedImage copy = new BufferedImage(tileSize, tileSize, img.getType());
527 copy.createGraphics().drawImage(img, 0, 0, tileSize, tileSize, 0, img.getHeight() - tileSize, tileSize, img.getHeight(), null);
528 ImageIO.write(copy, "png", imageFile);
529 totalFileSize += imageFile.length();
530 } else {
531 OutputStream os = new BufferedOutputStream(new FileOutputStream(imageFile));
532 try {
533 totalFileSize += Utils.copyStream(imageData, os);
534 } finally {
535 os.close();
536 }
537 }
538 }
539
540 public synchronized void cleanSmallFiles(int size) {
541 for (ProjectionEntries projectionEntries: entries.values()) {
542 Iterator<CacheEntry> it = projectionEntries.entries.iterator();
543 while (it.hasNext()) {
544 File file = getImageFile(projectionEntries, it.next());
545 long length = file.length();
546 if (length <= size) {
547 if (length == 0) {
548 totalFileSizeDirty = true; // File probably doesn't exist
549 }
550 totalFileSize -= size;
551 file.delete();
552 it.remove();
553 }
554 }
555 }
556 }
557
558 public static String printDate(Calendar c) {
559 return (new SimpleDateFormat("yyyy-MM-dd")).format(c.getTime());
560 }
561
562 private boolean isInsideAreaToCache(CacheEntry cacheEntry) {
563 for (ProjectionBounds b: areaToCache) {
564 if (cacheEntry.bounds.intersects(b))
565 return true;
566 }
567 return false;
568 }
569
570 public synchronized void setAreaToCache(Set<ProjectionBounds> areaToCache) {
571 this.areaToCache = areaToCache;
572 Iterator<CacheEntry> it = memoryCache.keySet().iterator();
573 while (it.hasNext()) {
574 if (!isInsideAreaToCache(it.next())) {
575 it.remove();
576 }
577 }
578 }
579}
Note: See TracBrowser for help on using the repository browser.