source: josm/trunk/src/org/openstreetmap/josm/data/imagery/WmsCache.java@ 4108

Last change on this file since 4108 was 4108, checked in by stoecker, 13 years ago

fix #6405 - patch by bilbo - wms cache accepting absolute filenames
  • Property svn:mime-type set to text/plain
File size: 21.4 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.imagery;
3
4
5import java.awt.Graphics2D;
6import java.awt.image.BufferedImage;
7import java.io.BufferedOutputStream;
8import java.io.File;
9import java.io.FileInputStream;
10import java.io.FileNotFoundException;
11import java.io.FileOutputStream;
12import java.io.IOException;
13import java.io.InputStream;
14import java.io.OutputStream;
15import java.lang.ref.SoftReference;
16import java.net.URLConnection;
17import java.text.SimpleDateFormat;
18import java.util.ArrayList;
19import java.util.Calendar;
20import java.util.Collections;
21import java.util.Comparator;
22import java.util.HashMap;
23import java.util.HashSet;
24import java.util.Iterator;
25import java.util.List;
26import java.util.Map;
27import java.util.Properties;
28import java.util.Set;
29
30import javax.imageio.ImageIO;
31import javax.xml.bind.JAXBContext;
32import javax.xml.bind.JAXBException;
33import javax.xml.bind.Marshaller;
34import javax.xml.bind.Unmarshaller;
35
36import org.openstreetmap.josm.Main;
37import org.openstreetmap.josm.data.ProjectionBounds;
38import org.openstreetmap.josm.data.coor.EastNorth;
39import org.openstreetmap.josm.data.coor.LatLon;
40import org.openstreetmap.josm.data.imagery.types.EntryType;
41import org.openstreetmap.josm.data.imagery.types.ProjectionType;
42import org.openstreetmap.josm.data.imagery.types.WmsCacheType;
43import org.openstreetmap.josm.data.preferences.StringProperty;
44import org.openstreetmap.josm.data.projection.Projection;
45import org.openstreetmap.josm.gui.NavigatableComponent;
46import org.openstreetmap.josm.tools.Utils;
47
48
49
50public class WmsCache {
51 //TODO Property for maximum cache size
52 //TODO Property for maximum age of tile, automatically remove old tiles
53 //TODO Measure time for partially loading from cache, compare with time to download tile. If slower, disable partial cache
54 //TODO Do loading from partial cache and downloading at the same time, don't wait for partical cache to load
55
56 private static final StringProperty PROP_CACHE_PATH = new StringProperty("imagery.wms-cache.path", "wms-cache");
57 private static final String INDEX_FILENAME = "index.xml";
58 private static final String LAYERS_INDEX_FILENAME = "layers.properties";
59
60 private static class CacheEntry {
61 final double pixelPerDegree;
62 final double east;
63 final double north;
64 final ProjectionBounds bounds;
65
66 long lastUsed;
67 long lastModified;
68 String filename;
69
70 CacheEntry(double pixelPerDegree, double east, double north, int tileSize) {
71 this.pixelPerDegree = pixelPerDegree;
72 this.east = east;
73 this.north = north;
74 this.bounds = new ProjectionBounds(east, north, east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree);
75 }
76 }
77
78 private static class ProjectionEntries {
79 final String projection;
80 final String cacheDirectory;
81 final List<CacheEntry> entries = new ArrayList<WmsCache.CacheEntry>();
82
83 ProjectionEntries(String projection, String cacheDirectory) {
84 this.projection = projection;
85 this.cacheDirectory = cacheDirectory;
86 }
87 }
88
89 private final Map<String, ProjectionEntries> entries = new HashMap<String, ProjectionEntries>();
90 private final File cacheDir;
91 private final int tileSize; // Should be always 500
92 private int totalFileSize;
93 private boolean totalFileSizeDirty; // Some file was missing - size needs to be recalculated
94 // No need for hashCode/equals on CacheEntry, object identity is enough. Comparing by values can lead to error - CacheEntry for wrong projection could be found
95 private Map<CacheEntry, SoftReference<BufferedImage>> memoryCache = new HashMap<WmsCache.CacheEntry, SoftReference<BufferedImage>>();
96 private Set<ProjectionBounds> areaToCache;
97
98 protected String cacheDirPath() {
99 String cPath = PROP_CACHE_PATH.get();
100 if (!(cPath.startsWith("/") || cPath.startsWith(":/",1))) {
101 //Not an absolute path
102 cPath = Main.pref.getPreferencesDir() + cPath;
103 }
104 return cPath;
105 }
106
107 public WmsCache(String url, int tileSize) {
108 File globalCacheDir = new File(cacheDirPath());
109 globalCacheDir.mkdirs();
110 cacheDir = new File(globalCacheDir, getCacheDirectory(url));
111 cacheDir.mkdirs();
112 this.tileSize = tileSize;
113 }
114
115 private String getCacheDirectory(String url) {
116 String cacheDirName = null;
117 InputStream fis = null;
118 OutputStream fos = null;
119 try {
120 Properties layersIndex = new Properties();
121 File layerIndexFile = new File(cacheDirPath(), LAYERS_INDEX_FILENAME);
122 try {
123 fis = new FileInputStream(layerIndexFile);
124 layersIndex.load(fis);
125 } catch (FileNotFoundException e) {
126 System.out.println("Unable to load layers index for wms cache (file " + layerIndexFile + " not found)");
127 } catch (IOException e) {
128 System.err.println("Unable to load layers index for wms cache");
129 e.printStackTrace();
130 }
131
132 for (Object propKey: layersIndex.keySet()) {
133 String s = (String)propKey;
134 if (url.equals(layersIndex.getProperty(s))) {
135 cacheDirName = s;
136 break;
137 }
138 }
139
140 if (cacheDirName == null) {
141 int counter = 0;
142 while (true) {
143 counter++;
144 if (!layersIndex.keySet().contains(String.valueOf(counter))) {
145 break;
146 }
147 }
148 cacheDirName = String.valueOf(counter);
149 layersIndex.setProperty(cacheDirName, url);
150 try {
151 fos = new FileOutputStream(layerIndexFile);
152 layersIndex.store(fos, "");
153 } catch (IOException e) {
154 System.err.println("Unable to save layer index for wms cache");
155 e.printStackTrace();
156 }
157 }
158 } finally {
159 try {
160 if (fis != null) {
161 fis.close();
162 }
163 if (fos != null) {
164 fos.close();
165 }
166 } catch (IOException e) {
167 e.printStackTrace();
168 }
169 }
170
171 return cacheDirName;
172 }
173
174 private ProjectionEntries getProjectionEntries(Projection projection) {
175 return getProjectionEntries(projection.toCode(), projection.getCacheDirectoryName());
176 }
177
178 private ProjectionEntries getProjectionEntries(String projection, String cacheDirectory) {
179 ProjectionEntries result = entries.get(projection);
180 if (result == null) {
181 result = new ProjectionEntries(projection, cacheDirectory);
182 entries.put(projection, result);
183 }
184
185 return result;
186 }
187
188 public synchronized void loadIndex() {
189 File indexFile = new File(cacheDir, INDEX_FILENAME);
190 try {
191 JAXBContext context = JAXBContext.newInstance(
192 WmsCacheType.class.getPackage().getName(),
193 WmsCacheType.class.getClassLoader());
194 Unmarshaller unmarshaller = context.createUnmarshaller();
195 WmsCacheType cacheEntries = (WmsCacheType)unmarshaller.unmarshal(new FileInputStream(indexFile));
196 totalFileSize = cacheEntries.getTotalFileSize();
197 if (cacheEntries.getTileSize() != tileSize) {
198 System.out.println("Cache created with different tileSize, cache will be discarded");
199 return;
200 }
201 for (ProjectionType projectionType: cacheEntries.getProjection()) {
202 ProjectionEntries projection = getProjectionEntries(projectionType.getName(), projectionType.getCacheDirectory());
203 for (EntryType entry: projectionType.getEntry()) {
204 CacheEntry ce = new CacheEntry(entry.getPixelPerDegree(), entry.getEast(), entry.getNorth(), tileSize);
205 ce.lastUsed = entry.getLastUsed().getTimeInMillis();
206 ce.filename = entry.getFilename();
207 ce.lastModified = entry.getLastModified().getTimeInMillis();
208 projection.entries.add(ce);
209 }
210 }
211 } catch (Exception e) {
212 if (indexFile.exists()) {
213 e.printStackTrace();
214 System.out.println("Unable to load index for wms-cache, new file will be created");
215 } else {
216 System.out.println("Index for wms-cache doesn't exist, new file will be created");
217 }
218 }
219
220 removeNonReferencedFiles();
221 }
222
223 private void removeNonReferencedFiles() {
224
225 Set<String> usedProjections = new HashSet<String>();
226
227 for (ProjectionEntries projectionEntries: entries.values()) {
228
229 usedProjections.add(projectionEntries.cacheDirectory);
230
231 File projectionDir = new File(cacheDir, projectionEntries.cacheDirectory);
232 Set<String> referencedFiles = new HashSet<String>();
233
234 for (CacheEntry ce: projectionEntries.entries) {
235 referencedFiles.add(ce.filename);
236 }
237
238 for (File file: projectionDir.listFiles()) {
239 if (!referencedFiles.contains(file.getName())) {
240 file.delete();
241 }
242 }
243 }
244
245 for (File projectionDir: cacheDir.listFiles()) {
246 if (projectionDir.isDirectory() && !usedProjections.contains(projectionDir.getName())) {
247 Utils.deleteDirectory(projectionDir);
248 }
249 }
250 }
251
252 private int calculateTotalFileSize() {
253 int result = 0;
254 for (ProjectionEntries projectionEntries: entries.values()) {
255 Iterator<CacheEntry> it = projectionEntries.entries.iterator();
256 while (it.hasNext()) {
257 CacheEntry entry = it.next();
258 File imageFile = getImageFile(projectionEntries, entry);
259 if (!imageFile.exists()) {
260 it.remove();
261 } else {
262 result += imageFile.length();
263 }
264 }
265 }
266 return result;
267 }
268
269 public synchronized void saveIndex() {
270 WmsCacheType index = new WmsCacheType();
271
272 if (totalFileSizeDirty) {
273 totalFileSize = calculateTotalFileSize();
274 }
275
276 index.setTileSize(tileSize);
277 index.setTotalFileSize(totalFileSize);
278 for (ProjectionEntries projectionEntries: entries.values()) {
279 if (projectionEntries.entries.size() > 0) {
280 ProjectionType projectionType = new ProjectionType();
281 projectionType.setName(projectionEntries.projection);
282 projectionType.setCacheDirectory(projectionEntries.cacheDirectory);
283 index.getProjection().add(projectionType);
284 for (CacheEntry ce: projectionEntries.entries) {
285 EntryType entry = new EntryType();
286 entry.setPixelPerDegree(ce.pixelPerDegree);
287 entry.setEast(ce.east);
288 entry.setNorth(ce.north);
289 Calendar c = Calendar.getInstance();
290 c.setTimeInMillis(ce.lastUsed);
291 entry.setLastUsed(c);
292 c = Calendar.getInstance();
293 c.setTimeInMillis(ce.lastModified);
294 entry.setLastModified(c);
295 entry.setFilename(ce.filename);
296 projectionType.getEntry().add(entry);
297 }
298 }
299 }
300 try {
301 JAXBContext context = JAXBContext.newInstance(
302 WmsCacheType.class.getPackage().getName(),
303 WmsCacheType.class.getClassLoader());
304 Marshaller marshaller = context.createMarshaller();
305 marshaller.marshal(index, new FileOutputStream(new File(cacheDir, INDEX_FILENAME)));
306 } catch (Exception e) {
307 System.err.println("Failed to save wms-cache file");
308 e.printStackTrace();
309 }
310 }
311
312 private File getImageFile(ProjectionEntries projection, CacheEntry entry) {
313 return new File(cacheDir, projection.cacheDirectory + "/" + entry.filename);
314 }
315
316 private BufferedImage loadImage(ProjectionEntries projectionEntries, CacheEntry entry) throws IOException {
317 entry.lastUsed = System.currentTimeMillis();
318
319 SoftReference<BufferedImage> memCache = memoryCache.get(entry);
320 if (memCache != null) {
321 BufferedImage result = memCache.get();
322 if (result != null)
323 return result;
324 }
325
326 try {
327 BufferedImage result = ImageIO.read(getImageFile(projectionEntries, entry));
328 if (result == null) {
329 projectionEntries.entries.remove(entry);
330 totalFileSizeDirty = true;
331 }
332 return result;
333 } catch (IOException e) {
334 projectionEntries.entries.remove(entry);
335 totalFileSizeDirty = true;
336 throw e;
337 }
338 }
339
340 private CacheEntry findEntry(ProjectionEntries projectionEntries, double pixelPerDegree, double east, double north) {
341 for (CacheEntry entry: projectionEntries.entries) {
342 if (entry.pixelPerDegree == pixelPerDegree && entry.east == east && entry.north == north)
343 return entry;
344 }
345 return null;
346 }
347
348 public synchronized BufferedImage getExactMatch(Projection projection, double pixelPerDegree, double east, double north) {
349 ProjectionEntries projectionEntries = getProjectionEntries(projection);
350 CacheEntry entry = findEntry(projectionEntries, pixelPerDegree, east, north);
351 if (entry != null) {
352 try {
353 entry.lastUsed = System.currentTimeMillis();
354 return loadImage(projectionEntries, entry);
355 } catch (IOException e) {
356 System.err.println("Unable to load file from wms cache");
357 e.printStackTrace();
358 return null;
359 }
360 }
361 return null;
362 }
363
364 public synchronized BufferedImage getPartialMatch(Projection projection, double pixelPerDegree, double east, double north) {
365 List<CacheEntry> matches = new ArrayList<WmsCache.CacheEntry>();
366
367 double minPPD = pixelPerDegree / 5;
368 double maxPPD = pixelPerDegree * 5;
369 ProjectionEntries projectionEntries = getProjectionEntries(projection);
370
371 ProjectionBounds bounds = new ProjectionBounds(east, north,
372 east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree);
373
374 //TODO Do not load tile if it is completely overlapped by other tile with better ppd
375 for (CacheEntry entry: projectionEntries.entries) {
376 if (entry.pixelPerDegree >= minPPD && entry.pixelPerDegree <= maxPPD && entry.bounds.intersects(bounds)) {
377 entry.lastUsed = System.currentTimeMillis();
378 matches.add(entry);
379 }
380 }
381
382 if (matches.isEmpty())
383 return null;
384
385
386 Collections.sort(matches, new Comparator<CacheEntry>() {
387 @Override
388 public int compare(CacheEntry o1, CacheEntry o2) {
389 return Double.compare(o2.pixelPerDegree, o1.pixelPerDegree);
390 }
391 });
392
393 //TODO Use alpha layer only when enabled on wms layer
394 BufferedImage result = new BufferedImage(tileSize, tileSize, BufferedImage.TYPE_4BYTE_ABGR);
395 Graphics2D g = result.createGraphics();
396
397 boolean drawAtLeastOnce = false;
398 for (CacheEntry ce: matches) {
399 BufferedImage img;
400 try {
401 img = loadImage(projectionEntries, ce);
402 memoryCache.put(ce, new SoftReference<BufferedImage>(img));
403 } catch (IOException e) {
404 continue;
405 }
406
407 drawAtLeastOnce = true;
408
409 int xDiff = (int)((ce.east - east) * pixelPerDegree);
410 int yDiff = (int)((ce.north - north) * pixelPerDegree);
411 int size = (int)(pixelPerDegree / ce.pixelPerDegree * tileSize);
412
413 int x = xDiff;
414 int y = -size + tileSize - yDiff;
415
416 g.drawImage(img, x, y, size, size, null);
417 }
418
419 if (drawAtLeastOnce)
420 return result;
421 else
422 return null;
423 }
424
425 private String generateFileName(ProjectionEntries projectionEntries, double pixelPerDegree, Projection projection, double east, double north, String mimeType) {
426 LatLon ll1 = projection.eastNorth2latlon(new EastNorth(east, north));
427 LatLon ll2 = projection.eastNorth2latlon(new EastNorth(east + 100 / pixelPerDegree, north));
428 LatLon ll3 = projection.eastNorth2latlon(new EastNorth(east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree));
429
430 double deltaLat = Math.abs(ll3.lat() - ll1.lat());
431 double deltaLon = Math.abs(ll3.lon() - ll1.lon());
432 int precisionLat = Math.max(0, -(int)Math.ceil(Math.log10(deltaLat)) + 1);
433 int precisionLon = Math.max(0, -(int)Math.ceil(Math.log10(deltaLon)) + 1);
434
435 String zoom = NavigatableComponent.METRIC_SOM.getDistText(ll1.greatCircleDistance(ll2));
436 String extension;
437 if ("image/jpeg".equals(mimeType) || "image/jpg".equals(mimeType)) {
438 extension = "jpg";
439 } else if ("image/png".equals(mimeType)) {
440 extension = "png";
441 } else if ("image/gif".equals(mimeType)) {
442 extension = "gif";
443 } else {
444 extension = "dat";
445 }
446
447 int counter = 0;
448 FILENAME_LOOP:
449 while (true) {
450 String result = String.format("%s_%." + precisionLat + "f_%." + precisionLon +"f%s.%s", zoom, ll1.lat(), ll1.lon(), counter==0?"":"_" + counter, extension);
451 for (CacheEntry entry: projectionEntries.entries) {
452 if (entry.filename.equals(result)) {
453 counter++;
454 continue FILENAME_LOOP;
455 }
456 }
457 return result;
458 }
459 }
460
461 /**
462 *
463 * @param img Used only when overlapping is used, when not used, used raw from imageData
464 * @param imageData
465 * @param projection
466 * @param pixelPerDegree
467 * @param east
468 * @param north
469 * @throws IOException
470 */
471 public synchronized void saveToCache(BufferedImage img, InputStream imageData, Projection projection, double pixelPerDegree, double east, double north) throws IOException {
472 ProjectionEntries projectionEntries = getProjectionEntries(projection);
473 CacheEntry entry = findEntry(projectionEntries, pixelPerDegree, east, north);
474 File imageFile;
475 if (entry == null) {
476 entry = new CacheEntry(pixelPerDegree, east, north, tileSize);
477 entry.lastUsed = System.currentTimeMillis();
478 entry.lastModified = entry.lastUsed;
479
480 String mimeType;
481 if (img != null) {
482 mimeType = "image/png";
483 } else {
484 mimeType = URLConnection.guessContentTypeFromStream(imageData);
485 }
486 entry.filename = generateFileName(projectionEntries, pixelPerDegree, projection, east, north, mimeType);
487 projectionEntries.entries.add(entry);
488 imageFile = getImageFile(projectionEntries, entry);
489 } else {
490 imageFile = getImageFile(projectionEntries, entry);
491 totalFileSize -= imageFile.length();
492 }
493
494 imageFile.getParentFile().mkdirs();
495
496 if (img != null) {
497 BufferedImage copy = new BufferedImage(tileSize, tileSize, img.getType());
498 copy.createGraphics().drawImage(img, 0, 0, tileSize, tileSize, 0, img.getHeight() - tileSize, tileSize, img.getHeight(), null);
499 ImageIO.write(copy, "png", imageFile);
500 totalFileSize += imageFile.length();
501 } else {
502 OutputStream os = new BufferedOutputStream(new FileOutputStream(imageFile));
503 try {
504 totalFileSize += Utils.copyStream(imageData, os);
505 } finally {
506 os.close();
507 }
508 }
509 }
510
511 public synchronized void cleanSmallFiles(int size) {
512 for (ProjectionEntries projectionEntries: entries.values()) {
513 Iterator<CacheEntry> it = projectionEntries.entries.iterator();
514 while (it.hasNext()) {
515 File file = getImageFile(projectionEntries, it.next());
516 long length = file.length();
517 if (length <= size) {
518 if (length == 0) {
519 totalFileSizeDirty = true; // File probably doesn't exist
520 }
521 totalFileSize -= size;
522 file.delete();
523 it.remove();
524 }
525 }
526 }
527 }
528
529 public static String printDate(Calendar c) {
530 return (new SimpleDateFormat("yyyy-MM-dd")).format(c.getTime());
531 }
532
533 private boolean isInsideAreaToCache(CacheEntry cacheEntry) {
534 for (ProjectionBounds b: areaToCache) {
535 if (cacheEntry.bounds.intersects(b))
536 return true;
537 }
538 return false;
539 }
540
541 public synchronized void setAreaToCache(Set<ProjectionBounds> areaToCache) {
542 this.areaToCache = areaToCache;
543 Iterator<CacheEntry> it = memoryCache.keySet().iterator();
544 while (it.hasNext()) {
545 if (!isInsideAreaToCache(it.next())) {
546 it.remove();
547 }
548 }
549 }
550}
Note: See TracBrowser for help on using the repository browser.