source: josm/trunk/src/org/openstreetmap/josm/data/imagery/WmsCache.java@ 4077

Last change on this file since 4077 was 4077, checked in by jttt, 13 years ago

Fix #6320 imagery: Concurrent Modification Exception using jahoo sat
  • Property svn:mime-type set to text/plain
File size: 21.2 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.imagery;
3
4
5import java.awt.Graphics2D;
6import java.awt.image.BufferedImage;
7import java.io.BufferedOutputStream;
8import java.io.File;
9import java.io.FileInputStream;
10import java.io.FileNotFoundException;
11import java.io.FileOutputStream;
12import java.io.IOException;
13import java.io.InputStream;
14import java.io.OutputStream;
15import java.lang.ref.SoftReference;
16import java.net.URLConnection;
17import java.text.SimpleDateFormat;
18import java.util.ArrayList;
19import java.util.Calendar;
20import java.util.Collections;
21import java.util.Comparator;
22import java.util.HashMap;
23import java.util.HashSet;
24import java.util.Iterator;
25import java.util.List;
26import java.util.Map;
27import java.util.Properties;
28import java.util.Set;
29
30import javax.imageio.ImageIO;
31import javax.xml.bind.JAXBContext;
32import javax.xml.bind.JAXBException;
33import javax.xml.bind.Marshaller;
34import javax.xml.bind.Unmarshaller;
35
36import org.openstreetmap.josm.Main;
37import org.openstreetmap.josm.data.ProjectionBounds;
38import org.openstreetmap.josm.data.coor.EastNorth;
39import org.openstreetmap.josm.data.coor.LatLon;
40import org.openstreetmap.josm.data.imagery.types.EntryType;
41import org.openstreetmap.josm.data.imagery.types.ProjectionType;
42import org.openstreetmap.josm.data.imagery.types.WmsCacheType;
43import org.openstreetmap.josm.data.preferences.StringProperty;
44import org.openstreetmap.josm.data.projection.Projection;
45import org.openstreetmap.josm.gui.NavigatableComponent;
46import org.openstreetmap.josm.tools.Utils;
47
48
49
50public class WmsCache {
51 //TODO Property for maximum cache size
52 //TODO Property for maximum age of tile, automatically remove old tiles
53 //TODO Measure time for partially loading from cache, compare with time to download tile. If slower, disable partial cache
54 //TODO Do loading from partial cache and downloading at the same time, don't wait for partical cache to load
55
56 private static final StringProperty PROP_CACHE_PATH = new StringProperty("imagery.wms-cache.path", "wms-cache");
57 private static final String INDEX_FILENAME = "index.xml";
58 private static final String LAYERS_INDEX_FILENAME = "layers.properties";
59
60 private static class CacheEntry {
61 final double pixelPerDegree;
62 final double east;
63 final double north;
64 final ProjectionBounds bounds;
65
66 long lastUsed;
67 long lastModified;
68 String filename;
69
70 CacheEntry(double pixelPerDegree, double east, double north, int tileSize) {
71 this.pixelPerDegree = pixelPerDegree;
72 this.east = east;
73 this.north = north;
74 this.bounds = new ProjectionBounds(east, north, east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree);
75 }
76 }
77
78 private static class ProjectionEntries {
79 final String projection;
80 final String cacheDirectory;
81 final List<CacheEntry> entries = new ArrayList<WmsCache.CacheEntry>();
82
83 ProjectionEntries(String projection, String cacheDirectory) {
84 this.projection = projection;
85 this.cacheDirectory = cacheDirectory;
86 }
87 }
88
89 private final Map<String, ProjectionEntries> entries = new HashMap<String, ProjectionEntries>();
90 private final File cacheDir;
91 private final int tileSize; // Should be always 500
92 private int totalFileSize;
93 private boolean totalFileSizeDirty; // Some file was missing - size needs to be recalculated
94 // No need for hashCode/equals on CacheEntry, object identity is enough. Comparing by values can lead to error - CacheEntry for wrong projection could be found
95 private Map<CacheEntry, SoftReference<BufferedImage>> memoryCache = new HashMap<WmsCache.CacheEntry, SoftReference<BufferedImage>>();
96 private Set<ProjectionBounds> areaToCache;
97
98 public WmsCache(String url, int tileSize) {
99 File globalCacheDir = new File(Main.pref.getPreferencesDir() + PROP_CACHE_PATH.get());
100 globalCacheDir.mkdirs();
101 cacheDir = new File(globalCacheDir, getCacheDirectory(url));
102 cacheDir.mkdirs();
103 this.tileSize = tileSize;
104 }
105
106 private String getCacheDirectory(String url) {
107 String cacheDirName = null;
108 InputStream fis = null;
109 OutputStream fos = null;
110 try {
111 Properties layersIndex = new Properties();
112 File layerIndexFile = new File(Main.pref.getPreferencesDir() + PROP_CACHE_PATH.get(), LAYERS_INDEX_FILENAME);
113 try {
114 fis = new FileInputStream(layerIndexFile);
115 layersIndex.load(fis);
116 } catch (FileNotFoundException e) {
117 System.out.println("Unable to load layers index for wms cache (file " + layerIndexFile + " not found)");
118 } catch (IOException e) {
119 System.err.println("Unable to load layers index for wms cache");
120 e.printStackTrace();
121 }
122
123 for (Object propKey: layersIndex.keySet()) {
124 String s = (String)propKey;
125 if (url.equals(layersIndex.getProperty(s))) {
126 cacheDirName = s;
127 break;
128 }
129 }
130
131 if (cacheDirName == null) {
132 int counter = 0;
133 while (true) {
134 counter++;
135 if (!layersIndex.keySet().contains(String.valueOf(counter))) {
136 break;
137 }
138 }
139 cacheDirName = String.valueOf(counter);
140 layersIndex.setProperty(cacheDirName, url);
141 try {
142 fos = new FileOutputStream(layerIndexFile);
143 layersIndex.store(fos, "");
144 } catch (IOException e) {
145 System.err.println("Unable to save layer index for wms cache");
146 e.printStackTrace();
147 }
148 }
149 } finally {
150 try {
151 if (fis != null) {
152 fis.close();
153 }
154 if (fos != null) {
155 fos.close();
156 }
157 } catch (IOException e) {
158 e.printStackTrace();
159 }
160 }
161
162 return cacheDirName;
163 }
164
165 private ProjectionEntries getProjectionEntries(Projection projection) {
166 return getProjectionEntries(projection.toCode(), projection.getCacheDirectoryName());
167 }
168
169 private ProjectionEntries getProjectionEntries(String projection, String cacheDirectory) {
170 ProjectionEntries result = entries.get(projection);
171 if (result == null) {
172 result = new ProjectionEntries(projection, cacheDirectory);
173 entries.put(projection, result);
174 }
175
176 return result;
177 }
178
179 public synchronized void loadIndex() {
180 File indexFile = new File(cacheDir, INDEX_FILENAME);
181 try {
182 JAXBContext context = JAXBContext.newInstance(
183 WmsCacheType.class.getPackage().getName(),
184 WmsCacheType.class.getClassLoader());
185 Unmarshaller unmarshaller = context.createUnmarshaller();
186 WmsCacheType cacheEntries = (WmsCacheType)unmarshaller.unmarshal(new FileInputStream(indexFile));
187 totalFileSize = cacheEntries.getTotalFileSize();
188 if (cacheEntries.getTileSize() != tileSize) {
189 System.out.println("Cache created with different tileSize, cache will be discarded");
190 return;
191 }
192 for (ProjectionType projectionType: cacheEntries.getProjection()) {
193 ProjectionEntries projection = getProjectionEntries(projectionType.getName(), projectionType.getCacheDirectory());
194 for (EntryType entry: projectionType.getEntry()) {
195 CacheEntry ce = new CacheEntry(entry.getPixelPerDegree(), entry.getEast(), entry.getNorth(), tileSize);
196 ce.lastUsed = entry.getLastUsed().getTimeInMillis();
197 ce.filename = entry.getFilename();
198 ce.lastModified = entry.getLastModified().getTimeInMillis();
199 projection.entries.add(ce);
200 }
201 }
202 } catch (Exception e) {
203 if (indexFile.exists()) {
204 e.printStackTrace();
205 System.out.println("Unable to load index for wms-cache, new file will be created");
206 } else {
207 System.out.println("Index for wms-cache doesn't exist, new file will be created");
208 }
209 }
210
211 removeNonReferencedFiles();
212 }
213
214 private void removeNonReferencedFiles() {
215
216 Set<String> usedProjections = new HashSet<String>();
217
218 for (ProjectionEntries projectionEntries: entries.values()) {
219
220 usedProjections.add(projectionEntries.cacheDirectory);
221
222 File projectionDir = new File(cacheDir, projectionEntries.cacheDirectory);
223 Set<String> referencedFiles = new HashSet<String>();
224
225 for (CacheEntry ce: projectionEntries.entries) {
226 referencedFiles.add(ce.filename);
227 }
228
229 for (File file: projectionDir.listFiles()) {
230 if (!referencedFiles.contains(file.getName())) {
231 file.delete();
232 }
233 }
234 }
235
236 for (File projectionDir: cacheDir.listFiles()) {
237 if (projectionDir.isDirectory() && !usedProjections.contains(projectionDir.getName())) {
238 Utils.deleteDirectory(projectionDir);
239 }
240 }
241 }
242
243 private int calculateTotalFileSize() {
244 int result = 0;
245 for (ProjectionEntries projectionEntries: entries.values()) {
246 Iterator<CacheEntry> it = projectionEntries.entries.iterator();
247 while (it.hasNext()) {
248 CacheEntry entry = it.next();
249 File imageFile = getImageFile(projectionEntries, entry);
250 if (!imageFile.exists()) {
251 it.remove();
252 } else {
253 result += imageFile.length();
254 }
255 }
256 }
257 return result;
258 }
259
260 public synchronized void saveIndex() {
261 WmsCacheType index = new WmsCacheType();
262
263 if (totalFileSizeDirty) {
264 totalFileSize = calculateTotalFileSize();
265 }
266
267 index.setTileSize(tileSize);
268 index.setTotalFileSize(totalFileSize);
269 for (ProjectionEntries projectionEntries: entries.values()) {
270 if (projectionEntries.entries.size() > 0) {
271 ProjectionType projectionType = new ProjectionType();
272 projectionType.setName(projectionEntries.projection);
273 projectionType.setCacheDirectory(projectionEntries.cacheDirectory);
274 index.getProjection().add(projectionType);
275 for (CacheEntry ce: projectionEntries.entries) {
276 EntryType entry = new EntryType();
277 entry.setPixelPerDegree(ce.pixelPerDegree);
278 entry.setEast(ce.east);
279 entry.setNorth(ce.north);
280 Calendar c = Calendar.getInstance();
281 c.setTimeInMillis(ce.lastUsed);
282 entry.setLastUsed(c);
283 c = Calendar.getInstance();
284 c.setTimeInMillis(ce.lastModified);
285 entry.setLastModified(c);
286 entry.setFilename(ce.filename);
287 projectionType.getEntry().add(entry);
288 }
289 }
290 }
291 try {
292 JAXBContext context = JAXBContext.newInstance(
293 WmsCacheType.class.getPackage().getName(),
294 WmsCacheType.class.getClassLoader());
295 Marshaller marshaller = context.createMarshaller();
296 marshaller.marshal(index, new File(cacheDir, INDEX_FILENAME));
297 } catch (JAXBException e) {
298 System.err.println("Failed to save wms-cache file");
299 e.printStackTrace();
300 }
301
302 }
303
304 private File getImageFile(ProjectionEntries projection, CacheEntry entry) {
305 return new File(cacheDir, projection.cacheDirectory + "/" + entry.filename);
306 }
307
308 private BufferedImage loadImage(ProjectionEntries projectionEntries, CacheEntry entry) throws IOException {
309 entry.lastUsed = System.currentTimeMillis();
310
311 SoftReference<BufferedImage> memCache = memoryCache.get(entry);
312 if (memCache != null) {
313 BufferedImage result = memCache.get();
314 if (result != null)
315 return result;
316 }
317
318 try {
319 BufferedImage result = ImageIO.read(getImageFile(projectionEntries, entry));
320 if (result == null) {
321 projectionEntries.entries.remove(entry);
322 totalFileSizeDirty = true;
323 }
324 return result;
325 } catch (IOException e) {
326 projectionEntries.entries.remove(entry);
327 totalFileSizeDirty = true;
328 throw e;
329 }
330 }
331
332 private CacheEntry findEntry(ProjectionEntries projectionEntries, double pixelPerDegree, double east, double north) {
333 for (CacheEntry entry: projectionEntries.entries) {
334 if (entry.pixelPerDegree == pixelPerDegree && entry.east == east && entry.north == north)
335 return entry;
336 }
337 return null;
338 }
339
340 public synchronized BufferedImage getExactMatch(Projection projection, double pixelPerDegree, double east, double north) {
341 ProjectionEntries projectionEntries = getProjectionEntries(projection);
342 CacheEntry entry = findEntry(projectionEntries, pixelPerDegree, east, north);
343 if (entry != null) {
344 try {
345 entry.lastUsed = System.currentTimeMillis();
346 return loadImage(projectionEntries, entry);
347 } catch (IOException e) {
348 System.err.println("Unable to load file from wms cache");
349 e.printStackTrace();
350 return null;
351 }
352 }
353 return null;
354 }
355
356 public synchronized BufferedImage getPartialMatch(Projection projection, double pixelPerDegree, double east, double north) {
357 List<CacheEntry> matches = new ArrayList<WmsCache.CacheEntry>();
358
359 double minPPD = pixelPerDegree / 5;
360 double maxPPD = pixelPerDegree * 5;
361 ProjectionEntries projectionEntries = getProjectionEntries(projection);
362
363 ProjectionBounds bounds = new ProjectionBounds(east, north,
364 east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree);
365
366 //TODO Do not load tile if it is completely overlapped by other tile with better ppd
367 for (CacheEntry entry: projectionEntries.entries) {
368 if (entry.pixelPerDegree >= minPPD && entry.pixelPerDegree <= maxPPD && entry.bounds.intersects(bounds)) {
369 entry.lastUsed = System.currentTimeMillis();
370 matches.add(entry);
371 }
372 }
373
374 if (matches.isEmpty())
375 return null;
376
377
378 Collections.sort(matches, new Comparator<CacheEntry>() {
379 @Override
380 public int compare(CacheEntry o1, CacheEntry o2) {
381 return Double.compare(o2.pixelPerDegree, o1.pixelPerDegree);
382 }
383 });
384
385 //TODO Use alpha layer only when enabled on wms layer
386 BufferedImage result = new BufferedImage(tileSize, tileSize, BufferedImage.TYPE_4BYTE_ABGR);
387 Graphics2D g = result.createGraphics();
388
389 boolean drawAtLeastOnce = false;
390 for (CacheEntry ce: matches) {
391 BufferedImage img;
392 try {
393 img = loadImage(projectionEntries, ce);
394 memoryCache.put(ce, new SoftReference<BufferedImage>(img));
395 } catch (IOException e) {
396 continue;
397 }
398
399 drawAtLeastOnce = true;
400
401 int xDiff = (int)((ce.east - east) * pixelPerDegree);
402 int yDiff = (int)((ce.north - north) * pixelPerDegree);
403 int size = (int)(pixelPerDegree / ce.pixelPerDegree * tileSize);
404
405 int x = xDiff;
406 int y = -size + tileSize - yDiff;
407
408 g.drawImage(img, x, y, size, size, null);
409 }
410
411 if (drawAtLeastOnce)
412 return result;
413 else
414 return null;
415 }
416
417 private String generateFileName(ProjectionEntries projectionEntries, double pixelPerDegree, Projection projection, double east, double north, String mimeType) {
418 LatLon ll1 = projection.eastNorth2latlon(new EastNorth(east, north));
419 LatLon ll2 = projection.eastNorth2latlon(new EastNorth(east + 100 / pixelPerDegree, north));
420 LatLon ll3 = projection.eastNorth2latlon(new EastNorth(east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree));
421
422 double deltaLat = Math.abs(ll3.lat() - ll1.lat());
423 double deltaLon = Math.abs(ll3.lon() - ll1.lon());
424 int precisionLat = Math.max(0, -(int)Math.ceil(Math.log10(deltaLat)) + 1);
425 int precisionLon = Math.max(0, -(int)Math.ceil(Math.log10(deltaLon)) + 1);
426
427 String zoom = NavigatableComponent.METRIC_SOM.getDistText(ll1.greatCircleDistance(ll2));
428 String extension;
429 if ("image/jpeg".equals(mimeType) || "image/jpg".equals(mimeType)) {
430 extension = "jpg";
431 } else if ("image/png".equals(mimeType)) {
432 extension = "png";
433 } else if ("image/gif".equals(mimeType)) {
434 extension = "gif";
435 } else {
436 extension = "dat";
437 }
438
439 int counter = 0;
440 FILENAME_LOOP:
441 while (true) {
442 String result = String.format("%s_%." + precisionLat + "f_%." + precisionLon +"f%s.%s", zoom, ll1.lat(), ll1.lon(), counter==0?"":"_" + counter, extension);
443 for (CacheEntry entry: projectionEntries.entries) {
444 if (entry.filename.equals(result)) {
445 counter++;
446 continue FILENAME_LOOP;
447 }
448 }
449 return result;
450 }
451 }
452
453 /**
454 *
455 * @param img Used only when overlapping is used, when not used, used raw from imageData
456 * @param imageData
457 * @param projection
458 * @param pixelPerDegree
459 * @param east
460 * @param north
461 * @throws IOException
462 */
463 public synchronized void saveToCache(BufferedImage img, InputStream imageData, Projection projection, double pixelPerDegree, double east, double north) throws IOException {
464 ProjectionEntries projectionEntries = getProjectionEntries(projection);
465 CacheEntry entry = findEntry(projectionEntries, pixelPerDegree, east, north);
466 File imageFile;
467 if (entry == null) {
468 entry = new CacheEntry(pixelPerDegree, east, north, tileSize);
469 entry.lastUsed = System.currentTimeMillis();
470 entry.lastModified = entry.lastUsed;
471
472 String mimeType;
473 if (img != null) {
474 mimeType = "image/png";
475 } else {
476 mimeType = URLConnection.guessContentTypeFromStream(imageData);
477 }
478 entry.filename = generateFileName(projectionEntries, pixelPerDegree, projection, east, north, mimeType);
479 projectionEntries.entries.add(entry);
480 imageFile = getImageFile(projectionEntries, entry);
481 } else {
482 imageFile = getImageFile(projectionEntries, entry);
483 totalFileSize -= imageFile.length();
484 }
485
486 imageFile.getParentFile().mkdirs();
487
488 if (img != null) {
489 BufferedImage copy = new BufferedImage(tileSize, tileSize, img.getType());
490 copy.createGraphics().drawImage(img, 0, 0, tileSize, tileSize, 0, img.getHeight() - tileSize, tileSize, img.getHeight(), null);
491 ImageIO.write(copy, "png", imageFile);
492 totalFileSize += imageFile.length();
493 } else {
494 OutputStream os = new BufferedOutputStream(new FileOutputStream(imageFile));
495 try {
496 totalFileSize += Utils.copyStream(imageData, os);
497 } finally {
498 os.close();
499 }
500 }
501 }
502
503 public synchronized void cleanSmallFiles(int size) {
504 for (ProjectionEntries projectionEntries: entries.values()) {
505 Iterator<CacheEntry> it = projectionEntries.entries.iterator();
506 while (it.hasNext()) {
507 File file = getImageFile(projectionEntries, it.next());
508 long length = file.length();
509 if (length <= size) {
510 if (length == 0) {
511 totalFileSizeDirty = true; // File probably doesn't exist
512 }
513 totalFileSize -= size;
514 file.delete();
515 it.remove();
516 }
517 }
518 }
519 }
520
521 public static String printDate(Calendar c) {
522 return (new SimpleDateFormat("yyyy-MM-dd")).format(c.getTime());
523 }
524
525 private boolean isInsideAreaToCache(CacheEntry cacheEntry) {
526 for (ProjectionBounds b: areaToCache) {
527 if (cacheEntry.bounds.intersects(b))
528 return true;
529 }
530 return false;
531 }
532
533 public synchronized void setAreaToCache(Set<ProjectionBounds> areaToCache) {
534 this.areaToCache = areaToCache;
535 Iterator<CacheEntry> it = memoryCache.keySet().iterator();
536 while (it.hasNext()) {
537 if (!isInsideAreaToCache(it.next())) {
538 it.remove();
539 }
540 }
541 }
542}
Note: See TracBrowser for help on using the repository browser.