source: josm/trunk/src/org/openstreetmap/josm/data/imagery/WmsCache.java@ 4065

Last change on this file since 4065 was 4065, checked in by jttt, 13 years ago

Improved wms cache

  • files saved in original format (no need to recode to png)
  • possibility to tile with different pos/ppd that overlaps current tile
  • Property svn:mime-type set to text/plain
File size: 21.1 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.imagery;
3
4
5import java.awt.Graphics2D;
6import java.awt.image.BufferedImage;
7import java.io.BufferedOutputStream;
8import java.io.File;
9import java.io.FileInputStream;
10import java.io.FileNotFoundException;
11import java.io.FileOutputStream;
12import java.io.IOException;
13import java.io.InputStream;
14import java.io.OutputStream;
15import java.lang.ref.SoftReference;
16import java.net.URLConnection;
17import java.text.SimpleDateFormat;
18import java.util.ArrayList;
19import java.util.Calendar;
20import java.util.Collections;
21import java.util.Comparator;
22import java.util.HashMap;
23import java.util.HashSet;
24import java.util.Iterator;
25import java.util.List;
26import java.util.Map;
27import java.util.Properties;
28import java.util.Set;
29
30import javax.imageio.ImageIO;
31import javax.xml.bind.JAXBContext;
32import javax.xml.bind.JAXBException;
33import javax.xml.bind.Marshaller;
34import javax.xml.bind.Unmarshaller;
35
36import org.openstreetmap.josm.Main;
37import org.openstreetmap.josm.data.ProjectionBounds;
38import org.openstreetmap.josm.data.coor.EastNorth;
39import org.openstreetmap.josm.data.coor.LatLon;
40import org.openstreetmap.josm.data.imagery.types.EntryType;
41import org.openstreetmap.josm.data.imagery.types.ProjectionType;
42import org.openstreetmap.josm.data.imagery.types.WmsCacheType;
43import org.openstreetmap.josm.data.preferences.StringProperty;
44import org.openstreetmap.josm.data.projection.Projection;
45import org.openstreetmap.josm.gui.NavigatableComponent;
46import org.openstreetmap.josm.tools.Utils;
47
48
49
50public class WmsCache {
51 //TODO Property for maximum cache size
52 //TODO Property for maximum age of tile, automatically remove old tiles
53 //TODO Measure time for partially loading from cache, compare with time to download tile. If slower, disable partial cache
54 //TODO Do loading from partial cache and downloading at the same time, don't wait for partical cache to load
55
56 private static final StringProperty PROP_CACHE_PATH = new StringProperty("imagery.wms-cache.path", "wms-cache");
57 private static final String INDEX_FILENAME = "index.xml";
58 private static final String LAYERS_INDEX_FILENAME = "layers.properties";
59
60 private static class CacheEntry {
61 final double pixelPerDegree;
62 final double east;
63 final double north;
64 final ProjectionBounds bounds;
65
66 long lastUsed;
67 long lastModified;
68 String filename;
69
70 CacheEntry(double pixelPerDegree, double east, double north, int tileSize) {
71 this.pixelPerDegree = pixelPerDegree;
72 this.east = east;
73 this.north = north;
74 this.bounds = new ProjectionBounds(east, north, east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree);
75 }
76 }
77
78 private static class ProjectionEntries {
79 final String projection;
80 final String cacheDirectory;
81 final List<CacheEntry> entries = new ArrayList<WmsCache.CacheEntry>();
82
83 ProjectionEntries(String projection, String cacheDirectory) {
84 this.projection = projection;
85 this.cacheDirectory = cacheDirectory;
86 }
87 }
88
89 private final Map<String, ProjectionEntries> entries = new HashMap<String, ProjectionEntries>();
90 private final File cacheDir;
91 private final int tileSize; // Should be always 500
92 private int totalFileSize;
93 private boolean totalFileSizeDirty; // Some file was missing - size needs to be recalculated
94 // No need for hashCode/equals on CacheEntry, object identity is enough. Comparing by values can lead to error - CacheEntry for wrong projection could be found
95 private Map<CacheEntry, SoftReference<BufferedImage>> memoryCache = new HashMap<WmsCache.CacheEntry, SoftReference<BufferedImage>>();
96 private Set<ProjectionBounds> areaToCache;
97
98 public WmsCache(String url, int tileSize) {
99 File globalCacheDir = new File(Main.pref.getPreferencesDir() + PROP_CACHE_PATH.get());
100 globalCacheDir.mkdirs();
101 cacheDir = new File(globalCacheDir, getCacheDirectory(url));
102 cacheDir.mkdirs();
103 this.tileSize = tileSize;
104 }
105
106 private String getCacheDirectory(String url) {
107 String cacheDirName = null;
108 InputStream fis = null;
109 OutputStream fos = null;
110 try {
111 Properties layersIndex = new Properties();
112 File layerIndexFile = new File(Main.pref.getPreferencesDir() + PROP_CACHE_PATH.get(), LAYERS_INDEX_FILENAME);
113 try {
114 fis = new FileInputStream(layerIndexFile);
115 layersIndex.load(fis);
116 } catch (FileNotFoundException e) {
117 System.out.println("Unable to load layers index for wms cache (file " + layerIndexFile + " not found)");
118 } catch (IOException e) {
119 System.err.println("Unable to load layers index for wms cache");
120 e.printStackTrace();
121 }
122
123 for (Object propKey: layersIndex.keySet()) {
124 String s = (String)propKey;
125 if (url.equals(layersIndex.getProperty(s))) {
126 cacheDirName = s;
127 break;
128 }
129 }
130
131 if (cacheDirName == null) {
132 int counter = 0;
133 while (true) {
134 counter++;
135 if (!layersIndex.keySet().contains(String.valueOf(counter))) {
136 break;
137 }
138 }
139 cacheDirName = String.valueOf(counter);
140 layersIndex.setProperty(cacheDirName, url);
141 try {
142 fos = new FileOutputStream(layerIndexFile);
143 layersIndex.store(fos, "");
144 } catch (IOException e) {
145 System.err.println("Unable to save layer index for wms cache");
146 e.printStackTrace();
147 }
148 }
149 } finally {
150 try {
151 if (fis != null) {
152 fis.close();
153 }
154 if (fos != null) {
155 fos.close();
156 }
157 } catch (IOException e) {
158 e.printStackTrace();
159 }
160 }
161
162 return cacheDirName;
163 }
164
165 private ProjectionEntries getProjectionEntries(Projection projection) {
166 return getProjectionEntries(projection.toCode(), projection.getCacheDirectoryName());
167 }
168
169 private ProjectionEntries getProjectionEntries(String projection, String cacheDirectory) {
170 ProjectionEntries result = entries.get(projection);
171 if (result == null) {
172 result = new ProjectionEntries(projection, cacheDirectory);
173 entries.put(projection, result);
174 }
175
176 return result;
177 }
178
179 public synchronized void loadIndex() {
180 try {
181 JAXBContext context = JAXBContext.newInstance(
182 WmsCacheType.class.getPackage().getName(),
183 WmsCacheType.class.getClassLoader());
184 Unmarshaller unmarshaller = context.createUnmarshaller();
185 WmsCacheType cacheEntries = (WmsCacheType)unmarshaller.unmarshal(
186 new FileInputStream(new File(cacheDir, INDEX_FILENAME)));
187 totalFileSize = cacheEntries.getTotalFileSize();
188 if (cacheEntries.getTileSize() != tileSize) {
189 System.out.println("Cache created with different tileSize, cache will be discarded");
190 return;
191 }
192 for (ProjectionType projectionType: cacheEntries.getProjection()) {
193 ProjectionEntries projection = getProjectionEntries(projectionType.getName(), projectionType.getCacheDirectory());
194 for (EntryType entry: projectionType.getEntry()) {
195 CacheEntry ce = new CacheEntry(entry.getPixelPerDegree(), entry.getEast(), entry.getNorth(), tileSize);
196 ce.lastUsed = entry.getLastUsed().getTimeInMillis();
197 ce.filename = entry.getFilename();
198 ce.lastModified = entry.getLastModified().getTimeInMillis();
199 projection.entries.add(ce);
200 }
201 }
202 } catch (Exception e) {
203 e.printStackTrace();
204 System.out.println("Unable to load index for wms-cache, new file will be created");
205 }
206
207 removeNonReferencedFiles();
208 }
209
210 private void removeNonReferencedFiles() {
211
212 Set<String> usedProjections = new HashSet<String>();
213
214 for (ProjectionEntries projectionEntries: entries.values()) {
215
216 usedProjections.add(projectionEntries.cacheDirectory);
217
218 File projectionDir = new File(cacheDir, projectionEntries.cacheDirectory);
219 Set<String> referencedFiles = new HashSet<String>();
220
221 for (CacheEntry ce: projectionEntries.entries) {
222 referencedFiles.add(ce.filename);
223 }
224
225 for (File file: projectionDir.listFiles()) {
226 if (!referencedFiles.contains(file.getName())) {
227 file.delete();
228 }
229 }
230 }
231
232 for (File projectionDir: cacheDir.listFiles()) {
233 if (projectionDir.isDirectory() && !usedProjections.contains(projectionDir.getName())) {
234 Utils.deleteDirectory(projectionDir);
235 }
236 }
237 }
238
239 private int calculateTotalFileSize() {
240 int result = 0;
241 for (ProjectionEntries projectionEntries: entries.values()) {
242 Iterator<CacheEntry> it = projectionEntries.entries.iterator();
243 while (it.hasNext()) {
244 CacheEntry entry = it.next();
245 File imageFile = getImageFile(projectionEntries, entry);
246 if (!imageFile.exists()) {
247 it.remove();
248 } else {
249 result += imageFile.length();
250 }
251 }
252 }
253 return result;
254 }
255
256 public synchronized void saveIndex() {
257 WmsCacheType index = new WmsCacheType();
258
259 if (totalFileSizeDirty) {
260 totalFileSize = calculateTotalFileSize();
261 }
262
263 index.setTileSize(tileSize);
264 index.setTotalFileSize(totalFileSize);
265 for (ProjectionEntries projectionEntries: entries.values()) {
266 if (projectionEntries.entries.size() > 0) {
267 ProjectionType projectionType = new ProjectionType();
268 projectionType.setName(projectionEntries.projection);
269 projectionType.setCacheDirectory(projectionEntries.cacheDirectory);
270 index.getProjection().add(projectionType);
271 for (CacheEntry ce: projectionEntries.entries) {
272 EntryType entry = new EntryType();
273 entry.setPixelPerDegree(ce.pixelPerDegree);
274 entry.setEast(ce.east);
275 entry.setNorth(ce.north);
276 Calendar c = Calendar.getInstance();
277 c.setTimeInMillis(ce.lastUsed);
278 entry.setLastUsed(c);
279 c = Calendar.getInstance();
280 c.setTimeInMillis(ce.lastModified);
281 entry.setLastModified(c);
282 entry.setFilename(ce.filename);
283 projectionType.getEntry().add(entry);
284 }
285 }
286 }
287 try {
288 JAXBContext context = JAXBContext.newInstance(
289 WmsCacheType.class.getPackage().getName(),
290 WmsCacheType.class.getClassLoader());
291 Marshaller marshaller = context.createMarshaller();
292 marshaller.marshal(index, new File(cacheDir, INDEX_FILENAME));
293 } catch (JAXBException e) {
294 System.err.println("Failed to save wms-cache file");
295 e.printStackTrace();
296 }
297
298 }
299
300 private File getImageFile(ProjectionEntries projection, CacheEntry entry) {
301 return new File(cacheDir, projection.cacheDirectory + "/" + entry.filename);
302 }
303
304 private BufferedImage loadImage(ProjectionEntries projectionEntries, CacheEntry entry) throws IOException {
305 entry.lastUsed = System.currentTimeMillis();
306
307 SoftReference<BufferedImage> memCache = memoryCache.get(entry);
308 if (memCache != null) {
309 BufferedImage result = memCache.get();
310 if (result != null)
311 return result;
312 }
313
314 try {
315 BufferedImage result = ImageIO.read(getImageFile(projectionEntries, entry));
316 if (result == null) {
317 projectionEntries.entries.remove(entry);
318 totalFileSizeDirty = true;
319 }
320 return result;
321 } catch (IOException e) {
322 projectionEntries.entries.remove(entry);
323 totalFileSizeDirty = true;
324 throw e;
325 }
326 }
327
328 private CacheEntry findEntry(ProjectionEntries projectionEntries, double pixelPerDegree, double east, double north) {
329 for (CacheEntry entry: projectionEntries.entries) {
330 if (entry.pixelPerDegree == pixelPerDegree && entry.east == east && entry.north == north)
331 return entry;
332 }
333 return null;
334 }
335
336 public synchronized BufferedImage getExactMatch(Projection projection, double pixelPerDegree, double east, double north) {
337 ProjectionEntries projectionEntries = getProjectionEntries(projection);
338 CacheEntry entry = findEntry(projectionEntries, pixelPerDegree, east, north);
339 if (entry != null) {
340 try {
341 entry.lastUsed = System.currentTimeMillis();
342 return loadImage(projectionEntries, entry);
343 } catch (IOException e) {
344 System.err.println("Unable to load file from wms cache");
345 e.printStackTrace();
346 return null;
347 }
348 }
349 return null;
350 }
351
352 public synchronized BufferedImage getPartialMatch(Projection projection, double pixelPerDegree, double east, double north) {
353 List<CacheEntry> matches = new ArrayList<WmsCache.CacheEntry>();
354
355 double minPPD = pixelPerDegree / 5;
356 double maxPPD = pixelPerDegree * 5;
357 ProjectionEntries projectionEntries = getProjectionEntries(projection);
358
359 ProjectionBounds bounds = new ProjectionBounds(east, north,
360 east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree);
361
362 //TODO Do not load tile if it is completely overlapped by other tile with better ppd
363 for (CacheEntry entry: projectionEntries.entries) {
364 if (entry.pixelPerDegree >= minPPD && entry.pixelPerDegree <= maxPPD && entry.bounds.intersects(bounds)) {
365 entry.lastUsed = System.currentTimeMillis();
366 matches.add(entry);
367 }
368 }
369
370 if (matches.isEmpty())
371 return null;
372
373
374 Collections.sort(matches, new Comparator<CacheEntry>() {
375 @Override
376 public int compare(CacheEntry o1, CacheEntry o2) {
377 return Double.compare(o2.pixelPerDegree, o1.pixelPerDegree);
378 }
379 });
380
381 //TODO Use alpha layer only when enabled on wms layer
382 BufferedImage result = new BufferedImage(tileSize, tileSize, BufferedImage.TYPE_4BYTE_ABGR);
383 Graphics2D g = result.createGraphics();
384
385 boolean drawAtLeastOnce = false;
386 for (CacheEntry ce: matches) {
387 BufferedImage img;
388 try {
389 img = loadImage(projectionEntries, ce);
390 memoryCache.put(ce, new SoftReference<BufferedImage>(img));
391 } catch (IOException e) {
392 continue;
393 }
394
395 drawAtLeastOnce = true;
396
397 int xDiff = (int)((ce.east - east) * pixelPerDegree);
398 int yDiff = (int)((ce.north - north) * pixelPerDegree);
399 int size = (int)(pixelPerDegree / ce.pixelPerDegree * tileSize);
400
401 int x = xDiff;
402 int y = -size + tileSize - yDiff;
403
404 g.drawImage(img, x, y, size, size, null);
405 }
406
407 if (drawAtLeastOnce)
408 return result;
409 else
410 return null;
411 }
412
413 private String generateFileName(ProjectionEntries projectionEntries, double pixelPerDegree, Projection projection, double east, double north, String mimeType) {
414 LatLon ll1 = projection.eastNorth2latlon(new EastNorth(east, north));
415 LatLon ll2 = projection.eastNorth2latlon(new EastNorth(east + 100 / pixelPerDegree, north));
416 LatLon ll3 = projection.eastNorth2latlon(new EastNorth(east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree));
417
418 double deltaLat = Math.abs(ll3.lat() - ll1.lat());
419 double deltaLon = Math.abs(ll3.lon() - ll1.lon());
420 int precisionLat = Math.max(0, -(int)Math.ceil(Math.log10(deltaLat)) + 1);
421 int precisionLon = Math.max(0, -(int)Math.ceil(Math.log10(deltaLon)) + 1);
422
423 String zoom = NavigatableComponent.METRIC_SOM.getDistText(ll1.greatCircleDistance(ll2));
424 String extension;
425 if ("image/jpeg".equals(mimeType) || "image/jpg".equals(mimeType)) {
426 extension = "jpg";
427 } else if ("image/png".equals(mimeType)) {
428 extension = "png";
429 } else if ("image/gif".equals(mimeType)) {
430 extension = "gif";
431 } else {
432 extension = "dat";
433 }
434
435 int counter = 0;
436 FILENAME_LOOP:
437 while (true) {
438 String result = String.format("%s_%." + precisionLat + "f_%." + precisionLon +"f%s.%s", zoom, ll1.lat(), ll1.lon(), counter==0?"":"_" + counter, extension);
439 for (CacheEntry entry: projectionEntries.entries) {
440 if (entry.filename.equals(result)) {
441 counter++;
442 continue FILENAME_LOOP;
443 }
444 }
445 return result;
446 }
447 }
448
449 /**
450 *
451 * @param img Used only when overlapping is used, when not used, used raw from imageData
452 * @param imageData
453 * @param projection
454 * @param pixelPerDegree
455 * @param east
456 * @param north
457 * @throws IOException
458 */
459 public synchronized void saveToCache(BufferedImage img, InputStream imageData, Projection projection, double pixelPerDegree, double east, double north) throws IOException {
460 ProjectionEntries projectionEntries = getProjectionEntries(projection);
461 CacheEntry entry = findEntry(projectionEntries, pixelPerDegree, east, north);
462 File imageFile;
463 if (entry == null) {
464 entry = new CacheEntry(pixelPerDegree, east, north, tileSize);
465 entry.lastUsed = System.currentTimeMillis();
466 entry.lastModified = entry.lastUsed;
467
468 String mimeType;
469 if (img != null) {
470 mimeType = "image/png";
471 } else {
472 mimeType = URLConnection.guessContentTypeFromStream(imageData);
473 }
474 entry.filename = generateFileName(projectionEntries, pixelPerDegree, projection, east, north, mimeType);
475 projectionEntries.entries.add(entry);
476 imageFile = getImageFile(projectionEntries, entry);
477 } else {
478 imageFile = getImageFile(projectionEntries, entry);
479 totalFileSize -= imageFile.length();
480 }
481
482 imageFile.getParentFile().mkdirs();
483
484 if (img != null) {
485 BufferedImage copy = new BufferedImage(tileSize, tileSize, img.getType());
486 copy.createGraphics().drawImage(img, 0, 0, tileSize, tileSize, 0, img.getHeight() - tileSize, tileSize, img.getHeight(), null);
487 ImageIO.write(copy, "png", imageFile);
488 totalFileSize += imageFile.length();
489 } else {
490 OutputStream os = new BufferedOutputStream(new FileOutputStream(imageFile));
491 try {
492 totalFileSize += Utils.copyStream(imageData, os);
493 } finally {
494 os.close();
495 }
496 }
497 }
498
499 public synchronized void cleanSmallFiles(int size) {
500 for (ProjectionEntries projectionEntries: entries.values()) {
501 Iterator<CacheEntry> it = projectionEntries.entries.iterator();
502 while (it.hasNext()) {
503 File file = getImageFile(projectionEntries, it.next());
504 long length = file.length();
505 if (length <= size) {
506 if (length == 0) {
507 totalFileSizeDirty = true; // File probably doesn't exist
508 }
509 totalFileSize -= size;
510 file.delete();
511 it.remove();
512 }
513 }
514 }
515 }
516
517 public static String printDate(Calendar c) {
518 return (new SimpleDateFormat("yyyy-MM-dd")).format(c.getTime());
519 }
520
521 private boolean isInsideAreaToCache(CacheEntry cacheEntry) {
522 for (ProjectionBounds b: areaToCache) {
523 if (cacheEntry.bounds.intersects(b))
524 return true;
525 }
526 return false;
527 }
528
529 public void setAreaToCache(Set<ProjectionBounds> areaToCache) {
530 this.areaToCache = areaToCache;
531 Iterator<CacheEntry> it = memoryCache.keySet().iterator();
532 while (it.hasNext()) {
533 if (!isInsideAreaToCache(it.next())) {
534 it.remove();
535 }
536 }
537 }
538
539 public Set<ProjectionBounds> getAreaToCache() {
540 return areaToCache;
541 }
542}
Note: See TracBrowser for help on using the repository browser.