source: josm/trunk/src/org/openstreetmap/josm/data/imagery/WmsCache.java@ 7033

Last change on this file since 7033 was 7033, checked in by Don-vip, 10 years ago

see #8465 - global use of try-with-resources, according to
File size: 22.0 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.imagery;
3
4import java.awt.Graphics2D;
5import java.awt.image.BufferedImage;
6import java.io.BufferedOutputStream;
7import java.io.File;
8import java.io.FileInputStream;
9import java.io.FileNotFoundException;
10import java.io.FileOutputStream;
11import java.io.IOException;
12import java.io.InputStream;
13import java.io.OutputStream;
14import java.lang.ref.SoftReference;
15import java.net.URLConnection;
16import java.text.SimpleDateFormat;
17import java.util.ArrayList;
18import java.util.Calendar;
19import java.util.Collections;
20import java.util.Comparator;
21import java.util.HashMap;
22import java.util.HashSet;
23import java.util.Iterator;
24import java.util.List;
25import java.util.Map;
26import java.util.Properties;
27import java.util.Set;
28
29import javax.imageio.ImageIO;
30import javax.xml.bind.JAXBContext;
31import javax.xml.bind.Marshaller;
32import javax.xml.bind.Unmarshaller;
33
34import org.openstreetmap.josm.Main;
35import org.openstreetmap.josm.data.ProjectionBounds;
36import org.openstreetmap.josm.data.SystemOfMeasurement;
37import org.openstreetmap.josm.data.coor.EastNorth;
38import org.openstreetmap.josm.data.coor.LatLon;
39import org.openstreetmap.josm.data.imagery.types.EntryType;
40import org.openstreetmap.josm.data.imagery.types.ProjectionType;
41import org.openstreetmap.josm.data.imagery.types.WmsCacheType;
42import org.openstreetmap.josm.data.preferences.StringProperty;
43import org.openstreetmap.josm.data.projection.Projection;
44import org.openstreetmap.josm.tools.Utils;
45
46public class WmsCache {
47 //TODO Property for maximum cache size
48 //TODO Property for maximum age of tile, automatically remove old tiles
49 //TODO Measure time for partially loading from cache, compare with time to download tile. If slower, disable partial cache
50 //TODO Do loading from partial cache and downloading at the same time, don't wait for partical cache to load
51
52 private static final StringProperty PROP_CACHE_PATH = new StringProperty("imagery.wms-cache.path", "wms");
53 private static final String INDEX_FILENAME = "index.xml";
54 private static final String LAYERS_INDEX_FILENAME = "layers.properties";
55
56 private static class CacheEntry {
57 final double pixelPerDegree;
58 final double east;
59 final double north;
60 final ProjectionBounds bounds;
61 final String filename;
62
63 long lastUsed;
64 long lastModified;
65
66 CacheEntry(double pixelPerDegree, double east, double north, int tileSize, String filename) {
67 this.pixelPerDegree = pixelPerDegree;
68 this.east = east;
69 this.north = north;
70 this.bounds = new ProjectionBounds(east, north, east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree);
71 this.filename = filename;
72 }
73 }
74
75 private static class ProjectionEntries {
76 final String projection;
77 final String cacheDirectory;
78 final List<CacheEntry> entries = new ArrayList<>();
79
80 ProjectionEntries(String projection, String cacheDirectory) {
81 this.projection = projection;
82 this.cacheDirectory = cacheDirectory;
83 }
84 }
85
86 private final Map<String, ProjectionEntries> entries = new HashMap<>();
87 private final File cacheDir;
88 private final int tileSize; // Should be always 500
89 private int totalFileSize;
90 private boolean totalFileSizeDirty; // Some file was missing - size needs to be recalculated
91 // No need for hashCode/equals on CacheEntry, object identity is enough. Comparing by values can lead to error - CacheEntry for wrong projection could be found
92 private Map<CacheEntry, SoftReference<BufferedImage>> memoryCache = new HashMap<>();
93 private Set<ProjectionBounds> areaToCache;
94
95 protected String cacheDirPath() {
96 String cPath = PROP_CACHE_PATH.get();
97 if (!(new File(cPath).isAbsolute())) {
98 cPath = Main.pref.getCacheDirectory() + File.separator + cPath;
99 }
100 return cPath;
101 }
102
103 public WmsCache(String url, int tileSize) {
104 File globalCacheDir = new File(cacheDirPath());
105 globalCacheDir.mkdirs();
106 cacheDir = new File(globalCacheDir, getCacheDirectory(url));
107 cacheDir.mkdirs();
108 this.tileSize = tileSize;
109 }
110
111 private String getCacheDirectory(String url) {
112 String cacheDirName = null;
113 Properties layersIndex = new Properties();
114 File layerIndexFile = new File(cacheDirPath(), LAYERS_INDEX_FILENAME);
115 try (InputStream fis = new FileInputStream(layerIndexFile)) {
116 layersIndex.load(fis);
117 } catch (FileNotFoundException e) {
118 Main.error("Unable to load layers index for wms cache (file " + layerIndexFile + " not found)");
119 } catch (IOException e) {
120 Main.error("Unable to load layers index for wms cache");
121 Main.error(e);
122 }
123
124 for (Object propKey: layersIndex.keySet()) {
125 String s = (String)propKey;
126 if (url.equals(layersIndex.getProperty(s))) {
127 cacheDirName = s;
128 break;
129 }
130 }
131
132 if (cacheDirName == null) {
133 int counter = 0;
134 while (true) {
135 counter++;
136 if (!layersIndex.keySet().contains(String.valueOf(counter))) {
137 break;
138 }
139 }
140 cacheDirName = String.valueOf(counter);
141 layersIndex.setProperty(cacheDirName, url);
142 try (OutputStream fos = new FileOutputStream(layerIndexFile)) {
143 layersIndex.store(fos, "");
144 } catch (IOException e) {
145 Main.error("Unable to save layer index for wms cache");
146 Main.error(e);
147 }
148 }
149
150 return cacheDirName;
151 }
152
153 private ProjectionEntries getProjectionEntries(Projection projection) {
154 return getProjectionEntries(projection.toCode(), projection.getCacheDirectoryName());
155 }
156
157 private ProjectionEntries getProjectionEntries(String projection, String cacheDirectory) {
158 ProjectionEntries result = entries.get(projection);
159 if (result == null) {
160 result = new ProjectionEntries(projection, cacheDirectory);
161 entries.put(projection, result);
162 }
163
164 return result;
165 }
166
167 public synchronized void loadIndex() {
168 File indexFile = new File(cacheDir, INDEX_FILENAME);
169 try {
170 JAXBContext context = JAXBContext.newInstance(
171 WmsCacheType.class.getPackage().getName(),
172 WmsCacheType.class.getClassLoader());
173 Unmarshaller unmarshaller = context.createUnmarshaller();
174 WmsCacheType cacheEntries;
175 try (InputStream is = new FileInputStream(indexFile)) {
176 cacheEntries = (WmsCacheType)unmarshaller.unmarshal(is);
177 }
178 totalFileSize = cacheEntries.getTotalFileSize();
179 if (cacheEntries.getTileSize() != tileSize) {
180 Main.info("Cache created with different tileSize, cache will be discarded");
181 return;
182 }
183 for (ProjectionType projectionType: cacheEntries.getProjection()) {
184 ProjectionEntries projection = getProjectionEntries(projectionType.getName(), projectionType.getCacheDirectory());
185 for (EntryType entry: projectionType.getEntry()) {
186 CacheEntry ce = new CacheEntry(entry.getPixelPerDegree(), entry.getEast(), entry.getNorth(), tileSize, entry.getFilename());
187 ce.lastUsed = entry.getLastUsed().getTimeInMillis();
188 ce.lastModified = entry.getLastModified().getTimeInMillis();
189 projection.entries.add(ce);
190 }
191 }
192 } catch (Exception e) {
193 if (indexFile.exists()) {
194 Main.error(e);
195 Main.info("Unable to load index for wms-cache, new file will be created");
196 } else {
197 Main.info("Index for wms-cache doesn't exist, new file will be created");
198 }
199 }
200
201 removeNonReferencedFiles();
202 }
203
204 private void removeNonReferencedFiles() {
205
206 Set<String> usedProjections = new HashSet<>();
207
208 for (ProjectionEntries projectionEntries: entries.values()) {
209
210 usedProjections.add(projectionEntries.cacheDirectory);
211
212 File projectionDir = new File(cacheDir, projectionEntries.cacheDirectory);
213 if (projectionDir.exists()) {
214 Set<String> referencedFiles = new HashSet<>();
215
216 for (CacheEntry ce: projectionEntries.entries) {
217 referencedFiles.add(ce.filename);
218 }
219
220 for (File file: projectionDir.listFiles()) {
221 if (!referencedFiles.contains(file.getName())) {
222 file.delete();
223 }
224 }
225 }
226 }
227
228 for (File projectionDir: cacheDir.listFiles()) {
229 if (projectionDir.isDirectory() && !usedProjections.contains(projectionDir.getName())) {
230 Utils.deleteDirectory(projectionDir);
231 }
232 }
233 }
234
235 private int calculateTotalFileSize() {
236 int result = 0;
237 for (ProjectionEntries projectionEntries: entries.values()) {
238 Iterator<CacheEntry> it = projectionEntries.entries.iterator();
239 while (it.hasNext()) {
240 CacheEntry entry = it.next();
241 File imageFile = getImageFile(projectionEntries, entry);
242 if (!imageFile.exists()) {
243 it.remove();
244 } else {
245 result += imageFile.length();
246 }
247 }
248 }
249 return result;
250 }
251
252 public synchronized void saveIndex() {
253 WmsCacheType index = new WmsCacheType();
254
255 if (totalFileSizeDirty) {
256 totalFileSize = calculateTotalFileSize();
257 }
258
259 index.setTileSize(tileSize);
260 index.setTotalFileSize(totalFileSize);
261 for (ProjectionEntries projectionEntries: entries.values()) {
262 if (!projectionEntries.entries.isEmpty()) {
263 ProjectionType projectionType = new ProjectionType();
264 projectionType.setName(projectionEntries.projection);
265 projectionType.setCacheDirectory(projectionEntries.cacheDirectory);
266 index.getProjection().add(projectionType);
267 for (CacheEntry ce: projectionEntries.entries) {
268 EntryType entry = new EntryType();
269 entry.setPixelPerDegree(ce.pixelPerDegree);
270 entry.setEast(ce.east);
271 entry.setNorth(ce.north);
272 Calendar c = Calendar.getInstance();
273 c.setTimeInMillis(ce.lastUsed);
274 entry.setLastUsed(c);
275 c = Calendar.getInstance();
276 c.setTimeInMillis(ce.lastModified);
277 entry.setLastModified(c);
278 entry.setFilename(ce.filename);
279 projectionType.getEntry().add(entry);
280 }
281 }
282 }
283 try {
284 JAXBContext context = JAXBContext.newInstance(
285 WmsCacheType.class.getPackage().getName(),
286 WmsCacheType.class.getClassLoader());
287 Marshaller marshaller = context.createMarshaller();
288 try (OutputStream fos = new FileOutputStream(new File(cacheDir, INDEX_FILENAME))) {
289 marshaller.marshal(index, fos);
290 }
291 } catch (Exception e) {
292 Main.error("Failed to save wms-cache file");
293 Main.error(e);
294 }
295 }
296
297 private File getImageFile(ProjectionEntries projection, CacheEntry entry) {
298 return new File(cacheDir, projection.cacheDirectory + "/" + entry.filename);
299 }
300
301 private BufferedImage loadImage(ProjectionEntries projectionEntries, CacheEntry entry) throws IOException {
302
303 synchronized (this) {
304 entry.lastUsed = System.currentTimeMillis();
305
306 SoftReference<BufferedImage> memCache = memoryCache.get(entry);
307 if (memCache != null) {
308 BufferedImage result = memCache.get();
309 if (result != null)
310 return result;
311 }
312 }
313
314 try {
315 // Reading can't be in synchronized section, it's too slow
316 BufferedImage result = ImageIO.read(getImageFile(projectionEntries, entry));
317 synchronized (this) {
318 if (result == null) {
319 projectionEntries.entries.remove(entry);
320 totalFileSizeDirty = true;
321 }
322 return result;
323 }
324 } catch (IOException e) {
325 synchronized (this) {
326 projectionEntries.entries.remove(entry);
327 totalFileSizeDirty = true;
328 throw e;
329 }
330 }
331 }
332
333 private CacheEntry findEntry(ProjectionEntries projectionEntries, double pixelPerDegree, double east, double north) {
334 for (CacheEntry entry: projectionEntries.entries) {
335 if (entry.pixelPerDegree == pixelPerDegree && entry.east == east && entry.north == north)
336 return entry;
337 }
338 return null;
339 }
340
341 public synchronized boolean hasExactMatch(Projection projection, double pixelPerDegree, double east, double north) {
342 ProjectionEntries projectionEntries = getProjectionEntries(projection);
343 CacheEntry entry = findEntry(projectionEntries, pixelPerDegree, east, north);
344 return (entry != null);
345 }
346
347 public BufferedImage getExactMatch(Projection projection, double pixelPerDegree, double east, double north) {
348 CacheEntry entry = null;
349 ProjectionEntries projectionEntries = null;
350 synchronized (this) {
351 projectionEntries = getProjectionEntries(projection);
352 entry = findEntry(projectionEntries, pixelPerDegree, east, north);
353 }
354 if (entry != null) {
355 try {
356 return loadImage(projectionEntries, entry);
357 } catch (IOException e) {
358 Main.error("Unable to load file from wms cache");
359 Main.error(e);
360 return null;
361 }
362 }
363 return null;
364 }
365
366 public BufferedImage getPartialMatch(Projection projection, double pixelPerDegree, double east, double north) {
367 ProjectionEntries projectionEntries;
368 List<CacheEntry> matches;
369 synchronized (this) {
370 matches = new ArrayList<>();
371
372 double minPPD = pixelPerDegree / 5;
373 double maxPPD = pixelPerDegree * 5;
374 projectionEntries = getProjectionEntries(projection);
375
376 double size2 = tileSize / pixelPerDegree;
377 double border = tileSize * 0.01; // Make sure not to load neighboring tiles that intersects this tile only slightly
378 ProjectionBounds bounds = new ProjectionBounds(east + border, north + border,
379 east + size2 - border, north + size2 - border);
380
381 //TODO Do not load tile if it is completely overlapped by other tile with better ppd
382 for (CacheEntry entry: projectionEntries.entries) {
383 if (entry.pixelPerDegree >= minPPD && entry.pixelPerDegree <= maxPPD && entry.bounds.intersects(bounds)) {
384 entry.lastUsed = System.currentTimeMillis();
385 matches.add(entry);
386 }
387 }
388
389 if (matches.isEmpty())
390 return null;
391
392
393 Collections.sort(matches, new Comparator<CacheEntry>() {
394 @Override
395 public int compare(CacheEntry o1, CacheEntry o2) {
396 return Double.compare(o2.pixelPerDegree, o1.pixelPerDegree);
397 }
398 });
399 }
400
401 //TODO Use alpha layer only when enabled on wms layer
402 BufferedImage result = new BufferedImage(tileSize, tileSize, BufferedImage.TYPE_4BYTE_ABGR);
403 Graphics2D g = result.createGraphics();
404
405
406 boolean drawAtLeastOnce = false;
407 Map<CacheEntry, SoftReference<BufferedImage>> localCache = new HashMap<>();
408 for (CacheEntry ce: matches) {
409 BufferedImage img;
410 try {
411 img = loadImage(projectionEntries, ce);
412 localCache.put(ce, new SoftReference<>(img));
413 } catch (IOException e) {
414 continue;
415 }
416
417 drawAtLeastOnce = true;
418
419 int xDiff = (int)((ce.east - east) * pixelPerDegree);
420 int yDiff = (int)((ce.north - north) * pixelPerDegree);
421 int size = (int)(pixelPerDegree / ce.pixelPerDegree * tileSize);
422
423 int x = xDiff;
424 int y = -size + tileSize - yDiff;
425
426 g.drawImage(img, x, y, size, size, null);
427 }
428
429 if (drawAtLeastOnce) {
430 synchronized (this) {
431 memoryCache.putAll(localCache);
432 }
433 return result;
434 } else
435 return null;
436 }
437
438 private String generateFileName(ProjectionEntries projectionEntries, double pixelPerDegree, Projection projection, double east, double north, String mimeType) {
439 LatLon ll1 = projection.eastNorth2latlon(new EastNorth(east, north));
440 LatLon ll2 = projection.eastNorth2latlon(new EastNorth(east + 100 / pixelPerDegree, north));
441 LatLon ll3 = projection.eastNorth2latlon(new EastNorth(east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree));
442
443 double deltaLat = Math.abs(ll3.lat() - ll1.lat());
444 double deltaLon = Math.abs(ll3.lon() - ll1.lon());
445 int precisionLat = Math.max(0, -(int)Math.ceil(Math.log10(deltaLat)) + 1);
446 int precisionLon = Math.max(0, -(int)Math.ceil(Math.log10(deltaLon)) + 1);
447
448 String zoom = SystemOfMeasurement.METRIC.getDistText(ll1.greatCircleDistance(ll2));
449 String extension = "dat";
450 if (mimeType != null) {
451 switch(mimeType) {
452 case "image/jpeg":
453 case "image/jpg":
454 extension = "jpg";
455 break;
456 case "image/png":
457 extension = "png";
458 break;
459 case "image/gif":
460 extension = "gif";
461 break;
462 default:
463 Main.warn("Unrecognized MIME type: "+mimeType);
464 }
465 }
466
467 int counter = 0;
468 FILENAME_LOOP:
469 while (true) {
470 String result = String.format("%s_%." + precisionLat + "f_%." + precisionLon +"f%s.%s", zoom, ll1.lat(), ll1.lon(), counter==0?"":"_" + counter, extension);
471 for (CacheEntry entry: projectionEntries.entries) {
472 if (entry.filename.equals(result)) {
473 counter++;
474 continue FILENAME_LOOP;
475 }
476 }
477 return result;
478 }
479 }
480
481 /**
482 *
483 * @param img Used only when overlapping is used, when not used, used raw from imageData
484 * @param imageData
485 * @param projection
486 * @param pixelPerDegree
487 * @param east
488 * @param north
489 * @throws IOException
490 */
491 public synchronized void saveToCache(BufferedImage img, InputStream imageData, Projection projection, double pixelPerDegree, double east, double north) throws IOException {
492 ProjectionEntries projectionEntries = getProjectionEntries(projection);
493 CacheEntry entry = findEntry(projectionEntries, pixelPerDegree, east, north);
494 File imageFile;
495 if (entry == null) {
496
497 String mimeType;
498 if (img != null) {
499 mimeType = "image/png";
500 } else {
501 mimeType = URLConnection.guessContentTypeFromStream(imageData);
502 }
503 entry = new CacheEntry(pixelPerDegree, east, north, tileSize,generateFileName(projectionEntries, pixelPerDegree, projection, east, north, mimeType));
504 entry.lastUsed = System.currentTimeMillis();
505 entry.lastModified = entry.lastUsed;
506 projectionEntries.entries.add(entry);
507 imageFile = getImageFile(projectionEntries, entry);
508 } else {
509 imageFile = getImageFile(projectionEntries, entry);
510 totalFileSize -= imageFile.length();
511 }
512
513 imageFile.getParentFile().mkdirs();
514
515 if (img != null) {
516 BufferedImage copy = new BufferedImage(tileSize, tileSize, img.getType());
517 copy.createGraphics().drawImage(img, 0, 0, tileSize, tileSize, 0, img.getHeight() - tileSize, tileSize, img.getHeight(), null);
518 ImageIO.write(copy, "png", imageFile);
519 totalFileSize += imageFile.length();
520 } else {
521 try (OutputStream os = new BufferedOutputStream(new FileOutputStream(imageFile))) {
522 totalFileSize += Utils.copyStream(imageData, os);
523 }
524 }
525 }
526
527 public synchronized void cleanSmallFiles(int size) {
528 for (ProjectionEntries projectionEntries: entries.values()) {
529 Iterator<CacheEntry> it = projectionEntries.entries.iterator();
530 while (it.hasNext()) {
531 File file = getImageFile(projectionEntries, it.next());
532 long length = file.length();
533 if (length <= size) {
534 if (length == 0) {
535 totalFileSizeDirty = true; // File probably doesn't exist
536 }
537 totalFileSize -= size;
538 file.delete();
539 it.remove();
540 }
541 }
542 }
543 }
544
545 public static String printDate(Calendar c) {
546 return (new SimpleDateFormat("yyyy-MM-dd")).format(c.getTime());
547 }
548
549 private boolean isInsideAreaToCache(CacheEntry cacheEntry) {
550 for (ProjectionBounds b: areaToCache) {
551 if (cacheEntry.bounds.intersects(b))
552 return true;
553 }
554 return false;
555 }
556
557 public synchronized void setAreaToCache(Set<ProjectionBounds> areaToCache) {
558 this.areaToCache = areaToCache;
559 Iterator<CacheEntry> it = memoryCache.keySet().iterator();
560 while (it.hasNext()) {
561 if (!isInsideAreaToCache(it.next())) {
562 it.remove();
563 }
564 }
565 }
566}
Note: See TracBrowser for help on using the repository browser.