source: josm/trunk/src/org/openstreetmap/josm/io/CachedFile.java@ 14397

Last change on this file since 14397 was 14367, checked in by Don-vip, 5 years ago

fix #16897 - catch InvalidPathException for invalid WMS GetCapabilities URL
  • Property svn:eol-style set to native
File size: 22.5 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.io;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.BufferedReader;
7import java.io.ByteArrayOutputStream;
8import java.io.Closeable;
9import java.io.File;
10import java.io.IOException;
11import java.io.InputStream;
12import java.math.BigInteger;
13import java.net.HttpURLConnection;
14import java.net.MalformedURLException;
15import java.net.URL;
16import java.nio.charset.StandardCharsets;
17import java.nio.file.Files;
18import java.nio.file.InvalidPathException;
19import java.nio.file.StandardCopyOption;
20import java.security.MessageDigest;
21import java.security.NoSuchAlgorithmException;
22import java.util.ArrayList;
23import java.util.Arrays;
24import java.util.Enumeration;
25import java.util.List;
26import java.util.Map;
27import java.util.concurrent.ConcurrentHashMap;
28import java.util.concurrent.TimeUnit;
29import java.util.zip.ZipEntry;
30import java.util.zip.ZipFile;
31
32import org.openstreetmap.josm.data.Preferences;
33import org.openstreetmap.josm.spi.preferences.Config;
34import org.openstreetmap.josm.tools.HttpClient;
35import org.openstreetmap.josm.tools.Logging;
36import org.openstreetmap.josm.tools.Pair;
37import org.openstreetmap.josm.tools.PlatformManager;
38import org.openstreetmap.josm.tools.Utils;
39
40/**
41 * Downloads a file and caches it on disk in order to reduce network load.
42 *
43 * Supports URLs, local files, and a custom scheme (<code>resource:</code>) to get
44 * resources from the current *.jar file. (Local caching is only done for URLs.)
45 * <p>
46 * The mirrored file is only downloaded if it has been more than 7 days since
47 * last download. (Time can be configured.)
48 * <p>
49 * The file content is normally accessed with {@link #getInputStream()}, but
50 * you can also get the mirrored copy with {@link #getFile()}.
51 */
52public class CachedFile implements Closeable {
53
54 /**
55 * Caching strategy.
56 */
57 public enum CachingStrategy {
58 /**
59 * If cached file on disk is older than a certain time (7 days by default),
60 * consider the cache stale and try to download the file again.
61 */
62 MaxAge,
63 /**
64 * Similar to MaxAge, considers the cache stale when a certain age is
65 * exceeded. In addition, a If-Modified-Since HTTP header is added.
66 * When the server replies "304 Not Modified", this is considered the same
67 * as a full download.
68 */
69 IfModifiedSince
70 }
71
72 protected String name;
73 protected long maxAge;
74 protected String destDir;
75 protected String httpAccept;
76 protected CachingStrategy cachingStrategy;
77
78 private boolean fastFail;
79 private HttpClient activeConnection;
80 protected File cacheFile;
81 protected boolean initialized;
82 protected String parameter;
83
84 public static final long DEFAULT_MAXTIME = -1L;
85 public static final long DAYS = TimeUnit.DAYS.toSeconds(1); // factor to get caching time in days
86
87 private final Map<String, String> httpHeaders = new ConcurrentHashMap<>();
88
89 /**
90 * Constructs a CachedFile object from a given filename, URL or internal resource.
91 *
92 * @param name can be:<ul>
93 * <li>relative or absolute file name</li>
94 * <li>{@code file:///SOME/FILE} the same as above</li>
95 * <li>{@code http://...} a URL. It will be cached on disk.</li>
96 * <li>{@code resource://SOME/FILE} file from the classpath (usually in the current *.jar)</li>
97 * <li>{@code josmdir://SOME/FILE} file inside josm user data directory (since r7058)</li>
98 * <li>{@code josmplugindir://SOME/FILE} file inside josm plugin directory (since r7834)</li></ul>
99 */
100 public CachedFile(String name) {
101 this.name = name;
102 }
103
104 /**
105 * Set the name of the resource.
106 * @param name can be:<ul>
107 * <li>relative or absolute file name</li>
108 * <li>{@code file:///SOME/FILE} the same as above</li>
109 * <li>{@code http://...} a URL. It will be cached on disk.</li>
110 * <li>{@code resource://SOME/FILE} file from the classpath (usually in the current *.jar)</li>
111 * <li>{@code josmdir://SOME/FILE} file inside josm user data directory (since r7058)</li>
112 * <li>{@code josmplugindir://SOME/FILE} file inside josm plugin directory (since r7834)</li></ul>
113 * @return this object
114 */
115 public CachedFile setName(String name) {
116 this.name = name;
117 return this;
118 }
119
120 /**
121 * Set maximum age of cache file. Only applies to URLs.
122 * When this time has passed after the last download of the file, the
123 * cache is considered stale and a new download will be attempted.
124 * @param maxAge the maximum cache age in seconds
125 * @return this object
126 */
127 public CachedFile setMaxAge(long maxAge) {
128 this.maxAge = maxAge;
129 return this;
130 }
131
132 /**
133 * Set the destination directory for the cache file. Only applies to URLs.
134 * @param destDir the destination directory
135 * @return this object
136 */
137 public CachedFile setDestDir(String destDir) {
138 this.destDir = destDir;
139 return this;
140 }
141
142 /**
143 * Set the accepted MIME types sent in the HTTP Accept header. Only applies to URLs.
144 * @param httpAccept the accepted MIME types
145 * @return this object
146 */
147 public CachedFile setHttpAccept(String httpAccept) {
148 this.httpAccept = httpAccept;
149 return this;
150 }
151
152 /**
153 * Set the caching strategy. Only applies to URLs.
154 * @param cachingStrategy caching strategy
155 * @return this object
156 */
157 public CachedFile setCachingStrategy(CachingStrategy cachingStrategy) {
158 this.cachingStrategy = cachingStrategy;
159 return this;
160 }
161
162 /**
163 * Sets the http headers. Only applies to URL pointing to http or https resources
164 * @param headers that should be sent together with request
165 * @return this object
166 */
167 public CachedFile setHttpHeaders(Map<String, String> headers) {
168 this.httpHeaders.putAll(headers);
169 return this;
170 }
171
172 /**
173 * Sets whether opening HTTP connections should fail fast, i.e., whether a
174 * {@link HttpClient#setConnectTimeout(int) low connect timeout} should be used.
175 * @param fastFail whether opening HTTP connections should fail fast
176 */
177 public void setFastFail(boolean fastFail) {
178 this.fastFail = fastFail;
179 }
180
181 /**
182 * Sets additional URL parameter (used e.g. for maps)
183 * @param parameter the URL parameter
184 * @since 13536
185 */
186 public void setParam(String parameter) {
187 this.parameter = parameter;
188 }
189
190 public String getName() {
191 if (parameter != null)
192 return name.replaceAll("%<(.*)>", "");
193 return name;
194 }
195
196 /**
197 * Returns maximum age of cache file. Only applies to URLs.
198 * When this time has passed after the last download of the file, the
199 * cache is considered stale and a new download will be attempted.
200 * @return the maximum cache age in seconds
201 */
202 public long getMaxAge() {
203 return maxAge;
204 }
205
206 public String getDestDir() {
207 return destDir;
208 }
209
210 public String getHttpAccept() {
211 return httpAccept;
212 }
213
214 public CachingStrategy getCachingStrategy() {
215 return cachingStrategy;
216 }
217
218 /**
219 * Get InputStream to the requested resource.
220 * @return the InputStream
221 * @throws IOException when the resource with the given name could not be retrieved
222 * @throws InvalidPathException if a Path object cannot be constructed from the inner file path
223 */
224 public InputStream getInputStream() throws IOException {
225 File file = getFile();
226 if (file == null) {
227 if (name != null && name.startsWith("resource://")) {
228 InputStream is = getClass().getResourceAsStream(
229 name.substring("resource:/".length()));
230 if (is == null)
231 throw new IOException(tr("Failed to open input stream for resource ''{0}''", name));
232 return is;
233 } else {
234 throw new IOException("No file found for: "+name);
235 }
236 }
237 return Files.newInputStream(file.toPath());
238 }
239
240 /**
241 * Get the full content of the requested resource as a byte array.
242 * @return the full content of the requested resource as byte array
243 * @throws IOException in case of an I/O error
244 */
245 public byte[] getByteContent() throws IOException {
246 try (InputStream is = getInputStream()) {
247 ByteArrayOutputStream buffer = new ByteArrayOutputStream();
248 int nRead;
249 byte[] data = new byte[8192];
250 while ((nRead = is.read(data, 0, data.length)) != -1) {
251 buffer.write(data, 0, nRead);
252 }
253 buffer.flush();
254 return buffer.toByteArray();
255 }
256 }
257
258 /**
259 * Returns {@link #getInputStream()} wrapped in a buffered reader.
260 * <p>
261 * Detects Unicode charset in use utilizing {@link UTFInputStreamReader}.
262 *
263 * @return buffered reader
264 * @throws IOException if any I/O error occurs
265 * @since 9411
266 */
267 public BufferedReader getContentReader() throws IOException {
268 return new BufferedReader(UTFInputStreamReader.create(getInputStream()));
269 }
270
271 /**
272 * Get local file for the requested resource.
273 * @return The local cache file for URLs. If the resource is a local file,
274 * returns just that file.
275 * @throws IOException when the resource with the given name could not be retrieved
276 */
277 public synchronized File getFile() throws IOException {
278 if (initialized)
279 return cacheFile;
280 initialized = true;
281 URL url;
282 try {
283 url = new URL(name);
284 if ("file".equals(url.getProtocol())) {
285 cacheFile = new File(name.substring("file:/".length() - 1));
286 if (!cacheFile.exists()) {
287 cacheFile = new File(name.substring("file://".length() - 1));
288 }
289 } else {
290 try {
291 cacheFile = checkLocal(url);
292 } catch (SecurityException e) {
293 throw new IOException(e);
294 }
295 }
296 } catch (MalformedURLException e) {
297 if (name == null || name.startsWith("resource://")) {
298 return null;
299 } else if (name.startsWith("josmdir://")) {
300 cacheFile = new File(Config.getDirs().getUserDataDirectory(false), name.substring("josmdir://".length()));
301 } else if (name.startsWith("josmplugindir://")) {
302 cacheFile = new File(Preferences.main().getPluginsDirectory(), name.substring("josmplugindir://".length()));
303 } else {
304 cacheFile = new File(name);
305 }
306 }
307 if (cacheFile == null)
308 throw new IOException("Unable to get cache file for "+getName());
309 return cacheFile;
310 }
311
312 /**
313 * Looks for a certain entry inside a zip file and returns the entry path.
314 *
315 * Replies a file in the top level directory of the ZIP file which has an
316 * extension <code>extension</code>. If more than one files have this
317 * extension, the last file whose name includes <code>namepart</code>
318 * is opened.
319 *
320 * @param extension the extension of the file we're looking for
321 * @param namepart the name part
322 * @return The zip entry path of the matching file. <code>null</code> if this cached file
323 * doesn't represent a zip file or if there was no matching
324 * file in the ZIP file.
325 */
326 public String findZipEntryPath(String extension, String namepart) {
327 Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart);
328 if (ze == null) return null;
329 return ze.a;
330 }
331
332 /**
333 * Like {@link #findZipEntryPath}, but returns the corresponding InputStream.
334 * @param extension the extension of the file we're looking for
335 * @param namepart the name part
336 * @return InputStream to the matching file. <code>null</code> if this cached file
337 * doesn't represent a zip file or if there was no matching
338 * file in the ZIP file.
339 * @since 6148
340 */
341 public InputStream findZipEntryInputStream(String extension, String namepart) {
342 Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart);
343 if (ze == null) return null;
344 return ze.b;
345 }
346
347 private Pair<String, InputStream> findZipEntryImpl(String extension, String namepart) {
348 File file = null;
349 try {
350 file = getFile();
351 } catch (IOException ex) {
352 Logging.log(Logging.LEVEL_WARN, ex);
353 }
354 if (file == null)
355 return null;
356 Pair<String, InputStream> res = null;
357 try {
358 ZipFile zipFile = new ZipFile(file, StandardCharsets.UTF_8);
359 ZipEntry resentry = null;
360 Enumeration<? extends ZipEntry> entries = zipFile.entries();
361 while (entries.hasMoreElements()) {
362 ZipEntry entry = entries.nextElement();
363 // choose any file with correct extension. When more than one file, prefer the one which matches namepart
364 if (entry.getName().endsWith('.' + extension) && (resentry == null || entry.getName().indexOf(namepart) >= 0)) {
365 resentry = entry;
366 }
367 }
368 if (resentry != null) {
369 InputStream is = zipFile.getInputStream(resentry);
370 res = Pair.create(resentry.getName(), is);
371 } else {
372 Utils.close(zipFile);
373 }
374 } catch (IOException e) {
375 if (file.getName().endsWith(".zip")) {
376 Logging.log(Logging.LEVEL_WARN,
377 tr("Failed to open file with extension ''{2}'' and namepart ''{3}'' in zip file ''{0}''. Exception was: {1}",
378 file.getName(), e.toString(), extension, namepart), e);
379 }
380 }
381 return res;
382 }
383
384 /**
385 * Clear the cache for the given resource.
386 * This forces a fresh download.
387 * @param name the URL
388 */
389 public static void cleanup(String name) {
390 cleanup(name, null);
391 }
392
393 /**
394 * Clear the cache for the given resource.
395 * This forces a fresh download.
396 * @param name the URL
397 * @param destDir the destination directory (see {@link #setDestDir(java.lang.String)})
398 */
399 public static void cleanup(String name, String destDir) {
400 URL url;
401 try {
402 url = new URL(name);
403 if (!"file".equals(url.getProtocol())) {
404 String prefKey = getPrefKey(url, destDir);
405 List<String> localPath = new ArrayList<>(Config.getPref().getList(prefKey));
406 if (localPath.size() == 2) {
407 File lfile = new File(localPath.get(1));
408 if (lfile.exists()) {
409 Utils.deleteFile(lfile);
410 }
411 }
412 Config.getPref().putList(prefKey, null);
413 }
414 } catch (MalformedURLException e) {
415 Logging.warn(e);
416 }
417 }
418
419 /**
420 * Get preference key to store the location and age of the cached file.
421 * 2 resources that point to the same url, but that are to be stored in different
422 * directories will not share a cache file.
423 * @param url URL
424 * @param destDir destination directory
425 * @return Preference key
426 */
427 private static String getPrefKey(URL url, String destDir) {
428 StringBuilder prefKey = new StringBuilder("mirror.");
429 if (destDir != null) {
430 prefKey.append(destDir).append('.');
431 }
432 prefKey.append(url.toString().replaceAll("%<(.*)>", ""));
433 return prefKey.toString().replaceAll("=", "_");
434 }
435
436 private File checkLocal(URL url) throws IOException {
437 String prefKey = getPrefKey(url, destDir);
438 String urlStr = url.toExternalForm();
439 if (parameter != null)
440 urlStr = urlStr.replaceAll("%<(.*)>", "");
441 long age = 0L;
442 long maxAgeMillis = TimeUnit.SECONDS.toMillis(maxAge);
443 Long ifModifiedSince = null;
444 File localFile = null;
445 List<String> localPathEntry = new ArrayList<>(Config.getPref().getList(prefKey));
446 boolean offline = false;
447 try {
448 checkOfflineAccess(urlStr);
449 } catch (OfflineAccessException e) {
450 Logging.trace(e);
451 offline = true;
452 }
453 if (localPathEntry.size() == 2) {
454 localFile = new File(localPathEntry.get(1));
455 if (!localFile.exists()) {
456 localFile = null;
457 } else {
458 if (maxAge == DEFAULT_MAXTIME
459 || maxAge <= 0 // arbitrary value <= 0 is deprecated
460 ) {
461 maxAgeMillis = TimeUnit.SECONDS.toMillis(Config.getPref().getLong("mirror.maxtime", TimeUnit.DAYS.toSeconds(7)));
462 }
463 age = System.currentTimeMillis() - Long.parseLong(localPathEntry.get(0));
464 if (offline || age < maxAgeMillis) {
465 return localFile;
466 }
467 if (cachingStrategy == CachingStrategy.IfModifiedSince) {
468 ifModifiedSince = Long.valueOf(localPathEntry.get(0));
469 }
470 }
471 }
472 if (destDir == null) {
473 destDir = Config.getDirs().getCacheDirectory(true).getPath();
474 }
475
476 File destDirFile = new File(destDir);
477 if (!destDirFile.exists()) {
478 Utils.mkDirs(destDirFile);
479 }
480
481 // No local file + offline => nothing to do
482 if (offline) {
483 return null;
484 }
485
486 if (parameter != null) {
487 String u = url.toExternalForm();
488 String uc;
489 if (parameter.isEmpty()) {
490 uc = u.replaceAll("%<(.*)>", "");
491 } else {
492 uc = u.replaceAll("%<(.*)>", "$1" + Utils.encodeUrl(parameter));
493 }
494 if (!uc.equals(u))
495 url = new URL(uc);
496 }
497
498 String a = urlStr.replaceAll("[^A-Za-z0-9_.-]", "_");
499 String localPath = "mirror_" + a;
500 localPath = truncatePath(destDir, localPath);
501 destDirFile = new File(destDir, localPath + ".tmp");
502 try {
503 activeConnection = HttpClient.create(url)
504 .setAccept(httpAccept)
505 .setIfModifiedSince(ifModifiedSince == null ? 0L : ifModifiedSince)
506 .setHeaders(httpHeaders);
507 if (fastFail) {
508 activeConnection.setReadTimeout(1000);
509 }
510 final HttpClient.Response con = activeConnection.connect();
511 if (ifModifiedSince != null && con.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) {
512 Logging.debug("304 Not Modified ({0})", urlStr);
513 if (localFile == null)
514 throw new AssertionError();
515 Config.getPref().putList(prefKey,
516 Arrays.asList(Long.toString(System.currentTimeMillis()), localPathEntry.get(1)));
517 return localFile;
518 } else if (con.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) {
519 throw new IOException(tr("The requested URL {0} was not found", urlStr));
520 }
521 try (InputStream is = con.getContent()) {
522 Files.copy(is, destDirFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
523 }
524 activeConnection = null;
525 localFile = new File(destDir, localPath);
526 if (PlatformManager.getPlatform().rename(destDirFile, localFile)) {
527 Config.getPref().putList(prefKey,
528 Arrays.asList(Long.toString(System.currentTimeMillis()), localFile.toString()));
529 } else {
530 Logging.warn(tr("Failed to rename file {0} to {1}.",
531 destDirFile.getPath(), localFile.getPath()));
532 }
533 } catch (IOException e) {
534 if (age >= maxAgeMillis && age < maxAgeMillis*2) {
535 Logging.warn(tr("Failed to load {0}, use cached file and retry next time: {1}", urlStr, e));
536 return localFile;
537 } else {
538 throw e;
539 }
540 }
541
542 return localFile;
543 }
544
545 private static void checkOfflineAccess(String urlString) {
546 OnlineResource.JOSM_WEBSITE.checkOfflineAccess(urlString, Config.getUrls().getJOSMWebsite());
547 OnlineResource.OSM_API.checkOfflineAccess(urlString, OsmApi.getOsmApi().getServerUrl());
548 }
549
550 private static String truncatePath(String directory, String fileName) {
551 if (directory.length() + fileName.length() > 255) {
552 // Windows doesn't support paths longer than 260, leave 5 chars as safe buffer, 4 will be used by ".tmp"
553 // TODO: what about filename size on other systems? 255?
554 if (directory.length() > 191 && PlatformManager.isPlatformWindows()) {
555 // digest length + name prefix == 64
556 // 255 - 64 = 191
557 // TODO: use this check only on Windows?
558 throw new IllegalArgumentException("Path " + directory + " too long to cached files");
559 }
560
561 MessageDigest md;
562 try {
563 md = MessageDigest.getInstance("SHA-256");
564 md.update(fileName.getBytes(StandardCharsets.UTF_8));
565 String digest = String.format("%064x", new BigInteger(1, md.digest()));
566 return fileName.substring(0, Math.min(fileName.length(), 32)) + digest.substring(0, 32);
567 } catch (NoSuchAlgorithmException e) {
568 Logging.error(e);
569 // TODO: what better can we do here?
570 throw new IllegalArgumentException("Missing digest algorithm SHA-256", e);
571 }
572 }
573 return fileName;
574 }
575
576 /**
577 * Attempts to disconnect an URL connection.
578 * @see HttpClient#disconnect()
579 * @since 9411
580 */
581 @Override
582 public void close() {
583 if (activeConnection != null) {
584 activeConnection.disconnect();
585 }
586 }
587
588 /**
589 * Clears the cached file
590 * @throws IOException if any I/O error occurs
591 * @since 10993
592 */
593 public void clear() throws IOException {
594 URL url;
595 try {
596 url = new URL(name);
597 if ("file".equals(url.getProtocol())) {
598 return; // this is local file - do not delete it
599 }
600 } catch (MalformedURLException e) {
601 return; // if it's not a URL, then it still might be a local file - better not to delete
602 }
603 File f = getFile();
604 if (f != null && f.exists()) {
605 Utils.deleteFile(f);
606 }
607 }
608}
Note: See TracBrowser for help on using the repository browser.