source: josm/trunk/src/org/openstreetmap/josm/io/CachedFile.java@ 14406

Last change on this file since 14406 was 14406, checked in by Don-vip, 5 years ago

see #16937 - Support user logins with space character on Java 11 (JOSM startup)
  • Property svn:eol-style set to native
File size: 23.1 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.io;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.BufferedReader;
7import java.io.ByteArrayOutputStream;
8import java.io.Closeable;
9import java.io.File;
10import java.io.IOException;
11import java.io.InputStream;
12import java.math.BigInteger;
13import java.net.HttpURLConnection;
14import java.net.MalformedURLException;
15import java.net.URL;
16import java.nio.charset.StandardCharsets;
17import java.nio.file.Files;
18import java.nio.file.InvalidPathException;
19import java.nio.file.StandardCopyOption;
20import java.security.MessageDigest;
21import java.security.NoSuchAlgorithmException;
22import java.util.ArrayList;
23import java.util.Arrays;
24import java.util.Enumeration;
25import java.util.List;
26import java.util.Map;
27import java.util.concurrent.ConcurrentHashMap;
28import java.util.concurrent.TimeUnit;
29import java.util.zip.ZipEntry;
30import java.util.zip.ZipFile;
31
32import org.openstreetmap.josm.data.Preferences;
33import org.openstreetmap.josm.spi.preferences.Config;
34import org.openstreetmap.josm.tools.HttpClient;
35import org.openstreetmap.josm.tools.Logging;
36import org.openstreetmap.josm.tools.Pair;
37import org.openstreetmap.josm.tools.PlatformManager;
38import org.openstreetmap.josm.tools.Utils;
39
40/**
41 * Downloads a file and caches it on disk in order to reduce network load.
42 *
43 * Supports URLs, local files, and a custom scheme (<code>resource:</code>) to get
44 * resources from the current *.jar file. (Local caching is only done for URLs.)
45 * <p>
46 * The mirrored file is only downloaded if it has been more than 7 days since
47 * last download. (Time can be configured.)
48 * <p>
49 * The file content is normally accessed with {@link #getInputStream()}, but
50 * you can also get the mirrored copy with {@link #getFile()}.
51 */
52public class CachedFile implements Closeable {
53
54 /**
55 * Caching strategy.
56 */
57 public enum CachingStrategy {
58 /**
59 * If cached file on disk is older than a certain time (7 days by default),
60 * consider the cache stale and try to download the file again.
61 */
62 MaxAge,
63 /**
64 * Similar to MaxAge, considers the cache stale when a certain age is
65 * exceeded. In addition, a If-Modified-Since HTTP header is added.
66 * When the server replies "304 Not Modified", this is considered the same
67 * as a full download.
68 */
69 IfModifiedSince
70 }
71
72 protected String name;
73 protected long maxAge;
74 protected String destDir;
75 protected String httpAccept;
76 protected CachingStrategy cachingStrategy;
77
78 private boolean fastFail;
79 private HttpClient activeConnection;
80 protected File cacheFile;
81 protected boolean initialized;
82 protected String parameter;
83
84 public static final long DEFAULT_MAXTIME = -1L;
85 public static final long DAYS = TimeUnit.DAYS.toSeconds(1); // factor to get caching time in days
86
87 private final Map<String, String> httpHeaders = new ConcurrentHashMap<>();
88
89 /**
90 * Constructs a CachedFile object from a given filename, URL or internal resource.
91 *
92 * @param name can be:<ul>
93 * <li>relative or absolute file name</li>
94 * <li>{@code file:///SOME/FILE} the same as above</li>
95 * <li>{@code http://...} a URL. It will be cached on disk.</li>
96 * <li>{@code resource://SOME/FILE} file from the classpath (usually in the current *.jar)</li>
97 * <li>{@code josmdir://SOME/FILE} file inside josm user data directory (since r7058)</li>
98 * <li>{@code josmplugindir://SOME/FILE} file inside josm plugin directory (since r7834)</li></ul>
99 */
100 public CachedFile(String name) {
101 this.name = name;
102 }
103
104 /**
105 * Set the name of the resource.
106 * @param name can be:<ul>
107 * <li>relative or absolute file name</li>
108 * <li>{@code file:///SOME/FILE} the same as above</li>
109 * <li>{@code http://...} a URL. It will be cached on disk.</li>
110 * <li>{@code resource://SOME/FILE} file from the classpath (usually in the current *.jar)</li>
111 * <li>{@code josmdir://SOME/FILE} file inside josm user data directory (since r7058)</li>
112 * <li>{@code josmplugindir://SOME/FILE} file inside josm plugin directory (since r7834)</li></ul>
113 * @return this object
114 */
115 public CachedFile setName(String name) {
116 this.name = name;
117 return this;
118 }
119
120 /**
121 * Set maximum age of cache file. Only applies to URLs.
122 * When this time has passed after the last download of the file, the
123 * cache is considered stale and a new download will be attempted.
124 * @param maxAge the maximum cache age in seconds
125 * @return this object
126 */
127 public CachedFile setMaxAge(long maxAge) {
128 this.maxAge = maxAge;
129 return this;
130 }
131
132 /**
133 * Set the destination directory for the cache file. Only applies to URLs.
134 * @param destDir the destination directory
135 * @return this object
136 */
137 public CachedFile setDestDir(String destDir) {
138 this.destDir = destDir;
139 return this;
140 }
141
142 /**
143 * Set the accepted MIME types sent in the HTTP Accept header. Only applies to URLs.
144 * @param httpAccept the accepted MIME types
145 * @return this object
146 */
147 public CachedFile setHttpAccept(String httpAccept) {
148 this.httpAccept = httpAccept;
149 return this;
150 }
151
152 /**
153 * Set the caching strategy. Only applies to URLs.
154 * @param cachingStrategy caching strategy
155 * @return this object
156 */
157 public CachedFile setCachingStrategy(CachingStrategy cachingStrategy) {
158 this.cachingStrategy = cachingStrategy;
159 return this;
160 }
161
162 /**
163 * Sets the http headers. Only applies to URL pointing to http or https resources
164 * @param headers that should be sent together with request
165 * @return this object
166 */
167 public CachedFile setHttpHeaders(Map<String, String> headers) {
168 this.httpHeaders.putAll(headers);
169 return this;
170 }
171
172 /**
173 * Sets whether opening HTTP connections should fail fast, i.e., whether a
174 * {@link HttpClient#setConnectTimeout(int) low connect timeout} should be used.
175 * @param fastFail whether opening HTTP connections should fail fast
176 */
177 public void setFastFail(boolean fastFail) {
178 this.fastFail = fastFail;
179 }
180
181 /**
182 * Sets additional URL parameter (used e.g. for maps)
183 * @param parameter the URL parameter
184 * @since 13536
185 */
186 public void setParam(String parameter) {
187 this.parameter = parameter;
188 }
189
190 public String getName() {
191 if (parameter != null)
192 return name.replaceAll("%<(.*)>", "");
193 return name;
194 }
195
196 /**
197 * Returns maximum age of cache file. Only applies to URLs.
198 * When this time has passed after the last download of the file, the
199 * cache is considered stale and a new download will be attempted.
200 * @return the maximum cache age in seconds
201 */
202 public long getMaxAge() {
203 return maxAge;
204 }
205
206 public String getDestDir() {
207 return destDir;
208 }
209
210 public String getHttpAccept() {
211 return httpAccept;
212 }
213
214 public CachingStrategy getCachingStrategy() {
215 return cachingStrategy;
216 }
217
218 /**
219 * Get InputStream to the requested resource.
220 * @return the InputStream
221 * @throws IOException when the resource with the given name could not be retrieved
222 * @throws InvalidPathException if a Path object cannot be constructed from the inner file path
223 */
224 public InputStream getInputStream() throws IOException {
225 File file = getFile();
226 if (file == null) {
227 if (name != null && name.startsWith("resource://")) {
228 String resourceName = name.substring("resource:/".length());
229 InputStream is = null;
230 try {
231 is = getClass().getResourceAsStream(resourceName);
232 } catch (InvalidPathException e) {
233 Logging.error("Cannot open {0}: {1}", resourceName, e.getMessage());
234 Logging.trace(e);
235 }
236 if (is == null) {
237 URL resource = getClass().getResource(resourceName);
238 if (resource != null) {
239 // More robust way to open stream
240 is = Utils.openStream(resource);
241 }
242 if (is == null) {
243 throw new IOException(tr("Failed to open input stream for resource ''{0}''", name));
244 }
245 }
246 return is;
247 } else {
248 throw new IOException("No file found for: "+name);
249 }
250 }
251 return Files.newInputStream(file.toPath());
252 }
253
254 /**
255 * Get the full content of the requested resource as a byte array.
256 * @return the full content of the requested resource as byte array
257 * @throws IOException in case of an I/O error
258 */
259 public byte[] getByteContent() throws IOException {
260 try (InputStream is = getInputStream()) {
261 ByteArrayOutputStream buffer = new ByteArrayOutputStream();
262 int nRead;
263 byte[] data = new byte[8192];
264 while ((nRead = is.read(data, 0, data.length)) != -1) {
265 buffer.write(data, 0, nRead);
266 }
267 buffer.flush();
268 return buffer.toByteArray();
269 }
270 }
271
272 /**
273 * Returns {@link #getInputStream()} wrapped in a buffered reader.
274 * <p>
275 * Detects Unicode charset in use utilizing {@link UTFInputStreamReader}.
276 *
277 * @return buffered reader
278 * @throws IOException if any I/O error occurs
279 * @since 9411
280 */
281 public BufferedReader getContentReader() throws IOException {
282 return new BufferedReader(UTFInputStreamReader.create(getInputStream()));
283 }
284
285 /**
286 * Get local file for the requested resource.
287 * @return The local cache file for URLs. If the resource is a local file,
288 * returns just that file.
289 * @throws IOException when the resource with the given name could not be retrieved
290 */
291 public synchronized File getFile() throws IOException {
292 if (initialized)
293 return cacheFile;
294 initialized = true;
295 URL url;
296 try {
297 url = new URL(name);
298 if ("file".equals(url.getProtocol())) {
299 cacheFile = new File(name.substring("file:/".length() - 1));
300 if (!cacheFile.exists()) {
301 cacheFile = new File(name.substring("file://".length() - 1));
302 }
303 } else {
304 try {
305 cacheFile = checkLocal(url);
306 } catch (SecurityException e) {
307 throw new IOException(e);
308 }
309 }
310 } catch (MalformedURLException e) {
311 if (name == null || name.startsWith("resource://")) {
312 return null;
313 } else if (name.startsWith("josmdir://")) {
314 cacheFile = new File(Config.getDirs().getUserDataDirectory(false), name.substring("josmdir://".length()));
315 } else if (name.startsWith("josmplugindir://")) {
316 cacheFile = new File(Preferences.main().getPluginsDirectory(), name.substring("josmplugindir://".length()));
317 } else {
318 cacheFile = new File(name);
319 }
320 }
321 if (cacheFile == null)
322 throw new IOException("Unable to get cache file for "+getName());
323 return cacheFile;
324 }
325
326 /**
327 * Looks for a certain entry inside a zip file and returns the entry path.
328 *
329 * Replies a file in the top level directory of the ZIP file which has an
330 * extension <code>extension</code>. If more than one files have this
331 * extension, the last file whose name includes <code>namepart</code>
332 * is opened.
333 *
334 * @param extension the extension of the file we're looking for
335 * @param namepart the name part
336 * @return The zip entry path of the matching file. <code>null</code> if this cached file
337 * doesn't represent a zip file or if there was no matching
338 * file in the ZIP file.
339 */
340 public String findZipEntryPath(String extension, String namepart) {
341 Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart);
342 if (ze == null) return null;
343 return ze.a;
344 }
345
346 /**
347 * Like {@link #findZipEntryPath}, but returns the corresponding InputStream.
348 * @param extension the extension of the file we're looking for
349 * @param namepart the name part
350 * @return InputStream to the matching file. <code>null</code> if this cached file
351 * doesn't represent a zip file or if there was no matching
352 * file in the ZIP file.
353 * @since 6148
354 */
355 public InputStream findZipEntryInputStream(String extension, String namepart) {
356 Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart);
357 if (ze == null) return null;
358 return ze.b;
359 }
360
361 private Pair<String, InputStream> findZipEntryImpl(String extension, String namepart) {
362 File file = null;
363 try {
364 file = getFile();
365 } catch (IOException ex) {
366 Logging.log(Logging.LEVEL_WARN, ex);
367 }
368 if (file == null)
369 return null;
370 Pair<String, InputStream> res = null;
371 try {
372 ZipFile zipFile = new ZipFile(file, StandardCharsets.UTF_8);
373 ZipEntry resentry = null;
374 Enumeration<? extends ZipEntry> entries = zipFile.entries();
375 while (entries.hasMoreElements()) {
376 ZipEntry entry = entries.nextElement();
377 // choose any file with correct extension. When more than one file, prefer the one which matches namepart
378 if (entry.getName().endsWith('.' + extension) && (resentry == null || entry.getName().indexOf(namepart) >= 0)) {
379 resentry = entry;
380 }
381 }
382 if (resentry != null) {
383 InputStream is = zipFile.getInputStream(resentry);
384 res = Pair.create(resentry.getName(), is);
385 } else {
386 Utils.close(zipFile);
387 }
388 } catch (IOException e) {
389 if (file.getName().endsWith(".zip")) {
390 Logging.log(Logging.LEVEL_WARN,
391 tr("Failed to open file with extension ''{2}'' and namepart ''{3}'' in zip file ''{0}''. Exception was: {1}",
392 file.getName(), e.toString(), extension, namepart), e);
393 }
394 }
395 return res;
396 }
397
398 /**
399 * Clear the cache for the given resource.
400 * This forces a fresh download.
401 * @param name the URL
402 */
403 public static void cleanup(String name) {
404 cleanup(name, null);
405 }
406
407 /**
408 * Clear the cache for the given resource.
409 * This forces a fresh download.
410 * @param name the URL
411 * @param destDir the destination directory (see {@link #setDestDir(java.lang.String)})
412 */
413 public static void cleanup(String name, String destDir) {
414 URL url;
415 try {
416 url = new URL(name);
417 if (!"file".equals(url.getProtocol())) {
418 String prefKey = getPrefKey(url, destDir);
419 List<String> localPath = new ArrayList<>(Config.getPref().getList(prefKey));
420 if (localPath.size() == 2) {
421 File lfile = new File(localPath.get(1));
422 if (lfile.exists()) {
423 Utils.deleteFile(lfile);
424 }
425 }
426 Config.getPref().putList(prefKey, null);
427 }
428 } catch (MalformedURLException e) {
429 Logging.warn(e);
430 }
431 }
432
433 /**
434 * Get preference key to store the location and age of the cached file.
435 * 2 resources that point to the same url, but that are to be stored in different
436 * directories will not share a cache file.
437 * @param url URL
438 * @param destDir destination directory
439 * @return Preference key
440 */
441 private static String getPrefKey(URL url, String destDir) {
442 StringBuilder prefKey = new StringBuilder("mirror.");
443 if (destDir != null) {
444 prefKey.append(destDir).append('.');
445 }
446 prefKey.append(url.toString().replaceAll("%<(.*)>", ""));
447 return prefKey.toString().replaceAll("=", "_");
448 }
449
450 private File checkLocal(URL url) throws IOException {
451 String prefKey = getPrefKey(url, destDir);
452 String urlStr = url.toExternalForm();
453 if (parameter != null)
454 urlStr = urlStr.replaceAll("%<(.*)>", "");
455 long age = 0L;
456 long maxAgeMillis = TimeUnit.SECONDS.toMillis(maxAge);
457 Long ifModifiedSince = null;
458 File localFile = null;
459 List<String> localPathEntry = new ArrayList<>(Config.getPref().getList(prefKey));
460 boolean offline = false;
461 try {
462 checkOfflineAccess(urlStr);
463 } catch (OfflineAccessException e) {
464 Logging.trace(e);
465 offline = true;
466 }
467 if (localPathEntry.size() == 2) {
468 localFile = new File(localPathEntry.get(1));
469 if (!localFile.exists()) {
470 localFile = null;
471 } else {
472 if (maxAge == DEFAULT_MAXTIME
473 || maxAge <= 0 // arbitrary value <= 0 is deprecated
474 ) {
475 maxAgeMillis = TimeUnit.SECONDS.toMillis(Config.getPref().getLong("mirror.maxtime", TimeUnit.DAYS.toSeconds(7)));
476 }
477 age = System.currentTimeMillis() - Long.parseLong(localPathEntry.get(0));
478 if (offline || age < maxAgeMillis) {
479 return localFile;
480 }
481 if (cachingStrategy == CachingStrategy.IfModifiedSince) {
482 ifModifiedSince = Long.valueOf(localPathEntry.get(0));
483 }
484 }
485 }
486 if (destDir == null) {
487 destDir = Config.getDirs().getCacheDirectory(true).getPath();
488 }
489
490 File destDirFile = new File(destDir);
491 if (!destDirFile.exists()) {
492 Utils.mkDirs(destDirFile);
493 }
494
495 // No local file + offline => nothing to do
496 if (offline) {
497 return null;
498 }
499
500 if (parameter != null) {
501 String u = url.toExternalForm();
502 String uc;
503 if (parameter.isEmpty()) {
504 uc = u.replaceAll("%<(.*)>", "");
505 } else {
506 uc = u.replaceAll("%<(.*)>", "$1" + Utils.encodeUrl(parameter));
507 }
508 if (!uc.equals(u))
509 url = new URL(uc);
510 }
511
512 String a = urlStr.replaceAll("[^A-Za-z0-9_.-]", "_");
513 String localPath = "mirror_" + a;
514 localPath = truncatePath(destDir, localPath);
515 destDirFile = new File(destDir, localPath + ".tmp");
516 try {
517 activeConnection = HttpClient.create(url)
518 .setAccept(httpAccept)
519 .setIfModifiedSince(ifModifiedSince == null ? 0L : ifModifiedSince)
520 .setHeaders(httpHeaders);
521 if (fastFail) {
522 activeConnection.setReadTimeout(1000);
523 }
524 final HttpClient.Response con = activeConnection.connect();
525 if (ifModifiedSince != null && con.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) {
526 Logging.debug("304 Not Modified ({0})", urlStr);
527 if (localFile == null)
528 throw new AssertionError();
529 Config.getPref().putList(prefKey,
530 Arrays.asList(Long.toString(System.currentTimeMillis()), localPathEntry.get(1)));
531 return localFile;
532 } else if (con.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) {
533 throw new IOException(tr("The requested URL {0} was not found", urlStr));
534 }
535 try (InputStream is = con.getContent()) {
536 Files.copy(is, destDirFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
537 }
538 activeConnection = null;
539 localFile = new File(destDir, localPath);
540 if (PlatformManager.getPlatform().rename(destDirFile, localFile)) {
541 Config.getPref().putList(prefKey,
542 Arrays.asList(Long.toString(System.currentTimeMillis()), localFile.toString()));
543 } else {
544 Logging.warn(tr("Failed to rename file {0} to {1}.",
545 destDirFile.getPath(), localFile.getPath()));
546 }
547 } catch (IOException e) {
548 if (age >= maxAgeMillis && age < maxAgeMillis*2) {
549 Logging.warn(tr("Failed to load {0}, use cached file and retry next time: {1}", urlStr, e));
550 return localFile;
551 } else {
552 throw e;
553 }
554 }
555
556 return localFile;
557 }
558
559 private static void checkOfflineAccess(String urlString) {
560 OnlineResource.JOSM_WEBSITE.checkOfflineAccess(urlString, Config.getUrls().getJOSMWebsite());
561 OnlineResource.OSM_API.checkOfflineAccess(urlString, OsmApi.getOsmApi().getServerUrl());
562 }
563
564 private static String truncatePath(String directory, String fileName) {
565 if (directory.length() + fileName.length() > 255) {
566 // Windows doesn't support paths longer than 260, leave 5 chars as safe buffer, 4 will be used by ".tmp"
567 // TODO: what about filename size on other systems? 255?
568 if (directory.length() > 191 && PlatformManager.isPlatformWindows()) {
569 // digest length + name prefix == 64
570 // 255 - 64 = 191
571 // TODO: use this check only on Windows?
572 throw new IllegalArgumentException("Path " + directory + " too long to cached files");
573 }
574
575 MessageDigest md;
576 try {
577 md = MessageDigest.getInstance("SHA-256");
578 md.update(fileName.getBytes(StandardCharsets.UTF_8));
579 String digest = String.format("%064x", new BigInteger(1, md.digest()));
580 return fileName.substring(0, Math.min(fileName.length(), 32)) + digest.substring(0, 32);
581 } catch (NoSuchAlgorithmException e) {
582 Logging.error(e);
583 // TODO: what better can we do here?
584 throw new IllegalArgumentException("Missing digest algorithm SHA-256", e);
585 }
586 }
587 return fileName;
588 }
589
590 /**
591 * Attempts to disconnect an URL connection.
592 * @see HttpClient#disconnect()
593 * @since 9411
594 */
595 @Override
596 public void close() {
597 if (activeConnection != null) {
598 activeConnection.disconnect();
599 }
600 }
601
602 /**
603 * Clears the cached file
604 * @throws IOException if any I/O error occurs
605 * @since 10993
606 */
607 public void clear() throws IOException {
608 URL url;
609 try {
610 url = new URL(name);
611 if ("file".equals(url.getProtocol())) {
612 return; // this is local file - do not delete it
613 }
614 } catch (MalformedURLException e) {
615 return; // if it's not a URL, then it still might be a local file - better not to delete
616 }
617 File f = getFile();
618 if (f != null && f.exists()) {
619 Utils.deleteFile(f);
620 }
621 }
622}
Note: See TracBrowser for help on using the repository browser.