source: josm/trunk/src/org/openstreetmap/josm/io/CachedFile.java@ 13652

Last change on this file since 13652 was 13647, checked in by Don-vip, 6 years ago

see #16204 - Allow to start and close JOSM in WebStart sandbox mode (where every external access is denied). This was very useful to reproduce some very tricky bugs that occured in real life but were almost impossible to diagnose.
  • Property svn:eol-style set to native
File size: 20.9 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.io;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.BufferedReader;
7import java.io.ByteArrayOutputStream;
8import java.io.Closeable;
9import java.io.File;
10import java.io.IOException;
11import java.io.InputStream;
12import java.net.HttpURLConnection;
13import java.net.MalformedURLException;
14import java.net.URL;
15import java.nio.charset.StandardCharsets;
16import java.nio.file.Files;
17import java.nio.file.StandardCopyOption;
18import java.util.ArrayList;
19import java.util.Arrays;
20import java.util.Enumeration;
21import java.util.List;
22import java.util.Map;
23import java.util.concurrent.ConcurrentHashMap;
24import java.util.concurrent.TimeUnit;
25import java.util.zip.ZipEntry;
26import java.util.zip.ZipFile;
27
28import org.openstreetmap.josm.Main;
29import org.openstreetmap.josm.spi.preferences.Config;
30import org.openstreetmap.josm.tools.HttpClient;
31import org.openstreetmap.josm.tools.Logging;
32import org.openstreetmap.josm.tools.Pair;
33import org.openstreetmap.josm.tools.Utils;
34
35/**
36 * Downloads a file and caches it on disk in order to reduce network load.
37 *
38 * Supports URLs, local files, and a custom scheme (<code>resource:</code>) to get
39 * resources from the current *.jar file. (Local caching is only done for URLs.)
40 * <p>
41 * The mirrored file is only downloaded if it has been more than 7 days since
42 * last download. (Time can be configured.)
43 * <p>
44 * The file content is normally accessed with {@link #getInputStream()}, but
45 * you can also get the mirrored copy with {@link #getFile()}.
46 */
47public class CachedFile implements Closeable {
48
49 /**
50 * Caching strategy.
51 */
52 public enum CachingStrategy {
53 /**
54 * If cached file on disk is older than a certain time (7 days by default),
55 * consider the cache stale and try to download the file again.
56 */
57 MaxAge,
58 /**
59 * Similar to MaxAge, considers the cache stale when a certain age is
60 * exceeded. In addition, a If-Modified-Since HTTP header is added.
61 * When the server replies "304 Not Modified", this is considered the same
62 * as a full download.
63 */
64 IfModifiedSince
65 }
66
67 protected String name;
68 protected long maxAge;
69 protected String destDir;
70 protected String httpAccept;
71 protected CachingStrategy cachingStrategy;
72
73 private boolean fastFail;
74 private HttpClient activeConnection;
75 protected File cacheFile;
76 protected boolean initialized;
77 protected String parameter;
78
79 public static final long DEFAULT_MAXTIME = -1L;
80 public static final long DAYS = TimeUnit.DAYS.toSeconds(1); // factor to get caching time in days
81
82 private final Map<String, String> httpHeaders = new ConcurrentHashMap<>();
83
84 /**
85 * Constructs a CachedFile object from a given filename, URL or internal resource.
86 *
87 * @param name can be:<ul>
88 * <li>relative or absolute file name</li>
89 * <li>{@code file:///SOME/FILE} the same as above</li>
90 * <li>{@code http://...} a URL. It will be cached on disk.</li>
91 * <li>{@code resource://SOME/FILE} file from the classpath (usually in the current *.jar)</li>
92 * <li>{@code josmdir://SOME/FILE} file inside josm user data directory (since r7058)</li>
93 * <li>{@code josmplugindir://SOME/FILE} file inside josm plugin directory (since r7834)</li></ul>
94 */
95 public CachedFile(String name) {
96 this.name = name;
97 }
98
99 /**
100 * Set the name of the resource.
101 * @param name can be:<ul>
102 * <li>relative or absolute file name</li>
103 * <li>{@code file:///SOME/FILE} the same as above</li>
104 * <li>{@code http://...} a URL. It will be cached on disk.</li>
105 * <li>{@code resource://SOME/FILE} file from the classpath (usually in the current *.jar)</li>
106 * <li>{@code josmdir://SOME/FILE} file inside josm user data directory (since r7058)</li>
107 * <li>{@code josmplugindir://SOME/FILE} file inside josm plugin directory (since r7834)</li></ul>
108 * @return this object
109 */
110 public CachedFile setName(String name) {
111 this.name = name;
112 return this;
113 }
114
115 /**
116 * Set maximum age of cache file. Only applies to URLs.
117 * When this time has passed after the last download of the file, the
118 * cache is considered stale and a new download will be attempted.
119 * @param maxAge the maximum cache age in seconds
120 * @return this object
121 */
122 public CachedFile setMaxAge(long maxAge) {
123 this.maxAge = maxAge;
124 return this;
125 }
126
127 /**
128 * Set the destination directory for the cache file. Only applies to URLs.
129 * @param destDir the destination directory
130 * @return this object
131 */
132 public CachedFile setDestDir(String destDir) {
133 this.destDir = destDir;
134 return this;
135 }
136
137 /**
138 * Set the accepted MIME types sent in the HTTP Accept header. Only applies to URLs.
139 * @param httpAccept the accepted MIME types
140 * @return this object
141 */
142 public CachedFile setHttpAccept(String httpAccept) {
143 this.httpAccept = httpAccept;
144 return this;
145 }
146
147 /**
148 * Set the caching strategy. Only applies to URLs.
149 * @param cachingStrategy caching strategy
150 * @return this object
151 */
152 public CachedFile setCachingStrategy(CachingStrategy cachingStrategy) {
153 this.cachingStrategy = cachingStrategy;
154 return this;
155 }
156
157 /**
158 * Sets the http headers. Only applies to URL pointing to http or https resources
159 * @param headers that should be sent together with request
160 * @return this object
161 */
162 public CachedFile setHttpHeaders(Map<String, String> headers) {
163 this.httpHeaders.putAll(headers);
164 return this;
165 }
166
167 /**
168 * Sets whether opening HTTP connections should fail fast, i.e., whether a
169 * {@link HttpClient#setConnectTimeout(int) low connect timeout} should be used.
170 * @param fastFail whether opening HTTP connections should fail fast
171 */
172 public void setFastFail(boolean fastFail) {
173 this.fastFail = fastFail;
174 }
175
176 /**
177 * Sets additional URL parameter (used e.g. for maps)
178 * @param parameter the URL parameter
179 * @since 13536
180 */
181 public void setParam(String parameter) {
182 this.parameter = parameter;
183 }
184
185 public String getName() {
186 if (parameter != null)
187 return name.replaceAll("%<(.*)>", "");
188 return name;
189 }
190
191 /**
192 * Returns maximum age of cache file. Only applies to URLs.
193 * When this time has passed after the last download of the file, the
194 * cache is considered stale and a new download will be attempted.
195 * @return the maximum cache age in seconds
196 */
197 public long getMaxAge() {
198 return maxAge;
199 }
200
201 public String getDestDir() {
202 return destDir;
203 }
204
205 public String getHttpAccept() {
206 return httpAccept;
207 }
208
209 public CachingStrategy getCachingStrategy() {
210 return cachingStrategy;
211 }
212
213 /**
214 * Get InputStream to the requested resource.
215 * @return the InputStream
216 * @throws IOException when the resource with the given name could not be retrieved
217 */
218 public InputStream getInputStream() throws IOException {
219 File file = getFile();
220 if (file == null) {
221 if (name != null && name.startsWith("resource://")) {
222 InputStream is = getClass().getResourceAsStream(
223 name.substring("resource:/".length()));
224 if (is == null)
225 throw new IOException(tr("Failed to open input stream for resource ''{0}''", name));
226 return is;
227 } else {
228 throw new IOException("No file found for: "+name);
229 }
230 }
231 return Files.newInputStream(file.toPath());
232 }
233
234 /**
235 * Get the full content of the requested resource as a byte array.
236 * @return the full content of the requested resource as byte array
237 * @throws IOException in case of an I/O error
238 */
239 public byte[] getByteContent() throws IOException {
240 try (InputStream is = getInputStream()) {
241 ByteArrayOutputStream buffer = new ByteArrayOutputStream();
242 int nRead;
243 byte[] data = new byte[8192];
244 while ((nRead = is.read(data, 0, data.length)) != -1) {
245 buffer.write(data, 0, nRead);
246 }
247 buffer.flush();
248 return buffer.toByteArray();
249 }
250 }
251
252 /**
253 * Returns {@link #getInputStream()} wrapped in a buffered reader.
254 * <p>
255 * Detects Unicode charset in use utilizing {@link UTFInputStreamReader}.
256 *
257 * @return buffered reader
258 * @throws IOException if any I/O error occurs
259 * @since 9411
260 */
261 public BufferedReader getContentReader() throws IOException {
262 return new BufferedReader(UTFInputStreamReader.create(getInputStream()));
263 }
264
265 /**
266 * Get local file for the requested resource.
267 * @return The local cache file for URLs. If the resource is a local file,
268 * returns just that file.
269 * @throws IOException when the resource with the given name could not be retrieved
270 */
271 public synchronized File getFile() throws IOException {
272 if (initialized)
273 return cacheFile;
274 initialized = true;
275 URL url;
276 try {
277 url = new URL(name);
278 if ("file".equals(url.getProtocol())) {
279 cacheFile = new File(name.substring("file:/".length() - 1));
280 if (!cacheFile.exists()) {
281 cacheFile = new File(name.substring("file://".length() - 1));
282 }
283 } else {
284 try {
285 cacheFile = checkLocal(url);
286 } catch (SecurityException e) {
287 throw new IOException(e);
288 }
289 }
290 } catch (MalformedURLException e) {
291 if (name == null || name.startsWith("resource://")) {
292 return null;
293 } else if (name.startsWith("josmdir://")) {
294 cacheFile = new File(Config.getDirs().getUserDataDirectory(false), name.substring("josmdir://".length()));
295 } else if (name.startsWith("josmplugindir://")) {
296 cacheFile = new File(Main.pref.getPluginsDirectory(), name.substring("josmplugindir://".length()));
297 } else {
298 cacheFile = new File(name);
299 }
300 }
301 if (cacheFile == null)
302 throw new IOException("Unable to get cache file for "+getName());
303 return cacheFile;
304 }
305
306 /**
307 * Looks for a certain entry inside a zip file and returns the entry path.
308 *
309 * Replies a file in the top level directory of the ZIP file which has an
310 * extension <code>extension</code>. If more than one files have this
311 * extension, the last file whose name includes <code>namepart</code>
312 * is opened.
313 *
314 * @param extension the extension of the file we're looking for
315 * @param namepart the name part
316 * @return The zip entry path of the matching file. <code>null</code> if this cached file
317 * doesn't represent a zip file or if there was no matching
318 * file in the ZIP file.
319 */
320 public String findZipEntryPath(String extension, String namepart) {
321 Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart);
322 if (ze == null) return null;
323 return ze.a;
324 }
325
326 /**
327 * Like {@link #findZipEntryPath}, but returns the corresponding InputStream.
328 * @param extension the extension of the file we're looking for
329 * @param namepart the name part
330 * @return InputStream to the matching file. <code>null</code> if this cached file
331 * doesn't represent a zip file or if there was no matching
332 * file in the ZIP file.
333 * @since 6148
334 */
335 public InputStream findZipEntryInputStream(String extension, String namepart) {
336 Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart);
337 if (ze == null) return null;
338 return ze.b;
339 }
340
341 private Pair<String, InputStream> findZipEntryImpl(String extension, String namepart) {
342 File file = null;
343 try {
344 file = getFile();
345 } catch (IOException ex) {
346 Logging.log(Logging.LEVEL_WARN, ex);
347 }
348 if (file == null)
349 return null;
350 Pair<String, InputStream> res = null;
351 try {
352 ZipFile zipFile = new ZipFile(file, StandardCharsets.UTF_8);
353 ZipEntry resentry = null;
354 Enumeration<? extends ZipEntry> entries = zipFile.entries();
355 while (entries.hasMoreElements()) {
356 ZipEntry entry = entries.nextElement();
357 // choose any file with correct extension. When more than one file, prefer the one which matches namepart
358 if (entry.getName().endsWith('.' + extension) && (resentry == null || entry.getName().indexOf(namepart) >= 0)) {
359 resentry = entry;
360 }
361 }
362 if (resentry != null) {
363 InputStream is = zipFile.getInputStream(resentry);
364 res = Pair.create(resentry.getName(), is);
365 } else {
366 Utils.close(zipFile);
367 }
368 } catch (IOException e) {
369 if (file.getName().endsWith(".zip")) {
370 Logging.log(Logging.LEVEL_WARN,
371 tr("Failed to open file with extension ''{2}'' and namepart ''{3}'' in zip file ''{0}''. Exception was: {1}",
372 file.getName(), e.toString(), extension, namepart), e);
373 }
374 }
375 return res;
376 }
377
378 /**
379 * Clear the cache for the given resource.
380 * This forces a fresh download.
381 * @param name the URL
382 */
383 public static void cleanup(String name) {
384 cleanup(name, null);
385 }
386
387 /**
388 * Clear the cache for the given resource.
389 * This forces a fresh download.
390 * @param name the URL
391 * @param destDir the destination directory (see {@link #setDestDir(java.lang.String)})
392 */
393 public static void cleanup(String name, String destDir) {
394 URL url;
395 try {
396 url = new URL(name);
397 if (!"file".equals(url.getProtocol())) {
398 String prefKey = getPrefKey(url, destDir, null);
399 List<String> localPath = new ArrayList<>(Config.getPref().getList(prefKey));
400 if (localPath.size() == 2) {
401 File lfile = new File(localPath.get(1));
402 if (lfile.exists()) {
403 Utils.deleteFile(lfile);
404 }
405 }
406 Config.getPref().putList(prefKey, null);
407 }
408 } catch (MalformedURLException e) {
409 Logging.warn(e);
410 }
411 }
412
413 /**
414 * Get preference key to store the location and age of the cached file.
415 * 2 resources that point to the same url, but that are to be stored in different
416 * directories will not share a cache file.
417 * @param url URL
418 * @param destDir destination directory
419 * @param parameter additional URL parameter (used e.g. for maps)
420 * @return Preference key
421 */
422 private static String getPrefKey(URL url, String destDir, String parameter) {
423 StringBuilder prefKey = new StringBuilder("mirror.");
424 if (destDir != null) {
425 prefKey.append(destDir).append('.');
426 }
427 if (parameter != null) {
428 prefKey.append(url.toString().replaceAll("%<(.*)>", ""));
429 } else {
430 prefKey.append(url.toString());
431 }
432 return prefKey.toString().replaceAll("=", "_");
433 }
434
435 private File checkLocal(URL url) throws IOException {
436 String prefKey = getPrefKey(url, destDir, parameter);
437 String urlStr = url.toExternalForm();
438 if (parameter != null)
439 urlStr = urlStr.replaceAll("%<(.*)>", "");
440 long age = 0L;
441 long maxAgeMillis = maxAge;
442 Long ifModifiedSince = null;
443 File localFile = null;
444 List<String> localPathEntry = new ArrayList<>(Config.getPref().getList(prefKey));
445 boolean offline = false;
446 try {
447 checkOfflineAccess(urlStr);
448 } catch (OfflineAccessException e) {
449 Logging.trace(e);
450 offline = true;
451 }
452 if (localPathEntry.size() == 2) {
453 localFile = new File(localPathEntry.get(1));
454 if (!localFile.exists()) {
455 localFile = null;
456 } else {
457 if (maxAge == DEFAULT_MAXTIME
458 || maxAge <= 0 // arbitrary value <= 0 is deprecated
459 ) {
460 maxAgeMillis = TimeUnit.SECONDS.toMillis(Config.getPref().getLong("mirror.maxtime", TimeUnit.DAYS.toSeconds(7)));
461 }
462 age = System.currentTimeMillis() - Long.parseLong(localPathEntry.get(0));
463 if (offline || age < maxAgeMillis) {
464 return localFile;
465 }
466 if (cachingStrategy == CachingStrategy.IfModifiedSince) {
467 ifModifiedSince = Long.valueOf(localPathEntry.get(0));
468 }
469 }
470 }
471 if (destDir == null) {
472 destDir = Config.getDirs().getCacheDirectory(true).getPath();
473 }
474
475 File destDirFile = new File(destDir);
476 if (!destDirFile.exists()) {
477 Utils.mkDirs(destDirFile);
478 }
479
480 // No local file + offline => nothing to do
481 if (offline) {
482 return null;
483 }
484
485 if (parameter != null) {
486 String u = url.toExternalForm();
487 String uc;
488 if (parameter.isEmpty()) {
489 uc = u.replaceAll("%<(.*)>", "");
490 } else {
491 uc = u.replaceAll("%<(.*)>", "$1"+parameter);
492 }
493 if (!uc.equals(u))
494 url = new URL(uc);
495 }
496
497 String a = urlStr.replaceAll("[^A-Za-z0-9_.-]", "_");
498 String localPath = "mirror_" + a;
499 destDirFile = new File(destDir, localPath + ".tmp");
500 try {
501 activeConnection = HttpClient.create(url)
502 .setAccept(httpAccept)
503 .setIfModifiedSince(ifModifiedSince == null ? 0L : ifModifiedSince)
504 .setHeaders(httpHeaders);
505 if (fastFail) {
506 activeConnection.setReadTimeout(1000);
507 }
508 final HttpClient.Response con = activeConnection.connect();
509 if (ifModifiedSince != null && con.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) {
510 Logging.debug("304 Not Modified ({0})", urlStr);
511 if (localFile == null)
512 throw new AssertionError();
513 Config.getPref().putList(prefKey,
514 Arrays.asList(Long.toString(System.currentTimeMillis()), localPathEntry.get(1)));
515 return localFile;
516 } else if (con.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) {
517 throw new IOException(tr("The requested URL {0} was not found", urlStr));
518 }
519 try (InputStream is = con.getContent()) {
520 Files.copy(is, destDirFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
521 }
522 activeConnection = null;
523 localFile = new File(destDir, localPath);
524 if (Main.platform.rename(destDirFile, localFile)) {
525 Config.getPref().putList(prefKey,
526 Arrays.asList(Long.toString(System.currentTimeMillis()), localFile.toString()));
527 } else {
528 Logging.warn(tr("Failed to rename file {0} to {1}.",
529 destDirFile.getPath(), localFile.getPath()));
530 }
531 } catch (IOException e) {
532 if (age >= maxAgeMillis && age < maxAgeMillis*2) {
533 Logging.warn(tr("Failed to load {0}, use cached file and retry next time: {1}", urlStr, e));
534 return localFile;
535 } else {
536 throw e;
537 }
538 }
539
540 return localFile;
541 }
542
543 private static void checkOfflineAccess(String urlString) {
544 OnlineResource.JOSM_WEBSITE.checkOfflineAccess(urlString, Main.getJOSMWebsite());
545 OnlineResource.OSM_API.checkOfflineAccess(urlString, OsmApi.getOsmApi().getServerUrl());
546 }
547
548 /**
549 * Attempts to disconnect an URL connection.
550 * @see HttpClient#disconnect()
551 * @since 9411
552 */
553 @Override
554 public void close() {
555 if (activeConnection != null) {
556 activeConnection.disconnect();
557 }
558 }
559
560 /**
561 * Clears the cached file
562 * @throws IOException if any I/O error occurs
563 * @since 10993
564 */
565 public void clear() throws IOException {
566 URL url;
567 try {
568 url = new URL(name);
569 if ("file".equals(url.getProtocol())) {
570 return; // this is local file - do not delete it
571 }
572 } catch (MalformedURLException e) {
573 return; // if it's not a URL, then it still might be a local file - better not to delete
574 }
575 File f = getFile();
576 if (f != null && f.exists()) {
577 Utils.deleteFile(f);
578 }
579 }
580}
Note: See TracBrowser for help on using the repository browser.