source: josm/trunk/src/org/openstreetmap/josm/io/CachedFile.java@ 13725

Last change on this file since 13725 was 13725, checked in by stoecker, 6 years ago

proper URL encoding
  • Property svn:eol-style set to native
File size: 21.0 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.io;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.BufferedReader;
7import java.io.ByteArrayOutputStream;
8import java.io.Closeable;
9import java.io.File;
10import java.io.IOException;
11import java.io.InputStream;
12import java.net.HttpURLConnection;
13import java.net.MalformedURLException;
14import java.net.URL;
15import java.net.URLEncoder;
16import java.nio.charset.StandardCharsets;
17import java.nio.file.Files;
18import java.nio.file.StandardCopyOption;
19import java.util.ArrayList;
20import java.util.Arrays;
21import java.util.Enumeration;
22import java.util.List;
23import java.util.Map;
24import java.util.concurrent.ConcurrentHashMap;
25import java.util.concurrent.TimeUnit;
26import java.util.zip.ZipEntry;
27import java.util.zip.ZipFile;
28
29import org.openstreetmap.josm.Main;
30import org.openstreetmap.josm.spi.preferences.Config;
31import org.openstreetmap.josm.tools.HttpClient;
32import org.openstreetmap.josm.tools.Logging;
33import org.openstreetmap.josm.tools.Pair;
34import org.openstreetmap.josm.tools.Utils;
35
36/**
37 * Downloads a file and caches it on disk in order to reduce network load.
38 *
39 * Supports URLs, local files, and a custom scheme (<code>resource:</code>) to get
40 * resources from the current *.jar file. (Local caching is only done for URLs.)
41 * <p>
42 * The mirrored file is only downloaded if it has been more than 7 days since
43 * last download. (Time can be configured.)
44 * <p>
45 * The file content is normally accessed with {@link #getInputStream()}, but
46 * you can also get the mirrored copy with {@link #getFile()}.
47 */
48public class CachedFile implements Closeable {
49
50 /**
51 * Caching strategy.
52 */
53 public enum CachingStrategy {
54 /**
55 * If cached file on disk is older than a certain time (7 days by default),
56 * consider the cache stale and try to download the file again.
57 */
58 MaxAge,
59 /**
60 * Similar to MaxAge, considers the cache stale when a certain age is
61 * exceeded. In addition, a If-Modified-Since HTTP header is added.
62 * When the server replies "304 Not Modified", this is considered the same
63 * as a full download.
64 */
65 IfModifiedSince
66 }
67
68 protected String name;
69 protected long maxAge;
70 protected String destDir;
71 protected String httpAccept;
72 protected CachingStrategy cachingStrategy;
73
74 private boolean fastFail;
75 private HttpClient activeConnection;
76 protected File cacheFile;
77 protected boolean initialized;
78 protected String parameter;
79
80 public static final long DEFAULT_MAXTIME = -1L;
81 public static final long DAYS = TimeUnit.DAYS.toSeconds(1); // factor to get caching time in days
82
83 private final Map<String, String> httpHeaders = new ConcurrentHashMap<>();
84
85 /**
86 * Constructs a CachedFile object from a given filename, URL or internal resource.
87 *
88 * @param name can be:<ul>
89 * <li>relative or absolute file name</li>
90 * <li>{@code file:///SOME/FILE} the same as above</li>
91 * <li>{@code http://...} a URL. It will be cached on disk.</li>
92 * <li>{@code resource://SOME/FILE} file from the classpath (usually in the current *.jar)</li>
93 * <li>{@code josmdir://SOME/FILE} file inside josm user data directory (since r7058)</li>
94 * <li>{@code josmplugindir://SOME/FILE} file inside josm plugin directory (since r7834)</li></ul>
95 */
96 public CachedFile(String name) {
97 this.name = name;
98 }
99
100 /**
101 * Set the name of the resource.
102 * @param name can be:<ul>
103 * <li>relative or absolute file name</li>
104 * <li>{@code file:///SOME/FILE} the same as above</li>
105 * <li>{@code http://...} a URL. It will be cached on disk.</li>
106 * <li>{@code resource://SOME/FILE} file from the classpath (usually in the current *.jar)</li>
107 * <li>{@code josmdir://SOME/FILE} file inside josm user data directory (since r7058)</li>
108 * <li>{@code josmplugindir://SOME/FILE} file inside josm plugin directory (since r7834)</li></ul>
109 * @return this object
110 */
111 public CachedFile setName(String name) {
112 this.name = name;
113 return this;
114 }
115
116 /**
117 * Set maximum age of cache file. Only applies to URLs.
118 * When this time has passed after the last download of the file, the
119 * cache is considered stale and a new download will be attempted.
120 * @param maxAge the maximum cache age in seconds
121 * @return this object
122 */
123 public CachedFile setMaxAge(long maxAge) {
124 this.maxAge = maxAge;
125 return this;
126 }
127
128 /**
129 * Set the destination directory for the cache file. Only applies to URLs.
130 * @param destDir the destination directory
131 * @return this object
132 */
133 public CachedFile setDestDir(String destDir) {
134 this.destDir = destDir;
135 return this;
136 }
137
138 /**
139 * Set the accepted MIME types sent in the HTTP Accept header. Only applies to URLs.
140 * @param httpAccept the accepted MIME types
141 * @return this object
142 */
143 public CachedFile setHttpAccept(String httpAccept) {
144 this.httpAccept = httpAccept;
145 return this;
146 }
147
148 /**
149 * Set the caching strategy. Only applies to URLs.
150 * @param cachingStrategy caching strategy
151 * @return this object
152 */
153 public CachedFile setCachingStrategy(CachingStrategy cachingStrategy) {
154 this.cachingStrategy = cachingStrategy;
155 return this;
156 }
157
158 /**
159 * Sets the http headers. Only applies to URL pointing to http or https resources
160 * @param headers that should be sent together with request
161 * @return this object
162 */
163 public CachedFile setHttpHeaders(Map<String, String> headers) {
164 this.httpHeaders.putAll(headers);
165 return this;
166 }
167
168 /**
169 * Sets whether opening HTTP connections should fail fast, i.e., whether a
170 * {@link HttpClient#setConnectTimeout(int) low connect timeout} should be used.
171 * @param fastFail whether opening HTTP connections should fail fast
172 */
173 public void setFastFail(boolean fastFail) {
174 this.fastFail = fastFail;
175 }
176
177 /**
178 * Sets additional URL parameter (used e.g. for maps)
179 * @param parameter the URL parameter
180 * @since 13536
181 */
182 public void setParam(String parameter) {
183 this.parameter = parameter;
184 }
185
186 public String getName() {
187 if (parameter != null)
188 return name.replaceAll("%<(.*)>", "");
189 return name;
190 }
191
192 /**
193 * Returns maximum age of cache file. Only applies to URLs.
194 * When this time has passed after the last download of the file, the
195 * cache is considered stale and a new download will be attempted.
196 * @return the maximum cache age in seconds
197 */
198 public long getMaxAge() {
199 return maxAge;
200 }
201
202 public String getDestDir() {
203 return destDir;
204 }
205
206 public String getHttpAccept() {
207 return httpAccept;
208 }
209
210 public CachingStrategy getCachingStrategy() {
211 return cachingStrategy;
212 }
213
214 /**
215 * Get InputStream to the requested resource.
216 * @return the InputStream
217 * @throws IOException when the resource with the given name could not be retrieved
218 */
219 public InputStream getInputStream() throws IOException {
220 File file = getFile();
221 if (file == null) {
222 if (name != null && name.startsWith("resource://")) {
223 InputStream is = getClass().getResourceAsStream(
224 name.substring("resource:/".length()));
225 if (is == null)
226 throw new IOException(tr("Failed to open input stream for resource ''{0}''", name));
227 return is;
228 } else {
229 throw new IOException("No file found for: "+name);
230 }
231 }
232 return Files.newInputStream(file.toPath());
233 }
234
235 /**
236 * Get the full content of the requested resource as a byte array.
237 * @return the full content of the requested resource as byte array
238 * @throws IOException in case of an I/O error
239 */
240 public byte[] getByteContent() throws IOException {
241 try (InputStream is = getInputStream()) {
242 ByteArrayOutputStream buffer = new ByteArrayOutputStream();
243 int nRead;
244 byte[] data = new byte[8192];
245 while ((nRead = is.read(data, 0, data.length)) != -1) {
246 buffer.write(data, 0, nRead);
247 }
248 buffer.flush();
249 return buffer.toByteArray();
250 }
251 }
252
253 /**
254 * Returns {@link #getInputStream()} wrapped in a buffered reader.
255 * <p>
256 * Detects Unicode charset in use utilizing {@link UTFInputStreamReader}.
257 *
258 * @return buffered reader
259 * @throws IOException if any I/O error occurs
260 * @since 9411
261 */
262 public BufferedReader getContentReader() throws IOException {
263 return new BufferedReader(UTFInputStreamReader.create(getInputStream()));
264 }
265
266 /**
267 * Get local file for the requested resource.
268 * @return The local cache file for URLs. If the resource is a local file,
269 * returns just that file.
270 * @throws IOException when the resource with the given name could not be retrieved
271 */
272 public synchronized File getFile() throws IOException {
273 if (initialized)
274 return cacheFile;
275 initialized = true;
276 URL url;
277 try {
278 url = new URL(name);
279 if ("file".equals(url.getProtocol())) {
280 cacheFile = new File(name.substring("file:/".length() - 1));
281 if (!cacheFile.exists()) {
282 cacheFile = new File(name.substring("file://".length() - 1));
283 }
284 } else {
285 try {
286 cacheFile = checkLocal(url);
287 } catch (SecurityException e) {
288 throw new IOException(e);
289 }
290 }
291 } catch (MalformedURLException e) {
292 if (name == null || name.startsWith("resource://")) {
293 return null;
294 } else if (name.startsWith("josmdir://")) {
295 cacheFile = new File(Config.getDirs().getUserDataDirectory(false), name.substring("josmdir://".length()));
296 } else if (name.startsWith("josmplugindir://")) {
297 cacheFile = new File(Main.pref.getPluginsDirectory(), name.substring("josmplugindir://".length()));
298 } else {
299 cacheFile = new File(name);
300 }
301 }
302 if (cacheFile == null)
303 throw new IOException("Unable to get cache file for "+getName());
304 return cacheFile;
305 }
306
307 /**
308 * Looks for a certain entry inside a zip file and returns the entry path.
309 *
310 * Replies a file in the top level directory of the ZIP file which has an
311 * extension <code>extension</code>. If more than one files have this
312 * extension, the last file whose name includes <code>namepart</code>
313 * is opened.
314 *
315 * @param extension the extension of the file we're looking for
316 * @param namepart the name part
317 * @return The zip entry path of the matching file. <code>null</code> if this cached file
318 * doesn't represent a zip file or if there was no matching
319 * file in the ZIP file.
320 */
321 public String findZipEntryPath(String extension, String namepart) {
322 Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart);
323 if (ze == null) return null;
324 return ze.a;
325 }
326
327 /**
328 * Like {@link #findZipEntryPath}, but returns the corresponding InputStream.
329 * @param extension the extension of the file we're looking for
330 * @param namepart the name part
331 * @return InputStream to the matching file. <code>null</code> if this cached file
332 * doesn't represent a zip file or if there was no matching
333 * file in the ZIP file.
334 * @since 6148
335 */
336 public InputStream findZipEntryInputStream(String extension, String namepart) {
337 Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart);
338 if (ze == null) return null;
339 return ze.b;
340 }
341
342 private Pair<String, InputStream> findZipEntryImpl(String extension, String namepart) {
343 File file = null;
344 try {
345 file = getFile();
346 } catch (IOException ex) {
347 Logging.log(Logging.LEVEL_WARN, ex);
348 }
349 if (file == null)
350 return null;
351 Pair<String, InputStream> res = null;
352 try {
353 ZipFile zipFile = new ZipFile(file, StandardCharsets.UTF_8);
354 ZipEntry resentry = null;
355 Enumeration<? extends ZipEntry> entries = zipFile.entries();
356 while (entries.hasMoreElements()) {
357 ZipEntry entry = entries.nextElement();
358 // choose any file with correct extension. When more than one file, prefer the one which matches namepart
359 if (entry.getName().endsWith('.' + extension) && (resentry == null || entry.getName().indexOf(namepart) >= 0)) {
360 resentry = entry;
361 }
362 }
363 if (resentry != null) {
364 InputStream is = zipFile.getInputStream(resentry);
365 res = Pair.create(resentry.getName(), is);
366 } else {
367 Utils.close(zipFile);
368 }
369 } catch (IOException e) {
370 if (file.getName().endsWith(".zip")) {
371 Logging.log(Logging.LEVEL_WARN,
372 tr("Failed to open file with extension ''{2}'' and namepart ''{3}'' in zip file ''{0}''. Exception was: {1}",
373 file.getName(), e.toString(), extension, namepart), e);
374 }
375 }
376 return res;
377 }
378
379 /**
380 * Clear the cache for the given resource.
381 * This forces a fresh download.
382 * @param name the URL
383 */
384 public static void cleanup(String name) {
385 cleanup(name, null);
386 }
387
388 /**
389 * Clear the cache for the given resource.
390 * This forces a fresh download.
391 * @param name the URL
392 * @param destDir the destination directory (see {@link #setDestDir(java.lang.String)})
393 */
394 public static void cleanup(String name, String destDir) {
395 URL url;
396 try {
397 url = new URL(name);
398 if (!"file".equals(url.getProtocol())) {
399 String prefKey = getPrefKey(url, destDir, null);
400 List<String> localPath = new ArrayList<>(Config.getPref().getList(prefKey));
401 if (localPath.size() == 2) {
402 File lfile = new File(localPath.get(1));
403 if (lfile.exists()) {
404 Utils.deleteFile(lfile);
405 }
406 }
407 Config.getPref().putList(prefKey, null);
408 }
409 } catch (MalformedURLException e) {
410 Logging.warn(e);
411 }
412 }
413
414 /**
415 * Get preference key to store the location and age of the cached file.
416 * 2 resources that point to the same url, but that are to be stored in different
417 * directories will not share a cache file.
418 * @param url URL
419 * @param destDir destination directory
420 * @param parameter additional URL parameter (used e.g. for maps)
421 * @return Preference key
422 */
423 private static String getPrefKey(URL url, String destDir, String parameter) {
424 StringBuilder prefKey = new StringBuilder("mirror.");
425 if (destDir != null) {
426 prefKey.append(destDir).append('.');
427 }
428 if (parameter != null) {
429 prefKey.append(url.toString().replaceAll("%<(.*)>", ""));
430 } else {
431 prefKey.append(url.toString());
432 }
433 return prefKey.toString().replaceAll("=", "_");
434 }
435
436 private File checkLocal(URL url) throws IOException {
437 String prefKey = getPrefKey(url, destDir, parameter);
438 String urlStr = url.toExternalForm();
439 if (parameter != null)
440 urlStr = urlStr.replaceAll("%<(.*)>", "");
441 long age = 0L;
442 long maxAgeMillis = maxAge;
443 Long ifModifiedSince = null;
444 File localFile = null;
445 List<String> localPathEntry = new ArrayList<>(Config.getPref().getList(prefKey));
446 boolean offline = false;
447 try {
448 checkOfflineAccess(urlStr);
449 } catch (OfflineAccessException e) {
450 Logging.trace(e);
451 offline = true;
452 }
453 if (localPathEntry.size() == 2) {
454 localFile = new File(localPathEntry.get(1));
455 if (!localFile.exists()) {
456 localFile = null;
457 } else {
458 if (maxAge == DEFAULT_MAXTIME
459 || maxAge <= 0 // arbitrary value <= 0 is deprecated
460 ) {
461 maxAgeMillis = TimeUnit.SECONDS.toMillis(Config.getPref().getLong("mirror.maxtime", TimeUnit.DAYS.toSeconds(7)));
462 }
463 age = System.currentTimeMillis() - Long.parseLong(localPathEntry.get(0));
464 if (offline || age < maxAgeMillis) {
465 return localFile;
466 }
467 if (cachingStrategy == CachingStrategy.IfModifiedSince) {
468 ifModifiedSince = Long.valueOf(localPathEntry.get(0));
469 }
470 }
471 }
472 if (destDir == null) {
473 destDir = Config.getDirs().getCacheDirectory(true).getPath();
474 }
475
476 File destDirFile = new File(destDir);
477 if (!destDirFile.exists()) {
478 Utils.mkDirs(destDirFile);
479 }
480
481 // No local file + offline => nothing to do
482 if (offline) {
483 return null;
484 }
485
486 if (parameter != null) {
487 String u = url.toExternalForm();
488 String uc;
489 if (parameter.isEmpty()) {
490 uc = u.replaceAll("%<(.*)>", "");
491 } else {
492 uc = u.replaceAll("%<(.*)>", "$1"+URLEncoder.encode(parameter, "UTF-8"));
493 }
494 if (!uc.equals(u))
495 url = new URL(uc);
496 }
497
498 String a = urlStr.replaceAll("[^A-Za-z0-9_.-]", "_");
499 String localPath = "mirror_" + a;
500 destDirFile = new File(destDir, localPath + ".tmp");
501 try {
502 activeConnection = HttpClient.create(url)
503 .setAccept(httpAccept)
504 .setIfModifiedSince(ifModifiedSince == null ? 0L : ifModifiedSince)
505 .setHeaders(httpHeaders);
506 if (fastFail) {
507 activeConnection.setReadTimeout(1000);
508 }
509 final HttpClient.Response con = activeConnection.connect();
510 if (ifModifiedSince != null && con.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) {
511 Logging.debug("304 Not Modified ({0})", urlStr);
512 if (localFile == null)
513 throw new AssertionError();
514 Config.getPref().putList(prefKey,
515 Arrays.asList(Long.toString(System.currentTimeMillis()), localPathEntry.get(1)));
516 return localFile;
517 } else if (con.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) {
518 throw new IOException(tr("The requested URL {0} was not found", urlStr));
519 }
520 try (InputStream is = con.getContent()) {
521 Files.copy(is, destDirFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
522 }
523 activeConnection = null;
524 localFile = new File(destDir, localPath);
525 if (Main.platform.rename(destDirFile, localFile)) {
526 Config.getPref().putList(prefKey,
527 Arrays.asList(Long.toString(System.currentTimeMillis()), localFile.toString()));
528 } else {
529 Logging.warn(tr("Failed to rename file {0} to {1}.",
530 destDirFile.getPath(), localFile.getPath()));
531 }
532 } catch (IOException e) {
533 if (age >= maxAgeMillis && age < maxAgeMillis*2) {
534 Logging.warn(tr("Failed to load {0}, use cached file and retry next time: {1}", urlStr, e));
535 return localFile;
536 } else {
537 throw e;
538 }
539 }
540
541 return localFile;
542 }
543
544 private static void checkOfflineAccess(String urlString) {
545 OnlineResource.JOSM_WEBSITE.checkOfflineAccess(urlString, Main.getJOSMWebsite());
546 OnlineResource.OSM_API.checkOfflineAccess(urlString, OsmApi.getOsmApi().getServerUrl());
547 }
548
549 /**
550 * Attempts to disconnect an URL connection.
551 * @see HttpClient#disconnect()
552 * @since 9411
553 */
554 @Override
555 public void close() {
556 if (activeConnection != null) {
557 activeConnection.disconnect();
558 }
559 }
560
561 /**
562 * Clears the cached file
563 * @throws IOException if any I/O error occurs
564 * @since 10993
565 */
566 public void clear() throws IOException {
567 URL url;
568 try {
569 url = new URL(name);
570 if ("file".equals(url.getProtocol())) {
571 return; // this is local file - do not delete it
572 }
573 } catch (MalformedURLException e) {
574 return; // if it's not a URL, then it still might be a local file - better not to delete
575 }
576 File f = getFile();
577 if (f != null && f.exists()) {
578 Utils.deleteFile(f);
579 }
580 }
581}
Note: See TracBrowser for help on using the repository browser.