source: josm/trunk/src/org/openstreetmap/josm/io/CachedFile.java@ 9185

Last change on this file since 9185 was 9168, checked in by simon04, 8 years ago

see #12231 - Uniform access to HTTP resources
  • Property svn:eol-style set to native
File size: 16.9 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.io;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.BufferedInputStream;
7import java.io.BufferedOutputStream;
8import java.io.File;
9import java.io.FileInputStream;
10import java.io.FileOutputStream;
11import java.io.IOException;
12import java.io.InputStream;
13import java.io.OutputStream;
14import java.net.HttpURLConnection;
15import java.net.MalformedURLException;
16import java.net.URL;
17import java.nio.charset.StandardCharsets;
18import java.util.ArrayList;
19import java.util.Arrays;
20import java.util.Enumeration;
21import java.util.List;
22import java.util.Map;
23import java.util.concurrent.ConcurrentHashMap;
24import java.util.zip.ZipEntry;
25import java.util.zip.ZipFile;
26
27import org.openstreetmap.josm.Main;
28import org.openstreetmap.josm.tools.HttpClient;
29import org.openstreetmap.josm.tools.Pair;
30import org.openstreetmap.josm.tools.Utils;
31
32/**
33 * Downloads a file and caches it on disk in order to reduce network load.
34 *
35 * Supports URLs, local files, and a custom scheme (<code>resource:</code>) to get
36 * resources from the current *.jar file. (Local caching is only done for URLs.)
37 * <p>
38 * The mirrored file is only downloaded if it has been more than 7 days since
39 * last download. (Time can be configured.)
40 * <p>
41 * The file content is normally accessed with {@link #getInputStream()}, but
42 * you can also get the mirrored copy with {@link #getFile()}.
43 */
44public class CachedFile {
45
46 /**
47 * Caching strategy.
48 */
49 public enum CachingStrategy {
50 /**
51 * If cached file on disk is older than a certain time (7 days by default),
52 * consider the cache stale and try to download the file again.
53 */
54 MaxAge,
55 /**
56 * Similar to MaxAge, considers the cache stale when a certain age is
57 * exceeded. In addition, a If-Modified-Since HTTP header is added.
58 * When the server replies "304 Not Modified", this is considered the same
59 * as a full download.
60 */
61 IfModifiedSince
62 }
63
64 protected String name;
65 protected long maxAge;
66 protected String destDir;
67 protected String httpAccept;
68 protected CachingStrategy cachingStrategy;
69
70 protected File cacheFile;
71 protected boolean initialized;
72
73 public static final long DEFAULT_MAXTIME = -1L;
74 public static final long DAYS = 24*60*60; // factor to get caching time in days
75
76 private final Map<String, String> httpHeaders = new ConcurrentHashMap<>();
77
78 /**
79 * Constructs a CachedFile object from a given filename, URL or internal resource.
80 *
81 * @param name can be:<ul>
82 * <li>relative or absolute file name</li>
83 * <li>{@code file:///SOME/FILE} the same as above</li>
84 * <li>{@code http://...} a URL. It will be cached on disk.</li>
85 * <li>{@code resource://SOME/FILE} file from the classpath (usually in the current *.jar)</li>
86 * <li>{@code josmdir://SOME/FILE} file inside josm user data directory (since r7058)</li>
87 * <li>{@code josmplugindir://SOME/FILE} file inside josm plugin directory (since r7834)</li></ul>
88 */
89 public CachedFile(String name) {
90 this.name = name;
91 }
92
93 /**
94 * Set the name of the resource.
95 * @param name can be:<ul>
96 * <li>relative or absolute file name</li>
97 * <li>{@code file:///SOME/FILE} the same as above</li>
98 * <li>{@code http://...} a URL. It will be cached on disk.</li>
99 * <li>{@code resource://SOME/FILE} file from the classpath (usually in the current *.jar)</li>
100 * <li>{@code josmdir://SOME/FILE} file inside josm user data directory (since r7058)</li>
101 * <li>{@code josmplugindir://SOME/FILE} file inside josm plugin directory (since r7834)</li></ul>
102 * @return this object
103 */
104 public CachedFile setName(String name) {
105 this.name = name;
106 return this;
107 }
108
109 /**
110 * Set maximum age of cache file. Only applies to URLs.
111 * When this time has passed after the last download of the file, the
112 * cache is considered stale and a new download will be attempted.
113 * @param maxAge the maximum cache age in seconds
114 * @return this object
115 */
116 public CachedFile setMaxAge(long maxAge) {
117 this.maxAge = maxAge;
118 return this;
119 }
120
121 /**
122 * Set the destination directory for the cache file. Only applies to URLs.
123 * @param destDir the destination directory
124 * @return this object
125 */
126 public CachedFile setDestDir(String destDir) {
127 this.destDir = destDir;
128 return this;
129 }
130
131 /**
132 * Set the accepted MIME types sent in the HTTP Accept header. Only applies to URLs.
133 * @param httpAccept the accepted MIME types
134 * @return this object
135 */
136 public CachedFile setHttpAccept(String httpAccept) {
137 this.httpAccept = httpAccept;
138 return this;
139 }
140
141 /**
142 * Set the caching strategy. Only applies to URLs.
143 * @param cachingStrategy caching strategy
144 * @return this object
145 */
146 public CachedFile setCachingStrategy(CachingStrategy cachingStrategy) {
147 this.cachingStrategy = cachingStrategy;
148 return this;
149 }
150
151 /**
152 * Sets the http headers. Only applies to URL pointing to http or https resources
153 * @param headers that should be sent together with request
154 * @return this object
155 */
156 public CachedFile setHttpHeaders(Map<String, String> headers) {
157 this.httpHeaders.putAll(headers);
158 return this;
159 }
160
161 public String getName() {
162 return name;
163 }
164
165 public long getMaxAge() {
166 return maxAge;
167 }
168
169 public String getDestDir() {
170 return destDir;
171 }
172
173 public String getHttpAccept() {
174 return httpAccept;
175 }
176
177 public CachingStrategy getCachingStrategy() {
178 return cachingStrategy;
179 }
180
181 /**
182 * Get InputStream to the requested resource.
183 * @return the InputStream
184 * @throws IOException when the resource with the given name could not be retrieved
185 */
186 public InputStream getInputStream() throws IOException {
187 File file = getFile();
188 if (file == null) {
189 if (name.startsWith("resource://")) {
190 InputStream is = getClass().getResourceAsStream(
191 name.substring("resource:/".length()));
192 if (is == null)
193 throw new IOException(tr("Failed to open input stream for resource ''{0}''", name));
194 return is;
195 } else {
196 throw new IOException("No file found for: "+name);
197 }
198 }
199 return new FileInputStream(file);
200 }
201
202 /**
203 * Get local file for the requested resource.
204 * @return The local cache file for URLs. If the resource is a local file,
205 * returns just that file.
206 * @throws IOException when the resource with the given name could not be retrieved
207 */
208 public synchronized File getFile() throws IOException {
209 if (initialized)
210 return cacheFile;
211 initialized = true;
212 URL url;
213 try {
214 url = new URL(name);
215 if ("file".equals(url.getProtocol())) {
216 cacheFile = new File(name.substring("file:/".length() - 1));
217 if (!cacheFile.exists()) {
218 cacheFile = new File(name.substring("file://".length() - 1));
219 }
220 } else {
221 cacheFile = checkLocal(url);
222 }
223 } catch (MalformedURLException e) {
224 if (name.startsWith("resource://")) {
225 return null;
226 } else if (name.startsWith("josmdir://")) {
227 cacheFile = new File(Main.pref.getUserDataDirectory(), name.substring("josmdir://".length()));
228 } else if (name.startsWith("josmplugindir://")) {
229 cacheFile = new File(Main.pref.getPluginsDirectory(), name.substring("josmplugindir://".length()));
230 } else {
231 cacheFile = new File(name);
232 }
233 }
234 if (cacheFile == null)
235 throw new IOException("Unable to get cache file for "+name);
236 return cacheFile;
237 }
238
239 /**
240 * Looks for a certain entry inside a zip file and returns the entry path.
241 *
242 * Replies a file in the top level directory of the ZIP file which has an
243 * extension <code>extension</code>. If more than one files have this
244 * extension, the last file whose name includes <code>namepart</code>
245 * is opened.
246 *
247 * @param extension the extension of the file we're looking for
248 * @param namepart the name part
249 * @return The zip entry path of the matching file. Null if this cached file
250 * doesn't represent a zip file or if there was no matching
251 * file in the ZIP file.
252 */
253 public String findZipEntryPath(String extension, String namepart) {
254 Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart);
255 if (ze == null) return null;
256 return ze.a;
257 }
258
259 /**
260 * Like {@link #findZipEntryPath}, but returns the corresponding InputStream.
261 * @param extension the extension of the file we're looking for
262 * @param namepart the name part
263 * @return InputStream to the matching file. Null if this cached file
264 * doesn't represent a zip file or if there was no matching
265 * file in the ZIP file.
266 * @since 6148
267 */
268 public InputStream findZipEntryInputStream(String extension, String namepart) {
269 Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart);
270 if (ze == null) return null;
271 return ze.b;
272 }
273
274 private Pair<String, InputStream> findZipEntryImpl(String extension, String namepart) {
275 File file = null;
276 try {
277 file = getFile();
278 } catch (IOException ex) {
279 Main.warn(ex, false);
280 }
281 if (file == null)
282 return null;
283 Pair<String, InputStream> res = null;
284 try {
285 ZipFile zipFile = new ZipFile(file, StandardCharsets.UTF_8);
286 ZipEntry resentry = null;
287 Enumeration<? extends ZipEntry> entries = zipFile.entries();
288 while (entries.hasMoreElements()) {
289 ZipEntry entry = entries.nextElement();
290 if (entry.getName().endsWith('.' + extension)) {
291 /* choose any file with correct extension. When more than
292 one file, prefer the one which matches namepart */
293 if (resentry == null || entry.getName().indexOf(namepart) >= 0) {
294 resentry = entry;
295 }
296 }
297 }
298 if (resentry != null) {
299 InputStream is = zipFile.getInputStream(resentry);
300 res = Pair.create(resentry.getName(), is);
301 } else {
302 Utils.close(zipFile);
303 }
304 } catch (Exception e) {
305 if (file.getName().endsWith(".zip")) {
306 Main.warn(tr("Failed to open file with extension ''{2}'' and namepart ''{3}'' in zip file ''{0}''. Exception was: {1}",
307 file.getName(), e.toString(), extension, namepart));
308 }
309 }
310 return res;
311 }
312
313 /**
314 * Clear the cache for the given resource.
315 * This forces a fresh download.
316 * @param name the URL
317 */
318 public static void cleanup(String name) {
319 cleanup(name, null);
320 }
321
322 /**
323 * Clear the cache for the given resource.
324 * This forces a fresh download.
325 * @param name the URL
326 * @param destDir the destination directory (see {@link #setDestDir(java.lang.String)})
327 */
328 public static void cleanup(String name, String destDir) {
329 URL url;
330 try {
331 url = new URL(name);
332 if (!"file".equals(url.getProtocol())) {
333 String prefKey = getPrefKey(url, destDir);
334 List<String> localPath = new ArrayList<>(Main.pref.getCollection(prefKey));
335 if (localPath.size() == 2) {
336 File lfile = new File(localPath.get(1));
337 if (lfile.exists()) {
338 lfile.delete();
339 }
340 }
341 Main.pref.putCollection(prefKey, null);
342 }
343 } catch (MalformedURLException e) {
344 Main.warn(e);
345 }
346 }
347
348 /**
349 * Get preference key to store the location and age of the cached file.
350 * 2 resources that point to the same url, but that are to be stored in different
351 * directories will not share a cache file.
352 * @param url URL
353 * @param destDir destination directory
354 * @return Preference key
355 */
356 private static String getPrefKey(URL url, String destDir) {
357 StringBuilder prefKey = new StringBuilder("mirror.");
358 if (destDir != null) {
359 prefKey.append(destDir).append('.');
360 }
361 prefKey.append(url.toString());
362 return prefKey.toString().replaceAll("=", "_");
363 }
364
365 private File checkLocal(URL url) throws IOException {
366 String prefKey = getPrefKey(url, destDir);
367 String urlStr = url.toExternalForm();
368 long age = 0L;
369 long lMaxAge = maxAge;
370 Long ifModifiedSince = null;
371 File localFile = null;
372 List<String> localPathEntry = new ArrayList<>(Main.pref.getCollection(prefKey));
373 boolean offline = false;
374 try {
375 checkOfflineAccess(urlStr);
376 } catch (OfflineAccessException e) {
377 offline = true;
378 }
379 if (localPathEntry.size() == 2) {
380 localFile = new File(localPathEntry.get(1));
381 if (!localFile.exists()) {
382 localFile = null;
383 } else {
384 if (maxAge == DEFAULT_MAXTIME
385 || maxAge <= 0 // arbitrary value <= 0 is deprecated
386 ) {
387 lMaxAge = Main.pref.getInteger("mirror.maxtime", 7*24*60*60); // one week
388 }
389 age = System.currentTimeMillis() - Long.parseLong(localPathEntry.get(0));
390 if (offline || age < lMaxAge*1000) {
391 return localFile;
392 }
393 if (cachingStrategy == CachingStrategy.IfModifiedSince) {
394 ifModifiedSince = Long.valueOf(localPathEntry.get(0));
395 }
396 }
397 }
398 if (destDir == null) {
399 destDir = Main.pref.getCacheDirectory().getPath();
400 }
401
402 File destDirFile = new File(destDir);
403 if (!destDirFile.exists()) {
404 destDirFile.mkdirs();
405 }
406
407 // No local file + offline => nothing to do
408 if (offline) {
409 return null;
410 }
411
412 String a = urlStr.replaceAll("[^A-Za-z0-9_.-]", "_");
413 String localPath = "mirror_" + a;
414 destDirFile = new File(destDir, localPath + ".tmp");
415 try {
416 final HttpClient.Response con = HttpClient.create(url)
417 .setAccept(httpAccept)
418 .setIfModifiedSince(ifModifiedSince == null ? 0L : ifModifiedSince)
419 .setHeaders(httpHeaders)
420 .connect();
421 if (ifModifiedSince != null && con.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) {
422 if (Main.isDebugEnabled()) {
423 Main.debug("304 Not Modified ("+urlStr+')');
424 }
425 if (localFile == null)
426 throw new AssertionError();
427 Main.pref.putCollection(prefKey,
428 Arrays.asList(Long.toString(System.currentTimeMillis()), localPathEntry.get(1)));
429 return localFile;
430 }
431 try (
432 InputStream bis = new BufferedInputStream(con.getContent());
433 OutputStream fos = new FileOutputStream(destDirFile);
434 OutputStream bos = new BufferedOutputStream(fos)
435 ) {
436 byte[] buffer = new byte[4096];
437 int length;
438 while ((length = bis.read(buffer)) > -1) {
439 bos.write(buffer, 0, length);
440 }
441 }
442 localFile = new File(destDir, localPath);
443 if (Main.platform.rename(destDirFile, localFile)) {
444 Main.pref.putCollection(prefKey,
445 Arrays.asList(Long.toString(System.currentTimeMillis()), localFile.toString()));
446 } else {
447 Main.warn(tr("Failed to rename file {0} to {1}.",
448 destDirFile.getPath(), localFile.getPath()));
449 }
450 } catch (IOException e) {
451 if (age >= lMaxAge*1000 && age < lMaxAge*1000*2) {
452 Main.warn(tr("Failed to load {0}, use cached file and retry next time: {1}", urlStr, e));
453 return localFile;
454 } else {
455 throw e;
456 }
457 }
458
459 return localFile;
460 }
461
462 private static void checkOfflineAccess(String urlString) {
463 OnlineResource.JOSM_WEBSITE.checkOfflineAccess(urlString, Main.getJOSMWebsite());
464 OnlineResource.OSM_API.checkOfflineAccess(urlString, Main.pref.get("osm-server.url", OsmApi.DEFAULT_API_URL));
465 }
466
467}
Note: See TracBrowser for help on using the repository browser.