source: josm/trunk/src/org/openstreetmap/josm/io/MirroredInputStream.java@ 6388

Last change on this file since 6388 was 6380, checked in by Don-vip, 10 years ago

update license/copyright information
  • Property svn:eol-style set to native
File size: 13.4 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.io;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.BufferedInputStream;
7import java.io.BufferedOutputStream;
8import java.io.File;
9import java.io.FileInputStream;
10import java.io.FileOutputStream;
11import java.io.IOException;
12import java.io.InputStream;
13import java.net.HttpURLConnection;
14import java.net.MalformedURLException;
15import java.net.URL;
16import java.util.ArrayList;
17import java.util.Arrays;
18import java.util.Enumeration;
19import java.util.List;
20import java.util.zip.ZipEntry;
21import java.util.zip.ZipFile;
22
23import org.openstreetmap.josm.Main;
24import org.openstreetmap.josm.tools.Pair;
25import org.openstreetmap.josm.tools.Utils;
26
27/**
28 * Mirrors a file to a local file.
29 * <p>
30 * The file mirrored is only downloaded if it has been more than 7 days since last download
31 */
32public class MirroredInputStream extends InputStream {
33 InputStream fs = null;
34 File file = null;
35
36 public final static long DEFAULT_MAXTIME = -1L;
37
38 public MirroredInputStream(String name) throws IOException {
39 this(name, null, DEFAULT_MAXTIME);
40 }
41
42 public MirroredInputStream(String name, long maxTime) throws IOException {
43 this(name, null, maxTime);
44 }
45
46 public MirroredInputStream(String name, String destDir) throws IOException {
47 this(name, destDir, DEFAULT_MAXTIME);
48 }
49
50 /**
51 * Get an inputstream from a given filename, url or internal resource.
52 * @param name can be
53 * - relative or absolute file name
54 * - file:///SOME/FILE the same as above
55 * - resource://SOME/FILE file from the classpath (usually in the current *.jar)
56 * - http://... a url. It will be cached on disk.
57 * @param destDir the destination directory for the cache file. only applies for urls.
58 * @param maxTime the maximum age of the cache file (in seconds)
59 * @throws IOException when the resource with the given name could not be retrieved
60 */
61 public MirroredInputStream(String name, String destDir, long maxTime) throws IOException {
62 URL url;
63 try {
64 url = new URL(name);
65 if (url.getProtocol().equals("file")) {
66 file = new File(name.substring("file:/".length()));
67 if (!file.exists()) {
68 file = new File(name.substring("file://".length()));
69 }
70 } else {
71 if (Main.applet) {
72 fs = new BufferedInputStream(Utils.openURL(url));
73 file = new File(url.getFile());
74 } else {
75 file = checkLocal(url, destDir, maxTime);
76 }
77 }
78 } catch (java.net.MalformedURLException e) {
79 if (name.startsWith("resource://")) {
80 fs = getClass().getResourceAsStream(
81 name.substring("resource:/".length()));
82 if (fs == null)
83 throw new IOException(tr("Failed to open input stream for resource ''{0}''", name));
84 return;
85 }
86 file = new File(name);
87 }
88 if (file == null)
89 throw new IOException();
90 fs = new FileInputStream(file);
91 }
92
93 /**
94 * Looks for a certain entry inside a zip file and returns the entry path.
95 *
96 * Replies a file in the top level directory of the ZIP file which has an
97 * extension <code>extension</code>. If more than one files have this
98 * extension, the last file whose name includes <code>namepart</code>
99 * is opened.
100 *
101 * @param extension the extension of the file we're looking for
102 * @param namepart the name part
103 * @return The zip entry path of the matching file. Null if this mirrored
104 * input stream doesn't represent a zip file or if there was no matching
105 * file in the ZIP file.
106 */
107 public String findZipEntryPath(String extension, String namepart) {
108 Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart);
109 if (ze == null) return null;
110 return ze.a;
111 }
112
113 /**
114 * Like {@link #findZipEntryPath}, but returns the corresponding InputStream.
115 */
116 public InputStream findZipEntryInputStream(String extension, String namepart) {
117 Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart);
118 if (ze == null) return null;
119 return ze.b;
120 }
121
122 @Deprecated // use findZipEntryInputStream
123 public InputStream getZipEntry(String extension, String namepart) {
124 return findZipEntryInputStream(extension, namepart);
125 }
126
127 private Pair<String, InputStream> findZipEntryImpl(String extension, String namepart) {
128 if (file == null)
129 return null;
130 Pair<String, InputStream> res = null;
131 try {
132 ZipFile zipFile = new ZipFile(file);
133 ZipEntry resentry = null;
134 Enumeration<? extends ZipEntry> entries = zipFile.entries();
135 while (entries.hasMoreElements()) {
136 ZipEntry entry = entries.nextElement();
137 if (entry.getName().endsWith("." + extension)) {
138 /* choose any file with correct extension. When more than
139 one file, prefer the one which matches namepart */
140 if (resentry == null || entry.getName().indexOf(namepart) >= 0) {
141 resentry = entry;
142 }
143 }
144 }
145 if (resentry != null) {
146 InputStream is = zipFile.getInputStream(resentry);
147 res = Pair.create(resentry.getName(), is);
148 } else {
149 Utils.close(zipFile);
150 }
151 } catch (Exception e) {
152 if (file.getName().endsWith(".zip")) {
153 Main.warn(tr("Failed to open file with extension ''{2}'' and namepart ''{3}'' in zip file ''{0}''. Exception was: {1}",
154 file.getName(), e.toString(), extension, namepart));
155 }
156 }
157 return res;
158 }
159
160 public File getFile() {
161 return file;
162 }
163
164 public static void cleanup(String name) {
165 cleanup(name, null);
166 }
167
168 public static void cleanup(String name, String destDir) {
169 URL url;
170 try {
171 url = new URL(name);
172 if (!url.getProtocol().equals("file")) {
173 String prefKey = getPrefKey(url, destDir);
174 List<String> localPath = new ArrayList<String>(Main.pref.getCollection(prefKey));
175 if (localPath.size() == 2) {
176 File lfile = new File(localPath.get(1));
177 if(lfile.exists()) {
178 lfile.delete();
179 }
180 }
181 Main.pref.putCollection(prefKey, null);
182 }
183 } catch (MalformedURLException e) {
184 Main.warn(e);
185 }
186 }
187
188 /**
189 * get preference key to store the location and age of the cached file.
190 * 2 resources that point to the same url, but that are to be stored in different
191 * directories will not share a cache file.
192 */
193 private static String getPrefKey(URL url, String destDir) {
194 StringBuilder prefKey = new StringBuilder("mirror.");
195 if (destDir != null) {
196 prefKey.append(destDir);
197 prefKey.append(".");
198 }
199 prefKey.append(url.toString());
200 return prefKey.toString().replaceAll("=","_");
201 }
202
203 private File checkLocal(URL url, String destDir, long maxTime) throws IOException {
204 String prefKey = getPrefKey(url, destDir);
205 long age = 0L;
206 File localFile = null;
207 List<String> localPathEntry = new ArrayList<String>(Main.pref.getCollection(prefKey));
208 if (localPathEntry.size() == 2) {
209 localFile = new File(localPathEntry.get(1));
210 if(!localFile.exists())
211 localFile = null;
212 else {
213 if ( maxTime == DEFAULT_MAXTIME
214 || maxTime <= 0 // arbitrary value <= 0 is deprecated
215 ) {
216 maxTime = Main.pref.getInteger("mirror.maxtime", 7*24*60*60); // one week
217 }
218 age = System.currentTimeMillis() - Long.parseLong(localPathEntry.get(0));
219 if (age < maxTime*1000) {
220 return localFile;
221 }
222 }
223 }
224 if (destDir == null) {
225 destDir = Main.pref.getCacheDirectory().getPath();
226 }
227
228 File destDirFile = new File(destDir);
229 if (!destDirFile.exists()) {
230 destDirFile.mkdirs();
231 }
232
233 String a = url.toString().replaceAll("[^A-Za-z0-9_.-]", "_");
234 String localPath = "mirror_" + a;
235 destDirFile = new File(destDir, localPath + ".tmp");
236 BufferedOutputStream bos = null;
237 BufferedInputStream bis = null;
238 try {
239 HttpURLConnection con = connectFollowingRedirect(url);
240 bis = new BufferedInputStream(con.getInputStream());
241 FileOutputStream fos = new FileOutputStream(destDirFile);
242 bos = new BufferedOutputStream(fos);
243 byte[] buffer = new byte[4096];
244 int length;
245 while ((length = bis.read(buffer)) > -1) {
246 bos.write(buffer, 0, length);
247 }
248 Utils.close(bos);
249 bos = null;
250 /* close fos as well to be sure! */
251 Utils.close(fos);
252 fos = null;
253 localFile = new File(destDir, localPath);
254 if(Main.platform.rename(destDirFile, localFile)) {
255 Main.pref.putCollection(prefKey, Arrays.asList(new String[]
256 {Long.toString(System.currentTimeMillis()), localFile.toString()}));
257 } else {
258 Main.warn(tr("Failed to rename file {0} to {1}.",
259 destDirFile.getPath(), localFile.getPath()));
260 }
261 } catch (IOException e) {
262 if (age >= maxTime*1000 && age < maxTime*1000*2) {
263 Main.warn(tr("Failed to load {0}, use cached file and retry next time: {1}", url, e));
264 return localFile;
265 } else {
266 throw e;
267 }
268 } finally {
269 Utils.close(bis);
270 Utils.close(bos);
271 }
272
273 return localFile;
274 }
275
276 /**
277 * Opens a connection for downloading a resource.
278 * <p>
279 * Manually follows redirects because
280 * {@link HttpURLConnection#setFollowRedirects(boolean)} fails if the redirect
281 * is going from a http to a https URL, see <a href="http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4620571">bug report</a>.
282 * <p>
283 * This can causes problems when downloading from certain GitHub URLs.
284 *
285 * @param downloadUrl The resource URL to download
286 * @return The HTTP connection effectively linked to the resource, after all potential redirections
287 * @throws MalformedURLException If a redirected URL is wrong
288 * @throws IOException If any I/O operation goes wrong
289 * @since 6073
290 */
291 public static HttpURLConnection connectFollowingRedirect(URL downloadUrl) throws MalformedURLException, IOException {
292 HttpURLConnection con = null;
293 int numRedirects = 0;
294 while(true) {
295 con = Utils.openHttpConnection(downloadUrl);
296 con.setInstanceFollowRedirects(false);
297 con.setConnectTimeout(Main.pref.getInteger("socket.timeout.connect",15)*1000);
298 con.setReadTimeout(Main.pref.getInteger("socket.timeout.read",30)*1000);
299 con.connect();
300 switch(con.getResponseCode()) {
301 case HttpURLConnection.HTTP_OK:
302 return con;
303 case HttpURLConnection.HTTP_MOVED_PERM:
304 case HttpURLConnection.HTTP_MOVED_TEMP:
305 case HttpURLConnection.HTTP_SEE_OTHER:
306 String redirectLocation = con.getHeaderField("Location");
307 if (downloadUrl == null) {
308 /* I18n: argument is HTTP response code */ String msg = tr("Unexpected response from HTTP server. Got {0} response without ''Location'' header. Can''t redirect. Aborting.", con.getResponseCode());
309 throw new IOException(msg);
310 }
311 downloadUrl = new URL(redirectLocation);
312 // keep track of redirect attempts to break a redirect loops if it happens
313 // to occur for whatever reason
314 numRedirects++;
315 if (numRedirects >= Main.pref.getInteger("socket.maxredirects", 5)) {
316 String msg = tr("Too many redirects to the download URL detected. Aborting.");
317 throw new IOException(msg);
318 }
319 Main.info(tr("Download redirected to ''{0}''", downloadUrl));
320 break;
321 default:
322 String msg = tr("Failed to read from ''{0}''. Server responded with status code {1}.", downloadUrl, con.getResponseCode());
323 throw new IOException(msg);
324 }
325 }
326 }
327
328 @Override
329 public int available() throws IOException
330 { return fs.available(); }
331 @Override
332 public void close() throws IOException
333 { Utils.close(fs); }
334 @Override
335 public int read() throws IOException
336 { return fs.read(); }
337 @Override
338 public int read(byte[] b) throws IOException
339 { return fs.read(b); }
340 @Override
341 public int read(byte[] b, int off, int len) throws IOException
342 { return fs.read(b,off, len); }
343 @Override
344 public long skip(long n) throws IOException
345 { return fs.skip(n); }
346}
Note: See TracBrowser for help on using the repository browser.