source: josm/trunk/src/org/openstreetmap/josm/io/MirroredInputStream.java@ 6248

Last change on this file since 6248 was 6248, checked in by Don-vip, 11 years ago

Rework console output:

  • new log level "error"
  • Replace nearly all calls to system.out and system.err to Main.(error|warn|info|debug)
  • Remove some unnecessary debug output
  • Some messages are modified (removal of "Info", "Warning", "Error" from the message itself -> notable i18n impact but limited to console error messages not seen by the majority of users, so that's ok)
  • Property svn:eol-style set to native
File size: 13.4 KB
Line 
1// License: GPL. Copyright 2007 by Immanuel Scholz and others
2package org.openstreetmap.josm.io;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.BufferedInputStream;
7import java.io.BufferedOutputStream;
8import java.io.File;
9import java.io.FileInputStream;
10import java.io.FileOutputStream;
11import java.io.IOException;
12import java.io.InputStream;
13import java.net.HttpURLConnection;
14import java.net.MalformedURLException;
15import java.net.URL;
16import java.util.ArrayList;
17import java.util.Arrays;
18import java.util.Enumeration;
19import java.util.List;
20import java.util.zip.ZipEntry;
21import java.util.zip.ZipFile;
22
23import org.openstreetmap.josm.Main;
24import org.openstreetmap.josm.tools.Pair;
25import org.openstreetmap.josm.tools.Utils;
26
27/**
28 * Mirrors a file to a local file.
29 * <p>
30 * The file mirrored is only downloaded if it has been more than 7 days since last download
31 */
32public class MirroredInputStream extends InputStream {
33 InputStream fs = null;
34 File file = null;
35
36 public final static long DEFAULT_MAXTIME = -1L;
37
38 public MirroredInputStream(String name) throws IOException {
39 this(name, null, DEFAULT_MAXTIME);
40 }
41
42 public MirroredInputStream(String name, long maxTime) throws IOException {
43 this(name, null, maxTime);
44 }
45
46 public MirroredInputStream(String name, String destDir) throws IOException {
47 this(name, destDir, DEFAULT_MAXTIME);
48 }
49
50 /**
51 * Get an inputstream from a given filename, url or internal resource.
52 * @param name can be
53 * - relative or absolute file name
54 * - file:///SOME/FILE the same as above
55 * - resource://SOME/FILE file from the classpath (usually in the current *.jar)
56 * - http://... a url. It will be cached on disk.
57 * @param destDir the destination directory for the cache file. only applies for urls.
58 * @param maxTime the maximum age of the cache file (in seconds)
59 * @throws IOException when the resource with the given name could not be retrieved
60 */
61 public MirroredInputStream(String name, String destDir, long maxTime) throws IOException {
62 URL url;
63 try {
64 url = new URL(name);
65 if (url.getProtocol().equals("file")) {
66 file = new File(name.substring("file:/".length()));
67 if (!file.exists()) {
68 file = new File(name.substring("file://".length()));
69 }
70 } else {
71 if (Main.applet) {
72 fs = new BufferedInputStream(Utils.openURL(url));
73 file = new File(url.getFile());
74 } else {
75 file = checkLocal(url, destDir, maxTime);
76 }
77 }
78 } catch (java.net.MalformedURLException e) {
79 if (name.startsWith("resource://")) {
80 fs = getClass().getResourceAsStream(
81 name.substring("resource:/".length()));
82 if (fs == null)
83 throw new IOException(tr("Failed to open input stream for resource ''{0}''", name));
84 return;
85 }
86 file = new File(name);
87 }
88 if (file == null)
89 throw new IOException();
90 fs = new FileInputStream(file);
91 }
92
93 /**
94 * Looks for a certain entry inside a zip file and returns the entry path.
95 *
96 * Replies a file in the top level directory of the ZIP file which has an
97 * extension <code>extension</code>. If more than one files have this
98 * extension, the last file whose name includes <code>namepart</code>
99 * is opened.
100 *
101 * @param extension the extension of the file we're looking for
102 * @param namepart the name part
103 * @return The zip entry path of the matching file. Null if this mirrored
104 * input stream doesn't represent a zip file or if there was no matching
105 * file in the ZIP file.
106 */
107 public String findZipEntryPath(String extension, String namepart) {
108 Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart);
109 if (ze == null) return null;
110 return ze.a;
111 }
112
113 /**
114 * Like {@link #findZipEntryPath}, but returns the corresponding InputStream.
115 */
116 public InputStream findZipEntryInputStream(String extension, String namepart) {
117 Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart);
118 if (ze == null) return null;
119 return ze.b;
120 }
121
122 @Deprecated // use findZipEntryInputStream
123 public InputStream getZipEntry(String extension, String namepart) {
124 return findZipEntryInputStream(extension, namepart);
125 }
126
127 private Pair<String, InputStream> findZipEntryImpl(String extension, String namepart) {
128 if (file == null)
129 return null;
130 Pair<String, InputStream> res = null;
131 try {
132 ZipFile zipFile = new ZipFile(file);
133 ZipEntry resentry = null;
134 Enumeration<? extends ZipEntry> entries = zipFile.entries();
135 while (entries.hasMoreElements()) {
136 ZipEntry entry = entries.nextElement();
137 if (entry.getName().endsWith("." + extension)) {
138 /* choose any file with correct extension. When more than
139 one file, prefer the one which matches namepart */
140 if (resentry == null || entry.getName().indexOf(namepart) >= 0) {
141 resentry = entry;
142 }
143 }
144 }
145 if (resentry != null) {
146 InputStream is = zipFile.getInputStream(resentry);
147 res = Pair.create(resentry.getName(), is);
148 } else {
149 Utils.close(zipFile);
150 }
151 } catch (Exception e) {
152 if (file.getName().endsWith(".zip")) {
153 Main.warn(tr("Failed to open file with extension ''{2}'' and namepart ''{3}'' in zip file ''{0}''. Exception was: {1}",
154 file.getName(), e.toString(), extension, namepart));
155 }
156 }
157 return res;
158 }
159
160 public File getFile()
161 {
162 return file;
163 }
164
165 static public void cleanup(String name)
166 {
167 cleanup(name, null);
168 }
169 static public void cleanup(String name, String destDir)
170 {
171 URL url;
172 try {
173 url = new URL(name);
174 if (!url.getProtocol().equals("file"))
175 {
176 String prefKey = getPrefKey(url, destDir);
177 List<String> localPath = new ArrayList<String>(Main.pref.getCollection(prefKey));
178 if (localPath.size() == 2) {
179 File lfile = new File(localPath.get(1));
180 if(lfile.exists()) {
181 lfile.delete();
182 }
183 }
184 Main.pref.putCollection(prefKey, null);
185 }
186 } catch (java.net.MalformedURLException e) {}
187 }
188
189 /**
190 * get preference key to store the location and age of the cached file.
191 * 2 resources that point to the same url, but that are to be stored in different
192 * directories will not share a cache file.
193 */
194 private static String getPrefKey(URL url, String destDir) {
195 StringBuilder prefKey = new StringBuilder("mirror.");
196 if (destDir != null) {
197 prefKey.append(destDir);
198 prefKey.append(".");
199 }
200 prefKey.append(url.toString());
201 return prefKey.toString().replaceAll("=","_");
202 }
203
204 private File checkLocal(URL url, String destDir, long maxTime) throws IOException {
205 String prefKey = getPrefKey(url, destDir);
206 long age = 0L;
207 File localFile = null;
208 List<String> localPathEntry = new ArrayList<String>(Main.pref.getCollection(prefKey));
209 if (localPathEntry.size() == 2) {
210 localFile = new File(localPathEntry.get(1));
211 if(!localFile.exists())
212 localFile = null;
213 else {
214 if ( maxTime == DEFAULT_MAXTIME
215 || maxTime <= 0 // arbitrary value <= 0 is deprecated
216 ) {
217 maxTime = Main.pref.getInteger("mirror.maxtime", 7*24*60*60); // one week
218 }
219 age = System.currentTimeMillis() - Long.parseLong(localPathEntry.get(0));
220 if (age < maxTime*1000) {
221 return localFile;
222 }
223 }
224 }
225 if (destDir == null) {
226 destDir = Main.pref.getCacheDirectory().getPath();
227 }
228
229 File destDirFile = new File(destDir);
230 if (!destDirFile.exists()) {
231 destDirFile.mkdirs();
232 }
233
234 String a = url.toString().replaceAll("[^A-Za-z0-9_.-]", "_");
235 String localPath = "mirror_" + a;
236 destDirFile = new File(destDir, localPath + ".tmp");
237 BufferedOutputStream bos = null;
238 BufferedInputStream bis = null;
239 try {
240 HttpURLConnection con = connectFollowingRedirect(url);
241 bis = new BufferedInputStream(con.getInputStream());
242 FileOutputStream fos = new FileOutputStream(destDirFile);
243 bos = new BufferedOutputStream(fos);
244 byte[] buffer = new byte[4096];
245 int length;
246 while ((length = bis.read(buffer)) > -1) {
247 bos.write(buffer, 0, length);
248 }
249 Utils.close(bos);
250 bos = null;
251 /* close fos as well to be sure! */
252 Utils.close(fos);
253 fos = null;
254 localFile = new File(destDir, localPath);
255 if(Main.platform.rename(destDirFile, localFile)) {
256 Main.pref.putCollection(prefKey, Arrays.asList(new String[]
257 {Long.toString(System.currentTimeMillis()), localFile.toString()}));
258 } else {
259 Main.warn(tr("Failed to rename file {0} to {1}.",
260 destDirFile.getPath(), localFile.getPath()));
261 }
262 } catch (IOException e) {
263 if (age >= maxTime*1000 && age < maxTime*1000*2) {
264 Main.warn(tr("Failed to load {0}, use cached file and retry next time: {1}", url, e));
265 return localFile;
266 } else {
267 throw e;
268 }
269 } finally {
270 Utils.close(bis);
271 Utils.close(bos);
272 }
273
274 return localFile;
275 }
276
277 /**
278 * Opens a connection for downloading a resource.
279 * <p>
280 * Manually follows redirects because
281 * {@link HttpURLConnection#setFollowRedirects(boolean)} fails if the redirect
282 * is going from a http to a https URL, see <a href="http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4620571">bug report</a>.
283 * <p>
284 * This can causes problems when downloading from certain GitHub URLs.
285 *
286 * @param downloadUrl The resource URL to download
287 * @return The HTTP connection effectively linked to the resource, after all potential redirections
288 * @throws MalformedURLException If a redirected URL is wrong
289 * @throws IOException If any I/O operation goes wrong
290 * @since 6073
291 */
292 public static HttpURLConnection connectFollowingRedirect(URL downloadUrl) throws MalformedURLException, IOException {
293 HttpURLConnection con = null;
294 int numRedirects = 0;
295 while(true) {
296 con = Utils.openHttpConnection(downloadUrl);
297 con.setInstanceFollowRedirects(false);
298 con.setConnectTimeout(Main.pref.getInteger("socket.timeout.connect",15)*1000);
299 con.setReadTimeout(Main.pref.getInteger("socket.timeout.read",30)*1000);
300 con.connect();
301 switch(con.getResponseCode()) {
302 case HttpURLConnection.HTTP_OK:
303 return con;
304 case HttpURLConnection.HTTP_MOVED_PERM:
305 case HttpURLConnection.HTTP_MOVED_TEMP:
306 case HttpURLConnection.HTTP_SEE_OTHER:
307 String redirectLocation = con.getHeaderField("Location");
308 if (downloadUrl == null) {
309 /* I18n: argument is HTTP response code */ String msg = tr("Unexpected response from HTTP server. Got {0} response without ''Location'' header. Can''t redirect. Aborting.", con.getResponseCode());
310 throw new IOException(msg);
311 }
312 downloadUrl = new URL(redirectLocation);
313 // keep track of redirect attempts to break a redirect loops if it happens
314 // to occur for whatever reason
315 numRedirects++;
316 if (numRedirects >= Main.pref.getInteger("socket.maxredirects", 5)) {
317 String msg = tr("Too many redirects to the download URL detected. Aborting.");
318 throw new IOException(msg);
319 }
320 Main.info(tr("Download redirected to ''{0}''", downloadUrl));
321 break;
322 default:
323 String msg = tr("Failed to read from ''{0}''. Server responded with status code {1}.", downloadUrl, con.getResponseCode());
324 throw new IOException(msg);
325 }
326 }
327 }
328
329 @Override
330 public int available() throws IOException
331 { return fs.available(); }
332 @Override
333 public void close() throws IOException
334 { Utils.close(fs); }
335 @Override
336 public int read() throws IOException
337 { return fs.read(); }
338 @Override
339 public int read(byte[] b) throws IOException
340 { return fs.read(b); }
341 @Override
342 public int read(byte[] b, int off, int len) throws IOException
343 { return fs.read(b,off, len); }
344 @Override
345 public long skip(long n) throws IOException
346 { return fs.skip(n); }
347}
Note: See TracBrowser for help on using the repository browser.