Ticket #12231: 12231-alpha.patch

src/org/openstreetmap/josm/gui/help/HelpContentReader.java

@@ -3 +3 @@
 
 import java.io.BufferedReader;
 import java.io.IOException;
-import java.io.InputStreamReader;
-import java.net.HttpURLConnection;
 import java.net.MalformedURLException;
 import java.net.URL;
-import java.nio.charset.StandardCharsets;
 
-import org.openstreetmap.josm.Main;
-import org.openstreetmap.josm.tools.Utils;
+import org.openstreetmap.josm.tools.HttpClient;
 import org.openstreetmap.josm.tools.WikiReader;
 
 /**
@@ -45 +41 @@
     public String fetchHelpTopicContent(String helpTopicUrl, boolean dotest) throws HelpContentReaderException {
         if (helpTopicUrl == null)
             throw new MissingHelpContentException("helpTopicUrl is null");
-        HttpURLConnection con = null;
+        HttpClient.Response con = null;
         try {
             URL u = new URL(helpTopicUrl);
-            con = Utils.openHttpConnection(u);
-            con.connect();
-            try (BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream(), StandardCharsets.UTF_8))) {
+            con = HttpClient.create(u).connect();
+            try (BufferedReader in = con.getContentReader()) {
                 return prepareHelpContent(in, dotest, u);
             }
         } catch (MalformedURLException e) {
@@ -58 +53 @@
         } catch (IOException e) {
             HelpContentReaderException ex = new HelpContentReaderException(e);
             if (con != null) {
-                try {
-                    ex.setResponseCode(con.getResponseCode());
-                } catch (IOException e1) {
-                    // ignore
-                    if (Main.isTraceEnabled()) {
-                        Main.trace(e1.getMessage());
-                    }
-                }
+                ex.setResponseCode(con.getResponseCode());
             }
             throw ex;
         }

src/org/openstreetmap/josm/io/CachedFile.java

@@ -20 +20 @@
 import java.util.Enumeration;
 import java.util.List;
 import java.util.Map;
-import java.util.Map.Entry;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
 
 import org.openstreetmap.josm.Main;
-import org.openstreetmap.josm.tools.CheckParameterUtil;
+import org.openstreetmap.josm.tools.HttpClient;
 import org.openstreetmap.josm.tools.Pair;
 import org.openstreetmap.josm.tools.Utils;
 
@@ -414 +413 @@
         String localPath = "mirror_" + a;
         destDirFile = new File(destDir, localPath + ".tmp");
         try {
-            HttpURLConnection con = connectFollowingRedirect(url, httpAccept, ifModifiedSince, httpHeaders);
+            final HttpClient.Response con = HttpClient.create(url)
+                    .setAccept(httpAccept)
+                    .setIfModifiedSince(ifModifiedSince == null ? 0L : ifModifiedSince)
+                    .setHeaders(httpHeaders)
+                    .connect();
             if (ifModifiedSince != null && con.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) {
                 if (Main.isDebugEnabled()) {
                     Main.debug("304 Not Modified ("+urlStr+')');
@@ -426 +429 @@
                 return localFile;
             }
             try (
-                InputStream bis = new BufferedInputStream(con.getInputStream());
+                InputStream bis = new BufferedInputStream(con.getContent());
                 OutputStream fos = new FileOutputStream(destDirFile);
                 OutputStream bos = new BufferedOutputStream(fos)
             ) {
@@ -461 +464 @@
         OnlineResource.OSM_API.checkOfflineAccess(urlString, Main.pref.get("osm-server.url", OsmApi.DEFAULT_API_URL));
     }
 
-    /**
-     * Opens a connection for downloading a resource.
-     * <p>
-     * Manually follows redirects because
-     * {@link HttpURLConnection#setFollowRedirects(boolean)} fails if the redirect
-     * is going from a http to a https URL, see <a href="https://bugs.openjdk.java.net/browse/JDK-4620571">bug report</a>.
-     * <p>
-     * This can cause problems when downloading from certain GitHub URLs.
-     *
-     * @param downloadUrl The resource URL to download
-     * @param httpAccept The accepted MIME types sent in the HTTP Accept header. Can be {@code null}
-     * @param ifModifiedSince The download time of the cache file, optional
-     * @return The HTTP connection effectively linked to the resource, after all potential redirections
-     * @throws MalformedURLException If a redirected URL is wrong
-     * @throws IOException If any I/O operation goes wrong
-     * @throws OfflineAccessException if resource is accessed in offline mode, in any protocol
-     * @since 6867
-     */
-    public static HttpURLConnection connectFollowingRedirect(URL downloadUrl, String httpAccept, Long ifModifiedSince)
-            throws MalformedURLException, IOException {
-        return connectFollowingRedirect(downloadUrl, httpAccept, ifModifiedSince, null);
-    }
-
-    /**
-     * Opens a connection for downloading a resource.
-     * <p>
-     * Manually follows redirects because
-     * {@link HttpURLConnection#setFollowRedirects(boolean)} fails if the redirect
-     * is going from a http to a https URL, see <a href="https://bugs.openjdk.java.net/browse/JDK-4620571">bug report</a>.
-     * <p>
-     * This can cause problems when downloading from certain GitHub URLs.
-     *
-     * @param downloadUrl The resource URL to download
-     * @param httpAccept The accepted MIME types sent in the HTTP Accept header. Can be {@code null}
-     * @param ifModifiedSince The download time of the cache file, optional
-     * @param headers http headers to be sent together with http request
-     * @return The HTTP connection effectively linked to the resource, after all potential redirections
-     * @throws MalformedURLException If a redirected URL is wrong
-     * @throws IOException If any I/O operation goes wrong
-     * @throws OfflineAccessException if resource is accessed in offline mode, in any protocol
-     * @since TODO
-     */
-    public static HttpURLConnection connectFollowingRedirect(URL downloadUrl, String httpAccept, Long ifModifiedSince,
-            Map<String, String> headers) throws MalformedURLException, IOException {
-        CheckParameterUtil.ensureParameterNotNull(downloadUrl, "downloadUrl");
-        String downloadString = downloadUrl.toExternalForm();
-
-        checkOfflineAccess(downloadString);
-
-        int numRedirects = 0;
-        while (true) {
-            HttpURLConnection con = Utils.openHttpConnection(downloadUrl);
-            if (ifModifiedSince != null) {
-                con.setIfModifiedSince(ifModifiedSince);
-            }
-            if (headers != null) {
-                for (Entry<String, String> header: headers.entrySet()) {
-                    con.setRequestProperty(header.getKey(), header.getValue());
-                }
-            }
-            con.setInstanceFollowRedirects(false);
-            con.setConnectTimeout(Main.pref.getInteger("socket.timeout.connect", 15)*1000);
-            con.setReadTimeout(Main.pref.getInteger("socket.timeout.read", 30)*1000);
-            if (Main.isDebugEnabled()) {
-                Main.debug("GET "+downloadString);
-            }
-            if (httpAccept != null) {
-                if (Main.isTraceEnabled()) {
-                    Main.trace("Accept: "+httpAccept);
-                }
-                con.setRequestProperty("Accept", httpAccept);
-            }
-            try {
-                con.connect();
-            } catch (IOException e) {
-                Main.addNetworkError(downloadUrl, Utils.getRootCause(e));
-                throw e;
-            }
-            switch(con.getResponseCode()) {
-            case HttpURLConnection.HTTP_OK:
-                return con;
-            case HttpURLConnection.HTTP_NOT_MODIFIED:
-                if (ifModifiedSince != null)
-                    return con;
-            case HttpURLConnection.HTTP_MOVED_PERM:
-            case HttpURLConnection.HTTP_MOVED_TEMP:
-            case HttpURLConnection.HTTP_SEE_OTHER:
-                String redirectLocation = con.getHeaderField("Location");
-                if (redirectLocation == null) {
-                    /* I18n: argument is HTTP response code */
-                    String msg = tr("Unexpected response from HTTP server. Got {0} response without ''Location'' header."+
-                            " Can''t redirect. Aborting.", con.getResponseCode());
-                    throw new IOException(msg);
-                }
-                downloadUrl = new URL(redirectLocation);
-                downloadString = downloadUrl.toExternalForm();
-                // keep track of redirect attempts to break a redirect loops if it happens
-                // to occur for whatever reason
-                numRedirects++;
-                if (numRedirects >= Main.pref.getInteger("socket.maxredirects", 5)) {
-                    String msg = tr("Too many redirects to the download URL detected. Aborting.");
-                    throw new IOException(msg);
-                }
-                Main.info(tr("Download redirected to ''{0}''", downloadString));
-                break;
-            default:
-                String msg = tr("Failed to read from ''{0}''. Server responded with status code {1}.", downloadString, con.getResponseCode());
-                throw new IOException(msg);
-            }
-        }
-    }
 }

src/org/openstreetmap/josm/plugins/PluginDownloadTask.java

@@ -9 +9 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
-import java.net.HttpURLConnection;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.Collection;
@@ -21 +20 @@
 import org.openstreetmap.josm.gui.PleaseWaitRunnable;
 import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
 import org.openstreetmap.josm.gui.progress.ProgressMonitor;
-import org.openstreetmap.josm.io.CachedFile;
 import org.openstreetmap.josm.tools.CheckParameterUtil;
+import org.openstreetmap.josm.tools.HttpClient;
 import org.xml.sax.SAXException;
 
 /**
@@ -44 +43 @@
     private final Collection<PluginInformation> failed = new LinkedList<>();
     private final Collection<PluginInformation> downloaded = new LinkedList<>();
     private boolean canceled;
-    private HttpURLConnection downloadConnection;
+    private HttpClient.Response downloadConnection;
 
     /**
      * Creates the download task
@@ -123 +122 @@
             }
             URL url = new URL(pi.downloadlink);
             synchronized (this) {
-                downloadConnection = CachedFile.connectFollowingRedirect(url, PLUGIN_MIME_TYPES, null);
+                downloadConnection = HttpClient.create(url)
+                        .setAccept(PLUGIN_MIME_TYPES)
+                        .connect();
             }
             try (
-                InputStream in = downloadConnection.getInputStream();
+                InputStream in = downloadConnection.getContent();
                 OutputStream out = new FileOutputStream(file)
             ) {
                 byte[] buffer = new byte[8192];

new file src/org/openstreetmap/josm/tools/HttpClient.java

@@ +1 @@
+// License: GPL. For details, see LICENSE file.
+package org.openstreetmap.josm.tools;
+
+import static org.openstreetmap.josm.tools.I18n.tr;
+
+import java.io.BufferedOutputStream;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.zip.GZIPInputStream;
+
+import org.openstreetmap.josm.Main;
+import org.openstreetmap.josm.data.Version;
+
+/**
+ * Provides a uniform access for a HTTP/HTTPS server. This class should be used in favour of {@link HttpURLConnection}.
+ */
+public class HttpClient {
+
+    private URL url;
+    private final String requestMethod;
+    private int connectTimeout = Main.pref.getInteger("socket.timeout.connect", 15) * 1000;
+    private int readTimeout = Main.pref.getInteger("socket.timeout.read", 30) * 1000;
+    private String accept;
+    private String contentType;
+    private String acceptEncoding = "gzip";
+    private long contentLength;
+    private byte[] requestBody;
+    private long ifModifiedSince;
+    private final Map<String, String> headers = new ConcurrentHashMap<>();
+    private int maxRedirects = Main.pref.getInteger("socket.maxredirects", 5);
+
+    private HttpClient(URL url, String requestMethod) {
+        this.url = url;
+        this.requestMethod = requestMethod;
+    }
+
+    public Response connect() throws IOException {
+        final HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+        connection.setRequestProperty("User-Agent", Version.getInstance().getFullAgentString());
+        connection.setConnectTimeout(connectTimeout);
+        connection.setReadTimeout(readTimeout);
+        if (accept != null) {
+            connection.setRequestProperty("Accept", accept);
+        }
+        if (contentType != null) {
+            connection.setRequestProperty("Content-Type", contentType);
+        }
+        if (acceptEncoding != null) {
+            connection.setRequestProperty("Accept-Encoding", acceptEncoding);
+        }
+        if (contentLength > 0) {
+            connection.setRequestProperty("Content-Length", String.valueOf(contentLength));
+        }
+        if ("PUT".equals(requestMethod) || "POST".equals(requestMethod) || "DELETE".equals(requestMethod)) {
+            connection.setDoOutput(true);
+            try (OutputStream out = new BufferedOutputStream(connection.getOutputStream())) {
+                out.write(requestBody);
+            }
+        }
+        if (ifModifiedSince > 0) {
+            connection.setIfModifiedSince(ifModifiedSince);
+        }
+        for (Map.Entry<String, String> header : headers.entrySet()) {
+            connection.setRequestProperty(header.getKey(), header.getValue());
+        }
+
+        boolean successfulConnection = false;
+        try {
+            try {
+                connection.connect();
+            } catch (IOException e) {
+                //noinspection ThrowableResultOfMethodCallIgnored
+                Main.addNetworkError(url, Utils.getRootCause(e));
+                throw e;
+            }
+            if (isRedirect(connection.getResponseCode())) {
+                final String redirectLocation = connection.getHeaderField("Location");
+                if (redirectLocation == null) {
+                    /* I18n: argument is HTTP response code */
+                    String msg = tr("Unexpected response from HTTP server. Got {0} response without ''Location'' header." +
+                            " Can''t redirect. Aborting.", connection.getResponseCode());
+                    throw new IOException(msg);
+                } else if (maxRedirects > 0) {
+                    url = new URL(redirectLocation);
+                    maxRedirects--;
+                    Main.info(tr("Download redirected to ''{0}''", redirectLocation));
+                    return connect();
+                } else {
+                    String msg = tr("Too many redirects to the download URL detected. Aborting.");
+                    throw new IOException(msg);
+                }
+            }
+            Response response = new Response(connection);
+            successfulConnection = true;
+            return response;
+        } finally {
+            if (!successfulConnection) {
+                connection.disconnect();
+            }
+        }
+    }
+
+    /**
+     * A wrapper for the HTTP response.
+     */
+    public static class Response {
+        private final HttpURLConnection connection;
+        private final int responseCode;
+
+        private Response(HttpURLConnection connection) throws IOException {
+            this.connection = connection;
+            this.responseCode = connection.getResponseCode();
+        }
+
+        /**
+         * Returns an input stream that reads from this HTTP connection, or,
+         * error stream if the connection failed but the server sent useful data.
+         *
+         * @see HttpURLConnection#getInputStream()
+         * @see HttpURLConnection#getErrorStream()
+         */
+        public InputStream getContent() throws IOException {
+            InputStream in;
+            try {
+                in = connection.getInputStream();
+            } catch (IOException ioe) {
+                in = connection.getErrorStream();
+            }
+            return "gzip".equalsIgnoreCase(getContentEncoding()) ? new GZIPInputStream(in) : in;
+        }
+
+        /**
+         * Returns {@link #getContent()} wrapped in a buffered reader
+         */
+        public BufferedReader getContentReader() throws IOException {
+            return new BufferedReader(new InputStreamReader(getContent(), StandardCharsets.UTF_8));
+        }
+
+        /**
+         * Gets the response code from this HTTP connection.
+         *
+         * @see HttpURLConnection#getResponseCode()
+         */
+        public int getResponseCode() {
+            return responseCode;
+        }
+
+        /**
+         * Returns the {@code Content-Encoding} header.
+         */
+        public String getContentEncoding() {
+            return connection.getContentEncoding();
+        }
+
+        /**
+         * Returns the {@code Content-Type} header.
+         */
+        public String getContentType() {
+            return connection.getHeaderField("Content-Type");
+        }
+
+        /**
+         * @see HttpURLConnection#disconnect()
+         */
+        public void disconnect() {
+            connection.disconnect();
+        }
+    }
+
+    /**
+     * Creates a new instance for the given URL and a {@code GET} request
+     *
+     * @param url the URL
+     * @return a new instance
+     */
+    public static HttpClient create(URL url) {
+        return create(url, "GET");
+    }
+
+    /**
+     * Creates a new instance for the given URL and a {@code GET} request
+     *
+     * @param url           the URL
+     * @param requestMethod the HTTP request method to perform when calling
+     * @return a new instance
+     */
+    public static HttpClient create(URL url, String requestMethod) {
+        return new HttpClient(url, requestMethod);
+    }
+
+    /**
+     * @return {@code this}
+     * @see HttpURLConnection#setConnectTimeout(int)
+     */
+    public HttpClient setConnectTimeout(int connectTimeout) {
+        this.connectTimeout = connectTimeout;
+        return this;
+    }
+
+    /**
+     * @return {@code this}
+     * @see HttpURLConnection#setReadTimeout(int) (int)
+     */
+
+    public HttpClient setReadTimeout(int readTimeout) {
+        this.readTimeout = readTimeout;
+        return this;
+    }
+
+    /**
+     * Sets the {@code Accept} header.
+     *
+     * @return {@code this}
+     */
+    public HttpClient setAccept(String accept) {
+        this.accept = accept;
+        return this;
+    }
+
+    /**
+     * Sets the {@code Content-Type} header.
+     *
+     * @return {@code this}
+     */
+    public HttpClient setContentType(String contentType) {
+        this.contentType = contentType;
+        return this;
+    }
+
+    /**
+     * Sets the {@code Accept-Encoding} header.
+     *
+     * @return {@code this}
+     */
+    public HttpClient setAcceptEncoding(String acceptEncoding) {
+        this.acceptEncoding = acceptEncoding;
+        return this;
+    }
+
+    /**
+     * Sets the {@code Content-Length} header for {@code PUT}/{@code POST} requests.
+     *
+     * @return {@code this}
+     */
+    public HttpClient setContentLength(long contentLength) {
+        this.contentLength = contentLength;
+        return this;
+    }
+
+    /**
+     * Sets the request body for {@code PUT}/{@code POST} requests.
+     *
+     * @return {@code this}
+     */
+    public HttpClient setRequestBody(byte[] requestBody) {
+        this.requestBody = requestBody;
+        return this;
+    }
+
+    /**
+     * Sets the {@code If-Modified-Since} header.
+     *
+     * @return {@code this}
+     */
+    public HttpClient setIfModifiedSince(long ifModifiedSince) {
+        this.ifModifiedSince = ifModifiedSince;
+        return this;
+    }
+
+    /**
+     * Sets the maximum number of redirections to follow.
+     *
+     * @return {@code this}
+     */
+    public HttpClient setMaxRedirects(int maxRedirects) {
+        this.maxRedirects = maxRedirects;
+        return this;
+    }
+
+    /**
+     * Sets an arbitrary HTTP header.
+     *
+     * @return {@code this}
+     */
+    public HttpClient setHeader(String key, String value) {
+        this.headers.put(key, value);
+        return this;
+    }
+
+    /**
+     * Sets arbitrary HTTP headers.
+     *
+     * @return {@code this}
+     */
+    public HttpClient setHeaders(Map<String, String> headers) {
+        this.headers.putAll(headers);
+        return this;
+    }
+
+    private static boolean isRedirect(final int statusCode) {
+        switch (statusCode) {
+            case HttpURLConnection.HTTP_MOVED_PERM: // 301
+            case HttpURLConnection.HTTP_MOVED_TEMP: // 302
+            case HttpURLConnection.HTTP_SEE_OTHER: // 303
+            case 307: // TEMPORARY_REDIRECT:
+            case 308: // PERMANENT_REDIRECT:
+                return true;
+            default:
+                return false;
+        }
+    }
+
+}

src/org/openstreetmap/josm/tools/WikiReader.java

@@ -52 +52 @@
      */
     public String read(String url) throws IOException {
         URL u = new URL(url);
-        try (BufferedReader in = Utils.openURLReader(u)) {
+        try (BufferedReader in = HttpClient.create(u).connect().getContentReader()) {
             boolean txt = url.endsWith("?format=txt");
             if (url.startsWith(getBaseUrlWiki()) && !txt)
                 return readFromTrac(in, u);
@@ -97 +97 @@
     }
 
     private String readLang(URL url) throws IOException {
-        try (BufferedReader in = Utils.openURLReader(url)) {
+        try (BufferedReader in = HttpClient.create(url).connect().getContentReader()) {
             return readFromTrac(in, url);
-        } catch (IOException e) {
-            Main.addNetworkError(url, Utils.getRootCause(e));
-            throw e;
         }
     }