Index: trunk/src/org/openstreetmap/josm/data/oauth/OAuthParameters.java =================================================================== --- trunk/src/org/openstreetmap/josm/data/oauth/OAuthParameters.java (revision 9171) +++ trunk/src/org/openstreetmap/josm/data/oauth/OAuthParameters.java (revision 9172) @@ -7,6 +7,4 @@ import oauth.signpost.OAuthConsumer; import oauth.signpost.OAuthProvider; -import oauth.signpost.basic.DefaultOAuthConsumer; -import oauth.signpost.basic.DefaultOAuthProvider; import org.openstreetmap.josm.Main; @@ -220,5 +218,5 @@ */ public OAuthConsumer buildConsumer() { - return new DefaultOAuthConsumer(consumerKey, consumerSecret); + return new SignpostAdapters.OAuthConsumer(consumerKey, consumerSecret); } @@ -232,5 +230,5 @@ public OAuthProvider buildProvider(OAuthConsumer consumer) { CheckParameterUtil.ensureParameterNotNull(consumer, "consumer"); - return new DefaultOAuthProvider( + return new SignpostAdapters.OAuthProvider( requestTokenUrl, accessTokenUrl, Index: trunk/src/org/openstreetmap/josm/data/oauth/SignpostAdapters.java =================================================================== --- trunk/src/org/openstreetmap/josm/data/oauth/SignpostAdapters.java (revision 9172) +++ trunk/src/org/openstreetmap/josm/data/oauth/SignpostAdapters.java (revision 9172) @@ -0,0 +1,138 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.oauth; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.util.Map; + +import oauth.signpost.AbstractOAuthConsumer; +import oauth.signpost.AbstractOAuthProvider; +import org.openstreetmap.josm.tools.HttpClient; + +/** + * Adapters to make {@link oauth.signpost} work with {@link HttpClient}. + */ +public class SignpostAdapters { + + private SignpostAdapters() { + } + + public static class OAuthProvider extends AbstractOAuthProvider { + + public OAuthProvider(String requestTokenEndpointUrl, String accessTokenEndpointUrl, String authorizationWebsiteUrl) { + super(requestTokenEndpointUrl, accessTokenEndpointUrl, authorizationWebsiteUrl); + } + + @Override + protected HttpRequest createRequest(String endpointUrl) throws Exception { + return new HttpRequest(HttpClient.create(new URL(endpointUrl), "GET")); + } + + @Override + protected HttpResponse sendRequest(oauth.signpost.http.HttpRequest request) throws Exception { + return new HttpResponse(((HttpRequest) request).request.connect()); + } + + @Override + protected void closeConnection(oauth.signpost.http.HttpRequest request, oauth.signpost.http.HttpResponse response) throws Exception { + if (response != null) { + ((HttpResponse) response).response.disconnect(); + } + } + } + + public static class OAuthConsumer extends AbstractOAuthConsumer { + + public OAuthConsumer(String consumerKey, String consumerSecret) { + super(consumerKey, consumerSecret); + } + + @Override + protected HttpRequest wrap(Object request) { + return new HttpRequest(((HttpClient) request)); + } + } + + private static class HttpRequest implements oauth.signpost.http.HttpRequest { + private final HttpClient request; + + public HttpRequest(HttpClient request) { + this.request = request; + } + + @Override + public void setHeader(String name, String value) { + request.setHeader(name, value); + } + + @Override + public String getMethod() { + return request.getRequestMethod(); + } + + @Override + public String getRequestUrl() { + return request.getURL().toExternalForm(); + } + + @Override + public String getContentType() { + return request.getRequestHeader("Content-Type"); + } + + @Override + public String getHeader(String name) { + return request.getRequestHeader(name); + } + + @Override + public InputStream getMessagePayload() throws IOException { + return null; + } + + @Override + public void setRequestUrl(String url) { + throw new IllegalStateException(); + } + + @Override + public Map getAllHeaders() { + throw new IllegalStateException(); + } + + @Override + public Object unwrap() { + throw new IllegalStateException(); + } + } + + private static class HttpResponse implements oauth.signpost.http.HttpResponse { + private final HttpClient.Response response; + + public HttpResponse(HttpClient.Response response) { + this.response = response; + } + + @Override + public int getStatusCode() throws IOException { + return response.getResponseCode(); + } + + @Override + public String getReasonPhrase() throws Exception { + return response.getResponseMessage(); + } + + @Override + public InputStream getContent() throws IOException { + return response.getContent(); + } + + @Override + public Object unwrap() { + throw new IllegalStateException(); + } + } + +} Index: trunk/src/org/openstreetmap/josm/gui/oauth/OsmOAuthAuthorizationClient.java =================================================================== --- trunk/src/org/openstreetmap/josm/gui/oauth/OsmOAuthAuthorizationClient.java (revision 9171) +++ trunk/src/org/openstreetmap/josm/gui/oauth/OsmOAuthAuthorizationClient.java (revision 9172) @@ -5,8 +5,5 @@ import java.io.BufferedReader; -import java.io.DataOutputStream; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; import java.lang.reflect.Field; import java.net.HttpURLConnection; @@ -25,5 +22,4 @@ import oauth.signpost.OAuthConsumer; import oauth.signpost.OAuthProvider; -import oauth.signpost.basic.DefaultOAuthProvider; import oauth.signpost.exception.OAuthException; @@ -36,4 +32,5 @@ import org.openstreetmap.josm.io.OsmTransferCanceledException; import org.openstreetmap.josm.tools.CheckParameterUtil; +import org.openstreetmap.josm.tools.HttpClient; import org.openstreetmap.josm.tools.Utils; @@ -47,5 +44,5 @@ private final OAuthProvider provider; private boolean canceled; - private HttpURLConnection connection; + private HttpClient.Response connection; private static class SessionId { @@ -99,11 +96,11 @@ */ public void cancel() { - DefaultOAuthProvider p = (DefaultOAuthProvider) provider; canceled = true; - if (p != null) { + if (provider != null) { try { - Field f = p.getClass().getDeclaredField("connection"); + // TODO + Field f = provider.getClass().getDeclaredField("connection"); f.setAccessible(true); - HttpURLConnection con = (HttpURLConnection) f.get(p); + HttpURLConnection con = (HttpURLConnection) f.get(provider); if (con != null) { con.disconnect(); @@ -195,9 +192,6 @@ } - protected String extractToken(HttpURLConnection connection) { - try ( - InputStream is = connection.getInputStream(); - BufferedReader r = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8)) - ) { + protected String extractToken() { + try (BufferedReader r = connection.getContentReader()) { String c; Pattern p = Pattern.compile(".*authenticity_token.*value=\"([^\"]+)\".*"); @@ -215,6 +209,6 @@ } - protected SessionId extractOsmSession(HttpURLConnection connection) { - List setCookies = connection.getHeaderFields().get("Set-Cookie"); + protected SessionId extractOsmSession() { + List setCookies = connection.getHeaderFields("Set-Cookie"); if (setCookies == null) // no cookies set @@ -234,5 +228,5 @@ if ("_osm_session".equals(kv[0])) { // osm session cookie found - String token = extractToken(connection); + String token = extractToken(); if (token == null) return null; @@ -309,11 +303,7 @@ URL url = new URL(sb.toString()); synchronized (this) { - connection = Utils.openHttpConnection(url); - } - connection.setRequestMethod("GET"); - connection.setDoInput(true); - connection.setDoOutput(false); - connection.connect(); - SessionId sessionId = extractOsmSession(connection); + connection = HttpClient.create(url).connect(); + } + SessionId sessionId = extractOsmSession(); if (sessionId == null) throw new OsmOAuthAuthorizationException( @@ -339,12 +329,9 @@ URL url = new URL(getAuthoriseUrl(requestToken)); synchronized (this) { - connection = Utils.openHttpConnection(url); - } - connection.setRequestMethod("GET"); - connection.setDoInput(true); - connection.setDoOutput(false); - connection.setRequestProperty("Cookie", "_osm_session=" + sessionId.id + "; _osm_username=" + sessionId.userName); - connection.connect(); - sessionId.token = extractToken(connection); + connection = HttpClient.create(url) + .setHeader("Cookie", "_osm_session=" + sessionId.id + "; _osm_username=" + sessionId.userName) + .connect(); + } + sessionId.token = extractToken(); if (sessionId.token == null) throw new OsmOAuthAuthorizationException(tr("OSM website did not return a session cookie in response to ''{0}'',", @@ -362,11 +349,5 @@ try { URL url = new URL(buildOsmLoginUrl()); - synchronized (this) { - connection = Utils.openHttpConnection(url); - } - connection.setRequestMethod("POST"); - connection.setDoInput(true); - connection.setDoOutput(true); - connection.setUseCaches(false); + final HttpClient client = HttpClient.create(url, "POST").useCache(false); Map parameters = new HashMap<>(); @@ -376,18 +357,13 @@ parameters.put("commit", "Login"); parameters.put("authenticity_token", sessionId.token); - - String request = buildPostRequest(parameters); - - connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); - connection.setRequestProperty("Content-Length", Integer.toString(request.length())); - connection.setRequestProperty("Cookie", "_osm_session=" + sessionId.id); + client.setRequestBody(buildPostRequest(parameters).getBytes(StandardCharsets.UTF_8)); + + client.setHeader("Content-Type", "application/x-www-form-urlencoded"); + client.setHeader("Cookie", "_osm_session=" + sessionId.id); // make sure we can catch 302 Moved Temporarily below - connection.setInstanceFollowRedirects(false); - - connection.connect(); - - try (DataOutputStream dout = new DataOutputStream(connection.getOutputStream())) { - dout.writeBytes(request); - dout.flush(); + client.setMaxRedirects(-1); + + synchronized (this) { + connection = client.connect(); } @@ -415,10 +391,6 @@ URL url = new URL(buildOsmLogoutUrl()); synchronized (this) { - connection = Utils.openHttpConnection(url); - } - connection.setRequestMethod("GET"); - connection.setDoInput(true); - connection.setDoOutput(false); - connection.connect(); + connection = HttpClient.create(url).connect(); + } } catch (IOException e) { throw new OsmOAuthAuthorizationException(e); @@ -461,21 +433,12 @@ try { URL url = new URL(oauthProviderParameters.getAuthoriseUrl()); - synchronized (this) { - connection = Utils.openHttpConnection(url); - } - connection.setRequestMethod("POST"); - connection.setDoInput(true); - connection.setDoOutput(true); - connection.setUseCaches(false); - connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); - connection.setRequestProperty("Content-Length", Integer.toString(request.length())); - connection.setRequestProperty("Cookie", "_osm_session=" + sessionId.id + "; _osm_username=" + sessionId.userName); - connection.setInstanceFollowRedirects(false); - - connection.connect(); - - try (DataOutputStream dout = new DataOutputStream(connection.getOutputStream())) { - dout.writeBytes(request); - dout.flush(); + final HttpClient client = HttpClient.create(url, "POST").useCache(false); + client.setHeader("Content-Type", "application/x-www-form-urlencoded"); + client.setHeader("Cookie", "_osm_session=" + sessionId.id + "; _osm_username=" + sessionId.userName); + client.setMaxRedirects(-1); + client.setRequestBody(request.getBytes(StandardCharsets.UTF_8)); + + synchronized (this) { + connection = client.connect(); } Index: trunk/src/org/openstreetmap/josm/gui/oauth/TestAccessTokenTask.java =================================================================== --- trunk/src/org/openstreetmap/josm/gui/oauth/TestAccessTokenTask.java (revision 9171) +++ trunk/src/org/openstreetmap/josm/gui/oauth/TestAccessTokenTask.java (revision 9172) @@ -25,5 +25,5 @@ import org.openstreetmap.josm.io.auth.DefaultAuthenticator; import org.openstreetmap.josm.tools.CheckParameterUtil; -import org.openstreetmap.josm.tools.Utils; +import org.openstreetmap.josm.tools.HttpClient; import org.openstreetmap.josm.tools.XmlParsingException; import org.w3c.dom.Document; @@ -46,5 +46,5 @@ private final Component parent; private final String apiUrl; - private HttpURLConnection connection; + private HttpClient.Response connection; /** @@ -80,5 +80,5 @@ protected void finish() {} - protected void sign(HttpURLConnection con) throws OAuthException { + protected void sign(HttpClient con) throws OAuthException { OAuthConsumer consumer = oauthParameters.buildConsumer(); consumer.setTokenWithSecret(token.getKey(), token.getSecret()); @@ -103,12 +103,10 @@ authenticatorEnabled = DefaultAuthenticator.getInstance().isEnabled(); DefaultAuthenticator.getInstance().setEnabled(false); + + final HttpClient client = HttpClient.create(url); + sign(client); synchronized (this) { - connection = Utils.openHttpConnection(url); + connection = client.connect(); } - - connection.setDoOutput(true); - connection.setRequestMethod("GET"); - sign(connection); - connection.connect(); if (connection.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) @@ -122,5 +120,5 @@ if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) throw new OsmApiException(connection.getResponseCode(), connection.getHeaderField("Error"), null); - Document d = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(connection.getInputStream()); + Document d = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(connection.getContent()); return OsmServerUserInfoReader.buildFromXML(d); } catch (SAXException | ParserConfigurationException e) { Index: trunk/src/org/openstreetmap/josm/gui/preferences/server/ApiUrlTestTask.java =================================================================== --- trunk/src/org/openstreetmap/josm/gui/preferences/server/ApiUrlTestTask.java (revision 9171) +++ trunk/src/org/openstreetmap/josm/gui/preferences/server/ApiUrlTestTask.java (revision 9172) @@ -20,5 +20,5 @@ import org.openstreetmap.josm.io.OsmTransferException; import org.openstreetmap.josm.tools.CheckParameterUtil; -import org.openstreetmap.josm.tools.Utils; +import org.openstreetmap.josm.tools.HttpClient; import org.xml.sax.InputSource; import org.xml.sax.SAXException; @@ -36,5 +36,5 @@ private boolean success; private final Component parent; - private HttpURLConnection connection; + private HttpClient.Response connection; /** @@ -177,10 +177,6 @@ synchronized (this) { - connection = Utils.openHttpConnection(capabilitiesUrl); - } - connection.setDoInput(true); - connection.setDoOutput(false); - connection.setRequestMethod("GET"); - connection.connect(); + connection = HttpClient.create(capabilitiesUrl).connect(); + } if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) { @@ -190,5 +186,5 @@ try { - Capabilities.CapabilitiesParser.parse(new InputSource(connection.getInputStream())); + Capabilities.CapabilitiesParser.parse(new InputSource(connection.getContent())); } catch (SAXException | ParserConfigurationException e) { Main.warn(e.getMessage()); Index: trunk/src/org/openstreetmap/josm/io/OsmApi.java =================================================================== --- trunk/src/org/openstreetmap/josm/io/OsmApi.java (revision 9171) +++ trunk/src/org/openstreetmap/josm/io/OsmApi.java (revision 9172) @@ -5,11 +5,5 @@ import static org.openstreetmap.josm.tools.I18n.trn; -import java.io.BufferedReader; -import java.io.BufferedWriter; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.io.StringReader; @@ -42,4 +36,5 @@ import org.openstreetmap.josm.io.Capabilities.CapabilitiesParser; import org.openstreetmap.josm.tools.CheckParameterUtil; +import org.openstreetmap.josm.tools.HttpClient; import org.openstreetmap.josm.tools.Utils; import org.openstreetmap.josm.tools.XmlParsingException; @@ -617,5 +612,4 @@ protected final String sendRequest(String requestMethod, String urlSuffix, String requestBody, ProgressMonitor monitor, boolean doAuthenticate, boolean fastFail) throws OsmTransferException { - StringBuilder responseBody = new StringBuilder(); int retries = fastFail ? 0 : getMaxRetries(); @@ -623,37 +617,23 @@ try { url = new URL(new URL(getBaseUrl()), urlSuffix); - Main.info(requestMethod + ' ' + url + "... "); - Main.debug(requestBody); - // fix #5369, see http://www.tikalk.com/java/forums/httpurlconnection-disable-keep-alive - activeConnection = Utils.openHttpConnection(url, false); - activeConnection.setConnectTimeout(fastFail ? 1000 : Main.pref.getInteger("socket.timeout.connect", 15)*1000); + final HttpClient client = HttpClient.create(url, requestMethod).keepAlive(false); if (fastFail) { - activeConnection.setReadTimeout(1000); + client.setReadTimeout(1000); } - activeConnection.setRequestMethod(requestMethod); if (doAuthenticate) { - addAuth(activeConnection); + addAuth(client); } if ("PUT".equals(requestMethod) || "POST".equals(requestMethod) || "DELETE".equals(requestMethod)) { - activeConnection.setDoOutput(true); - activeConnection.setRequestProperty("Content-type", "text/xml"); - try (OutputStream out = activeConnection.getOutputStream()) { - // It seems that certain bits of the Ruby API are very unhappy upon - // receipt of a PUT/POST message without a Content-length header, - // even if the request has no payload. - // Since Java will not generate a Content-length header unless - // we use the output stream, we create an output stream for PUT/POST - // even if there is no payload. - if (requestBody != null) { - try (BufferedWriter bwr = new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8))) { - bwr.write(requestBody); - bwr.flush(); - } - } - } + // It seems that certain bits of the Ruby API are very unhappy upon + // receipt of a PUT/POST message without a Content-length header, + // even if the request has no payload. + // Since Java will not generate a Content-length header unless + // we use the output stream, we create an output stream for PUT/POST + // even if there is no payload. + client.setRequestBody(requestBody.getBytes(StandardCharsets.UTF_8)); } - activeConnection.connect(); + activeConnection = client.connect(); Main.info(activeConnection.getResponseMessage()); int retCode = activeConnection.getResponseCode(); @@ -667,20 +647,6 @@ } - // populate return fields. - responseBody.setLength(0); - - // If the API returned an error code like 403 forbidden, getInputStream will fail with an IOException. - InputStream i = getConnectionStream(); - if (i != null) { - // the input stream can be null if both the input and the error stream - // are null. Seems to be the case if the OSM server replies a 401 Unauthorized, see #3887. - String s; - try (BufferedReader in = new BufferedReader(new InputStreamReader(i, StandardCharsets.UTF_8))) { - while ((s = in.readLine()) != null) { - responseBody.append(s); - responseBody.append('\n'); - } - } - } + final String responseBody = activeConnection.fetchContent(); + String errorHeader = null; // Look for a detailed error message from the server @@ -693,13 +659,9 @@ activeConnection.disconnect(); - if (Main.isDebugEnabled()) { - Main.debug("RESPONSE: "+ activeConnection.getHeaderFields()); - } - errorHeader = errorHeader == null ? null : errorHeader.trim(); - String errorBody = responseBody.length() == 0 ? null : responseBody.toString().trim(); + String errorBody = responseBody.length() == 0 ? null : responseBody.trim(); switch(retCode) { case HttpURLConnection.HTTP_OK: - return responseBody.toString(); + return responseBody; case HttpURLConnection.HTTP_GONE: throw new OsmApiPrimitiveGoneException(errorHeader, errorBody); @@ -729,13 +691,4 @@ } - private InputStream getConnectionStream() { - try { - return activeConnection.getInputStream(); - } catch (IOException ioe) { - Main.warn(ioe); - return activeConnection.getErrorStream(); - } - } - /** * Replies the API capabilities. Index: trunk/src/org/openstreetmap/josm/io/OsmConnection.java =================================================================== --- trunk/src/org/openstreetmap/josm/io/OsmConnection.java (revision 9171) +++ trunk/src/org/openstreetmap/josm/io/OsmConnection.java (revision 9172) @@ -5,5 +5,4 @@ import java.net.Authenticator.RequestorType; -import java.net.HttpURLConnection; import java.nio.ByteBuffer; import java.nio.CharBuffer; @@ -19,4 +18,5 @@ import org.openstreetmap.josm.io.auth.CredentialsManager; import org.openstreetmap.josm.tools.Base64; +import org.openstreetmap.josm.tools.HttpClient; import oauth.signpost.OAuthConsumer; @@ -31,17 +31,6 @@ public class OsmConnection { protected boolean cancel; - protected HttpURLConnection activeConnection; + protected HttpClient.Response activeConnection; protected OAuthParameters oauthParameters; - - /** - * Initialize the http defaults and the authenticator. - */ - static { - try { - HttpURLConnection.setFollowRedirects(true); - } catch (SecurityException e) { - Main.error(e); - } - } /** @@ -50,16 +39,4 @@ public void cancel() { cancel = true; - synchronized (this) { - if (activeConnection != null) { - activeConnection.setConnectTimeout(100); - activeConnection.setReadTimeout(100); - } - } - try { - Thread.sleep(100); - } catch (InterruptedException ex) { - Main.warn("InterruptedException in "+getClass().getSimpleName()+" during cancel"); - } - synchronized (this) { if (activeConnection != null) { @@ -75,5 +52,5 @@ * @throws OsmTransferException if something went wrong. Check for nested exceptions */ - protected void addBasicAuthorizationHeader(HttpURLConnection con) throws OsmTransferException { + protected void addBasicAuthorizationHeader(HttpClient con) throws OsmTransferException { CharsetEncoder encoder = StandardCharsets.UTF_8.newEncoder(); CredentialsAgentResponse response; @@ -98,5 +75,5 @@ try { ByteBuffer bytes = encoder.encode(CharBuffer.wrap(token)); - con.addRequestProperty("Authorization", "Basic "+Base64.encode(bytes)); + con.setHeader("Authorization", "Basic "+Base64.encode(bytes)); } catch (CharacterCodingException e) { throw new OsmTransferException(e); @@ -113,5 +90,5 @@ * @throws OsmTransferException if signing fails */ - protected void addOAuthAuthorizationHeader(HttpURLConnection connection) throws OsmTransferException { + protected void addOAuthAuthorizationHeader(HttpClient connection) throws OsmTransferException { if (oauthParameters == null) { oauthParameters = OAuthParameters.createFromPreferences(Main.pref); @@ -129,5 +106,5 @@ } - protected void addAuth(HttpURLConnection connection) throws OsmTransferException { + protected void addAuth(HttpClient connection) throws OsmTransferException { String authMethod = Main.pref.get("osm-server.auth-method", "basic"); if ("basic".equals(authMethod)) { Index: trunk/src/org/openstreetmap/josm/io/OsmServerReader.java =================================================================== --- trunk/src/org/openstreetmap/josm/io/OsmServerReader.java (revision 9171) +++ trunk/src/org/openstreetmap/josm/io/OsmServerReader.java (revision 9172) @@ -4,17 +4,9 @@ import static org.openstreetmap.josm.tools.I18n.tr; -import java.io.BufferedReader; -import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; -import java.nio.charset.StandardCharsets; import java.util.List; -import java.util.Map; -import java.util.zip.GZIPInputStream; -import java.util.zip.Inflater; -import java.util.zip.InflaterInputStream; import org.openstreetmap.josm.Main; @@ -23,5 +15,5 @@ import org.openstreetmap.josm.data.osm.DataSet; import org.openstreetmap.josm.gui.progress.ProgressMonitor; -import org.openstreetmap.josm.tools.Utils; +import org.openstreetmap.josm.tools.HttpClient; /** @@ -129,32 +121,15 @@ throw new OsmTransferException(e); } - try { - // fix #7640, see http://www.tikalk.com/java/forums/httpurlconnection-disable-keep-alive - activeConnection = Utils.openHttpConnection(url, false); - } catch (Exception e) { - throw new OsmTransferException(tr("Failed to open connection to API {0}.", url.toExternalForm()), e); - } - Utils.setupURLConnection(activeConnection); - if (cancel) { - activeConnection.disconnect(); - return null; - } - + + final HttpClient client = HttpClient.create(url); + client.setReasonForRequest(reason); if (doAuthenticate) { - addAuth(activeConnection); + addAuth(client); } if (cancel) throw new OsmTransferCanceledException("Operation canceled"); - if (Main.pref.getBoolean("osm-server.use-compression", true)) { - activeConnection.setRequestProperty("Accept-Encoding", "gzip, deflate"); - } try { - if (reason != null && !reason.isEmpty()) { - Main.info("GET " + url + " (" + reason + ')'); - } else { - Main.info("GET " + url); - } - activeConnection.connect(); + activeConnection = client.connect(); } catch (Exception e) { Main.error(e); @@ -165,7 +140,4 @@ } try { - if (Main.isDebugEnabled()) { - Main.debug("RESPONSE: "+activeConnection.getHeaderFields()); - } if (activeConnection.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) throw new OsmApiException(HttpURLConnection.HTTP_UNAUTHORIZED, null, null); @@ -174,30 +146,15 @@ throw new OsmTransferCanceledException("Proxy Authentication Required"); - String encoding = activeConnection.getContentEncoding(); if (activeConnection.getResponseCode() != HttpURLConnection.HTTP_OK) { String errorHeader = activeConnection.getHeaderField("Error"); - StringBuilder errorBody = new StringBuilder(); - try { - InputStream i = fixEncoding(activeConnection.getErrorStream(), encoding); - if (i != null) { - BufferedReader in = new BufferedReader(new InputStreamReader(i, StandardCharsets.UTF_8)); - String s; - while ((s = in.readLine()) != null) { - errorBody.append(s); - errorBody.append('\n'); - } - } - } catch (Exception e) { - errorBody.append(tr("Reading error text failed.")); - } - - throw new OsmApiException(activeConnection.getResponseCode(), errorHeader, errorBody.toString(), url.toString()); + final String errorBody = activeConnection.fetchContent(); + throw new OsmApiException(activeConnection.getResponseCode(), errorHeader, errorBody, url.toString()); } InputStream in = new ProgressInputStream(activeConnection, progressMonitor); if (uncompressAccordingToContentDisposition) { - in = uncompressAccordingToContentDisposition(in, activeConnection.getHeaderFields()); + activeConnection.uncompressAccordingToContentDisposition(true); } - return fixEncoding(in, encoding); + return in; } catch (OsmTransferException e) { throw e; @@ -210,24 +167,4 @@ } - private static InputStream fixEncoding(InputStream stream, String encoding) throws IOException { - if ("gzip".equalsIgnoreCase(encoding)) { - stream = new GZIPInputStream(stream); - } else if ("deflate".equalsIgnoreCase(encoding)) { - stream = new InflaterInputStream(stream, new Inflater(true)); - } - return stream; - } - - private InputStream uncompressAccordingToContentDisposition(InputStream stream, Map> headerFields) throws IOException { - List field = headerFields.get("Content-Disposition"); - if (field != null && field.toString().contains(".gz\"")) { - return Compression.GZIP.getUncompressedInputStream(stream); - } else if (field != null && field.toString().contains(".bz2\"")) { - return Compression.BZIP2.getUncompressedInputStream(stream); - } else { - return stream; - } - } - /** * Download OSM files from somewhere Index: trunk/src/org/openstreetmap/josm/io/ProgressInputStream.java =================================================================== --- trunk/src/org/openstreetmap/josm/io/ProgressInputStream.java (revision 9171) +++ trunk/src/org/openstreetmap/josm/io/ProgressInputStream.java (revision 9172) @@ -10,4 +10,5 @@ import org.openstreetmap.josm.gui.progress.NullProgressMonitor; import org.openstreetmap.josm.gui.progress.ProgressMonitor; +import org.openstreetmap.josm.tools.HttpClient; /** @@ -18,12 +19,26 @@ private final InputStream in; + private final long size; private int readSoFar; private int lastDialogUpdate; - private boolean sizeKnown; - private final URLConnection connection; private final ProgressMonitor progressMonitor; + public ProgressInputStream(InputStream in, long size, ProgressMonitor progressMonitor) { + if (progressMonitor == null) { + progressMonitor = NullProgressMonitor.INSTANCE; + } + this.in = in; + this.size = size; + this.progressMonitor = progressMonitor; + progressMonitor.beginTask(tr("Contacting OSM Server..."), 1); + progressMonitor.indeterminateSubTask(null); + initProgressMonitor(); + } + + public ProgressInputStream(HttpClient.Response response, ProgressMonitor progressMonitor) throws IOException { + this(response.getContent(), response.getContentLength(), progressMonitor); + } + public ProgressInputStream(URLConnection con, ProgressMonitor progressMonitor) throws OsmTransferException { - this.connection = con; if (progressMonitor == null) { progressMonitor = NullProgressMonitor.INSTANCE; @@ -35,4 +50,5 @@ try { this.in = con.getInputStream(); + this.size = con.getContentLength(); } catch (IOException e) { progressMonitor.finishTask(); @@ -41,7 +57,12 @@ throw new OsmTransferException(e); } + initProgressMonitor(); + } - updateSize(); - if (!sizeKnown) { + protected void initProgressMonitor() { + if (size > 0) { + progressMonitor.subTask(tr("Downloading OSM data...")); + progressMonitor.setTicksCount((int) size); + } else { progressMonitor.indeterminateSubTask(tr("Downloading OSM data...")); } @@ -82,9 +103,8 @@ private void advanceTicker(int amount) { readSoFar += amount; - updateSize(); if (readSoFar / 1024 != lastDialogUpdate) { lastDialogUpdate++; - if (sizeKnown) { + if (size > 0) { progressMonitor.setTicks(readSoFar); } @@ -92,11 +112,3 @@ } } - - private void updateSize() { - if (!sizeKnown && connection.getContentLength() > 0) { - sizeKnown = true; - progressMonitor.subTask(tr("Downloading OSM data...")); - progressMonitor.setTicksCount(connection.getContentLength()); - } - } } Index: trunk/src/org/openstreetmap/josm/tools/HttpClient.java =================================================================== --- trunk/src/org/openstreetmap/josm/tools/HttpClient.java (revision 9171) +++ trunk/src/org/openstreetmap/josm/tools/HttpClient.java (revision 9172) @@ -11,7 +11,10 @@ import java.net.HttpURLConnection; import java.net.URL; +import java.util.List; import java.util.Map; import java.util.Scanner; -import java.util.concurrent.ConcurrentHashMap; +import java.util.TreeMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.zip.GZIPInputStream; @@ -30,55 +33,43 @@ private int connectTimeout = Main.pref.getInteger("socket.timeout.connect", 15) * 1000; private int readTimeout = Main.pref.getInteger("socket.timeout.read", 30) * 1000; - private String accept; - private String contentType; - private String acceptEncoding = "gzip"; - private long contentLength; private byte[] requestBody; private long ifModifiedSince; - private final Map headers = new ConcurrentHashMap<>(); + private final Map headers = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); private int maxRedirects = Main.pref.getInteger("socket.maxredirects", 5); private boolean useCache; - private boolean keepAlive; + private String reasonForRequest; private HttpClient(URL url, String requestMethod) { this.url = url; this.requestMethod = requestMethod; + this.headers.put("Accept-Encoding", "gzip"); } public Response connect() throws IOException { final HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestMethod(requestMethod); connection.setRequestProperty("User-Agent", Version.getInstance().getFullAgentString()); connection.setConnectTimeout(connectTimeout); connection.setReadTimeout(readTimeout); - if (accept != null) { - connection.setRequestProperty("Accept", accept); - } - if (contentType != null) { - connection.setRequestProperty("Content-Type", contentType); - } - if (acceptEncoding != null) { - connection.setRequestProperty("Accept-Encoding", acceptEncoding); - } - if (contentLength > 0) { - connection.setRequestProperty("Content-Length", String.valueOf(contentLength)); - } + connection.setInstanceFollowRedirects(maxRedirects > 0); + if (ifModifiedSince > 0) { + connection.setIfModifiedSince(ifModifiedSince); + } + connection.setUseCaches(useCache); + if (!useCache) { + connection.setRequestProperty("Cache-Control", "no-cache"); + } + for (Map.Entry header : headers.entrySet()) { + if (header.getValue() != null) { + connection.setRequestProperty(header.getKey(), header.getValue()); + } + } + if ("PUT".equals(requestMethod) || "POST".equals(requestMethod) || "DELETE".equals(requestMethod)) { + headers.put("Content-Length", String.valueOf(requestBody.length)); connection.setDoOutput(true); try (OutputStream out = new BufferedOutputStream(connection.getOutputStream())) { out.write(requestBody); } - } - if (ifModifiedSince > 0) { - connection.setIfModifiedSince(ifModifiedSince); - } - connection.setUseCaches(useCache); - if (!useCache) { - connection.setRequestProperty("Cache-Control", "no-cache"); - } - if (!keepAlive) { - connection.setRequestProperty("Connection", "close"); - } - for (Map.Entry header : headers.entrySet()) { - connection.setRequestProperty(header.getKey(), header.getValue()); } @@ -87,5 +78,12 @@ try { connection.connect(); - Main.info("{0} {1} => {2}", requestMethod, url, connection.getResponseCode()); + if (reasonForRequest != null && "".equalsIgnoreCase(reasonForRequest)) { + Main.info("{0} {1} ({2}) -> {3}", requestMethod, url, reasonForRequest, connection.getResponseCode()); + } else { + Main.info("{0} {1} -> {2}", requestMethod, url, connection.getResponseCode()); + } + if (Main.isDebugEnabled()) { + Main.debug("RESPONSE: " + connection.getHeaderFields()); + } } catch (IOException e) { //noinspection ThrowableResultOfMethodCallIgnored @@ -105,5 +103,5 @@ Main.info(tr("Download redirected to ''{0}''", redirectLocation)); return connect(); - } else { + } else if (maxRedirects == 0) { String msg = tr("Too many redirects to the download URL detected. Aborting."); throw new IOException(msg); @@ -126,9 +124,13 @@ private final HttpURLConnection connection; private final int responseCode; + private final String responseMessage; private boolean uncompress; + private boolean uncompressAccordingToContentDisposition; private Response(HttpURLConnection connection) throws IOException { + CheckParameterUtil.ensureParameterNotNull(connection, "connection"); this.connection = connection; this.responseCode = connection.getResponseCode(); + this.responseMessage = connection.getResponseMessage(); } @@ -144,7 +146,29 @@ } + public Response uncompressAccordingToContentDisposition(boolean uncompressAccordingToContentDisposition) { + this.uncompressAccordingToContentDisposition = uncompressAccordingToContentDisposition; + return this; + } + + /** + * @see HttpURLConnection#getURL() + */ + public URL getURL() { + return connection.getURL(); + } + + /** + * @see HttpURLConnection#getRequestMethod() + */ + public String getRequestMethod() { + return connection.getRequestMethod(); + } + /** * Returns an input stream that reads from this HTTP connection, or, * error stream if the connection failed but the server sent useful data. + * + * Note: the return value can be null, if both the input and the error stream are null. + * Seems to be the case if the OSM server replies a 401 Unauthorized, see #3887 * * @see HttpURLConnection#getInputStream() @@ -160,8 +184,17 @@ in = "gzip".equalsIgnoreCase(getContentEncoding()) ? new GZIPInputStream(in) : in; if (uncompress) { - return Compression.forContentType(getContentType()).getUncompressedInputStream(in); - } else { - return in; - } + final String contentType = getContentType(); + Main.debug("Uncompressing input stream according to Content-Type header: {0}", contentType); + in = Compression.forContentType(contentType).getUncompressedInputStream(in); + } + if (uncompressAccordingToContentDisposition) { + final String contentDisposition = getHeaderField("Content-Disposition"); + final Matcher matcher = Pattern.compile("filename=\"([^\"]+)\"").matcher(contentDisposition); + if (matcher.find()) { + Main.debug("Uncompressing input stream according to Content-Disposition header: {0}", contentDisposition); + in = Compression.byExtension(matcher.group(1)).getUncompressedInputStream(in); + } + } + return in; } @@ -183,6 +216,6 @@ */ public String fetchContent() throws IOException { - try (Scanner scanner = new Scanner(getContentReader())) { - return scanner.useDelimiter("\\A").next(); + try (Scanner scanner = new Scanner(getContentReader()).useDelimiter("\\A")) { + return scanner.hasNext() ? scanner.next() : ""; } } @@ -198,4 +231,13 @@ /** + * Gets the response message from this HTTP connection. + * + * @see HttpURLConnection#getResponseMessage() + */ + public String getResponseMessage() { + return responseMessage; + } + + /** * Returns the {@code Content-Encoding} header. */ @@ -219,7 +261,31 @@ /** + * @see HttpURLConnection#getHeaderField(String) + */ + public String getHeaderField(String name) { + return connection.getHeaderField(name); + } + + /** + * @see HttpURLConnection#getHeaderFields() + */ + public List getHeaderFields(String name) { + return connection.getHeaderFields().get(name); + } + + /** * @see HttpURLConnection#disconnect() */ public void disconnect() { + // TODO is this block necessary for disconnecting? + // Fix upload aborts - see #263 + connection.setConnectTimeout(100); + connection.setReadTimeout(100); + try { + Thread.sleep(100); + } catch (InterruptedException ex) { + Main.warn("InterruptedException in " + getClass().getSimpleName() + " during cancel"); + } + connection.disconnect(); } @@ -239,5 +305,5 @@ * Creates a new instance for the given URL and a {@code GET} request * - * @param url the URL + * @param url the URL * @param requestMethod the HTTP request method to perform when calling * @return a new instance @@ -245,4 +311,28 @@ public static HttpClient create(URL url, String requestMethod) { return new HttpClient(url, requestMethod); + } + + /** + * Returns the URL set for this connection. + * @see #create(URL) + * @see #create(URL, String) + */ + public URL getURL() { + return url; + } + + /** + * Returns the request method set for this connection. + * @see #create(URL, String) + */ + public String getRequestMethod() { + return requestMethod; + } + + /** + * Returns the set value for the given {@code header}. + */ + public String getRequestHeader(String header) { + return headers.get(header); } @@ -268,6 +358,5 @@ */ public HttpClient keepAlive(boolean keepAlive) { - this.keepAlive = keepAlive; - return this; + return setHeader("Connection", keepAlive ? null : "close"); } @@ -297,36 +386,5 @@ */ public HttpClient setAccept(String accept) { - this.accept = accept; - return this; - } - - /** - * Sets the {@code Content-Type} header. - * - * @return {@code this} - */ - public HttpClient setContentType(String contentType) { - this.contentType = contentType; - return this; - } - - /** - * Sets the {@code Accept-Encoding} header. - * - * @return {@code this} - */ - public HttpClient setAcceptEncoding(String acceptEncoding) { - this.acceptEncoding = acceptEncoding; - return this; - } - - /** - * Sets the {@code Content-Length} header for {@code PUT}/{@code POST} requests. - * - * @return {@code this} - */ - public HttpClient setContentLength(long contentLength) { - this.contentLength = contentLength; - return this; + return setHeader("Accept", accept); } @@ -354,4 +412,7 @@ * Sets the maximum number of redirections to follow. * + * Set {@code maxRedirects} to {@code -1} in order to ignore redirects, i.e., + * to not throw an {@link IOException} in {@link #connect()}. + * * @return {@code this} */ @@ -378,4 +439,12 @@ public HttpClient setHeaders(Map headers) { this.headers.putAll(headers); + return this; + } + + /** + * Sets a reason to show on console. Can be {@code null} if no reason is given. + */ + public HttpClient setReasonForRequest(String reasonForRequest) { + this.reasonForRequest = reasonForRequest; return this; }