source: josm/trunk/src/org/openstreetmap/josm/tools/HttpClient.java@ 9454

Last change on this file since 9454 was 9396, checked in by simon04, 8 years ago

fix #7122 - Improve HTTP authentication for parallel requests (TMS/WMS imagery)
File size: 23.1 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.tools;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.BufferedOutputStream;
7import java.io.BufferedReader;
8import java.io.IOException;
9import java.io.InputStream;
10import java.io.OutputStream;
11import java.net.HttpRetryException;
12import java.net.HttpURLConnection;
13import java.net.URL;
14import java.util.List;
15import java.util.Locale;
16import java.util.Map;
17import java.util.Scanner;
18import java.util.TreeMap;
19import java.util.regex.Matcher;
20import java.util.regex.Pattern;
21import java.util.zip.GZIPInputStream;
22
23import org.openstreetmap.josm.Main;
24import org.openstreetmap.josm.data.Version;
25import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
26import org.openstreetmap.josm.gui.progress.ProgressMonitor;
27import org.openstreetmap.josm.io.Compression;
28import org.openstreetmap.josm.io.ProgressInputStream;
29import org.openstreetmap.josm.io.ProgressOutputStream;
30import org.openstreetmap.josm.io.UTFInputStreamReader;
31import org.openstreetmap.josm.io.auth.DefaultAuthenticator;
32
33/**
34 * Provides a uniform access for a HTTP/HTTPS server. This class should be used in favour of {@link HttpURLConnection}.
35 * @since 9168
36 */
37public final class HttpClient {
38
39 private URL url;
40 private final String requestMethod;
41 private int connectTimeout = Main.pref.getInteger("socket.timeout.connect", 15) * 1000;
42 private int readTimeout = Main.pref.getInteger("socket.timeout.read", 30) * 1000;
43 private byte[] requestBody;
44 private long ifModifiedSince;
45 private final Map<String, String> headers = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
46 private int maxRedirects = Main.pref.getInteger("socket.maxredirects", 5);
47 private boolean useCache;
48 private String reasonForRequest;
49 private transient HttpURLConnection connection; // to allow disconnecting before `response` is set
50 private transient Response response;
51
52 private HttpClient(URL url, String requestMethod) {
53 this.url = url;
54 this.requestMethod = requestMethod;
55 this.headers.put("Accept-Encoding", "gzip");
56 }
57
58 /**
59 * Opens the HTTP connection.
60 * @return HTTP response
61 * @throws IOException if any I/O error occurs
62 */
63 public Response connect() throws IOException {
64 return connect(null);
65 }
66
67 /**
68 * Opens the HTTP connection.
69 * @param progressMonitor progress monitor
70 * @return HTTP response
71 * @throws IOException if any I/O error occurs
72 * @since 9179
73 */
74 public Response connect(ProgressMonitor progressMonitor) throws IOException {
75 if (progressMonitor == null) {
76 progressMonitor = NullProgressMonitor.INSTANCE;
77 }
78 final HttpURLConnection connection = (HttpURLConnection) url.openConnection();
79 this.connection = connection;
80 connection.setRequestMethod(requestMethod);
81 connection.setRequestProperty("User-Agent", Version.getInstance().getFullAgentString());
82 connection.setConnectTimeout(connectTimeout);
83 connection.setReadTimeout(readTimeout);
84 connection.setInstanceFollowRedirects(false); // we do that ourselves
85 if (ifModifiedSince > 0) {
86 connection.setIfModifiedSince(ifModifiedSince);
87 }
88 connection.setUseCaches(useCache);
89 if (!useCache) {
90 connection.setRequestProperty("Cache-Control", "no-cache");
91 }
92 for (Map.Entry<String, String> header : headers.entrySet()) {
93 if (header.getValue() != null) {
94 connection.setRequestProperty(header.getKey(), header.getValue());
95 }
96 }
97
98 progressMonitor.beginTask(tr("Contacting Server..."), 1);
99 progressMonitor.indeterminateSubTask(null);
100
101 if ("PUT".equals(requestMethod) || "POST".equals(requestMethod) || "DELETE".equals(requestMethod)) {
102 Main.info("{0} {1} ({2}) ...", requestMethod, url, Utils.getSizeString(requestBody.length, Locale.getDefault()));
103 connection.setFixedLengthStreamingMode(requestBody.length);
104 connection.setDoOutput(true);
105 try (OutputStream out = new BufferedOutputStream(
106 new ProgressOutputStream(connection.getOutputStream(), requestBody.length, progressMonitor))) {
107 out.write(requestBody);
108 }
109 }
110
111 boolean successfulConnection = false;
112 try {
113 try {
114 connection.connect();
115 final boolean hasReason = reasonForRequest != null && !reasonForRequest.isEmpty();
116 Main.info("{0} {1}{2} -> {3}{4}",
117 requestMethod, url, hasReason ? " (" + reasonForRequest + ")" : "",
118 connection.getResponseCode(),
119 connection.getContentLengthLong() > 0
120 ? " (" + Utils.getSizeString(connection.getContentLengthLong(), Locale.getDefault()) + ")"
121 : ""
122 );
123 if (Main.isDebugEnabled()) {
124 Main.debug("RESPONSE: " + connection.getHeaderFields());
125 }
126 if (DefaultAuthenticator.getInstance().isEnabled() && connection.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) {
127 DefaultAuthenticator.getInstance().addFailedCredentialHost(url.getHost());
128 }
129 } catch (IOException e) {
130 Main.info("{0} {1} -> !!!", requestMethod, url);
131 Main.warn(e);
132 //noinspection ThrowableResultOfMethodCallIgnored
133 Main.addNetworkError(url, Utils.getRootCause(e));
134 throw e;
135 }
136 if (isRedirect(connection.getResponseCode())) {
137 final String redirectLocation = connection.getHeaderField("Location");
138 if (redirectLocation == null) {
139 /* I18n: argument is HTTP response code */
140 String msg = tr("Unexpected response from HTTP server. Got {0} response without ''Location'' header." +
141 " Can''t redirect. Aborting.", connection.getResponseCode());
142 throw new IOException(msg);
143 } else if (maxRedirects > 0) {
144 url = new URL(url, redirectLocation);
145 maxRedirects--;
146 Main.info(tr("Download redirected to ''{0}''", redirectLocation));
147 return connect();
148 } else if (maxRedirects == 0) {
149 String msg = tr("Too many redirects to the download URL detected. Aborting.");
150 throw new IOException(msg);
151 }
152 }
153 response = new Response(connection, progressMonitor);
154 successfulConnection = true;
155 return response;
156 } finally {
157 if (!successfulConnection) {
158 connection.disconnect();
159 }
160 }
161 }
162
163 /**
164 * Returns the HTTP response which is set only after calling {@link #connect()}.
165 * Calling this method again, returns the identical object (unless another {@link #connect()} is performed).
166 *
167 * @return the HTTP response
168 * @since 9309
169 */
170 public Response getResponse() {
171 return response;
172 }
173
174 /**
175 * A wrapper for the HTTP response.
176 */
177 public static final class Response {
178 private final HttpURLConnection connection;
179 private final ProgressMonitor monitor;
180 private final int responseCode;
181 private final String responseMessage;
182 private boolean uncompress;
183 private boolean uncompressAccordingToContentDisposition;
184
185 private Response(HttpURLConnection connection, ProgressMonitor monitor) throws IOException {
186 CheckParameterUtil.ensureParameterNotNull(connection, "connection");
187 CheckParameterUtil.ensureParameterNotNull(monitor, "monitor");
188 this.connection = connection;
189 this.monitor = monitor;
190 this.responseCode = connection.getResponseCode();
191 this.responseMessage = connection.getResponseMessage();
192 }
193
194 /**
195 * Sets whether {@link #getContent()} should uncompress the input stream if necessary.
196 *
197 * @param uncompress whether the input stream should be uncompressed if necessary
198 * @return {@code this}
199 */
200 public Response uncompress(boolean uncompress) {
201 this.uncompress = uncompress;
202 return this;
203 }
204
205 /**
206 * Sets whether {@link #getContent()} should uncompress the input stream according to {@code Content-Disposition}
207 * HTTP header.
208 * @param uncompressAccordingToContentDisposition whether the input stream should be uncompressed according to
209 * {@code Content-Disposition}
210 * @return {@code this}
211 * @since 9172
212 */
213 public Response uncompressAccordingToContentDisposition(boolean uncompressAccordingToContentDisposition) {
214 this.uncompressAccordingToContentDisposition = uncompressAccordingToContentDisposition;
215 return this;
216 }
217
218 /**
219 * Returns the URL.
220 * @return the URL
221 * @see HttpURLConnection#getURL()
222 * @since 9172
223 */
224 public URL getURL() {
225 return connection.getURL();
226 }
227
228 /**
229 * Returns the request method.
230 * @return the HTTP request method
231 * @see HttpURLConnection#getRequestMethod()
232 * @since 9172
233 */
234 public String getRequestMethod() {
235 return connection.getRequestMethod();
236 }
237
238 /**
239 * Returns an input stream that reads from this HTTP connection, or,
240 * error stream if the connection failed but the server sent useful data.
241 * <p>
242 * Note: the return value can be null, if both the input and the error stream are null.
243 * Seems to be the case if the OSM server replies a 401 Unauthorized, see #3887
244 * @return input or error stream
245 * @throws IOException if any I/O error occurs
246 *
247 * @see HttpURLConnection#getInputStream()
248 * @see HttpURLConnection#getErrorStream()
249 */
250 @SuppressWarnings("resource")
251 public InputStream getContent() throws IOException {
252 InputStream in;
253 try {
254 in = connection.getInputStream();
255 } catch (IOException ioe) {
256 in = connection.getErrorStream();
257 }
258 if (in != null) {
259 in = new ProgressInputStream(in, getContentLength(), monitor);
260 in = "gzip".equalsIgnoreCase(getContentEncoding()) ? new GZIPInputStream(in) : in;
261 Compression compression = Compression.NONE;
262 if (uncompress) {
263 final String contentType = getContentType();
264 Main.debug("Uncompressing input stream according to Content-Type header: {0}", contentType);
265 compression = Compression.forContentType(contentType);
266 }
267 if (uncompressAccordingToContentDisposition && Compression.NONE.equals(compression)) {
268 final String contentDisposition = getHeaderField("Content-Disposition");
269 final Matcher matcher = Pattern.compile("filename=\"([^\"]+)\"").matcher(contentDisposition);
270 if (matcher.find()) {
271 Main.debug("Uncompressing input stream according to Content-Disposition header: {0}", contentDisposition);
272 compression = Compression.byExtension(matcher.group(1));
273 }
274 }
275 in = compression.getUncompressedInputStream(in);
276 }
277 return in;
278 }
279
280 /**
281 * Returns {@link #getContent()} wrapped in a buffered reader.
282 *
283 * Detects Unicode charset in use utilizing {@link UTFInputStreamReader}.
284 * @return buffered reader
285 * @throws IOException if any I/O error occurs
286 */
287 public BufferedReader getContentReader() throws IOException {
288 return new BufferedReader(
289 UTFInputStreamReader.create(getContent())
290 );
291 }
292
293 /**
294 * Fetches the HTTP response as String.
295 * @return the response
296 * @throws IOException if any I/O error occurs
297 */
298 @SuppressWarnings("resource")
299 public String fetchContent() throws IOException {
300 try (Scanner scanner = new Scanner(getContentReader()).useDelimiter("\\A")) {
301 return scanner.hasNext() ? scanner.next() : "";
302 }
303 }
304
305 /**
306 * Gets the response code from this HTTP connection.
307 * @return HTTP response code
308 *
309 * @see HttpURLConnection#getResponseCode()
310 */
311 public int getResponseCode() {
312 return responseCode;
313 }
314
315 /**
316 * Gets the response message from this HTTP connection.
317 * @return HTTP response message
318 *
319 * @see HttpURLConnection#getResponseMessage()
320 * @since 9172
321 */
322 public String getResponseMessage() {
323 return responseMessage;
324 }
325
326 /**
327 * Returns the {@code Content-Encoding} header.
328 * @return {@code Content-Encoding} HTTP header
329 * @see HttpURLConnection#getContentEncoding()
330 */
331 public String getContentEncoding() {
332 return connection.getContentEncoding();
333 }
334
335 /**
336 * Returns the {@code Content-Type} header.
337 * @return {@code Content-Type} HTTP header
338 */
339 public String getContentType() {
340 return connection.getHeaderField("Content-Type");
341 }
342
343 /**
344 * Returns the {@code Expire} header.
345 * @return {@code Expire} HTTP header
346 * @see HttpURLConnection#getExpiration()
347 * @since 9232
348 */
349 public long getExpiration() {
350 return connection.getExpiration();
351 }
352
353 /**
354 * Returns the {@code Last-Modified} header.
355 * @return {@code Last-Modified} HTTP header
356 * @see HttpURLConnection#getLastModified()
357 * @since 9232
358 */
359 public long getLastModified() {
360 return connection.getLastModified();
361 }
362
363 /**
364 * Returns the {@code Content-Length} header.
365 * @return {@code Content-Length} HTTP header
366 * @see HttpURLConnection#getContentLengthLong()
367 */
368 public long getContentLength() {
369 return connection.getContentLengthLong();
370 }
371
372 /**
373 * Returns the value of the named header field.
374 * @param name the name of a header field
375 * @return the value of the named header field, or {@code null} if there is no such field in the header
376 * @see HttpURLConnection#getHeaderField(String)
377 * @since 9172
378 */
379 public String getHeaderField(String name) {
380 return connection.getHeaderField(name);
381 }
382
383 /**
384 * Returns an unmodifiable Map mapping header keys to a List of header values.
385 * @return unmodifiable Map mapping header keys to a List of header values
386 * @see HttpURLConnection#getHeaderFields()
387 * @since 9232
388 */
389 public Map<String, List<String>> getHeaderFields() {
390 return connection.getHeaderFields();
391 }
392
393 /**
394 * @see HttpURLConnection#disconnect()
395 */
396 public void disconnect() {
397 HttpClient.disconnect(connection);
398 }
399 }
400
401 /**
402 * Creates a new instance for the given URL and a {@code GET} request
403 *
404 * @param url the URL
405 * @return a new instance
406 */
407 public static HttpClient create(URL url) {
408 return create(url, "GET");
409 }
410
411 /**
412 * Creates a new instance for the given URL and a {@code GET} request
413 *
414 * @param url the URL
415 * @param requestMethod the HTTP request method to perform when calling
416 * @return a new instance
417 */
418 public static HttpClient create(URL url, String requestMethod) {
419 return new HttpClient(url, requestMethod);
420 }
421
422 /**
423 * Returns the URL set for this connection.
424 * @return the URL
425 * @see #create(URL)
426 * @see #create(URL, String)
427 * @since 9172
428 */
429 public URL getURL() {
430 return url;
431 }
432
433 /**
434 * Returns the request method set for this connection.
435 * @return the HTTP request method
436 * @see #create(URL, String)
437 * @since 9172
438 */
439 public String getRequestMethod() {
440 return requestMethod;
441 }
442
443 /**
444 * Returns the set value for the given {@code header}.
445 * @param header HTTP header name
446 * @return HTTP header value
447 * @since 9172
448 */
449 public String getRequestHeader(String header) {
450 return headers.get(header);
451 }
452
453 /**
454 * Sets whether not to set header {@code Cache-Control=no-cache}
455 *
456 * @param useCache whether not to set header {@code Cache-Control=no-cache}
457 * @return {@code this}
458 * @see HttpURLConnection#setUseCaches(boolean)
459 */
460 public HttpClient useCache(boolean useCache) {
461 this.useCache = useCache;
462 return this;
463 }
464
465 /**
466 * Sets whether not to set header {@code Connection=close}
467 * <p>
468 * This might fix #7640, see
469 * <a href='https://web.archive.org/web/20140118201501/http://www.tikalk.com/java/forums/httpurlconnection-disable-keep-alive'>here</a>.
470 *
471 * @param keepAlive whether not to set header {@code Connection=close}
472 * @return {@code this}
473 */
474 public HttpClient keepAlive(boolean keepAlive) {
475 return setHeader("Connection", keepAlive ? null : "close");
476 }
477
478 /**
479 * Sets a specified timeout value, in milliseconds, to be used when opening a communications link to the resource referenced
480 * by this URLConnection. If the timeout expires before the connection can be established, a
481 * {@link java.net.SocketTimeoutException} is raised. A timeout of zero is interpreted as an infinite timeout.
482 * @param connectTimeout an {@code int} that specifies the connect timeout value in milliseconds
483 * @return {@code this}
484 * @see HttpURLConnection#setConnectTimeout(int)
485 */
486 public HttpClient setConnectTimeout(int connectTimeout) {
487 this.connectTimeout = connectTimeout;
488 return this;
489 }
490
491 /**
492 * Sets the read timeout to a specified timeout, in milliseconds. A non-zero value specifies the timeout when reading from
493 * input stream when a connection is established to a resource. If the timeout expires before there is data available for
494 * read, a {@link java.net.SocketTimeoutException} is raised. A timeout of zero is interpreted as an infinite timeout.
495 * @param readTimeout an {@code int} that specifies the read timeout value in milliseconds
496 * @return {@code this}
497 * @see HttpURLConnection#setReadTimeout(int)
498 */
499 public HttpClient setReadTimeout(int readTimeout) {
500 this.readTimeout = readTimeout;
501 return this;
502 }
503
504 /**
505 * This method is used to enable streaming of a HTTP request body without internal buffering,
506 * when the content length is known in advance.
507 * <p>
508 * An exception will be thrown if the application attempts to write more data than the indicated content-length,
509 * or if the application closes the OutputStream before writing the indicated amount.
510 * <p>
511 * When output streaming is enabled, authentication and redirection cannot be handled automatically.
512 * A {@linkplain HttpRetryException} will be thrown when reading the response if authentication or redirection
513 * are required. This exception can be queried for the details of the error.
514 *
515 * @param contentLength The number of bytes which will be written to the OutputStream
516 * @return {@code this}
517 * @see HttpURLConnection#setFixedLengthStreamingMode(long)
518 * @since 9178
519 * @deprecated Submitting data via POST, PUT, DELETE automatically sets this property on the connection
520 */
521 @Deprecated
522 public HttpClient setFixedLengthStreamingMode(long contentLength) {
523 return this;
524 }
525
526 /**
527 * Sets the {@code Accept} header.
528 * @param accept header value
529 *
530 * @return {@code this}
531 */
532 public HttpClient setAccept(String accept) {
533 return setHeader("Accept", accept);
534 }
535
536 /**
537 * Sets the request body for {@code PUT}/{@code POST} requests.
538 * @param requestBody request body
539 *
540 * @return {@code this}
541 */
542 public HttpClient setRequestBody(byte[] requestBody) {
543 this.requestBody = requestBody;
544 return this;
545 }
546
547 /**
548 * Sets the {@code If-Modified-Since} header.
549 * @param ifModifiedSince header value
550 *
551 * @return {@code this}
552 */
553 public HttpClient setIfModifiedSince(long ifModifiedSince) {
554 this.ifModifiedSince = ifModifiedSince;
555 return this;
556 }
557
558 /**
559 * Sets the maximum number of redirections to follow.
560 *
561 * Set {@code maxRedirects} to {@code -1} in order to ignore redirects, i.e.,
562 * to not throw an {@link IOException} in {@link #connect()}.
563 * @param maxRedirects header value
564 *
565 * @return {@code this}
566 */
567 public HttpClient setMaxRedirects(int maxRedirects) {
568 this.maxRedirects = maxRedirects;
569 return this;
570 }
571
572 /**
573 * Sets an arbitrary HTTP header.
574 * @param key header name
575 * @param value header value
576 *
577 * @return {@code this}
578 */
579 public HttpClient setHeader(String key, String value) {
580 this.headers.put(key, value);
581 return this;
582 }
583
584 /**
585 * Sets arbitrary HTTP headers.
586 * @param headers HTTP headers
587 *
588 * @return {@code this}
589 */
590 public HttpClient setHeaders(Map<String, String> headers) {
591 this.headers.putAll(headers);
592 return this;
593 }
594
595 /**
596 * Sets a reason to show on console. Can be {@code null} if no reason is given.
597 * @param reasonForRequest Reason to show
598 * @return {@code this}
599 * @since 9172
600 */
601 public HttpClient setReasonForRequest(String reasonForRequest) {
602 this.reasonForRequest = reasonForRequest;
603 return this;
604 }
605
606 private static boolean isRedirect(final int statusCode) {
607 switch (statusCode) {
608 case HttpURLConnection.HTTP_MOVED_PERM: // 301
609 case HttpURLConnection.HTTP_MOVED_TEMP: // 302
610 case HttpURLConnection.HTTP_SEE_OTHER: // 303
611 case 307: // TEMPORARY_REDIRECT:
612 case 308: // PERMANENT_REDIRECT:
613 return true;
614 default:
615 return false;
616 }
617 }
618
619 /**
620 * @see HttpURLConnection#disconnect()
621 * @since 9309
622 */
623 public void disconnect() {
624 HttpClient.disconnect(connection);
625 }
626
627 private static void disconnect(final HttpURLConnection connection) {
628 // Fix upload aborts - see #263
629 connection.setConnectTimeout(100);
630 connection.setReadTimeout(100);
631 try {
632 Thread.sleep(100);
633 } catch (InterruptedException ex) {
634 Main.warn("InterruptedException in " + HttpClient.class + " during cancel");
635 }
636 connection.disconnect();
637 }
638}
Note: See TracBrowser for help on using the repository browser.