source: josm/trunk/src/org/openstreetmap/josm/tools/HttpClient.java@ 11319

Last change on this file since 11319 was 11288, checked in by simon04, 7 years ago

see #13376 - Use TimeUnit instead of combinations of 1000/60/60/24
File size: 24.5 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.tools;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.BufferedOutputStream;
7import java.io.BufferedReader;
8import java.io.ByteArrayInputStream;
9import java.io.IOException;
10import java.io.InputStream;
11import java.io.OutputStream;
12import java.net.CookieHandler;
13import java.net.CookieManager;
14import java.net.HttpURLConnection;
15import java.net.URL;
16import java.util.Collections;
17import java.util.List;
18import java.util.Locale;
19import java.util.Map;
20import java.util.Map.Entry;
21import java.util.Scanner;
22import java.util.TreeMap;
23import java.util.concurrent.TimeUnit;
24import java.util.regex.Matcher;
25import java.util.regex.Pattern;
26import java.util.zip.GZIPInputStream;
27
28import org.openstreetmap.josm.Main;
29import org.openstreetmap.josm.data.Version;
30import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
31import org.openstreetmap.josm.gui.progress.ProgressMonitor;
32import org.openstreetmap.josm.io.Compression;
33import org.openstreetmap.josm.io.ProgressInputStream;
34import org.openstreetmap.josm.io.ProgressOutputStream;
35import org.openstreetmap.josm.io.UTFInputStreamReader;
36import org.openstreetmap.josm.io.auth.DefaultAuthenticator;
37
38/**
39 * Provides a uniform access for a HTTP/HTTPS server. This class should be used in favour of {@link HttpURLConnection}.
40 * @since 9168
41 */
42public final class HttpClient {
43
44 private URL url;
45 private final String requestMethod;
46 private int connectTimeout = (int) TimeUnit.SECONDS.toMillis(Main.pref.getInteger("socket.timeout.connect", 15));
47 private int readTimeout = (int) TimeUnit.SECONDS.toMillis(Main.pref.getInteger("socket.timeout.read", 30));
48 private byte[] requestBody;
49 private long ifModifiedSince;
50 private final Map<String, String> headers = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
51 private int maxRedirects = Main.pref.getInteger("socket.maxredirects", 5);
52 private boolean useCache;
53 private String reasonForRequest;
54 private HttpURLConnection connection; // to allow disconnecting before `response` is set
55 private Response response;
56 private boolean finishOnCloseOutput = true;
57
58 static {
59 CookieHandler.setDefault(new CookieManager());
60 }
61
62 private HttpClient(URL url, String requestMethod) {
63 this.url = url;
64 this.requestMethod = requestMethod;
65 this.headers.put("Accept-Encoding", "gzip");
66 }
67
68 /**
69 * Opens the HTTP connection.
70 * @return HTTP response
71 * @throws IOException if any I/O error occurs
72 */
73 public Response connect() throws IOException {
74 return connect(null);
75 }
76
77 /**
78 * Opens the HTTP connection.
79 * @param progressMonitor progress monitor
80 * @return HTTP response
81 * @throws IOException if any I/O error occurs
82 * @since 9179
83 */
84 public Response connect(ProgressMonitor progressMonitor) throws IOException {
85 if (progressMonitor == null) {
86 progressMonitor = NullProgressMonitor.INSTANCE;
87 }
88 final HttpURLConnection connection = (HttpURLConnection) url.openConnection();
89 this.connection = connection;
90 connection.setRequestMethod(requestMethod);
91 connection.setRequestProperty("User-Agent", Version.getInstance().getFullAgentString());
92 connection.setConnectTimeout(connectTimeout);
93 connection.setReadTimeout(readTimeout);
94 connection.setInstanceFollowRedirects(false); // we do that ourselves
95 if (ifModifiedSince > 0) {
96 connection.setIfModifiedSince(ifModifiedSince);
97 }
98 connection.setUseCaches(useCache);
99 if (!useCache) {
100 connection.setRequestProperty("Cache-Control", "no-cache");
101 }
102 for (Map.Entry<String, String> header : headers.entrySet()) {
103 if (header.getValue() != null) {
104 connection.setRequestProperty(header.getKey(), header.getValue());
105 }
106 }
107
108 progressMonitor.beginTask(tr("Contacting Server..."), 1);
109 progressMonitor.indeterminateSubTask(null);
110
111 if ("PUT".equals(requestMethod) || "POST".equals(requestMethod) || "DELETE".equals(requestMethod)) {
112 Main.info("{0} {1} ({2}) ...", requestMethod, url, Utils.getSizeString(requestBody.length, Locale.getDefault()));
113 connection.setFixedLengthStreamingMode(requestBody.length);
114 connection.setDoOutput(true);
115 try (OutputStream out = new BufferedOutputStream(
116 new ProgressOutputStream(connection.getOutputStream(), requestBody.length, progressMonitor, finishOnCloseOutput))) {
117 out.write(requestBody);
118 }
119 }
120
121 boolean successfulConnection = false;
122 try {
123 try {
124 connection.connect();
125 final boolean hasReason = reasonForRequest != null && !reasonForRequest.isEmpty();
126 Main.info("{0} {1}{2} -> {3}{4}",
127 requestMethod, url, hasReason ? (" (" + reasonForRequest + ')') : "",
128 connection.getResponseCode(),
129 connection.getContentLengthLong() > 0
130 ? (" (" + Utils.getSizeString(connection.getContentLengthLong(), Locale.getDefault()) + ')')
131 : ""
132 );
133 if (Main.isDebugEnabled()) {
134 Main.debug("RESPONSE: " + connection.getHeaderFields());
135 }
136 if (DefaultAuthenticator.getInstance().isEnabled() && connection.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) {
137 DefaultAuthenticator.getInstance().addFailedCredentialHost(url.getHost());
138 }
139 } catch (IOException e) {
140 Main.info("{0} {1} -> !!!", requestMethod, url);
141 Main.warn(e);
142 //noinspection ThrowableResultOfMethodCallIgnored
143 Main.addNetworkError(url, Utils.getRootCause(e));
144 throw e;
145 }
146 if (isRedirect(connection.getResponseCode())) {
147 final String redirectLocation = connection.getHeaderField("Location");
148 if (redirectLocation == null) {
149 /* I18n: argument is HTTP response code */
150 String msg = tr("Unexpected response from HTTP server. Got {0} response without ''Location'' header." +
151 " Can''t redirect. Aborting.", connection.getResponseCode());
152 throw new IOException(msg);
153 } else if (maxRedirects > 0) {
154 url = new URL(url, redirectLocation);
155 maxRedirects--;
156 Main.info(tr("Download redirected to ''{0}''", redirectLocation));
157 return connect();
158 } else if (maxRedirects == 0) {
159 String msg = tr("Too many redirects to the download URL detected. Aborting.");
160 throw new IOException(msg);
161 }
162 }
163 response = new Response(connection, progressMonitor);
164 successfulConnection = true;
165 return response;
166 } finally {
167 if (!successfulConnection) {
168 connection.disconnect();
169 }
170 }
171 }
172
173 /**
174 * Returns the HTTP response which is set only after calling {@link #connect()}.
175 * Calling this method again, returns the identical object (unless another {@link #connect()} is performed).
176 *
177 * @return the HTTP response
178 * @since 9309
179 */
180 public Response getResponse() {
181 return response;
182 }
183
184 /**
185 * A wrapper for the HTTP response.
186 */
187 public static final class Response {
188 private final HttpURLConnection connection;
189 private final ProgressMonitor monitor;
190 private final int responseCode;
191 private final String responseMessage;
192 private boolean uncompress;
193 private boolean uncompressAccordingToContentDisposition;
194 private String responseData;
195
196 private Response(HttpURLConnection connection, ProgressMonitor monitor) throws IOException {
197 CheckParameterUtil.ensureParameterNotNull(connection, "connection");
198 CheckParameterUtil.ensureParameterNotNull(monitor, "monitor");
199 this.connection = connection;
200 this.monitor = monitor;
201 this.responseCode = connection.getResponseCode();
202 this.responseMessage = connection.getResponseMessage();
203 if (this.responseCode >= 300) {
204 String contentType = getContentType();
205 if (contentType == null || (
206 contentType.contains("text") ||
207 contentType.contains("html") ||
208 contentType.contains("xml"))
209 ) {
210 String content = this.fetchContent();
211 if (content == null || content.isEmpty()) {
212 Main.debug("Server did not return any body");
213 } else {
214 Main.debug("Response body: ");
215 Main.debug(this.fetchContent());
216 }
217 } else {
218 Main.debug("Server returned content: {0} of length: {1}. Not printing.", contentType, this.getContentLength());
219 }
220 }
221 }
222
223 /**
224 * Sets whether {@link #getContent()} should uncompress the input stream if necessary.
225 *
226 * @param uncompress whether the input stream should be uncompressed if necessary
227 * @return {@code this}
228 */
229 public Response uncompress(boolean uncompress) {
230 this.uncompress = uncompress;
231 return this;
232 }
233
234 /**
235 * Sets whether {@link #getContent()} should uncompress the input stream according to {@code Content-Disposition}
236 * HTTP header.
237 * @param uncompressAccordingToContentDisposition whether the input stream should be uncompressed according to
238 * {@code Content-Disposition}
239 * @return {@code this}
240 * @since 9172
241 */
242 public Response uncompressAccordingToContentDisposition(boolean uncompressAccordingToContentDisposition) {
243 this.uncompressAccordingToContentDisposition = uncompressAccordingToContentDisposition;
244 return this;
245 }
246
247 /**
248 * Returns the URL.
249 * @return the URL
250 * @see HttpURLConnection#getURL()
251 * @since 9172
252 */
253 public URL getURL() {
254 return connection.getURL();
255 }
256
257 /**
258 * Returns the request method.
259 * @return the HTTP request method
260 * @see HttpURLConnection#getRequestMethod()
261 * @since 9172
262 */
263 public String getRequestMethod() {
264 return connection.getRequestMethod();
265 }
266
267 /**
268 * Returns an input stream that reads from this HTTP connection, or,
269 * error stream if the connection failed but the server sent useful data.
270 * <p>
271 * Note: the return value can be null, if both the input and the error stream are null.
272 * Seems to be the case if the OSM server replies a 401 Unauthorized, see #3887
273 * @return input or error stream
274 * @throws IOException if any I/O error occurs
275 *
276 * @see HttpURLConnection#getInputStream()
277 * @see HttpURLConnection#getErrorStream()
278 */
279 @SuppressWarnings("resource")
280 public InputStream getContent() throws IOException {
281 InputStream in;
282 try {
283 in = connection.getInputStream();
284 } catch (IOException ioe) {
285 Main.debug(ioe);
286 in = connection.getErrorStream();
287 if (in == null) {
288 in = new ByteArrayInputStream(new byte[]{});
289 }
290 }
291 if (in != null) {
292 in = new ProgressInputStream(in, getContentLength(), monitor);
293 in = "gzip".equalsIgnoreCase(getContentEncoding()) ? new GZIPInputStream(in) : in;
294 Compression compression = Compression.NONE;
295 if (uncompress) {
296 final String contentType = getContentType();
297 Main.debug("Uncompressing input stream according to Content-Type header: {0}", contentType);
298 compression = Compression.forContentType(contentType);
299 }
300 if (uncompressAccordingToContentDisposition && Compression.NONE.equals(compression)) {
301 final String contentDisposition = getHeaderField("Content-Disposition");
302 final Matcher matcher = Pattern.compile("filename=\"([^\"]+)\"").matcher(
303 contentDisposition != null ? contentDisposition : "");
304 if (matcher.find()) {
305 Main.debug("Uncompressing input stream according to Content-Disposition header: {0}", contentDisposition);
306 compression = Compression.byExtension(matcher.group(1));
307 }
308 }
309 in = compression.getUncompressedInputStream(in);
310 }
311 return in;
312 }
313
314 /**
315 * Returns {@link #getContent()} wrapped in a buffered reader.
316 *
317 * Detects Unicode charset in use utilizing {@link UTFInputStreamReader}.
318 * @return buffered reader
319 * @throws IOException if any I/O error occurs
320 */
321 public BufferedReader getContentReader() throws IOException {
322 return new BufferedReader(
323 UTFInputStreamReader.create(getContent())
324 );
325 }
326
327 /**
328 * Fetches the HTTP response as String.
329 * @return the response
330 * @throws IOException if any I/O error occurs
331 */
332 public synchronized String fetchContent() throws IOException {
333 if (responseData == null) {
334 try (Scanner scanner = new Scanner(getContentReader()).useDelimiter("\\A")) { // \A - beginning of input
335 responseData = scanner.hasNext() ? scanner.next() : "";
336 }
337 }
338 return responseData;
339 }
340
341 /**
342 * Gets the response code from this HTTP connection.
343 * @return HTTP response code
344 *
345 * @see HttpURLConnection#getResponseCode()
346 */
347 public int getResponseCode() {
348 return responseCode;
349 }
350
351 /**
352 * Gets the response message from this HTTP connection.
353 * @return HTTP response message
354 *
355 * @see HttpURLConnection#getResponseMessage()
356 * @since 9172
357 */
358 public String getResponseMessage() {
359 return responseMessage;
360 }
361
362 /**
363 * Returns the {@code Content-Encoding} header.
364 * @return {@code Content-Encoding} HTTP header
365 * @see HttpURLConnection#getContentEncoding()
366 */
367 public String getContentEncoding() {
368 return connection.getContentEncoding();
369 }
370
371 /**
372 * Returns the {@code Content-Type} header.
373 * @return {@code Content-Type} HTTP header
374 */
375 public String getContentType() {
376 return connection.getHeaderField("Content-Type");
377 }
378
379 /**
380 * Returns the {@code Expire} header.
381 * @return {@code Expire} HTTP header
382 * @see HttpURLConnection#getExpiration()
383 * @since 9232
384 */
385 public long getExpiration() {
386 return connection.getExpiration();
387 }
388
389 /**
390 * Returns the {@code Last-Modified} header.
391 * @return {@code Last-Modified} HTTP header
392 * @see HttpURLConnection#getLastModified()
393 * @since 9232
394 */
395 public long getLastModified() {
396 return connection.getLastModified();
397 }
398
399 /**
400 * Returns the {@code Content-Length} header.
401 * @return {@code Content-Length} HTTP header
402 * @see HttpURLConnection#getContentLengthLong()
403 */
404 public long getContentLength() {
405 return connection.getContentLengthLong();
406 }
407
408 /**
409 * Returns the value of the named header field.
410 * @param name the name of a header field
411 * @return the value of the named header field, or {@code null} if there is no such field in the header
412 * @see HttpURLConnection#getHeaderField(String)
413 * @since 9172
414 */
415 public String getHeaderField(String name) {
416 return connection.getHeaderField(name);
417 }
418
419 /**
420 * Returns an unmodifiable Map mapping header keys to a List of header values.
421 * As per RFC 2616, section 4.2 header names are case insensitive, so returned map is also case insensitive
422 * @return unmodifiable Map mapping header keys to a List of header values
423 * @see HttpURLConnection#getHeaderFields()
424 * @since 9232
425 */
426 public Map<String, List<String>> getHeaderFields() {
427 // returned map from HttpUrlConnection is case sensitive, use case insensitive TreeMap to conform to RFC 2616
428 Map<String, List<String>> ret = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
429 for (Entry<String, List<String>> e: connection.getHeaderFields().entrySet()) {
430 if (e.getKey() != null) {
431 ret.put(e.getKey(), e.getValue());
432 }
433 }
434 return Collections.unmodifiableMap(ret);
435 }
436
437 /**
438 * @see HttpURLConnection#disconnect()
439 */
440 public void disconnect() {
441 HttpClient.disconnect(connection);
442 }
443 }
444
445 /**
446 * Creates a new instance for the given URL and a {@code GET} request
447 *
448 * @param url the URL
449 * @return a new instance
450 */
451 public static HttpClient create(URL url) {
452 return create(url, "GET");
453 }
454
455 /**
456 * Creates a new instance for the given URL and a {@code GET} request
457 *
458 * @param url the URL
459 * @param requestMethod the HTTP request method to perform when calling
460 * @return a new instance
461 */
462 public static HttpClient create(URL url, String requestMethod) {
463 return new HttpClient(url, requestMethod);
464 }
465
466 /**
467 * Returns the URL set for this connection.
468 * @return the URL
469 * @see #create(URL)
470 * @see #create(URL, String)
471 * @since 9172
472 */
473 public URL getURL() {
474 return url;
475 }
476
477 /**
478 * Returns the request method set for this connection.
479 * @return the HTTP request method
480 * @see #create(URL, String)
481 * @since 9172
482 */
483 public String getRequestMethod() {
484 return requestMethod;
485 }
486
487 /**
488 * Returns the set value for the given {@code header}.
489 * @param header HTTP header name
490 * @return HTTP header value
491 * @since 9172
492 */
493 public String getRequestHeader(String header) {
494 return headers.get(header);
495 }
496
497 /**
498 * Sets whether not to set header {@code Cache-Control=no-cache}
499 *
500 * @param useCache whether not to set header {@code Cache-Control=no-cache}
501 * @return {@code this}
502 * @see HttpURLConnection#setUseCaches(boolean)
503 */
504 public HttpClient useCache(boolean useCache) {
505 this.useCache = useCache;
506 return this;
507 }
508
509 /**
510 * Sets whether not to set header {@code Connection=close}
511 * <p>
512 * This might fix #7640, see
513 * <a href='https://web.archive.org/web/20140118201501/http://www.tikalk.com/java/forums/httpurlconnection-disable-keep-alive'>here</a>.
514 *
515 * @param keepAlive whether not to set header {@code Connection=close}
516 * @return {@code this}
517 */
518 public HttpClient keepAlive(boolean keepAlive) {
519 return setHeader("Connection", keepAlive ? null : "close");
520 }
521
522 /**
523 * Sets a specified timeout value, in milliseconds, to be used when opening a communications link to the resource referenced
524 * by this URLConnection. If the timeout expires before the connection can be established, a
525 * {@link java.net.SocketTimeoutException} is raised. A timeout of zero is interpreted as an infinite timeout.
526 * @param connectTimeout an {@code int} that specifies the connect timeout value in milliseconds
527 * @return {@code this}
528 * @see HttpURLConnection#setConnectTimeout(int)
529 */
530 public HttpClient setConnectTimeout(int connectTimeout) {
531 this.connectTimeout = connectTimeout;
532 return this;
533 }
534
535 /**
536 * Sets the read timeout to a specified timeout, in milliseconds. A non-zero value specifies the timeout when reading from
537 * input stream when a connection is established to a resource. If the timeout expires before there is data available for
538 * read, a {@link java.net.SocketTimeoutException} is raised. A timeout of zero is interpreted as an infinite timeout.
539 * @param readTimeout an {@code int} that specifies the read timeout value in milliseconds
540 * @return {@code this}
541 * @see HttpURLConnection#setReadTimeout(int)
542 */
543 public HttpClient setReadTimeout(int readTimeout) {
544 this.readTimeout = readTimeout;
545 return this;
546 }
547
548 /**
549 * Sets the {@code Accept} header.
550 * @param accept header value
551 *
552 * @return {@code this}
553 */
554 public HttpClient setAccept(String accept) {
555 return setHeader("Accept", accept);
556 }
557
558 /**
559 * Sets the request body for {@code PUT}/{@code POST} requests.
560 * @param requestBody request body
561 *
562 * @return {@code this}
563 */
564 public HttpClient setRequestBody(byte[] requestBody) {
565 this.requestBody = requestBody;
566 return this;
567 }
568
569 /**
570 * Sets the {@code If-Modified-Since} header.
571 * @param ifModifiedSince header value
572 *
573 * @return {@code this}
574 */
575 public HttpClient setIfModifiedSince(long ifModifiedSince) {
576 this.ifModifiedSince = ifModifiedSince;
577 return this;
578 }
579
580 /**
581 * Sets the maximum number of redirections to follow.
582 *
583 * Set {@code maxRedirects} to {@code -1} in order to ignore redirects, i.e.,
584 * to not throw an {@link IOException} in {@link #connect()}.
585 * @param maxRedirects header value
586 *
587 * @return {@code this}
588 */
589 public HttpClient setMaxRedirects(int maxRedirects) {
590 this.maxRedirects = maxRedirects;
591 return this;
592 }
593
594 /**
595 * Sets an arbitrary HTTP header.
596 * @param key header name
597 * @param value header value
598 *
599 * @return {@code this}
600 */
601 public HttpClient setHeader(String key, String value) {
602 this.headers.put(key, value);
603 return this;
604 }
605
606 /**
607 * Sets arbitrary HTTP headers.
608 * @param headers HTTP headers
609 *
610 * @return {@code this}
611 */
612 public HttpClient setHeaders(Map<String, String> headers) {
613 this.headers.putAll(headers);
614 return this;
615 }
616
617 /**
618 * Sets a reason to show on console. Can be {@code null} if no reason is given.
619 * @param reasonForRequest Reason to show
620 * @return {@code this}
621 * @since 9172
622 */
623 public HttpClient setReasonForRequest(String reasonForRequest) {
624 this.reasonForRequest = reasonForRequest;
625 return this;
626 }
627
628 /**
629 * Sets whether the progress monitor task will be finished when the output stream is closed. This is {@code true} by default.
630 * @param finishOnCloseOutput whether the progress monitor task will be finished when the output stream is closed
631 * @return {@code this}
632 * @since 10302
633 */
634 public HttpClient setFinishOnCloseOutput(boolean finishOnCloseOutput) {
635 this.finishOnCloseOutput = finishOnCloseOutput;
636 return this;
637 }
638
639 private static boolean isRedirect(final int statusCode) {
640 switch (statusCode) {
641 case HttpURLConnection.HTTP_MOVED_PERM: // 301
642 case HttpURLConnection.HTTP_MOVED_TEMP: // 302
643 case HttpURLConnection.HTTP_SEE_OTHER: // 303
644 case 307: // TEMPORARY_REDIRECT:
645 case 308: // PERMANENT_REDIRECT:
646 return true;
647 default:
648 return false;
649 }
650 }
651
652 /**
653 * @see HttpURLConnection#disconnect()
654 * @since 9309
655 */
656 public void disconnect() {
657 HttpClient.disconnect(connection);
658 }
659
660 private static void disconnect(final HttpURLConnection connection) {
661 // Fix upload aborts - see #263
662 connection.setConnectTimeout(100);
663 connection.setReadTimeout(100);
664 try {
665 Thread.sleep(100);
666 } catch (InterruptedException ex) {
667 Main.warn("InterruptedException in " + HttpClient.class + " during cancel");
668 }
669 connection.disconnect();
670 }
671}
Note: See TracBrowser for help on using the repository browser.