source: josm/trunk/src/org/openstreetmap/josm/tools/HttpClient.java@ 12867

Last change on this file since 12867 was 12846, checked in by bastiK, 7 years ago

see #15229 - use Config.getPref() wherever possible
File size: 25.2 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.tools;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.BufferedOutputStream;
7import java.io.BufferedReader;
8import java.io.ByteArrayInputStream;
9import java.io.IOException;
10import java.io.InputStream;
11import java.io.OutputStream;
12import java.net.CookieHandler;
13import java.net.CookieManager;
14import java.net.HttpURLConnection;
15import java.net.URL;
16import java.util.Collections;
17import java.util.List;
18import java.util.Locale;
19import java.util.Map;
20import java.util.Map.Entry;
21import java.util.Optional;
22import java.util.Scanner;
23import java.util.TreeMap;
24import java.util.concurrent.TimeUnit;
25import java.util.regex.Matcher;
26import java.util.regex.Pattern;
27import java.util.zip.GZIPInputStream;
28
29import org.openstreetmap.josm.Main;
30import org.openstreetmap.josm.data.Version;
31import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
32import org.openstreetmap.josm.gui.progress.ProgressMonitor;
33import org.openstreetmap.josm.io.Compression;
34import org.openstreetmap.josm.io.ProgressInputStream;
35import org.openstreetmap.josm.io.ProgressOutputStream;
36import org.openstreetmap.josm.io.UTFInputStreamReader;
37import org.openstreetmap.josm.io.auth.DefaultAuthenticator;
38import org.openstreetmap.josm.spi.preferences.Config;
39
40/**
41 * Provides a uniform access for a HTTP/HTTPS server. This class should be used in favour of {@link HttpURLConnection}.
42 * @since 9168
43 */
44public final class HttpClient {
45
46 private URL url;
47 private final String requestMethod;
48 private int connectTimeout = (int) TimeUnit.SECONDS.toMillis(Config.getPref().getInt("socket.timeout.connect", 15));
49 private int readTimeout = (int) TimeUnit.SECONDS.toMillis(Config.getPref().getInt("socket.timeout.read", 30));
50 private byte[] requestBody;
51 private long ifModifiedSince;
52 private final Map<String, String> headers = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
53 private int maxRedirects = Config.getPref().getInt("socket.maxredirects", 5);
54 private boolean useCache;
55 private String reasonForRequest;
56 private String outputMessage = tr("Uploading data ...");
57 private HttpURLConnection connection; // to allow disconnecting before `response` is set
58 private Response response;
59 private boolean finishOnCloseOutput = true;
60
61 static {
62 CookieHandler.setDefault(new CookieManager());
63 }
64
65 private HttpClient(URL url, String requestMethod) {
66 this.url = url;
67 this.requestMethod = requestMethod;
68 this.headers.put("Accept-Encoding", "gzip");
69 }
70
71 /**
72 * Opens the HTTP connection.
73 * @return HTTP response
74 * @throws IOException if any I/O error occurs
75 */
76 public Response connect() throws IOException {
77 return connect(null);
78 }
79
80 /**
81 * Opens the HTTP connection.
82 * @param progressMonitor progress monitor
83 * @return HTTP response
84 * @throws IOException if any I/O error occurs
85 * @since 9179
86 */
87 public Response connect(ProgressMonitor progressMonitor) throws IOException {
88 if (progressMonitor == null) {
89 progressMonitor = NullProgressMonitor.INSTANCE;
90 }
91 final HttpURLConnection connection = (HttpURLConnection) url.openConnection();
92 this.connection = connection;
93 connection.setRequestMethod(requestMethod);
94 connection.setRequestProperty("User-Agent", Version.getInstance().getFullAgentString());
95 connection.setConnectTimeout(connectTimeout);
96 connection.setReadTimeout(readTimeout);
97 connection.setInstanceFollowRedirects(false); // we do that ourselves
98 if (ifModifiedSince > 0) {
99 connection.setIfModifiedSince(ifModifiedSince);
100 }
101 connection.setUseCaches(useCache);
102 if (!useCache) {
103 connection.setRequestProperty("Cache-Control", "no-cache");
104 }
105 for (Map.Entry<String, String> header : headers.entrySet()) {
106 if (header.getValue() != null) {
107 connection.setRequestProperty(header.getKey(), header.getValue());
108 }
109 }
110
111 progressMonitor.beginTask(tr("Contacting Server..."), 1);
112 progressMonitor.indeterminateSubTask(null);
113
114 if ("PUT".equals(requestMethod) || "POST".equals(requestMethod) || "DELETE".equals(requestMethod)) {
115 Logging.info("{0} {1} ({2}) ...", requestMethod, url, Utils.getSizeString(requestBody.length, Locale.getDefault()));
116 connection.setFixedLengthStreamingMode(requestBody.length);
117 connection.setDoOutput(true);
118 try (OutputStream out = new BufferedOutputStream(
119 new ProgressOutputStream(connection.getOutputStream(), requestBody.length,
120 progressMonitor, outputMessage, finishOnCloseOutput))) {
121 out.write(requestBody);
122 }
123 }
124
125 boolean successfulConnection = false;
126 try {
127 try {
128 connection.connect();
129 final boolean hasReason = reasonForRequest != null && !reasonForRequest.isEmpty();
130 Logging.info("{0} {1}{2} -> {3}{4}",
131 requestMethod, url, hasReason ? (" (" + reasonForRequest + ')') : "",
132 connection.getResponseCode(),
133 connection.getContentLengthLong() > 0
134 ? (" (" + Utils.getSizeString(connection.getContentLengthLong(), Locale.getDefault()) + ')')
135 : ""
136 );
137 if (Logging.isDebugEnabled()) {
138 Logging.debug("RESPONSE: {0}", connection.getHeaderFields());
139 }
140 if (DefaultAuthenticator.getInstance().isEnabled() && connection.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) {
141 DefaultAuthenticator.getInstance().addFailedCredentialHost(url.getHost());
142 }
143 } catch (IOException e) {
144 Logging.info("{0} {1} -> !!!", requestMethod, url);
145 Logging.warn(e);
146 //noinspection ThrowableResultOfMethodCallIgnored
147 Main.addNetworkError(url, Utils.getRootCause(e));
148 throw e;
149 }
150 if (isRedirect(connection.getResponseCode())) {
151 final String redirectLocation = connection.getHeaderField("Location");
152 if (redirectLocation == null) {
153 /* I18n: argument is HTTP response code */
154 throw new IOException(tr("Unexpected response from HTTP server. Got {0} response without ''Location'' header." +
155 " Can''t redirect. Aborting.", connection.getResponseCode()));
156 } else if (maxRedirects > 0) {
157 url = new URL(url, redirectLocation);
158 maxRedirects--;
159 Logging.info(tr("Download redirected to ''{0}''", redirectLocation));
160 return connect();
161 } else if (maxRedirects == 0) {
162 String msg = tr("Too many redirects to the download URL detected. Aborting.");
163 throw new IOException(msg);
164 }
165 }
166 response = new Response(connection, progressMonitor);
167 successfulConnection = true;
168 return response;
169 } finally {
170 if (!successfulConnection) {
171 connection.disconnect();
172 }
173 }
174 }
175
176 /**
177 * Returns the HTTP response which is set only after calling {@link #connect()}.
178 * Calling this method again, returns the identical object (unless another {@link #connect()} is performed).
179 *
180 * @return the HTTP response
181 * @since 9309
182 */
183 public Response getResponse() {
184 return response;
185 }
186
187 /**
188 * A wrapper for the HTTP response.
189 */
190 public static final class Response {
191 private final HttpURLConnection connection;
192 private final ProgressMonitor monitor;
193 private final int responseCode;
194 private final String responseMessage;
195 private boolean uncompress;
196 private boolean uncompressAccordingToContentDisposition;
197 private String responseData;
198
199 private Response(HttpURLConnection connection, ProgressMonitor monitor) throws IOException {
200 CheckParameterUtil.ensureParameterNotNull(connection, "connection");
201 CheckParameterUtil.ensureParameterNotNull(monitor, "monitor");
202 this.connection = connection;
203 this.monitor = monitor;
204 this.responseCode = connection.getResponseCode();
205 this.responseMessage = connection.getResponseMessage();
206 if (this.responseCode >= 300) {
207 String contentType = getContentType();
208 if (contentType == null || (
209 contentType.contains("text") ||
210 contentType.contains("html") ||
211 contentType.contains("xml"))
212 ) {
213 String content = this.fetchContent();
214 if (content.isEmpty()) {
215 Logging.debug("Server did not return any body");
216 } else {
217 Logging.debug("Response body: ");
218 Logging.debug(this.fetchContent());
219 }
220 } else {
221 Logging.debug("Server returned content: {0} of length: {1}. Not printing.", contentType, this.getContentLength());
222 }
223 }
224 }
225
226 /**
227 * Sets whether {@link #getContent()} should uncompress the input stream if necessary.
228 *
229 * @param uncompress whether the input stream should be uncompressed if necessary
230 * @return {@code this}
231 */
232 public Response uncompress(boolean uncompress) {
233 this.uncompress = uncompress;
234 return this;
235 }
236
237 /**
238 * Sets whether {@link #getContent()} should uncompress the input stream according to {@code Content-Disposition}
239 * HTTP header.
240 * @param uncompressAccordingToContentDisposition whether the input stream should be uncompressed according to
241 * {@code Content-Disposition}
242 * @return {@code this}
243 * @since 9172
244 */
245 public Response uncompressAccordingToContentDisposition(boolean uncompressAccordingToContentDisposition) {
246 this.uncompressAccordingToContentDisposition = uncompressAccordingToContentDisposition;
247 return this;
248 }
249
250 /**
251 * Returns the URL.
252 * @return the URL
253 * @see HttpURLConnection#getURL()
254 * @since 9172
255 */
256 public URL getURL() {
257 return connection.getURL();
258 }
259
260 /**
261 * Returns the request method.
262 * @return the HTTP request method
263 * @see HttpURLConnection#getRequestMethod()
264 * @since 9172
265 */
266 public String getRequestMethod() {
267 return connection.getRequestMethod();
268 }
269
270 /**
271 * Returns an input stream that reads from this HTTP connection, or,
272 * error stream if the connection failed but the server sent useful data.
273 * <p>
274 * Note: the return value can be null, if both the input and the error stream are null.
275 * Seems to be the case if the OSM server replies a 401 Unauthorized, see #3887
276 * @return input or error stream
277 * @throws IOException if any I/O error occurs
278 *
279 * @see HttpURLConnection#getInputStream()
280 * @see HttpURLConnection#getErrorStream()
281 */
282 @SuppressWarnings("resource")
283 public InputStream getContent() throws IOException {
284 InputStream in;
285 try {
286 in = connection.getInputStream();
287 } catch (IOException ioe) {
288 Logging.debug(ioe);
289 in = Optional.ofNullable(connection.getErrorStream()).orElseGet(() -> new ByteArrayInputStream(new byte[]{}));
290 }
291 in = new ProgressInputStream(in, getContentLength(), monitor);
292 in = "gzip".equalsIgnoreCase(getContentEncoding()) ? new GZIPInputStream(in) : in;
293 Compression compression = Compression.NONE;
294 if (uncompress) {
295 final String contentType = getContentType();
296 Logging.debug("Uncompressing input stream according to Content-Type header: {0}", contentType);
297 compression = Compression.forContentType(contentType);
298 }
299 if (uncompressAccordingToContentDisposition && Compression.NONE.equals(compression)) {
300 final String contentDisposition = getHeaderField("Content-Disposition");
301 final Matcher matcher = Pattern.compile("filename=\"([^\"]+)\"").matcher(
302 contentDisposition != null ? contentDisposition : "");
303 if (matcher.find()) {
304 Logging.debug("Uncompressing input stream according to Content-Disposition header: {0}", contentDisposition);
305 compression = Compression.byExtension(matcher.group(1));
306 }
307 }
308 in = compression.getUncompressedInputStream(in);
309 return in;
310 }
311
312 /**
313 * Returns {@link #getContent()} wrapped in a buffered reader.
314 *
315 * Detects Unicode charset in use utilizing {@link UTFInputStreamReader}.
316 * @return buffered reader
317 * @throws IOException if any I/O error occurs
318 */
319 public BufferedReader getContentReader() throws IOException {
320 return new BufferedReader(
321 UTFInputStreamReader.create(getContent())
322 );
323 }
324
325 /**
326 * Fetches the HTTP response as String.
327 * @return the response
328 * @throws IOException if any I/O error occurs
329 */
330 public synchronized String fetchContent() throws IOException {
331 if (responseData == null) {
332 try (Scanner scanner = new Scanner(getContentReader()).useDelimiter("\\A")) { // \A - beginning of input
333 responseData = scanner.hasNext() ? scanner.next() : "";
334 }
335 }
336 return responseData;
337 }
338
339 /**
340 * Gets the response code from this HTTP connection.
341 * @return HTTP response code
342 *
343 * @see HttpURLConnection#getResponseCode()
344 */
345 public int getResponseCode() {
346 return responseCode;
347 }
348
349 /**
350 * Gets the response message from this HTTP connection.
351 * @return HTTP response message
352 *
353 * @see HttpURLConnection#getResponseMessage()
354 * @since 9172
355 */
356 public String getResponseMessage() {
357 return responseMessage;
358 }
359
360 /**
361 * Returns the {@code Content-Encoding} header.
362 * @return {@code Content-Encoding} HTTP header
363 * @see HttpURLConnection#getContentEncoding()
364 */
365 public String getContentEncoding() {
366 return connection.getContentEncoding();
367 }
368
369 /**
370 * Returns the {@code Content-Type} header.
371 * @return {@code Content-Type} HTTP header
372 */
373 public String getContentType() {
374 return connection.getHeaderField("Content-Type");
375 }
376
377 /**
378 * Returns the {@code Expire} header.
379 * @return {@code Expire} HTTP header
380 * @see HttpURLConnection#getExpiration()
381 * @since 9232
382 */
383 public long getExpiration() {
384 return connection.getExpiration();
385 }
386
387 /**
388 * Returns the {@code Last-Modified} header.
389 * @return {@code Last-Modified} HTTP header
390 * @see HttpURLConnection#getLastModified()
391 * @since 9232
392 */
393 public long getLastModified() {
394 return connection.getLastModified();
395 }
396
397 /**
398 * Returns the {@code Content-Length} header.
399 * @return {@code Content-Length} HTTP header
400 * @see HttpURLConnection#getContentLengthLong()
401 */
402 public long getContentLength() {
403 return connection.getContentLengthLong();
404 }
405
406 /**
407 * Returns the value of the named header field.
408 * @param name the name of a header field
409 * @return the value of the named header field, or {@code null} if there is no such field in the header
410 * @see HttpURLConnection#getHeaderField(String)
411 * @since 9172
412 */
413 public String getHeaderField(String name) {
414 return connection.getHeaderField(name);
415 }
416
417 /**
418 * Returns an unmodifiable Map mapping header keys to a List of header values.
419 * As per RFC 2616, section 4.2 header names are case insensitive, so returned map is also case insensitive
420 * @return unmodifiable Map mapping header keys to a List of header values
421 * @see HttpURLConnection#getHeaderFields()
422 * @since 9232
423 */
424 public Map<String, List<String>> getHeaderFields() {
425 // returned map from HttpUrlConnection is case sensitive, use case insensitive TreeMap to conform to RFC 2616
426 Map<String, List<String>> ret = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
427 for (Entry<String, List<String>> e: connection.getHeaderFields().entrySet()) {
428 if (e.getKey() != null) {
429 ret.put(e.getKey(), e.getValue());
430 }
431 }
432 return Collections.unmodifiableMap(ret);
433 }
434
435 /**
436 * @see HttpURLConnection#disconnect()
437 */
438 public void disconnect() {
439 HttpClient.disconnect(connection);
440 }
441 }
442
443 /**
444 * Creates a new instance for the given URL and a {@code GET} request
445 *
446 * @param url the URL
447 * @return a new instance
448 */
449 public static HttpClient create(URL url) {
450 return create(url, "GET");
451 }
452
453 /**
454 * Creates a new instance for the given URL and a {@code GET} request
455 *
456 * @param url the URL
457 * @param requestMethod the HTTP request method to perform when calling
458 * @return a new instance
459 */
460 public static HttpClient create(URL url, String requestMethod) {
461 return new HttpClient(url, requestMethod);
462 }
463
464 /**
465 * Returns the URL set for this connection.
466 * @return the URL
467 * @see #create(URL)
468 * @see #create(URL, String)
469 * @since 9172
470 */
471 public URL getURL() {
472 return url;
473 }
474
475 /**
476 * Returns the request method set for this connection.
477 * @return the HTTP request method
478 * @see #create(URL, String)
479 * @since 9172
480 */
481 public String getRequestMethod() {
482 return requestMethod;
483 }
484
485 /**
486 * Returns the set value for the given {@code header}.
487 * @param header HTTP header name
488 * @return HTTP header value
489 * @since 9172
490 */
491 public String getRequestHeader(String header) {
492 return headers.get(header);
493 }
494
495 /**
496 * Sets whether not to set header {@code Cache-Control=no-cache}
497 *
498 * @param useCache whether not to set header {@code Cache-Control=no-cache}
499 * @return {@code this}
500 * @see HttpURLConnection#setUseCaches(boolean)
501 */
502 public HttpClient useCache(boolean useCache) {
503 this.useCache = useCache;
504 return this;
505 }
506
507 /**
508 * Sets whether not to set header {@code Connection=close}
509 * <p>
510 * This might fix #7640, see
511 * <a href='https://web.archive.org/web/20140118201501/http://www.tikalk.com/java/forums/httpurlconnection-disable-keep-alive'>here</a>.
512 *
513 * @param keepAlive whether not to set header {@code Connection=close}
514 * @return {@code this}
515 */
516 public HttpClient keepAlive(boolean keepAlive) {
517 return setHeader("Connection", keepAlive ? null : "close");
518 }
519
520 /**
521 * Sets a specified timeout value, in milliseconds, to be used when opening a communications link to the resource referenced
522 * by this URLConnection. If the timeout expires before the connection can be established, a
523 * {@link java.net.SocketTimeoutException} is raised. A timeout of zero is interpreted as an infinite timeout.
524 * @param connectTimeout an {@code int} that specifies the connect timeout value in milliseconds
525 * @return {@code this}
526 * @see HttpURLConnection#setConnectTimeout(int)
527 */
528 public HttpClient setConnectTimeout(int connectTimeout) {
529 this.connectTimeout = connectTimeout;
530 return this;
531 }
532
533 /**
534 * Sets the read timeout to a specified timeout, in milliseconds. A non-zero value specifies the timeout when reading from
535 * input stream when a connection is established to a resource. If the timeout expires before there is data available for
536 * read, a {@link java.net.SocketTimeoutException} is raised. A timeout of zero is interpreted as an infinite timeout.
537 * @param readTimeout an {@code int} that specifies the read timeout value in milliseconds
538 * @return {@code this}
539 * @see HttpURLConnection#setReadTimeout(int)
540 */
541 public HttpClient setReadTimeout(int readTimeout) {
542 this.readTimeout = readTimeout;
543 return this;
544 }
545
546 /**
547 * Sets the {@code Accept} header.
548 * @param accept header value
549 *
550 * @return {@code this}
551 */
552 public HttpClient setAccept(String accept) {
553 return setHeader("Accept", accept);
554 }
555
556 /**
557 * Sets the request body for {@code PUT}/{@code POST} requests.
558 * @param requestBody request body
559 *
560 * @return {@code this}
561 */
562 public HttpClient setRequestBody(byte[] requestBody) {
563 this.requestBody = Utils.copyArray(requestBody);
564 return this;
565 }
566
567 /**
568 * Sets the {@code If-Modified-Since} header.
569 * @param ifModifiedSince header value
570 *
571 * @return {@code this}
572 */
573 public HttpClient setIfModifiedSince(long ifModifiedSince) {
574 this.ifModifiedSince = ifModifiedSince;
575 return this;
576 }
577
578 /**
579 * Sets the maximum number of redirections to follow.
580 *
581 * Set {@code maxRedirects} to {@code -1} in order to ignore redirects, i.e.,
582 * to not throw an {@link IOException} in {@link #connect()}.
583 * @param maxRedirects header value
584 *
585 * @return {@code this}
586 */
587 public HttpClient setMaxRedirects(int maxRedirects) {
588 this.maxRedirects = maxRedirects;
589 return this;
590 }
591
592 /**
593 * Sets an arbitrary HTTP header.
594 * @param key header name
595 * @param value header value
596 *
597 * @return {@code this}
598 */
599 public HttpClient setHeader(String key, String value) {
600 this.headers.put(key, value);
601 return this;
602 }
603
604 /**
605 * Sets arbitrary HTTP headers.
606 * @param headers HTTP headers
607 *
608 * @return {@code this}
609 */
610 public HttpClient setHeaders(Map<String, String> headers) {
611 this.headers.putAll(headers);
612 return this;
613 }
614
615 /**
616 * Sets a reason to show on console. Can be {@code null} if no reason is given.
617 * @param reasonForRequest Reason to show
618 * @return {@code this}
619 * @since 9172
620 */
621 public HttpClient setReasonForRequest(String reasonForRequest) {
622 this.reasonForRequest = reasonForRequest;
623 return this;
624 }
625
626 /**
627 * Sets the output message to be displayed in progress monitor for {@code PUT}, {@code POST} and {@code DELETE} methods.
628 * Defaults to "Uploading data ..." (translated). Has no effect for {@code GET} or any other method.
629 * @param outputMessage message to be displayed in progress monitor
630 * @return {@code this}
631 * @since 12711
632 */
633 public HttpClient setOutputMessage(String outputMessage) {
634 this.outputMessage = outputMessage;
635 return this;
636 }
637
638 /**
639 * Sets whether the progress monitor task will be finished when the output stream is closed. This is {@code true} by default.
640 * @param finishOnCloseOutput whether the progress monitor task will be finished when the output stream is closed
641 * @return {@code this}
642 * @since 10302
643 */
644 public HttpClient setFinishOnCloseOutput(boolean finishOnCloseOutput) {
645 this.finishOnCloseOutput = finishOnCloseOutput;
646 return this;
647 }
648
649 private static boolean isRedirect(final int statusCode) {
650 switch (statusCode) {
651 case HttpURLConnection.HTTP_MOVED_PERM: // 301
652 case HttpURLConnection.HTTP_MOVED_TEMP: // 302
653 case HttpURLConnection.HTTP_SEE_OTHER: // 303
654 case 307: // TEMPORARY_REDIRECT:
655 case 308: // PERMANENT_REDIRECT:
656 return true;
657 default:
658 return false;
659 }
660 }
661
662 /**
663 * @see HttpURLConnection#disconnect()
664 * @since 9309
665 */
666 public void disconnect() {
667 HttpClient.disconnect(connection);
668 }
669
670 private static void disconnect(final HttpURLConnection connection) {
671 if (connection != null) {
672 // Fix upload aborts - see #263
673 connection.setConnectTimeout(100);
674 connection.setReadTimeout(100);
675 try {
676 Thread.sleep(100);
677 } catch (InterruptedException ex) {
678 Logging.warn("InterruptedException in " + HttpClient.class + " during cancel");
679 Thread.currentThread().interrupt();
680 }
681 connection.disconnect();
682 }
683 }
684}
Note: See TracBrowser for help on using the repository browser.