source: josm/trunk/src/org/openstreetmap/josm/tools/HttpClient.java@ 12649

Last change on this file since 12649 was 12620, checked in by Don-vip, 7 years ago

see #15182 - deprecate all Main logging methods and introduce suitable replacements in Logging for most of them
File size: 24.5 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.tools;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.BufferedOutputStream;
7import java.io.BufferedReader;
8import java.io.ByteArrayInputStream;
9import java.io.IOException;
10import java.io.InputStream;
11import java.io.OutputStream;
12import java.net.CookieHandler;
13import java.net.CookieManager;
14import java.net.HttpURLConnection;
15import java.net.URL;
16import java.util.Collections;
17import java.util.List;
18import java.util.Locale;
19import java.util.Map;
20import java.util.Map.Entry;
21import java.util.Optional;
22import java.util.Scanner;
23import java.util.TreeMap;
24import java.util.concurrent.TimeUnit;
25import java.util.regex.Matcher;
26import java.util.regex.Pattern;
27import java.util.zip.GZIPInputStream;
28
29import org.openstreetmap.josm.Main;
30import org.openstreetmap.josm.data.Version;
31import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
32import org.openstreetmap.josm.gui.progress.ProgressMonitor;
33import org.openstreetmap.josm.io.Compression;
34import org.openstreetmap.josm.io.ProgressInputStream;
35import org.openstreetmap.josm.io.ProgressOutputStream;
36import org.openstreetmap.josm.io.UTFInputStreamReader;
37import org.openstreetmap.josm.io.auth.DefaultAuthenticator;
38
39/**
40 * Provides a uniform access for a HTTP/HTTPS server. This class should be used in favour of {@link HttpURLConnection}.
41 * @since 9168
42 */
43public final class HttpClient {
44
45 private URL url;
46 private final String requestMethod;
47 private int connectTimeout = (int) TimeUnit.SECONDS.toMillis(Main.pref.getInteger("socket.timeout.connect", 15));
48 private int readTimeout = (int) TimeUnit.SECONDS.toMillis(Main.pref.getInteger("socket.timeout.read", 30));
49 private byte[] requestBody;
50 private long ifModifiedSince;
51 private final Map<String, String> headers = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
52 private int maxRedirects = Main.pref.getInteger("socket.maxredirects", 5);
53 private boolean useCache;
54 private String reasonForRequest;
55 private HttpURLConnection connection; // to allow disconnecting before `response` is set
56 private Response response;
57 private boolean finishOnCloseOutput = true;
58
59 static {
60 CookieHandler.setDefault(new CookieManager());
61 }
62
63 private HttpClient(URL url, String requestMethod) {
64 this.url = url;
65 this.requestMethod = requestMethod;
66 this.headers.put("Accept-Encoding", "gzip");
67 }
68
69 /**
70 * Opens the HTTP connection.
71 * @return HTTP response
72 * @throws IOException if any I/O error occurs
73 */
74 public Response connect() throws IOException {
75 return connect(null);
76 }
77
78 /**
79 * Opens the HTTP connection.
80 * @param progressMonitor progress monitor
81 * @return HTTP response
82 * @throws IOException if any I/O error occurs
83 * @since 9179
84 */
85 public Response connect(ProgressMonitor progressMonitor) throws IOException {
86 if (progressMonitor == null) {
87 progressMonitor = NullProgressMonitor.INSTANCE;
88 }
89 final HttpURLConnection connection = (HttpURLConnection) url.openConnection();
90 this.connection = connection;
91 connection.setRequestMethod(requestMethod);
92 connection.setRequestProperty("User-Agent", Version.getInstance().getFullAgentString());
93 connection.setConnectTimeout(connectTimeout);
94 connection.setReadTimeout(readTimeout);
95 connection.setInstanceFollowRedirects(false); // we do that ourselves
96 if (ifModifiedSince > 0) {
97 connection.setIfModifiedSince(ifModifiedSince);
98 }
99 connection.setUseCaches(useCache);
100 if (!useCache) {
101 connection.setRequestProperty("Cache-Control", "no-cache");
102 }
103 for (Map.Entry<String, String> header : headers.entrySet()) {
104 if (header.getValue() != null) {
105 connection.setRequestProperty(header.getKey(), header.getValue());
106 }
107 }
108
109 progressMonitor.beginTask(tr("Contacting Server..."), 1);
110 progressMonitor.indeterminateSubTask(null);
111
112 if ("PUT".equals(requestMethod) || "POST".equals(requestMethod) || "DELETE".equals(requestMethod)) {
113 Logging.info("{0} {1} ({2}) ...", requestMethod, url, Utils.getSizeString(requestBody.length, Locale.getDefault()));
114 connection.setFixedLengthStreamingMode(requestBody.length);
115 connection.setDoOutput(true);
116 try (OutputStream out = new BufferedOutputStream(
117 new ProgressOutputStream(connection.getOutputStream(), requestBody.length, progressMonitor, finishOnCloseOutput))) {
118 out.write(requestBody);
119 }
120 }
121
122 boolean successfulConnection = false;
123 try {
124 try {
125 connection.connect();
126 final boolean hasReason = reasonForRequest != null && !reasonForRequest.isEmpty();
127 Logging.info("{0} {1}{2} -> {3}{4}",
128 requestMethod, url, hasReason ? (" (" + reasonForRequest + ')') : "",
129 connection.getResponseCode(),
130 connection.getContentLengthLong() > 0
131 ? (" (" + Utils.getSizeString(connection.getContentLengthLong(), Locale.getDefault()) + ')')
132 : ""
133 );
134 if (Logging.isDebugEnabled()) {
135 Logging.debug("RESPONSE: {0}", connection.getHeaderFields());
136 }
137 if (DefaultAuthenticator.getInstance().isEnabled() && connection.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) {
138 DefaultAuthenticator.getInstance().addFailedCredentialHost(url.getHost());
139 }
140 } catch (IOException e) {
141 Logging.info("{0} {1} -> !!!", requestMethod, url);
142 Logging.warn(e);
143 //noinspection ThrowableResultOfMethodCallIgnored
144 Main.addNetworkError(url, Utils.getRootCause(e));
145 throw e;
146 }
147 if (isRedirect(connection.getResponseCode())) {
148 final String redirectLocation = connection.getHeaderField("Location");
149 if (redirectLocation == null) {
150 /* I18n: argument is HTTP response code */
151 throw new IOException(tr("Unexpected response from HTTP server. Got {0} response without ''Location'' header." +
152 " Can''t redirect. Aborting.", connection.getResponseCode()));
153 } else if (maxRedirects > 0) {
154 url = new URL(url, redirectLocation);
155 maxRedirects--;
156 Logging.info(tr("Download redirected to ''{0}''", redirectLocation));
157 return connect();
158 } else if (maxRedirects == 0) {
159 String msg = tr("Too many redirects to the download URL detected. Aborting.");
160 throw new IOException(msg);
161 }
162 }
163 response = new Response(connection, progressMonitor);
164 successfulConnection = true;
165 return response;
166 } finally {
167 if (!successfulConnection) {
168 connection.disconnect();
169 }
170 }
171 }
172
173 /**
174 * Returns the HTTP response which is set only after calling {@link #connect()}.
175 * Calling this method again, returns the identical object (unless another {@link #connect()} is performed).
176 *
177 * @return the HTTP response
178 * @since 9309
179 */
180 public Response getResponse() {
181 return response;
182 }
183
184 /**
185 * A wrapper for the HTTP response.
186 */
187 public static final class Response {
188 private final HttpURLConnection connection;
189 private final ProgressMonitor monitor;
190 private final int responseCode;
191 private final String responseMessage;
192 private boolean uncompress;
193 private boolean uncompressAccordingToContentDisposition;
194 private String responseData;
195
196 private Response(HttpURLConnection connection, ProgressMonitor monitor) throws IOException {
197 CheckParameterUtil.ensureParameterNotNull(connection, "connection");
198 CheckParameterUtil.ensureParameterNotNull(monitor, "monitor");
199 this.connection = connection;
200 this.monitor = monitor;
201 this.responseCode = connection.getResponseCode();
202 this.responseMessage = connection.getResponseMessage();
203 if (this.responseCode >= 300) {
204 String contentType = getContentType();
205 if (contentType == null || (
206 contentType.contains("text") ||
207 contentType.contains("html") ||
208 contentType.contains("xml"))
209 ) {
210 String content = this.fetchContent();
211 if (content.isEmpty()) {
212 Logging.debug("Server did not return any body");
213 } else {
214 Logging.debug("Response body: ");
215 Logging.debug(this.fetchContent());
216 }
217 } else {
218 Logging.debug("Server returned content: {0} of length: {1}. Not printing.", contentType, this.getContentLength());
219 }
220 }
221 }
222
223 /**
224 * Sets whether {@link #getContent()} should uncompress the input stream if necessary.
225 *
226 * @param uncompress whether the input stream should be uncompressed if necessary
227 * @return {@code this}
228 */
229 public Response uncompress(boolean uncompress) {
230 this.uncompress = uncompress;
231 return this;
232 }
233
234 /**
235 * Sets whether {@link #getContent()} should uncompress the input stream according to {@code Content-Disposition}
236 * HTTP header.
237 * @param uncompressAccordingToContentDisposition whether the input stream should be uncompressed according to
238 * {@code Content-Disposition}
239 * @return {@code this}
240 * @since 9172
241 */
242 public Response uncompressAccordingToContentDisposition(boolean uncompressAccordingToContentDisposition) {
243 this.uncompressAccordingToContentDisposition = uncompressAccordingToContentDisposition;
244 return this;
245 }
246
247 /**
248 * Returns the URL.
249 * @return the URL
250 * @see HttpURLConnection#getURL()
251 * @since 9172
252 */
253 public URL getURL() {
254 return connection.getURL();
255 }
256
257 /**
258 * Returns the request method.
259 * @return the HTTP request method
260 * @see HttpURLConnection#getRequestMethod()
261 * @since 9172
262 */
263 public String getRequestMethod() {
264 return connection.getRequestMethod();
265 }
266
267 /**
268 * Returns an input stream that reads from this HTTP connection, or,
269 * error stream if the connection failed but the server sent useful data.
270 * <p>
271 * Note: the return value can be null, if both the input and the error stream are null.
272 * Seems to be the case if the OSM server replies a 401 Unauthorized, see #3887
273 * @return input or error stream
274 * @throws IOException if any I/O error occurs
275 *
276 * @see HttpURLConnection#getInputStream()
277 * @see HttpURLConnection#getErrorStream()
278 */
279 @SuppressWarnings("resource")
280 public InputStream getContent() throws IOException {
281 InputStream in;
282 try {
283 in = connection.getInputStream();
284 } catch (IOException ioe) {
285 Logging.debug(ioe);
286 in = Optional.ofNullable(connection.getErrorStream()).orElseGet(() -> new ByteArrayInputStream(new byte[]{}));
287 }
288 in = new ProgressInputStream(in, getContentLength(), monitor);
289 in = "gzip".equalsIgnoreCase(getContentEncoding()) ? new GZIPInputStream(in) : in;
290 Compression compression = Compression.NONE;
291 if (uncompress) {
292 final String contentType = getContentType();
293 Logging.debug("Uncompressing input stream according to Content-Type header: {0}", contentType);
294 compression = Compression.forContentType(contentType);
295 }
296 if (uncompressAccordingToContentDisposition && Compression.NONE.equals(compression)) {
297 final String contentDisposition = getHeaderField("Content-Disposition");
298 final Matcher matcher = Pattern.compile("filename=\"([^\"]+)\"").matcher(
299 contentDisposition != null ? contentDisposition : "");
300 if (matcher.find()) {
301 Logging.debug("Uncompressing input stream according to Content-Disposition header: {0}", contentDisposition);
302 compression = Compression.byExtension(matcher.group(1));
303 }
304 }
305 in = compression.getUncompressedInputStream(in);
306 return in;
307 }
308
309 /**
310 * Returns {@link #getContent()} wrapped in a buffered reader.
311 *
312 * Detects Unicode charset in use utilizing {@link UTFInputStreamReader}.
313 * @return buffered reader
314 * @throws IOException if any I/O error occurs
315 */
316 public BufferedReader getContentReader() throws IOException {
317 return new BufferedReader(
318 UTFInputStreamReader.create(getContent())
319 );
320 }
321
322 /**
323 * Fetches the HTTP response as String.
324 * @return the response
325 * @throws IOException if any I/O error occurs
326 */
327 public synchronized String fetchContent() throws IOException {
328 if (responseData == null) {
329 try (Scanner scanner = new Scanner(getContentReader()).useDelimiter("\\A")) { // \A - beginning of input
330 responseData = scanner.hasNext() ? scanner.next() : "";
331 }
332 }
333 return responseData;
334 }
335
336 /**
337 * Gets the response code from this HTTP connection.
338 * @return HTTP response code
339 *
340 * @see HttpURLConnection#getResponseCode()
341 */
342 public int getResponseCode() {
343 return responseCode;
344 }
345
346 /**
347 * Gets the response message from this HTTP connection.
348 * @return HTTP response message
349 *
350 * @see HttpURLConnection#getResponseMessage()
351 * @since 9172
352 */
353 public String getResponseMessage() {
354 return responseMessage;
355 }
356
357 /**
358 * Returns the {@code Content-Encoding} header.
359 * @return {@code Content-Encoding} HTTP header
360 * @see HttpURLConnection#getContentEncoding()
361 */
362 public String getContentEncoding() {
363 return connection.getContentEncoding();
364 }
365
366 /**
367 * Returns the {@code Content-Type} header.
368 * @return {@code Content-Type} HTTP header
369 */
370 public String getContentType() {
371 return connection.getHeaderField("Content-Type");
372 }
373
374 /**
375 * Returns the {@code Expire} header.
376 * @return {@code Expire} HTTP header
377 * @see HttpURLConnection#getExpiration()
378 * @since 9232
379 */
380 public long getExpiration() {
381 return connection.getExpiration();
382 }
383
384 /**
385 * Returns the {@code Last-Modified} header.
386 * @return {@code Last-Modified} HTTP header
387 * @see HttpURLConnection#getLastModified()
388 * @since 9232
389 */
390 public long getLastModified() {
391 return connection.getLastModified();
392 }
393
394 /**
395 * Returns the {@code Content-Length} header.
396 * @return {@code Content-Length} HTTP header
397 * @see HttpURLConnection#getContentLengthLong()
398 */
399 public long getContentLength() {
400 return connection.getContentLengthLong();
401 }
402
403 /**
404 * Returns the value of the named header field.
405 * @param name the name of a header field
406 * @return the value of the named header field, or {@code null} if there is no such field in the header
407 * @see HttpURLConnection#getHeaderField(String)
408 * @since 9172
409 */
410 public String getHeaderField(String name) {
411 return connection.getHeaderField(name);
412 }
413
414 /**
415 * Returns an unmodifiable Map mapping header keys to a List of header values.
416 * As per RFC 2616, section 4.2 header names are case insensitive, so returned map is also case insensitive
417 * @return unmodifiable Map mapping header keys to a List of header values
418 * @see HttpURLConnection#getHeaderFields()
419 * @since 9232
420 */
421 public Map<String, List<String>> getHeaderFields() {
422 // returned map from HttpUrlConnection is case sensitive, use case insensitive TreeMap to conform to RFC 2616
423 Map<String, List<String>> ret = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
424 for (Entry<String, List<String>> e: connection.getHeaderFields().entrySet()) {
425 if (e.getKey() != null) {
426 ret.put(e.getKey(), e.getValue());
427 }
428 }
429 return Collections.unmodifiableMap(ret);
430 }
431
432 /**
433 * @see HttpURLConnection#disconnect()
434 */
435 public void disconnect() {
436 HttpClient.disconnect(connection);
437 }
438 }
439
440 /**
441 * Creates a new instance for the given URL and a {@code GET} request
442 *
443 * @param url the URL
444 * @return a new instance
445 */
446 public static HttpClient create(URL url) {
447 return create(url, "GET");
448 }
449
450 /**
451 * Creates a new instance for the given URL and a {@code GET} request
452 *
453 * @param url the URL
454 * @param requestMethod the HTTP request method to perform when calling
455 * @return a new instance
456 */
457 public static HttpClient create(URL url, String requestMethod) {
458 return new HttpClient(url, requestMethod);
459 }
460
461 /**
462 * Returns the URL set for this connection.
463 * @return the URL
464 * @see #create(URL)
465 * @see #create(URL, String)
466 * @since 9172
467 */
468 public URL getURL() {
469 return url;
470 }
471
472 /**
473 * Returns the request method set for this connection.
474 * @return the HTTP request method
475 * @see #create(URL, String)
476 * @since 9172
477 */
478 public String getRequestMethod() {
479 return requestMethod;
480 }
481
482 /**
483 * Returns the set value for the given {@code header}.
484 * @param header HTTP header name
485 * @return HTTP header value
486 * @since 9172
487 */
488 public String getRequestHeader(String header) {
489 return headers.get(header);
490 }
491
492 /**
493 * Sets whether not to set header {@code Cache-Control=no-cache}
494 *
495 * @param useCache whether not to set header {@code Cache-Control=no-cache}
496 * @return {@code this}
497 * @see HttpURLConnection#setUseCaches(boolean)
498 */
499 public HttpClient useCache(boolean useCache) {
500 this.useCache = useCache;
501 return this;
502 }
503
504 /**
505 * Sets whether not to set header {@code Connection=close}
506 * <p>
507 * This might fix #7640, see
508 * <a href='https://web.archive.org/web/20140118201501/http://www.tikalk.com/java/forums/httpurlconnection-disable-keep-alive'>here</a>.
509 *
510 * @param keepAlive whether not to set header {@code Connection=close}
511 * @return {@code this}
512 */
513 public HttpClient keepAlive(boolean keepAlive) {
514 return setHeader("Connection", keepAlive ? null : "close");
515 }
516
517 /**
518 * Sets a specified timeout value, in milliseconds, to be used when opening a communications link to the resource referenced
519 * by this URLConnection. If the timeout expires before the connection can be established, a
520 * {@link java.net.SocketTimeoutException} is raised. A timeout of zero is interpreted as an infinite timeout.
521 * @param connectTimeout an {@code int} that specifies the connect timeout value in milliseconds
522 * @return {@code this}
523 * @see HttpURLConnection#setConnectTimeout(int)
524 */
525 public HttpClient setConnectTimeout(int connectTimeout) {
526 this.connectTimeout = connectTimeout;
527 return this;
528 }
529
530 /**
531 * Sets the read timeout to a specified timeout, in milliseconds. A non-zero value specifies the timeout when reading from
532 * input stream when a connection is established to a resource. If the timeout expires before there is data available for
533 * read, a {@link java.net.SocketTimeoutException} is raised. A timeout of zero is interpreted as an infinite timeout.
534 * @param readTimeout an {@code int} that specifies the read timeout value in milliseconds
535 * @return {@code this}
536 * @see HttpURLConnection#setReadTimeout(int)
537 */
538 public HttpClient setReadTimeout(int readTimeout) {
539 this.readTimeout = readTimeout;
540 return this;
541 }
542
543 /**
544 * Sets the {@code Accept} header.
545 * @param accept header value
546 *
547 * @return {@code this}
548 */
549 public HttpClient setAccept(String accept) {
550 return setHeader("Accept", accept);
551 }
552
553 /**
554 * Sets the request body for {@code PUT}/{@code POST} requests.
555 * @param requestBody request body
556 *
557 * @return {@code this}
558 */
559 public HttpClient setRequestBody(byte[] requestBody) {
560 this.requestBody = Utils.copyArray(requestBody);
561 return this;
562 }
563
564 /**
565 * Sets the {@code If-Modified-Since} header.
566 * @param ifModifiedSince header value
567 *
568 * @return {@code this}
569 */
570 public HttpClient setIfModifiedSince(long ifModifiedSince) {
571 this.ifModifiedSince = ifModifiedSince;
572 return this;
573 }
574
575 /**
576 * Sets the maximum number of redirections to follow.
577 *
578 * Set {@code maxRedirects} to {@code -1} in order to ignore redirects, i.e.,
579 * to not throw an {@link IOException} in {@link #connect()}.
580 * @param maxRedirects header value
581 *
582 * @return {@code this}
583 */
584 public HttpClient setMaxRedirects(int maxRedirects) {
585 this.maxRedirects = maxRedirects;
586 return this;
587 }
588
589 /**
590 * Sets an arbitrary HTTP header.
591 * @param key header name
592 * @param value header value
593 *
594 * @return {@code this}
595 */
596 public HttpClient setHeader(String key, String value) {
597 this.headers.put(key, value);
598 return this;
599 }
600
601 /**
602 * Sets arbitrary HTTP headers.
603 * @param headers HTTP headers
604 *
605 * @return {@code this}
606 */
607 public HttpClient setHeaders(Map<String, String> headers) {
608 this.headers.putAll(headers);
609 return this;
610 }
611
612 /**
613 * Sets a reason to show on console. Can be {@code null} if no reason is given.
614 * @param reasonForRequest Reason to show
615 * @return {@code this}
616 * @since 9172
617 */
618 public HttpClient setReasonForRequest(String reasonForRequest) {
619 this.reasonForRequest = reasonForRequest;
620 return this;
621 }
622
623 /**
624 * Sets whether the progress monitor task will be finished when the output stream is closed. This is {@code true} by default.
625 * @param finishOnCloseOutput whether the progress monitor task will be finished when the output stream is closed
626 * @return {@code this}
627 * @since 10302
628 */
629 public HttpClient setFinishOnCloseOutput(boolean finishOnCloseOutput) {
630 this.finishOnCloseOutput = finishOnCloseOutput;
631 return this;
632 }
633
634 private static boolean isRedirect(final int statusCode) {
635 switch (statusCode) {
636 case HttpURLConnection.HTTP_MOVED_PERM: // 301
637 case HttpURLConnection.HTTP_MOVED_TEMP: // 302
638 case HttpURLConnection.HTTP_SEE_OTHER: // 303
639 case 307: // TEMPORARY_REDIRECT:
640 case 308: // PERMANENT_REDIRECT:
641 return true;
642 default:
643 return false;
644 }
645 }
646
647 /**
648 * @see HttpURLConnection#disconnect()
649 * @since 9309
650 */
651 public void disconnect() {
652 HttpClient.disconnect(connection);
653 }
654
655 private static void disconnect(final HttpURLConnection connection) {
656 if (connection != null) {
657 // Fix upload aborts - see #263
658 connection.setConnectTimeout(100);
659 connection.setReadTimeout(100);
660 try {
661 Thread.sleep(100);
662 } catch (InterruptedException ex) {
663 Logging.warn("InterruptedException in " + HttpClient.class + " during cancel");
664 Thread.currentThread().interrupt();
665 }
666 connection.disconnect();
667 }
668 }
669}
Note: See TracBrowser for help on using the repository browser.