source: josm/trunk/src/org/openstreetmap/josm/tools/HttpClient.java@ 13137

Last change on this file since 13137 was 13099, checked in by Don-vip, 6 years ago

fix #15531 - trace request body for HTTP POST, PUT and DELETE
File size: 25.4 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.tools;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.BufferedOutputStream;
7import java.io.BufferedReader;
8import java.io.ByteArrayInputStream;
9import java.io.IOException;
10import java.io.InputStream;
11import java.io.OutputStream;
12import java.net.CookieHandler;
13import java.net.CookieManager;
14import java.net.HttpURLConnection;
15import java.net.URL;
16import java.nio.charset.StandardCharsets;
17import java.util.Collections;
18import java.util.List;
19import java.util.Locale;
20import java.util.Map;
21import java.util.Map.Entry;
22import java.util.Optional;
23import java.util.Scanner;
24import java.util.TreeMap;
25import java.util.concurrent.TimeUnit;
26import java.util.regex.Matcher;
27import java.util.regex.Pattern;
28import java.util.zip.GZIPInputStream;
29
30import org.openstreetmap.josm.Main;
31import org.openstreetmap.josm.data.Version;
32import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
33import org.openstreetmap.josm.gui.progress.ProgressMonitor;
34import org.openstreetmap.josm.io.Compression;
35import org.openstreetmap.josm.io.ProgressInputStream;
36import org.openstreetmap.josm.io.ProgressOutputStream;
37import org.openstreetmap.josm.io.UTFInputStreamReader;
38import org.openstreetmap.josm.io.auth.DefaultAuthenticator;
39import org.openstreetmap.josm.spi.preferences.Config;
40
41/**
42 * Provides a uniform access for a HTTP/HTTPS server. This class should be used in favour of {@link HttpURLConnection}.
43 * @since 9168
44 */
45public final class HttpClient {
46
47 private URL url;
48 private final String requestMethod;
49 private int connectTimeout = (int) TimeUnit.SECONDS.toMillis(Config.getPref().getInt("socket.timeout.connect", 15));
50 private int readTimeout = (int) TimeUnit.SECONDS.toMillis(Config.getPref().getInt("socket.timeout.read", 30));
51 private byte[] requestBody;
52 private long ifModifiedSince;
53 private final Map<String, String> headers = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
54 private int maxRedirects = Config.getPref().getInt("socket.maxredirects", 5);
55 private boolean useCache;
56 private String reasonForRequest;
57 private String outputMessage = tr("Uploading data ...");
58 private HttpURLConnection connection; // to allow disconnecting before `response` is set
59 private Response response;
60 private boolean finishOnCloseOutput = true;
61
62 static {
63 CookieHandler.setDefault(new CookieManager());
64 }
65
66 private HttpClient(URL url, String requestMethod) {
67 this.url = url;
68 this.requestMethod = requestMethod;
69 this.headers.put("Accept-Encoding", "gzip");
70 }
71
72 /**
73 * Opens the HTTP connection.
74 * @return HTTP response
75 * @throws IOException if any I/O error occurs
76 */
77 public Response connect() throws IOException {
78 return connect(null);
79 }
80
81 /**
82 * Opens the HTTP connection.
83 * @param progressMonitor progress monitor
84 * @return HTTP response
85 * @throws IOException if any I/O error occurs
86 * @since 9179
87 */
88 public Response connect(ProgressMonitor progressMonitor) throws IOException {
89 if (progressMonitor == null) {
90 progressMonitor = NullProgressMonitor.INSTANCE;
91 }
92 final HttpURLConnection connection = (HttpURLConnection) url.openConnection();
93 this.connection = connection;
94 connection.setRequestMethod(requestMethod);
95 connection.setRequestProperty("User-Agent", Version.getInstance().getFullAgentString());
96 connection.setConnectTimeout(connectTimeout);
97 connection.setReadTimeout(readTimeout);
98 connection.setInstanceFollowRedirects(false); // we do that ourselves
99 if (ifModifiedSince > 0) {
100 connection.setIfModifiedSince(ifModifiedSince);
101 }
102 connection.setUseCaches(useCache);
103 if (!useCache) {
104 connection.setRequestProperty("Cache-Control", "no-cache");
105 }
106 for (Map.Entry<String, String> header : headers.entrySet()) {
107 if (header.getValue() != null) {
108 connection.setRequestProperty(header.getKey(), header.getValue());
109 }
110 }
111
112 progressMonitor.beginTask(tr("Contacting Server..."), 1);
113 progressMonitor.indeterminateSubTask(null);
114
115 if ("PUT".equals(requestMethod) || "POST".equals(requestMethod) || "DELETE".equals(requestMethod)) {
116 Logging.info("{0} {1} ({2}) ...", requestMethod, url, Utils.getSizeString(requestBody.length, Locale.getDefault()));
117 if (Logging.isTraceEnabled() && requestBody.length > 0) {
118 Logging.trace("BODY: {0}", new String(requestBody, StandardCharsets.UTF_8));
119 }
120 connection.setFixedLengthStreamingMode(requestBody.length);
121 connection.setDoOutput(true);
122 try (OutputStream out = new BufferedOutputStream(
123 new ProgressOutputStream(connection.getOutputStream(), requestBody.length,
124 progressMonitor, outputMessage, finishOnCloseOutput))) {
125 out.write(requestBody);
126 }
127 }
128
129 boolean successfulConnection = false;
130 try {
131 try {
132 connection.connect();
133 final boolean hasReason = reasonForRequest != null && !reasonForRequest.isEmpty();
134 Logging.info("{0} {1}{2} -> {3}{4}",
135 requestMethod, url, hasReason ? (" (" + reasonForRequest + ')') : "",
136 connection.getResponseCode(),
137 connection.getContentLengthLong() > 0
138 ? (" (" + Utils.getSizeString(connection.getContentLengthLong(), Locale.getDefault()) + ')')
139 : ""
140 );
141 if (Logging.isDebugEnabled()) {
142 Logging.debug("RESPONSE: {0}", connection.getHeaderFields());
143 }
144 if (DefaultAuthenticator.getInstance().isEnabled() && connection.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) {
145 DefaultAuthenticator.getInstance().addFailedCredentialHost(url.getHost());
146 }
147 } catch (IOException | IllegalArgumentException e) {
148 Logging.info("{0} {1} -> !!!", requestMethod, url);
149 Logging.warn(e);
150 //noinspection ThrowableResultOfMethodCallIgnored
151 Main.addNetworkError(url, Utils.getRootCause(e));
152 throw e;
153 }
154 if (isRedirect(connection.getResponseCode())) {
155 final String redirectLocation = connection.getHeaderField("Location");
156 if (redirectLocation == null) {
157 /* I18n: argument is HTTP response code */
158 throw new IOException(tr("Unexpected response from HTTP server. Got {0} response without ''Location'' header." +
159 " Can''t redirect. Aborting.", connection.getResponseCode()));
160 } else if (maxRedirects > 0) {
161 url = new URL(url, redirectLocation);
162 maxRedirects--;
163 Logging.info(tr("Download redirected to ''{0}''", redirectLocation));
164 return connect();
165 } else if (maxRedirects == 0) {
166 String msg = tr("Too many redirects to the download URL detected. Aborting.");
167 throw new IOException(msg);
168 }
169 }
170 response = new Response(connection, progressMonitor);
171 successfulConnection = true;
172 return response;
173 } finally {
174 if (!successfulConnection) {
175 connection.disconnect();
176 }
177 }
178 }
179
180 /**
181 * Returns the HTTP response which is set only after calling {@link #connect()}.
182 * Calling this method again, returns the identical object (unless another {@link #connect()} is performed).
183 *
184 * @return the HTTP response
185 * @since 9309
186 */
187 public Response getResponse() {
188 return response;
189 }
190
191 /**
192 * A wrapper for the HTTP response.
193 */
194 public static final class Response {
195 private final HttpURLConnection connection;
196 private final ProgressMonitor monitor;
197 private final int responseCode;
198 private final String responseMessage;
199 private boolean uncompress;
200 private boolean uncompressAccordingToContentDisposition;
201 private String responseData;
202
203 private Response(HttpURLConnection connection, ProgressMonitor monitor) throws IOException {
204 CheckParameterUtil.ensureParameterNotNull(connection, "connection");
205 CheckParameterUtil.ensureParameterNotNull(monitor, "monitor");
206 this.connection = connection;
207 this.monitor = monitor;
208 this.responseCode = connection.getResponseCode();
209 this.responseMessage = connection.getResponseMessage();
210 if (this.responseCode >= 300) {
211 String contentType = getContentType();
212 if (contentType == null || (
213 contentType.contains("text") ||
214 contentType.contains("html") ||
215 contentType.contains("xml"))
216 ) {
217 String content = this.fetchContent();
218 if (content.isEmpty()) {
219 Logging.debug("Server did not return any body");
220 } else {
221 Logging.debug("Response body: ");
222 Logging.debug(this.fetchContent());
223 }
224 } else {
225 Logging.debug("Server returned content: {0} of length: {1}. Not printing.", contentType, this.getContentLength());
226 }
227 }
228 }
229
230 /**
231 * Sets whether {@link #getContent()} should uncompress the input stream if necessary.
232 *
233 * @param uncompress whether the input stream should be uncompressed if necessary
234 * @return {@code this}
235 */
236 public Response uncompress(boolean uncompress) {
237 this.uncompress = uncompress;
238 return this;
239 }
240
241 /**
242 * Sets whether {@link #getContent()} should uncompress the input stream according to {@code Content-Disposition}
243 * HTTP header.
244 * @param uncompressAccordingToContentDisposition whether the input stream should be uncompressed according to
245 * {@code Content-Disposition}
246 * @return {@code this}
247 * @since 9172
248 */
249 public Response uncompressAccordingToContentDisposition(boolean uncompressAccordingToContentDisposition) {
250 this.uncompressAccordingToContentDisposition = uncompressAccordingToContentDisposition;
251 return this;
252 }
253
254 /**
255 * Returns the URL.
256 * @return the URL
257 * @see HttpURLConnection#getURL()
258 * @since 9172
259 */
260 public URL getURL() {
261 return connection.getURL();
262 }
263
264 /**
265 * Returns the request method.
266 * @return the HTTP request method
267 * @see HttpURLConnection#getRequestMethod()
268 * @since 9172
269 */
270 public String getRequestMethod() {
271 return connection.getRequestMethod();
272 }
273
274 /**
275 * Returns an input stream that reads from this HTTP connection, or,
276 * error stream if the connection failed but the server sent useful data.
277 * <p>
278 * Note: the return value can be null, if both the input and the error stream are null.
279 * Seems to be the case if the OSM server replies a 401 Unauthorized, see #3887
280 * @return input or error stream
281 * @throws IOException if any I/O error occurs
282 *
283 * @see HttpURLConnection#getInputStream()
284 * @see HttpURLConnection#getErrorStream()
285 */
286 @SuppressWarnings("resource")
287 public InputStream getContent() throws IOException {
288 InputStream in;
289 try {
290 in = connection.getInputStream();
291 } catch (IOException ioe) {
292 Logging.debug(ioe);
293 in = Optional.ofNullable(connection.getErrorStream()).orElseGet(() -> new ByteArrayInputStream(new byte[]{}));
294 }
295 in = new ProgressInputStream(in, getContentLength(), monitor);
296 in = "gzip".equalsIgnoreCase(getContentEncoding()) ? new GZIPInputStream(in) : in;
297 Compression compression = Compression.NONE;
298 if (uncompress) {
299 final String contentType = getContentType();
300 Logging.debug("Uncompressing input stream according to Content-Type header: {0}", contentType);
301 compression = Compression.forContentType(contentType);
302 }
303 if (uncompressAccordingToContentDisposition && Compression.NONE.equals(compression)) {
304 final String contentDisposition = getHeaderField("Content-Disposition");
305 final Matcher matcher = Pattern.compile("filename=\"([^\"]+)\"").matcher(
306 contentDisposition != null ? contentDisposition : "");
307 if (matcher.find()) {
308 Logging.debug("Uncompressing input stream according to Content-Disposition header: {0}", contentDisposition);
309 compression = Compression.byExtension(matcher.group(1));
310 }
311 }
312 in = compression.getUncompressedInputStream(in);
313 return in;
314 }
315
316 /**
317 * Returns {@link #getContent()} wrapped in a buffered reader.
318 *
319 * Detects Unicode charset in use utilizing {@link UTFInputStreamReader}.
320 * @return buffered reader
321 * @throws IOException if any I/O error occurs
322 */
323 public BufferedReader getContentReader() throws IOException {
324 return new BufferedReader(
325 UTFInputStreamReader.create(getContent())
326 );
327 }
328
329 /**
330 * Fetches the HTTP response as String.
331 * @return the response
332 * @throws IOException if any I/O error occurs
333 */
334 public synchronized String fetchContent() throws IOException {
335 if (responseData == null) {
336 try (Scanner scanner = new Scanner(getContentReader()).useDelimiter("\\A")) { // \A - beginning of input
337 responseData = scanner.hasNext() ? scanner.next() : "";
338 }
339 }
340 return responseData;
341 }
342
343 /**
344 * Gets the response code from this HTTP connection.
345 * @return HTTP response code
346 *
347 * @see HttpURLConnection#getResponseCode()
348 */
349 public int getResponseCode() {
350 return responseCode;
351 }
352
353 /**
354 * Gets the response message from this HTTP connection.
355 * @return HTTP response message
356 *
357 * @see HttpURLConnection#getResponseMessage()
358 * @since 9172
359 */
360 public String getResponseMessage() {
361 return responseMessage;
362 }
363
364 /**
365 * Returns the {@code Content-Encoding} header.
366 * @return {@code Content-Encoding} HTTP header
367 * @see HttpURLConnection#getContentEncoding()
368 */
369 public String getContentEncoding() {
370 return connection.getContentEncoding();
371 }
372
373 /**
374 * Returns the {@code Content-Type} header.
375 * @return {@code Content-Type} HTTP header
376 */
377 public String getContentType() {
378 return connection.getHeaderField("Content-Type");
379 }
380
381 /**
382 * Returns the {@code Expire} header.
383 * @return {@code Expire} HTTP header
384 * @see HttpURLConnection#getExpiration()
385 * @since 9232
386 */
387 public long getExpiration() {
388 return connection.getExpiration();
389 }
390
391 /**
392 * Returns the {@code Last-Modified} header.
393 * @return {@code Last-Modified} HTTP header
394 * @see HttpURLConnection#getLastModified()
395 * @since 9232
396 */
397 public long getLastModified() {
398 return connection.getLastModified();
399 }
400
401 /**
402 * Returns the {@code Content-Length} header.
403 * @return {@code Content-Length} HTTP header
404 * @see HttpURLConnection#getContentLengthLong()
405 */
406 public long getContentLength() {
407 return connection.getContentLengthLong();
408 }
409
410 /**
411 * Returns the value of the named header field.
412 * @param name the name of a header field
413 * @return the value of the named header field, or {@code null} if there is no such field in the header
414 * @see HttpURLConnection#getHeaderField(String)
415 * @since 9172
416 */
417 public String getHeaderField(String name) {
418 return connection.getHeaderField(name);
419 }
420
421 /**
422 * Returns an unmodifiable Map mapping header keys to a List of header values.
423 * As per RFC 2616, section 4.2 header names are case insensitive, so returned map is also case insensitive
424 * @return unmodifiable Map mapping header keys to a List of header values
425 * @see HttpURLConnection#getHeaderFields()
426 * @since 9232
427 */
428 public Map<String, List<String>> getHeaderFields() {
429 // returned map from HttpUrlConnection is case sensitive, use case insensitive TreeMap to conform to RFC 2616
430 Map<String, List<String>> ret = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
431 for (Entry<String, List<String>> e: connection.getHeaderFields().entrySet()) {
432 if (e.getKey() != null) {
433 ret.put(e.getKey(), e.getValue());
434 }
435 }
436 return Collections.unmodifiableMap(ret);
437 }
438
439 /**
440 * @see HttpURLConnection#disconnect()
441 */
442 public void disconnect() {
443 HttpClient.disconnect(connection);
444 }
445 }
446
447 /**
448 * Creates a new instance for the given URL and a {@code GET} request
449 *
450 * @param url the URL
451 * @return a new instance
452 */
453 public static HttpClient create(URL url) {
454 return create(url, "GET");
455 }
456
457 /**
458 * Creates a new instance for the given URL and a {@code GET} request
459 *
460 * @param url the URL
461 * @param requestMethod the HTTP request method to perform when calling
462 * @return a new instance
463 */
464 public static HttpClient create(URL url, String requestMethod) {
465 return new HttpClient(url, requestMethod);
466 }
467
468 /**
469 * Returns the URL set for this connection.
470 * @return the URL
471 * @see #create(URL)
472 * @see #create(URL, String)
473 * @since 9172
474 */
475 public URL getURL() {
476 return url;
477 }
478
479 /**
480 * Returns the request method set for this connection.
481 * @return the HTTP request method
482 * @see #create(URL, String)
483 * @since 9172
484 */
485 public String getRequestMethod() {
486 return requestMethod;
487 }
488
489 /**
490 * Returns the set value for the given {@code header}.
491 * @param header HTTP header name
492 * @return HTTP header value
493 * @since 9172
494 */
495 public String getRequestHeader(String header) {
496 return headers.get(header);
497 }
498
499 /**
500 * Sets whether not to set header {@code Cache-Control=no-cache}
501 *
502 * @param useCache whether not to set header {@code Cache-Control=no-cache}
503 * @return {@code this}
504 * @see HttpURLConnection#setUseCaches(boolean)
505 */
506 public HttpClient useCache(boolean useCache) {
507 this.useCache = useCache;
508 return this;
509 }
510
511 /**
512 * Sets whether not to set header {@code Connection=close}
513 * <p>
514 * This might fix #7640, see
515 * <a href='https://web.archive.org/web/20140118201501/http://www.tikalk.com/java/forums/httpurlconnection-disable-keep-alive'>here</a>.
516 *
517 * @param keepAlive whether not to set header {@code Connection=close}
518 * @return {@code this}
519 */
520 public HttpClient keepAlive(boolean keepAlive) {
521 return setHeader("Connection", keepAlive ? null : "close");
522 }
523
524 /**
525 * Sets a specified timeout value, in milliseconds, to be used when opening a communications link to the resource referenced
526 * by this URLConnection. If the timeout expires before the connection can be established, a
527 * {@link java.net.SocketTimeoutException} is raised. A timeout of zero is interpreted as an infinite timeout.
528 * @param connectTimeout an {@code int} that specifies the connect timeout value in milliseconds
529 * @return {@code this}
530 * @see HttpURLConnection#setConnectTimeout(int)
531 */
532 public HttpClient setConnectTimeout(int connectTimeout) {
533 this.connectTimeout = connectTimeout;
534 return this;
535 }
536
537 /**
538 * Sets the read timeout to a specified timeout, in milliseconds. A non-zero value specifies the timeout when reading from
539 * input stream when a connection is established to a resource. If the timeout expires before there is data available for
540 * read, a {@link java.net.SocketTimeoutException} is raised. A timeout of zero is interpreted as an infinite timeout.
541 * @param readTimeout an {@code int} that specifies the read timeout value in milliseconds
542 * @return {@code this}
543 * @see HttpURLConnection#setReadTimeout(int)
544 */
545 public HttpClient setReadTimeout(int readTimeout) {
546 this.readTimeout = readTimeout;
547 return this;
548 }
549
550 /**
551 * Sets the {@code Accept} header.
552 * @param accept header value
553 *
554 * @return {@code this}
555 */
556 public HttpClient setAccept(String accept) {
557 return setHeader("Accept", accept);
558 }
559
560 /**
561 * Sets the request body for {@code PUT}/{@code POST} requests.
562 * @param requestBody request body
563 *
564 * @return {@code this}
565 */
566 public HttpClient setRequestBody(byte[] requestBody) {
567 this.requestBody = Utils.copyArray(requestBody);
568 return this;
569 }
570
571 /**
572 * Sets the {@code If-Modified-Since} header.
573 * @param ifModifiedSince header value
574 *
575 * @return {@code this}
576 */
577 public HttpClient setIfModifiedSince(long ifModifiedSince) {
578 this.ifModifiedSince = ifModifiedSince;
579 return this;
580 }
581
582 /**
583 * Sets the maximum number of redirections to follow.
584 *
585 * Set {@code maxRedirects} to {@code -1} in order to ignore redirects, i.e.,
586 * to not throw an {@link IOException} in {@link #connect()}.
587 * @param maxRedirects header value
588 *
589 * @return {@code this}
590 */
591 public HttpClient setMaxRedirects(int maxRedirects) {
592 this.maxRedirects = maxRedirects;
593 return this;
594 }
595
596 /**
597 * Sets an arbitrary HTTP header.
598 * @param key header name
599 * @param value header value
600 *
601 * @return {@code this}
602 */
603 public HttpClient setHeader(String key, String value) {
604 this.headers.put(key, value);
605 return this;
606 }
607
608 /**
609 * Sets arbitrary HTTP headers.
610 * @param headers HTTP headers
611 *
612 * @return {@code this}
613 */
614 public HttpClient setHeaders(Map<String, String> headers) {
615 this.headers.putAll(headers);
616 return this;
617 }
618
619 /**
620 * Sets a reason to show on console. Can be {@code null} if no reason is given.
621 * @param reasonForRequest Reason to show
622 * @return {@code this}
623 * @since 9172
624 */
625 public HttpClient setReasonForRequest(String reasonForRequest) {
626 this.reasonForRequest = reasonForRequest;
627 return this;
628 }
629
630 /**
631 * Sets the output message to be displayed in progress monitor for {@code PUT}, {@code POST} and {@code DELETE} methods.
632 * Defaults to "Uploading data ..." (translated). Has no effect for {@code GET} or any other method.
633 * @param outputMessage message to be displayed in progress monitor
634 * @return {@code this}
635 * @since 12711
636 */
637 public HttpClient setOutputMessage(String outputMessage) {
638 this.outputMessage = outputMessage;
639 return this;
640 }
641
642 /**
643 * Sets whether the progress monitor task will be finished when the output stream is closed. This is {@code true} by default.
644 * @param finishOnCloseOutput whether the progress monitor task will be finished when the output stream is closed
645 * @return {@code this}
646 * @since 10302
647 */
648 public HttpClient setFinishOnCloseOutput(boolean finishOnCloseOutput) {
649 this.finishOnCloseOutput = finishOnCloseOutput;
650 return this;
651 }
652
653 private static boolean isRedirect(final int statusCode) {
654 switch (statusCode) {
655 case HttpURLConnection.HTTP_MOVED_PERM: // 301
656 case HttpURLConnection.HTTP_MOVED_TEMP: // 302
657 case HttpURLConnection.HTTP_SEE_OTHER: // 303
658 case 307: // TEMPORARY_REDIRECT:
659 case 308: // PERMANENT_REDIRECT:
660 return true;
661 default:
662 return false;
663 }
664 }
665
666 /**
667 * @see HttpURLConnection#disconnect()
668 * @since 9309
669 */
670 public void disconnect() {
671 HttpClient.disconnect(connection);
672 }
673
674 private static void disconnect(final HttpURLConnection connection) {
675 if (connection != null) {
676 // Fix upload aborts - see #263
677 connection.setConnectTimeout(100);
678 connection.setReadTimeout(100);
679 try {
680 Thread.sleep(100);
681 } catch (InterruptedException ex) {
682 Logging.warn("InterruptedException in " + HttpClient.class + " during cancel");
683 Thread.currentThread().interrupt();
684 }
685 connection.disconnect();
686 }
687 }
688}
Note: See TracBrowser for help on using the repository browser.