source: josm/trunk/src/org/openstreetmap/josm/tools/HttpClient.java@ 13186

Last change on this file since 13186 was 13179, checked in by Don-vip, 6 years ago

see #15609 - catch NoSuchElementException when performing HTTP call
File size: 25.5 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.tools;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.BufferedOutputStream;
7import java.io.BufferedReader;
8import java.io.ByteArrayInputStream;
9import java.io.IOException;
10import java.io.InputStream;
11import java.io.OutputStream;
12import java.net.CookieHandler;
13import java.net.CookieManager;
14import java.net.HttpURLConnection;
15import java.net.URL;
16import java.nio.charset.StandardCharsets;
17import java.util.Collections;
18import java.util.List;
19import java.util.Locale;
20import java.util.Map;
21import java.util.Map.Entry;
22import java.util.NoSuchElementException;
23import java.util.Optional;
24import java.util.Scanner;
25import java.util.TreeMap;
26import java.util.concurrent.TimeUnit;
27import java.util.regex.Matcher;
28import java.util.regex.Pattern;
29import java.util.zip.GZIPInputStream;
30
31import org.openstreetmap.josm.Main;
32import org.openstreetmap.josm.data.Version;
33import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
34import org.openstreetmap.josm.gui.progress.ProgressMonitor;
35import org.openstreetmap.josm.io.Compression;
36import org.openstreetmap.josm.io.ProgressInputStream;
37import org.openstreetmap.josm.io.ProgressOutputStream;
38import org.openstreetmap.josm.io.UTFInputStreamReader;
39import org.openstreetmap.josm.io.auth.DefaultAuthenticator;
40import org.openstreetmap.josm.spi.preferences.Config;
41
42/**
43 * Provides a uniform access for a HTTP/HTTPS server. This class should be used in favour of {@link HttpURLConnection}.
44 * @since 9168
45 */
46public final class HttpClient {
47
48 private URL url;
49 private final String requestMethod;
50 private int connectTimeout = (int) TimeUnit.SECONDS.toMillis(Config.getPref().getInt("socket.timeout.connect", 15));
51 private int readTimeout = (int) TimeUnit.SECONDS.toMillis(Config.getPref().getInt("socket.timeout.read", 30));
52 private byte[] requestBody;
53 private long ifModifiedSince;
54 private final Map<String, String> headers = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
55 private int maxRedirects = Config.getPref().getInt("socket.maxredirects", 5);
56 private boolean useCache;
57 private String reasonForRequest;
58 private String outputMessage = tr("Uploading data ...");
59 private HttpURLConnection connection; // to allow disconnecting before `response` is set
60 private Response response;
61 private boolean finishOnCloseOutput = true;
62
63 static {
64 CookieHandler.setDefault(new CookieManager());
65 }
66
67 private HttpClient(URL url, String requestMethod) {
68 this.url = url;
69 this.requestMethod = requestMethod;
70 this.headers.put("Accept-Encoding", "gzip");
71 }
72
73 /**
74 * Opens the HTTP connection.
75 * @return HTTP response
76 * @throws IOException if any I/O error occurs
77 */
78 public Response connect() throws IOException {
79 return connect(null);
80 }
81
82 /**
83 * Opens the HTTP connection.
84 * @param progressMonitor progress monitor
85 * @return HTTP response
86 * @throws IOException if any I/O error occurs
87 * @since 9179
88 */
89 public Response connect(ProgressMonitor progressMonitor) throws IOException {
90 if (progressMonitor == null) {
91 progressMonitor = NullProgressMonitor.INSTANCE;
92 }
93 final HttpURLConnection connection = (HttpURLConnection) url.openConnection();
94 this.connection = connection;
95 connection.setRequestMethod(requestMethod);
96 connection.setRequestProperty("User-Agent", Version.getInstance().getFullAgentString());
97 connection.setConnectTimeout(connectTimeout);
98 connection.setReadTimeout(readTimeout);
99 connection.setInstanceFollowRedirects(false); // we do that ourselves
100 if (ifModifiedSince > 0) {
101 connection.setIfModifiedSince(ifModifiedSince);
102 }
103 connection.setUseCaches(useCache);
104 if (!useCache) {
105 connection.setRequestProperty("Cache-Control", "no-cache");
106 }
107 for (Map.Entry<String, String> header : headers.entrySet()) {
108 if (header.getValue() != null) {
109 connection.setRequestProperty(header.getKey(), header.getValue());
110 }
111 }
112
113 progressMonitor.beginTask(tr("Contacting Server..."), 1);
114 progressMonitor.indeterminateSubTask(null);
115
116 if ("PUT".equals(requestMethod) || "POST".equals(requestMethod) || "DELETE".equals(requestMethod)) {
117 Logging.info("{0} {1} ({2}) ...", requestMethod, url, Utils.getSizeString(requestBody.length, Locale.getDefault()));
118 if (Logging.isTraceEnabled() && requestBody.length > 0) {
119 Logging.trace("BODY: {0}", new String(requestBody, StandardCharsets.UTF_8));
120 }
121 connection.setFixedLengthStreamingMode(requestBody.length);
122 connection.setDoOutput(true);
123 try (OutputStream out = new BufferedOutputStream(
124 new ProgressOutputStream(connection.getOutputStream(), requestBody.length,
125 progressMonitor, outputMessage, finishOnCloseOutput))) {
126 out.write(requestBody);
127 }
128 }
129
130 boolean successfulConnection = false;
131 try {
132 try {
133 connection.connect();
134 final boolean hasReason = reasonForRequest != null && !reasonForRequest.isEmpty();
135 Logging.info("{0} {1}{2} -> {3}{4}",
136 requestMethod, url, hasReason ? (" (" + reasonForRequest + ')') : "",
137 connection.getResponseCode(),
138 connection.getContentLengthLong() > 0
139 ? (" (" + Utils.getSizeString(connection.getContentLengthLong(), Locale.getDefault()) + ')')
140 : ""
141 );
142 if (Logging.isDebugEnabled()) {
143 Logging.debug("RESPONSE: {0}", connection.getHeaderFields());
144 }
145 if (DefaultAuthenticator.getInstance().isEnabled() && connection.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) {
146 DefaultAuthenticator.getInstance().addFailedCredentialHost(url.getHost());
147 }
148 } catch (IOException | IllegalArgumentException | NoSuchElementException e) {
149 Logging.info("{0} {1} -> !!!", requestMethod, url);
150 Logging.warn(e);
151 //noinspection ThrowableResultOfMethodCallIgnored
152 Main.addNetworkError(url, Utils.getRootCause(e));
153 throw e;
154 }
155 if (isRedirect(connection.getResponseCode())) {
156 final String redirectLocation = connection.getHeaderField("Location");
157 if (redirectLocation == null) {
158 /* I18n: argument is HTTP response code */
159 throw new IOException(tr("Unexpected response from HTTP server. Got {0} response without ''Location'' header." +
160 " Can''t redirect. Aborting.", connection.getResponseCode()));
161 } else if (maxRedirects > 0) {
162 url = new URL(url, redirectLocation);
163 maxRedirects--;
164 Logging.info(tr("Download redirected to ''{0}''", redirectLocation));
165 return connect();
166 } else if (maxRedirects == 0) {
167 String msg = tr("Too many redirects to the download URL detected. Aborting.");
168 throw new IOException(msg);
169 }
170 }
171 response = new Response(connection, progressMonitor);
172 successfulConnection = true;
173 return response;
174 } finally {
175 if (!successfulConnection) {
176 connection.disconnect();
177 }
178 }
179 }
180
181 /**
182 * Returns the HTTP response which is set only after calling {@link #connect()}.
183 * Calling this method again, returns the identical object (unless another {@link #connect()} is performed).
184 *
185 * @return the HTTP response
186 * @since 9309
187 */
188 public Response getResponse() {
189 return response;
190 }
191
192 /**
193 * A wrapper for the HTTP response.
194 */
195 public static final class Response {
196 private final HttpURLConnection connection;
197 private final ProgressMonitor monitor;
198 private final int responseCode;
199 private final String responseMessage;
200 private boolean uncompress;
201 private boolean uncompressAccordingToContentDisposition;
202 private String responseData;
203
204 private Response(HttpURLConnection connection, ProgressMonitor monitor) throws IOException {
205 CheckParameterUtil.ensureParameterNotNull(connection, "connection");
206 CheckParameterUtil.ensureParameterNotNull(monitor, "monitor");
207 this.connection = connection;
208 this.monitor = monitor;
209 this.responseCode = connection.getResponseCode();
210 this.responseMessage = connection.getResponseMessage();
211 if (this.responseCode >= 300) {
212 String contentType = getContentType();
213 if (contentType == null || (
214 contentType.contains("text") ||
215 contentType.contains("html") ||
216 contentType.contains("xml"))
217 ) {
218 String content = this.fetchContent();
219 if (content.isEmpty()) {
220 Logging.debug("Server did not return any body");
221 } else {
222 Logging.debug("Response body: ");
223 Logging.debug(this.fetchContent());
224 }
225 } else {
226 Logging.debug("Server returned content: {0} of length: {1}. Not printing.", contentType, this.getContentLength());
227 }
228 }
229 }
230
231 /**
232 * Sets whether {@link #getContent()} should uncompress the input stream if necessary.
233 *
234 * @param uncompress whether the input stream should be uncompressed if necessary
235 * @return {@code this}
236 */
237 public Response uncompress(boolean uncompress) {
238 this.uncompress = uncompress;
239 return this;
240 }
241
242 /**
243 * Sets whether {@link #getContent()} should uncompress the input stream according to {@code Content-Disposition}
244 * HTTP header.
245 * @param uncompressAccordingToContentDisposition whether the input stream should be uncompressed according to
246 * {@code Content-Disposition}
247 * @return {@code this}
248 * @since 9172
249 */
250 public Response uncompressAccordingToContentDisposition(boolean uncompressAccordingToContentDisposition) {
251 this.uncompressAccordingToContentDisposition = uncompressAccordingToContentDisposition;
252 return this;
253 }
254
255 /**
256 * Returns the URL.
257 * @return the URL
258 * @see HttpURLConnection#getURL()
259 * @since 9172
260 */
261 public URL getURL() {
262 return connection.getURL();
263 }
264
265 /**
266 * Returns the request method.
267 * @return the HTTP request method
268 * @see HttpURLConnection#getRequestMethod()
269 * @since 9172
270 */
271 public String getRequestMethod() {
272 return connection.getRequestMethod();
273 }
274
275 /**
276 * Returns an input stream that reads from this HTTP connection, or,
277 * error stream if the connection failed but the server sent useful data.
278 * <p>
279 * Note: the return value can be null, if both the input and the error stream are null.
280 * Seems to be the case if the OSM server replies a 401 Unauthorized, see #3887
281 * @return input or error stream
282 * @throws IOException if any I/O error occurs
283 *
284 * @see HttpURLConnection#getInputStream()
285 * @see HttpURLConnection#getErrorStream()
286 */
287 @SuppressWarnings("resource")
288 public InputStream getContent() throws IOException {
289 InputStream in;
290 try {
291 in = connection.getInputStream();
292 } catch (IOException ioe) {
293 Logging.debug(ioe);
294 in = Optional.ofNullable(connection.getErrorStream()).orElseGet(() -> new ByteArrayInputStream(new byte[]{}));
295 }
296 in = new ProgressInputStream(in, getContentLength(), monitor);
297 in = "gzip".equalsIgnoreCase(getContentEncoding()) ? new GZIPInputStream(in) : in;
298 Compression compression = Compression.NONE;
299 if (uncompress) {
300 final String contentType = getContentType();
301 Logging.debug("Uncompressing input stream according to Content-Type header: {0}", contentType);
302 compression = Compression.forContentType(contentType);
303 }
304 if (uncompressAccordingToContentDisposition && Compression.NONE.equals(compression)) {
305 final String contentDisposition = getHeaderField("Content-Disposition");
306 final Matcher matcher = Pattern.compile("filename=\"([^\"]+)\"").matcher(
307 contentDisposition != null ? contentDisposition : "");
308 if (matcher.find()) {
309 Logging.debug("Uncompressing input stream according to Content-Disposition header: {0}", contentDisposition);
310 compression = Compression.byExtension(matcher.group(1));
311 }
312 }
313 in = compression.getUncompressedInputStream(in);
314 return in;
315 }
316
317 /**
318 * Returns {@link #getContent()} wrapped in a buffered reader.
319 *
320 * Detects Unicode charset in use utilizing {@link UTFInputStreamReader}.
321 * @return buffered reader
322 * @throws IOException if any I/O error occurs
323 */
324 public BufferedReader getContentReader() throws IOException {
325 return new BufferedReader(
326 UTFInputStreamReader.create(getContent())
327 );
328 }
329
330 /**
331 * Fetches the HTTP response as String.
332 * @return the response
333 * @throws IOException if any I/O error occurs
334 */
335 public synchronized String fetchContent() throws IOException {
336 if (responseData == null) {
337 try (Scanner scanner = new Scanner(getContentReader()).useDelimiter("\\A")) { // \A - beginning of input
338 responseData = scanner.hasNext() ? scanner.next() : "";
339 }
340 }
341 return responseData;
342 }
343
344 /**
345 * Gets the response code from this HTTP connection.
346 * @return HTTP response code
347 *
348 * @see HttpURLConnection#getResponseCode()
349 */
350 public int getResponseCode() {
351 return responseCode;
352 }
353
354 /**
355 * Gets the response message from this HTTP connection.
356 * @return HTTP response message
357 *
358 * @see HttpURLConnection#getResponseMessage()
359 * @since 9172
360 */
361 public String getResponseMessage() {
362 return responseMessage;
363 }
364
365 /**
366 * Returns the {@code Content-Encoding} header.
367 * @return {@code Content-Encoding} HTTP header
368 * @see HttpURLConnection#getContentEncoding()
369 */
370 public String getContentEncoding() {
371 return connection.getContentEncoding();
372 }
373
374 /**
375 * Returns the {@code Content-Type} header.
376 * @return {@code Content-Type} HTTP header
377 */
378 public String getContentType() {
379 return connection.getHeaderField("Content-Type");
380 }
381
382 /**
383 * Returns the {@code Expire} header.
384 * @return {@code Expire} HTTP header
385 * @see HttpURLConnection#getExpiration()
386 * @since 9232
387 */
388 public long getExpiration() {
389 return connection.getExpiration();
390 }
391
392 /**
393 * Returns the {@code Last-Modified} header.
394 * @return {@code Last-Modified} HTTP header
395 * @see HttpURLConnection#getLastModified()
396 * @since 9232
397 */
398 public long getLastModified() {
399 return connection.getLastModified();
400 }
401
402 /**
403 * Returns the {@code Content-Length} header.
404 * @return {@code Content-Length} HTTP header
405 * @see HttpURLConnection#getContentLengthLong()
406 */
407 public long getContentLength() {
408 return connection.getContentLengthLong();
409 }
410
411 /**
412 * Returns the value of the named header field.
413 * @param name the name of a header field
414 * @return the value of the named header field, or {@code null} if there is no such field in the header
415 * @see HttpURLConnection#getHeaderField(String)
416 * @since 9172
417 */
418 public String getHeaderField(String name) {
419 return connection.getHeaderField(name);
420 }
421
422 /**
423 * Returns an unmodifiable Map mapping header keys to a List of header values.
424 * As per RFC 2616, section 4.2 header names are case insensitive, so returned map is also case insensitive
425 * @return unmodifiable Map mapping header keys to a List of header values
426 * @see HttpURLConnection#getHeaderFields()
427 * @since 9232
428 */
429 public Map<String, List<String>> getHeaderFields() {
430 // returned map from HttpUrlConnection is case sensitive, use case insensitive TreeMap to conform to RFC 2616
431 Map<String, List<String>> ret = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
432 for (Entry<String, List<String>> e: connection.getHeaderFields().entrySet()) {
433 if (e.getKey() != null) {
434 ret.put(e.getKey(), e.getValue());
435 }
436 }
437 return Collections.unmodifiableMap(ret);
438 }
439
440 /**
441 * @see HttpURLConnection#disconnect()
442 */
443 public void disconnect() {
444 HttpClient.disconnect(connection);
445 }
446 }
447
448 /**
449 * Creates a new instance for the given URL and a {@code GET} request
450 *
451 * @param url the URL
452 * @return a new instance
453 */
454 public static HttpClient create(URL url) {
455 return create(url, "GET");
456 }
457
458 /**
459 * Creates a new instance for the given URL and a {@code GET} request
460 *
461 * @param url the URL
462 * @param requestMethod the HTTP request method to perform when calling
463 * @return a new instance
464 */
465 public static HttpClient create(URL url, String requestMethod) {
466 return new HttpClient(url, requestMethod);
467 }
468
469 /**
470 * Returns the URL set for this connection.
471 * @return the URL
472 * @see #create(URL)
473 * @see #create(URL, String)
474 * @since 9172
475 */
476 public URL getURL() {
477 return url;
478 }
479
480 /**
481 * Returns the request method set for this connection.
482 * @return the HTTP request method
483 * @see #create(URL, String)
484 * @since 9172
485 */
486 public String getRequestMethod() {
487 return requestMethod;
488 }
489
490 /**
491 * Returns the set value for the given {@code header}.
492 * @param header HTTP header name
493 * @return HTTP header value
494 * @since 9172
495 */
496 public String getRequestHeader(String header) {
497 return headers.get(header);
498 }
499
500 /**
501 * Sets whether not to set header {@code Cache-Control=no-cache}
502 *
503 * @param useCache whether not to set header {@code Cache-Control=no-cache}
504 * @return {@code this}
505 * @see HttpURLConnection#setUseCaches(boolean)
506 */
507 public HttpClient useCache(boolean useCache) {
508 this.useCache = useCache;
509 return this;
510 }
511
512 /**
513 * Sets whether not to set header {@code Connection=close}
514 * <p>
515 * This might fix #7640, see
516 * <a href='https://web.archive.org/web/20140118201501/http://www.tikalk.com/java/forums/httpurlconnection-disable-keep-alive'>here</a>.
517 *
518 * @param keepAlive whether not to set header {@code Connection=close}
519 * @return {@code this}
520 */
521 public HttpClient keepAlive(boolean keepAlive) {
522 return setHeader("Connection", keepAlive ? null : "close");
523 }
524
525 /**
526 * Sets a specified timeout value, in milliseconds, to be used when opening a communications link to the resource referenced
527 * by this URLConnection. If the timeout expires before the connection can be established, a
528 * {@link java.net.SocketTimeoutException} is raised. A timeout of zero is interpreted as an infinite timeout.
529 * @param connectTimeout an {@code int} that specifies the connect timeout value in milliseconds
530 * @return {@code this}
531 * @see HttpURLConnection#setConnectTimeout(int)
532 */
533 public HttpClient setConnectTimeout(int connectTimeout) {
534 this.connectTimeout = connectTimeout;
535 return this;
536 }
537
538 /**
539 * Sets the read timeout to a specified timeout, in milliseconds. A non-zero value specifies the timeout when reading from
540 * input stream when a connection is established to a resource. If the timeout expires before there is data available for
541 * read, a {@link java.net.SocketTimeoutException} is raised. A timeout of zero is interpreted as an infinite timeout.
542 * @param readTimeout an {@code int} that specifies the read timeout value in milliseconds
543 * @return {@code this}
544 * @see HttpURLConnection#setReadTimeout(int)
545 */
546 public HttpClient setReadTimeout(int readTimeout) {
547 this.readTimeout = readTimeout;
548 return this;
549 }
550
551 /**
552 * Sets the {@code Accept} header.
553 * @param accept header value
554 *
555 * @return {@code this}
556 */
557 public HttpClient setAccept(String accept) {
558 return setHeader("Accept", accept);
559 }
560
561 /**
562 * Sets the request body for {@code PUT}/{@code POST} requests.
563 * @param requestBody request body
564 *
565 * @return {@code this}
566 */
567 public HttpClient setRequestBody(byte[] requestBody) {
568 this.requestBody = Utils.copyArray(requestBody);
569 return this;
570 }
571
572 /**
573 * Sets the {@code If-Modified-Since} header.
574 * @param ifModifiedSince header value
575 *
576 * @return {@code this}
577 */
578 public HttpClient setIfModifiedSince(long ifModifiedSince) {
579 this.ifModifiedSince = ifModifiedSince;
580 return this;
581 }
582
583 /**
584 * Sets the maximum number of redirections to follow.
585 *
586 * Set {@code maxRedirects} to {@code -1} in order to ignore redirects, i.e.,
587 * to not throw an {@link IOException} in {@link #connect()}.
588 * @param maxRedirects header value
589 *
590 * @return {@code this}
591 */
592 public HttpClient setMaxRedirects(int maxRedirects) {
593 this.maxRedirects = maxRedirects;
594 return this;
595 }
596
597 /**
598 * Sets an arbitrary HTTP header.
599 * @param key header name
600 * @param value header value
601 *
602 * @return {@code this}
603 */
604 public HttpClient setHeader(String key, String value) {
605 this.headers.put(key, value);
606 return this;
607 }
608
609 /**
610 * Sets arbitrary HTTP headers.
611 * @param headers HTTP headers
612 *
613 * @return {@code this}
614 */
615 public HttpClient setHeaders(Map<String, String> headers) {
616 this.headers.putAll(headers);
617 return this;
618 }
619
620 /**
621 * Sets a reason to show on console. Can be {@code null} if no reason is given.
622 * @param reasonForRequest Reason to show
623 * @return {@code this}
624 * @since 9172
625 */
626 public HttpClient setReasonForRequest(String reasonForRequest) {
627 this.reasonForRequest = reasonForRequest;
628 return this;
629 }
630
631 /**
632 * Sets the output message to be displayed in progress monitor for {@code PUT}, {@code POST} and {@code DELETE} methods.
633 * Defaults to "Uploading data ..." (translated). Has no effect for {@code GET} or any other method.
634 * @param outputMessage message to be displayed in progress monitor
635 * @return {@code this}
636 * @since 12711
637 */
638 public HttpClient setOutputMessage(String outputMessage) {
639 this.outputMessage = outputMessage;
640 return this;
641 }
642
643 /**
644 * Sets whether the progress monitor task will be finished when the output stream is closed. This is {@code true} by default.
645 * @param finishOnCloseOutput whether the progress monitor task will be finished when the output stream is closed
646 * @return {@code this}
647 * @since 10302
648 */
649 public HttpClient setFinishOnCloseOutput(boolean finishOnCloseOutput) {
650 this.finishOnCloseOutput = finishOnCloseOutput;
651 return this;
652 }
653
654 private static boolean isRedirect(final int statusCode) {
655 switch (statusCode) {
656 case HttpURLConnection.HTTP_MOVED_PERM: // 301
657 case HttpURLConnection.HTTP_MOVED_TEMP: // 302
658 case HttpURLConnection.HTTP_SEE_OTHER: // 303
659 case 307: // TEMPORARY_REDIRECT:
660 case 308: // PERMANENT_REDIRECT:
661 return true;
662 default:
663 return false;
664 }
665 }
666
667 /**
668 * @see HttpURLConnection#disconnect()
669 * @since 9309
670 */
671 public void disconnect() {
672 HttpClient.disconnect(connection);
673 }
674
675 private static void disconnect(final HttpURLConnection connection) {
676 if (connection != null) {
677 // Fix upload aborts - see #263
678 connection.setConnectTimeout(100);
679 connection.setReadTimeout(100);
680 try {
681 Thread.sleep(100);
682 } catch (InterruptedException ex) {
683 Logging.warn("InterruptedException in " + HttpClient.class + " during cancel");
684 Thread.currentThread().interrupt();
685 }
686 connection.disconnect();
687 }
688 }
689}
Note: See TracBrowser for help on using the repository browser.