source: josm/trunk/src/org/openstreetmap/josm/tools/HttpClient.java@ 10305

Last change on this file since 10305 was 10302, checked in by Don-vip, 8 years ago

fix #12583 - fix unit tests by adding a new mode to HttpClient. Don't know if it's the best way to do it...
File size: 23.3 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.tools;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.BufferedOutputStream;
7import java.io.BufferedReader;
8import java.io.IOException;
9import java.io.InputStream;
10import java.io.OutputStream;
11import java.net.CookieHandler;
12import java.net.CookieManager;
13import java.net.HttpURLConnection;
14import java.net.URL;
15import java.util.Collections;
16import java.util.List;
17import java.util.Locale;
18import java.util.Map;
19import java.util.Map.Entry;
20import java.util.Scanner;
21import java.util.TreeMap;
22import java.util.regex.Matcher;
23import java.util.regex.Pattern;
24import java.util.zip.GZIPInputStream;
25
26import org.openstreetmap.josm.Main;
27import org.openstreetmap.josm.data.Version;
28import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
29import org.openstreetmap.josm.gui.progress.ProgressMonitor;
30import org.openstreetmap.josm.io.Compression;
31import org.openstreetmap.josm.io.ProgressInputStream;
32import org.openstreetmap.josm.io.ProgressOutputStream;
33import org.openstreetmap.josm.io.UTFInputStreamReader;
34import org.openstreetmap.josm.io.auth.DefaultAuthenticator;
35
36/**
37 * Provides a uniform access for a HTTP/HTTPS server. This class should be used in favour of {@link HttpURLConnection}.
38 * @since 9168
39 */
40public final class HttpClient {
41
42 private URL url;
43 private final String requestMethod;
44 private int connectTimeout = Main.pref.getInteger("socket.timeout.connect", 15) * 1000;
45 private int readTimeout = Main.pref.getInteger("socket.timeout.read", 30) * 1000;
46 private byte[] requestBody;
47 private long ifModifiedSince;
48 private final Map<String, String> headers = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
49 private int maxRedirects = Main.pref.getInteger("socket.maxredirects", 5);
50 private boolean useCache;
51 private String reasonForRequest;
52 private HttpURLConnection connection; // to allow disconnecting before `response` is set
53 private Response response;
54 private boolean finishOnCloseOutput = true;
55
56 static {
57 CookieHandler.setDefault(new CookieManager());
58 }
59
60 private HttpClient(URL url, String requestMethod) {
61 this.url = url;
62 this.requestMethod = requestMethod;
63 this.headers.put("Accept-Encoding", "gzip");
64 }
65
66 /**
67 * Opens the HTTP connection.
68 * @return HTTP response
69 * @throws IOException if any I/O error occurs
70 */
71 public Response connect() throws IOException {
72 return connect(null);
73 }
74
75 /**
76 * Opens the HTTP connection.
77 * @param progressMonitor progress monitor
78 * @return HTTP response
79 * @throws IOException if any I/O error occurs
80 * @since 9179
81 */
82 public Response connect(ProgressMonitor progressMonitor) throws IOException {
83 if (progressMonitor == null) {
84 progressMonitor = NullProgressMonitor.INSTANCE;
85 }
86 final HttpURLConnection connection = (HttpURLConnection) url.openConnection();
87 this.connection = connection;
88 connection.setRequestMethod(requestMethod);
89 connection.setRequestProperty("User-Agent", Version.getInstance().getFullAgentString());
90 connection.setConnectTimeout(connectTimeout);
91 connection.setReadTimeout(readTimeout);
92 connection.setInstanceFollowRedirects(false); // we do that ourselves
93 if (ifModifiedSince > 0) {
94 connection.setIfModifiedSince(ifModifiedSince);
95 }
96 connection.setUseCaches(useCache);
97 if (!useCache) {
98 connection.setRequestProperty("Cache-Control", "no-cache");
99 }
100 for (Map.Entry<String, String> header : headers.entrySet()) {
101 if (header.getValue() != null) {
102 connection.setRequestProperty(header.getKey(), header.getValue());
103 }
104 }
105
106 progressMonitor.beginTask(tr("Contacting Server..."), 1);
107 progressMonitor.indeterminateSubTask(null);
108
109 if ("PUT".equals(requestMethod) || "POST".equals(requestMethod) || "DELETE".equals(requestMethod)) {
110 Main.info("{0} {1} ({2}) ...", requestMethod, url, Utils.getSizeString(requestBody.length, Locale.getDefault()));
111 connection.setFixedLengthStreamingMode(requestBody.length);
112 connection.setDoOutput(true);
113 try (OutputStream out = new BufferedOutputStream(
114 new ProgressOutputStream(connection.getOutputStream(), requestBody.length, progressMonitor, finishOnCloseOutput))) {
115 out.write(requestBody);
116 }
117 }
118
119 boolean successfulConnection = false;
120 try {
121 try {
122 connection.connect();
123 final boolean hasReason = reasonForRequest != null && !reasonForRequest.isEmpty();
124 Main.info("{0} {1}{2} -> {3}{4}",
125 requestMethod, url, hasReason ? (" (" + reasonForRequest + ')') : "",
126 connection.getResponseCode(),
127 connection.getContentLengthLong() > 0
128 ? (" (" + Utils.getSizeString(connection.getContentLengthLong(), Locale.getDefault()) + ')')
129 : ""
130 );
131 if (Main.isDebugEnabled()) {
132 Main.debug("RESPONSE: " + connection.getHeaderFields());
133 }
134 if (DefaultAuthenticator.getInstance().isEnabled() && connection.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) {
135 DefaultAuthenticator.getInstance().addFailedCredentialHost(url.getHost());
136 }
137 } catch (IOException e) {
138 Main.info("{0} {1} -> !!!", requestMethod, url);
139 Main.warn(e);
140 //noinspection ThrowableResultOfMethodCallIgnored
141 Main.addNetworkError(url, Utils.getRootCause(e));
142 throw e;
143 }
144 if (isRedirect(connection.getResponseCode())) {
145 final String redirectLocation = connection.getHeaderField("Location");
146 if (redirectLocation == null) {
147 /* I18n: argument is HTTP response code */
148 String msg = tr("Unexpected response from HTTP server. Got {0} response without ''Location'' header." +
149 " Can''t redirect. Aborting.", connection.getResponseCode());
150 throw new IOException(msg);
151 } else if (maxRedirects > 0) {
152 url = new URL(url, redirectLocation);
153 maxRedirects--;
154 Main.info(tr("Download redirected to ''{0}''", redirectLocation));
155 return connect();
156 } else if (maxRedirects == 0) {
157 String msg = tr("Too many redirects to the download URL detected. Aborting.");
158 throw new IOException(msg);
159 }
160 }
161 response = new Response(connection, progressMonitor);
162 successfulConnection = true;
163 return response;
164 } finally {
165 if (!successfulConnection) {
166 connection.disconnect();
167 }
168 }
169 }
170
171 /**
172 * Returns the HTTP response which is set only after calling {@link #connect()}.
173 * Calling this method again, returns the identical object (unless another {@link #connect()} is performed).
174 *
175 * @return the HTTP response
176 * @since 9309
177 */
178 public Response getResponse() {
179 return response;
180 }
181
182 /**
183 * A wrapper for the HTTP response.
184 */
185 public static final class Response {
186 private final HttpURLConnection connection;
187 private final ProgressMonitor monitor;
188 private final int responseCode;
189 private final String responseMessage;
190 private boolean uncompress;
191 private boolean uncompressAccordingToContentDisposition;
192
193 private Response(HttpURLConnection connection, ProgressMonitor monitor) throws IOException {
194 CheckParameterUtil.ensureParameterNotNull(connection, "connection");
195 CheckParameterUtil.ensureParameterNotNull(monitor, "monitor");
196 this.connection = connection;
197 this.monitor = monitor;
198 this.responseCode = connection.getResponseCode();
199 this.responseMessage = connection.getResponseMessage();
200 }
201
202 /**
203 * Sets whether {@link #getContent()} should uncompress the input stream if necessary.
204 *
205 * @param uncompress whether the input stream should be uncompressed if necessary
206 * @return {@code this}
207 */
208 public Response uncompress(boolean uncompress) {
209 this.uncompress = uncompress;
210 return this;
211 }
212
213 /**
214 * Sets whether {@link #getContent()} should uncompress the input stream according to {@code Content-Disposition}
215 * HTTP header.
216 * @param uncompressAccordingToContentDisposition whether the input stream should be uncompressed according to
217 * {@code Content-Disposition}
218 * @return {@code this}
219 * @since 9172
220 */
221 public Response uncompressAccordingToContentDisposition(boolean uncompressAccordingToContentDisposition) {
222 this.uncompressAccordingToContentDisposition = uncompressAccordingToContentDisposition;
223 return this;
224 }
225
226 /**
227 * Returns the URL.
228 * @return the URL
229 * @see HttpURLConnection#getURL()
230 * @since 9172
231 */
232 public URL getURL() {
233 return connection.getURL();
234 }
235
236 /**
237 * Returns the request method.
238 * @return the HTTP request method
239 * @see HttpURLConnection#getRequestMethod()
240 * @since 9172
241 */
242 public String getRequestMethod() {
243 return connection.getRequestMethod();
244 }
245
246 /**
247 * Returns an input stream that reads from this HTTP connection, or,
248 * error stream if the connection failed but the server sent useful data.
249 * <p>
250 * Note: the return value can be null, if both the input and the error stream are null.
251 * Seems to be the case if the OSM server replies a 401 Unauthorized, see #3887
252 * @return input or error stream
253 * @throws IOException if any I/O error occurs
254 *
255 * @see HttpURLConnection#getInputStream()
256 * @see HttpURLConnection#getErrorStream()
257 */
258 @SuppressWarnings("resource")
259 public InputStream getContent() throws IOException {
260 InputStream in;
261 try {
262 in = connection.getInputStream();
263 } catch (IOException ioe) {
264 in = connection.getErrorStream();
265 }
266 if (in != null) {
267 in = new ProgressInputStream(in, getContentLength(), monitor);
268 in = "gzip".equalsIgnoreCase(getContentEncoding()) ? new GZIPInputStream(in) : in;
269 Compression compression = Compression.NONE;
270 if (uncompress) {
271 final String contentType = getContentType();
272 Main.debug("Uncompressing input stream according to Content-Type header: {0}", contentType);
273 compression = Compression.forContentType(contentType);
274 }
275 if (uncompressAccordingToContentDisposition && Compression.NONE.equals(compression)) {
276 final String contentDisposition = getHeaderField("Content-Disposition");
277 final Matcher matcher = Pattern.compile("filename=\"([^\"]+)\"").matcher(
278 contentDisposition != null ? contentDisposition : "");
279 if (matcher.find()) {
280 Main.debug("Uncompressing input stream according to Content-Disposition header: {0}", contentDisposition);
281 compression = Compression.byExtension(matcher.group(1));
282 }
283 }
284 in = compression.getUncompressedInputStream(in);
285 }
286 return in;
287 }
288
289 /**
290 * Returns {@link #getContent()} wrapped in a buffered reader.
291 *
292 * Detects Unicode charset in use utilizing {@link UTFInputStreamReader}.
293 * @return buffered reader
294 * @throws IOException if any I/O error occurs
295 */
296 public BufferedReader getContentReader() throws IOException {
297 return new BufferedReader(
298 UTFInputStreamReader.create(getContent())
299 );
300 }
301
302 /**
303 * Fetches the HTTP response as String.
304 * @return the response
305 * @throws IOException if any I/O error occurs
306 */
307 @SuppressWarnings("resource")
308 public String fetchContent() throws IOException {
309 try (Scanner scanner = new Scanner(getContentReader()).useDelimiter("\\A")) {
310 return scanner.hasNext() ? scanner.next() : "";
311 }
312 }
313
314 /**
315 * Gets the response code from this HTTP connection.
316 * @return HTTP response code
317 *
318 * @see HttpURLConnection#getResponseCode()
319 */
320 public int getResponseCode() {
321 return responseCode;
322 }
323
324 /**
325 * Gets the response message from this HTTP connection.
326 * @return HTTP response message
327 *
328 * @see HttpURLConnection#getResponseMessage()
329 * @since 9172
330 */
331 public String getResponseMessage() {
332 return responseMessage;
333 }
334
335 /**
336 * Returns the {@code Content-Encoding} header.
337 * @return {@code Content-Encoding} HTTP header
338 * @see HttpURLConnection#getContentEncoding()
339 */
340 public String getContentEncoding() {
341 return connection.getContentEncoding();
342 }
343
344 /**
345 * Returns the {@code Content-Type} header.
346 * @return {@code Content-Type} HTTP header
347 */
348 public String getContentType() {
349 return connection.getHeaderField("Content-Type");
350 }
351
352 /**
353 * Returns the {@code Expire} header.
354 * @return {@code Expire} HTTP header
355 * @see HttpURLConnection#getExpiration()
356 * @since 9232
357 */
358 public long getExpiration() {
359 return connection.getExpiration();
360 }
361
362 /**
363 * Returns the {@code Last-Modified} header.
364 * @return {@code Last-Modified} HTTP header
365 * @see HttpURLConnection#getLastModified()
366 * @since 9232
367 */
368 public long getLastModified() {
369 return connection.getLastModified();
370 }
371
372 /**
373 * Returns the {@code Content-Length} header.
374 * @return {@code Content-Length} HTTP header
375 * @see HttpURLConnection#getContentLengthLong()
376 */
377 public long getContentLength() {
378 return connection.getContentLengthLong();
379 }
380
381 /**
382 * Returns the value of the named header field.
383 * @param name the name of a header field
384 * @return the value of the named header field, or {@code null} if there is no such field in the header
385 * @see HttpURLConnection#getHeaderField(String)
386 * @since 9172
387 */
388 public String getHeaderField(String name) {
389 return connection.getHeaderField(name);
390 }
391
392 /**
393 * Returns an unmodifiable Map mapping header keys to a List of header values.
394 * As per RFC 2616, section 4.2 header names are case insensitive, so returned map is also case insensitive
395 * @return unmodifiable Map mapping header keys to a List of header values
396 * @see HttpURLConnection#getHeaderFields()
397 * @since 9232
398 */
399 public Map<String, List<String>> getHeaderFields() {
400 // returned map from HttpUrlConnection is case sensitive, use case insensitive TreeMap to conform to RFC 2616
401 Map<String, List<String>> ret = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
402 for (Entry<String, List<String>> e: connection.getHeaderFields().entrySet()) {
403 if (e.getKey() != null) {
404 ret.put(e.getKey(), e.getValue());
405 }
406 }
407 return Collections.unmodifiableMap(ret);
408 }
409
410 /**
411 * @see HttpURLConnection#disconnect()
412 */
413 public void disconnect() {
414 HttpClient.disconnect(connection);
415 }
416 }
417
418 /**
419 * Creates a new instance for the given URL and a {@code GET} request
420 *
421 * @param url the URL
422 * @return a new instance
423 */
424 public static HttpClient create(URL url) {
425 return create(url, "GET");
426 }
427
428 /**
429 * Creates a new instance for the given URL and a {@code GET} request
430 *
431 * @param url the URL
432 * @param requestMethod the HTTP request method to perform when calling
433 * @return a new instance
434 */
435 public static HttpClient create(URL url, String requestMethod) {
436 return new HttpClient(url, requestMethod);
437 }
438
439 /**
440 * Returns the URL set for this connection.
441 * @return the URL
442 * @see #create(URL)
443 * @see #create(URL, String)
444 * @since 9172
445 */
446 public URL getURL() {
447 return url;
448 }
449
450 /**
451 * Returns the request method set for this connection.
452 * @return the HTTP request method
453 * @see #create(URL, String)
454 * @since 9172
455 */
456 public String getRequestMethod() {
457 return requestMethod;
458 }
459
460 /**
461 * Returns the set value for the given {@code header}.
462 * @param header HTTP header name
463 * @return HTTP header value
464 * @since 9172
465 */
466 public String getRequestHeader(String header) {
467 return headers.get(header);
468 }
469
470 /**
471 * Sets whether not to set header {@code Cache-Control=no-cache}
472 *
473 * @param useCache whether not to set header {@code Cache-Control=no-cache}
474 * @return {@code this}
475 * @see HttpURLConnection#setUseCaches(boolean)
476 */
477 public HttpClient useCache(boolean useCache) {
478 this.useCache = useCache;
479 return this;
480 }
481
482 /**
483 * Sets whether not to set header {@code Connection=close}
484 * <p>
485 * This might fix #7640, see
486 * <a href='https://web.archive.org/web/20140118201501/http://www.tikalk.com/java/forums/httpurlconnection-disable-keep-alive'>here</a>.
487 *
488 * @param keepAlive whether not to set header {@code Connection=close}
489 * @return {@code this}
490 */
491 public HttpClient keepAlive(boolean keepAlive) {
492 return setHeader("Connection", keepAlive ? null : "close");
493 }
494
495 /**
496 * Sets a specified timeout value, in milliseconds, to be used when opening a communications link to the resource referenced
497 * by this URLConnection. If the timeout expires before the connection can be established, a
498 * {@link java.net.SocketTimeoutException} is raised. A timeout of zero is interpreted as an infinite timeout.
499 * @param connectTimeout an {@code int} that specifies the connect timeout value in milliseconds
500 * @return {@code this}
501 * @see HttpURLConnection#setConnectTimeout(int)
502 */
503 public HttpClient setConnectTimeout(int connectTimeout) {
504 this.connectTimeout = connectTimeout;
505 return this;
506 }
507
508 /**
509 * Sets the read timeout to a specified timeout, in milliseconds. A non-zero value specifies the timeout when reading from
510 * input stream when a connection is established to a resource. If the timeout expires before there is data available for
511 * read, a {@link java.net.SocketTimeoutException} is raised. A timeout of zero is interpreted as an infinite timeout.
512 * @param readTimeout an {@code int} that specifies the read timeout value in milliseconds
513 * @return {@code this}
514 * @see HttpURLConnection#setReadTimeout(int)
515 */
516 public HttpClient setReadTimeout(int readTimeout) {
517 this.readTimeout = readTimeout;
518 return this;
519 }
520
521 /**
522 * Sets the {@code Accept} header.
523 * @param accept header value
524 *
525 * @return {@code this}
526 */
527 public HttpClient setAccept(String accept) {
528 return setHeader("Accept", accept);
529 }
530
531 /**
532 * Sets the request body for {@code PUT}/{@code POST} requests.
533 * @param requestBody request body
534 *
535 * @return {@code this}
536 */
537 public HttpClient setRequestBody(byte[] requestBody) {
538 this.requestBody = requestBody;
539 return this;
540 }
541
542 /**
543 * Sets the {@code If-Modified-Since} header.
544 * @param ifModifiedSince header value
545 *
546 * @return {@code this}
547 */
548 public HttpClient setIfModifiedSince(long ifModifiedSince) {
549 this.ifModifiedSince = ifModifiedSince;
550 return this;
551 }
552
553 /**
554 * Sets the maximum number of redirections to follow.
555 *
556 * Set {@code maxRedirects} to {@code -1} in order to ignore redirects, i.e.,
557 * to not throw an {@link IOException} in {@link #connect()}.
558 * @param maxRedirects header value
559 *
560 * @return {@code this}
561 */
562 public HttpClient setMaxRedirects(int maxRedirects) {
563 this.maxRedirects = maxRedirects;
564 return this;
565 }
566
567 /**
568 * Sets an arbitrary HTTP header.
569 * @param key header name
570 * @param value header value
571 *
572 * @return {@code this}
573 */
574 public HttpClient setHeader(String key, String value) {
575 this.headers.put(key, value);
576 return this;
577 }
578
579 /**
580 * Sets arbitrary HTTP headers.
581 * @param headers HTTP headers
582 *
583 * @return {@code this}
584 */
585 public HttpClient setHeaders(Map<String, String> headers) {
586 this.headers.putAll(headers);
587 return this;
588 }
589
590 /**
591 * Sets a reason to show on console. Can be {@code null} if no reason is given.
592 * @param reasonForRequest Reason to show
593 * @return {@code this}
594 * @since 9172
595 */
596 public HttpClient setReasonForRequest(String reasonForRequest) {
597 this.reasonForRequest = reasonForRequest;
598 return this;
599 }
600
601 /**
602 * Sets whether the progress monitor task will be finished when the output stream is closed. This is {@code true} by default.
603 * @param finishOnCloseOutput whether the progress monitor task will be finished when the output stream is closed
604 * @return {@code this}
605 * @since 10302
606 */
607 public HttpClient setFinishOnCloseOutput(boolean finishOnCloseOutput) {
608 this.finishOnCloseOutput = finishOnCloseOutput;
609 return this;
610 }
611
612 private static boolean isRedirect(final int statusCode) {
613 switch (statusCode) {
614 case HttpURLConnection.HTTP_MOVED_PERM: // 301
615 case HttpURLConnection.HTTP_MOVED_TEMP: // 302
616 case HttpURLConnection.HTTP_SEE_OTHER: // 303
617 case 307: // TEMPORARY_REDIRECT:
618 case 308: // PERMANENT_REDIRECT:
619 return true;
620 default:
621 return false;
622 }
623 }
624
625 /**
626 * @see HttpURLConnection#disconnect()
627 * @since 9309
628 */
629 public void disconnect() {
630 HttpClient.disconnect(connection);
631 }
632
633 private static void disconnect(final HttpURLConnection connection) {
634 // Fix upload aborts - see #263
635 connection.setConnectTimeout(100);
636 connection.setReadTimeout(100);
637 try {
638 Thread.sleep(100);
639 } catch (InterruptedException ex) {
640 Main.warn("InterruptedException in " + HttpClient.class + " during cancel");
641 }
642 connection.disconnect();
643 }
644}
Note: See TracBrowser for help on using the repository browser.