source: josm/trunk/src/org/openstreetmap/josm/io/OsmServerReader.java@ 12596

Last change on this file since 12596 was 12596, checked in by Don-vip, 7 years ago

fix #15141 - Make HTTP POST requests to Overpass API - allows longer/more complex queries
  • Property svn:eol-style set to native
File size: 17.8 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.io;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.IOException;
7import java.io.InputStream;
8import java.net.HttpURLConnection;
9import java.net.MalformedURLException;
10import java.net.URL;
11import java.util.List;
12
13import javax.xml.parsers.ParserConfigurationException;
14
15import org.openstreetmap.josm.Main;
16import org.openstreetmap.josm.data.gpx.GpxData;
17import org.openstreetmap.josm.data.notes.Note;
18import org.openstreetmap.josm.data.osm.DataSet;
19import org.openstreetmap.josm.gui.progress.ProgressMonitor;
20import org.openstreetmap.josm.io.auth.CredentialsAgentException;
21import org.openstreetmap.josm.io.auth.CredentialsManager;
22import org.openstreetmap.josm.tools.HttpClient;
23import org.openstreetmap.josm.tools.Utils;
24import org.openstreetmap.josm.tools.XmlParsingException;
25import org.w3c.dom.Document;
26import org.w3c.dom.Node;
27import org.xml.sax.SAXException;
28
29/**
30 * This DataReader reads directly from the REST API of the osm server.
31 *
32 * It supports plain text transfer as well as gzip or deflate encoded transfers;
33 * if compressed transfers are unwanted, set property osm-server.use-compression
34 * to false.
35 *
36 * @author imi
37 */
38public abstract class OsmServerReader extends OsmConnection {
39 private final OsmApi api = OsmApi.getOsmApi();
40 private boolean doAuthenticate;
41 protected boolean gpxParsedProperly;
42
43 /**
44 * Constructs a new {@code OsmServerReader}.
45 */
46 public OsmServerReader() {
47 try {
48 doAuthenticate = OsmApi.isUsingOAuth() && CredentialsManager.getInstance().lookupOAuthAccessToken() != null;
49 } catch (CredentialsAgentException e) {
50 Main.warn(e);
51 }
52 }
53
54 /**
55 * Open a connection to the given url and return a reader on the input stream
56 * from that connection. In case of user cancel, return <code>null</code>.
57 * Relative URL's are directed to API base URL.
58 * @param urlStr The url to connect to.
59 * @param progressMonitor progress monitoring and abort handler
60 * @return A reader reading the input stream (servers answer) or <code>null</code>.
61 * @throws OsmTransferException if data transfer errors occur
62 */
63 protected InputStream getInputStream(String urlStr, ProgressMonitor progressMonitor) throws OsmTransferException {
64 return getInputStream(urlStr, progressMonitor, null);
65 }
66
67 /**
68 * Open a connection to the given url and return a reader on the input stream
69 * from that connection. In case of user cancel, return <code>null</code>.
70 * Relative URL's are directed to API base URL.
71 * @param urlStr The url to connect to.
72 * @param progressMonitor progress monitoring and abort handler
73 * @param reason The reason to show on console. Can be {@code null} if no reason is given
74 * @return A reader reading the input stream (servers answer) or <code>null</code>.
75 * @throws OsmTransferException if data transfer errors occur
76 */
77 protected InputStream getInputStream(String urlStr, ProgressMonitor progressMonitor, String reason) throws OsmTransferException {
78 try {
79 api.initialize(progressMonitor);
80 String url = urlStr.startsWith("http") ? urlStr : (getBaseUrl() + urlStr);
81 return getInputStreamRaw(url, progressMonitor, reason);
82 } finally {
83 progressMonitor.invalidate();
84 }
85 }
86
87 /**
88 * Return the base URL for relative URL requests
89 * @return base url of API
90 */
91 protected String getBaseUrl() {
92 return api.getBaseUrl();
93 }
94
95 /**
96 * Open a connection to the given url and return a reader on the input stream
97 * from that connection. In case of user cancel, return <code>null</code>.
98 * @param urlStr The exact url to connect to.
99 * @param progressMonitor progress monitoring and abort handler
100 * @return An reader reading the input stream (servers answer) or <code>null</code>.
101 * @throws OsmTransferException if data transfer errors occur
102 */
103 protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor) throws OsmTransferException {
104 return getInputStreamRaw(urlStr, progressMonitor, null);
105 }
106
107 /**
108 * Open a connection to the given url and return a reader on the input stream
109 * from that connection. In case of user cancel, return <code>null</code>.
110 * @param urlStr The exact url to connect to.
111 * @param progressMonitor progress monitoring and abort handler
112 * @param reason The reason to show on console. Can be {@code null} if no reason is given
113 * @return An reader reading the input stream (servers answer) or <code>null</code>.
114 * @throws OsmTransferException if data transfer errors occur
115 */
116 protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason) throws OsmTransferException {
117 return getInputStreamRaw(urlStr, progressMonitor, reason, false);
118 }
119
120 /**
121 * Open a connection to the given url (if HTTP, trough a GET request) and return a reader on the input stream
122 * from that connection. In case of user cancel, return <code>null</code>.
123 * @param urlStr The exact url to connect to.
124 * @param progressMonitor progress monitoring and abort handler
125 * @param reason The reason to show on console. Can be {@code null} if no reason is given
126 * @param uncompressAccordingToContentDisposition Whether to inspect the HTTP header {@code Content-Disposition}
127 * for {@code filename} and uncompress a gzip/bzip2 stream.
128 * @return An reader reading the input stream (servers answer) or <code>null</code>.
129 * @throws OsmTransferException if data transfer errors occur
130 */
131 protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason,
132 boolean uncompressAccordingToContentDisposition) throws OsmTransferException {
133 return getInputStreamRaw(urlStr, progressMonitor, reason, uncompressAccordingToContentDisposition, "GET", null);
134 }
135
136 /**
137 * Open a connection to the given url (if HTTP, with the specified method) and return a reader on the input stream
138 * from that connection. In case of user cancel, return <code>null</code>.
139 * @param urlStr The exact url to connect to.
140 * @param progressMonitor progress monitoring and abort handler
141 * @param reason The reason to show on console. Can be {@code null} if no reason is given
142 * @param uncompressAccordingToContentDisposition Whether to inspect the HTTP header {@code Content-Disposition}
143 * for {@code filename} and uncompress a gzip/bzip2 stream.
144 * @param httpMethod HTTP method ("GET", "POST" or "PUT")
145 * @param requestBody HTTP request body (for "POST" and "PUT" methods only). Must be null for "GET" method.
146 * @return An reader reading the input stream (servers answer) or <code>null</code>.
147 * @throws OsmTransferException if data transfer errors occur
148 * @since 12596
149 */
150 @SuppressWarnings("resource")
151 protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason,
152 boolean uncompressAccordingToContentDisposition, String httpMethod, byte[] requestBody) throws OsmTransferException {
153 try {
154 OnlineResource.JOSM_WEBSITE.checkOfflineAccess(urlStr, Main.getJOSMWebsite());
155 OnlineResource.OSM_API.checkOfflineAccess(urlStr, OsmApi.getOsmApi().getServerUrl());
156
157 URL url = null;
158 try {
159 url = new URL(urlStr.replace(" ", "%20"));
160 } catch (MalformedURLException e) {
161 throw new OsmTransferException(e);
162 }
163
164 if ("file".equals(url.getProtocol())) {
165 try {
166 return url.openStream();
167 } catch (IOException e) {
168 throw new OsmTransferException(e);
169 }
170 }
171
172 final HttpClient client = HttpClient.create(url, httpMethod)
173 .setFinishOnCloseOutput(false)
174 .setReasonForRequest(reason)
175 .setRequestBody(requestBody);
176 activeConnection = client;
177 adaptRequest(client);
178 if (doAuthenticate) {
179 addAuth(client);
180 }
181 if (cancel)
182 throw new OsmTransferCanceledException("Operation canceled");
183
184 final HttpClient.Response response;
185 try {
186 response = client.connect(progressMonitor);
187 } catch (IOException e) {
188 Main.error(e);
189 OsmTransferException ote = new OsmTransferException(
190 tr("Could not connect to the OSM server. Please check your internet connection."), e);
191 ote.setUrl(url.toString());
192 throw ote;
193 }
194 try {
195 if (response.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED)
196 throw new OsmApiException(HttpURLConnection.HTTP_UNAUTHORIZED, null, null);
197
198 if (response.getResponseCode() == HttpURLConnection.HTTP_PROXY_AUTH)
199 throw new OsmTransferCanceledException("Proxy Authentication Required");
200
201 if (response.getResponseCode() != HttpURLConnection.HTTP_OK) {
202 String errorHeader = response.getHeaderField("Error");
203 String errorBody = fetchResponseText(response);
204 throw new OsmApiException(response.getResponseCode(), errorHeader, errorBody, url.toString());
205 }
206
207 response.uncompressAccordingToContentDisposition(uncompressAccordingToContentDisposition);
208 return response.getContent();
209 } catch (OsmTransferException e) {
210 throw e;
211 } catch (IOException e) {
212 throw new OsmTransferException(e);
213 }
214 } finally {
215 progressMonitor.invalidate();
216 }
217 }
218
219 private static String fetchResponseText(final HttpClient.Response response) {
220 try {
221 return response.fetchContent();
222 } catch (IOException e) {
223 Main.error(e);
224 return tr("Reading error text failed.");
225 }
226 }
227
228 /**
229 * Allows subclasses to modify the request.
230 * @param request the prepared request
231 * @since 9308
232 */
233 protected void adaptRequest(HttpClient request) {
234 }
235
236 /**
237 * Download OSM files from somewhere
238 * @param progressMonitor The progress monitor
239 * @return The corresponding dataset
240 * @throws OsmTransferException if any error occurs
241 */
242 public abstract DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException;
243
244 /**
245 * Download OSM Change files from somewhere
246 * @param progressMonitor The progress monitor
247 * @return The corresponding dataset
248 * @throws OsmTransferException if any error occurs
249 */
250 public DataSet parseOsmChange(final ProgressMonitor progressMonitor) throws OsmTransferException {
251 return null;
252 }
253
254 /**
255 * Download BZip2-compressed OSM Change files from somewhere
256 * @param progressMonitor The progress monitor
257 * @return The corresponding dataset
258 * @throws OsmTransferException if any error occurs
259 */
260 public DataSet parseOsmChangeBzip2(final ProgressMonitor progressMonitor) throws OsmTransferException {
261 return null;
262 }
263
264 /**
265 * Download GZip-compressed OSM Change files from somewhere
266 * @param progressMonitor The progress monitor
267 * @return The corresponding dataset
268 * @throws OsmTransferException if any error occurs
269 */
270 public DataSet parseOsmChangeGzip(final ProgressMonitor progressMonitor) throws OsmTransferException {
271 return null;
272 }
273
274 /**
275 * Retrieve raw gps waypoints from the server API.
276 * @param progressMonitor The progress monitor
277 * @return The corresponding GPX tracks
278 * @throws OsmTransferException if any error occurs
279 */
280 public GpxData parseRawGps(final ProgressMonitor progressMonitor) throws OsmTransferException {
281 return null;
282 }
283
284 /**
285 * Retrieve BZip2-compressed GPX files from somewhere.
286 * @param progressMonitor The progress monitor
287 * @return The corresponding GPX tracks
288 * @throws OsmTransferException if any error occurs
289 * @since 6244
290 */
291 public GpxData parseRawGpsBzip2(final ProgressMonitor progressMonitor) throws OsmTransferException {
292 return null;
293 }
294
295 /**
296 * Download BZip2-compressed OSM files from somewhere
297 * @param progressMonitor The progress monitor
298 * @return The corresponding dataset
299 * @throws OsmTransferException if any error occurs
300 */
301 public DataSet parseOsmBzip2(final ProgressMonitor progressMonitor) throws OsmTransferException {
302 return null;
303 }
304
305 /**
306 * Download GZip-compressed OSM files from somewhere
307 * @param progressMonitor The progress monitor
308 * @return The corresponding dataset
309 * @throws OsmTransferException if any error occurs
310 */
311 public DataSet parseOsmGzip(final ProgressMonitor progressMonitor) throws OsmTransferException {
312 return null;
313 }
314
315 /**
316 * Download Zip-compressed OSM files from somewhere
317 * @param progressMonitor The progress monitor
318 * @return The corresponding dataset
319 * @throws OsmTransferException if any error occurs
320 * @since 6882
321 */
322 public DataSet parseOsmZip(final ProgressMonitor progressMonitor) throws OsmTransferException {
323 return null;
324 }
325
326 /**
327 * Returns true if this reader is adding authentication credentials to the read
328 * request sent to the server.
329 *
330 * @return true if this reader is adding authentication credentials to the read
331 * request sent to the server
332 */
333 public boolean isDoAuthenticate() {
334 return doAuthenticate;
335 }
336
337 /**
338 * Sets whether this reader adds authentication credentials to the read
339 * request sent to the server.
340 *
341 * @param doAuthenticate true if this reader adds authentication credentials to the read
342 * request sent to the server
343 */
344 public void setDoAuthenticate(boolean doAuthenticate) {
345 this.doAuthenticate = doAuthenticate;
346 }
347
348 /**
349 * Determines if the GPX data has been parsed properly.
350 * @return true if the GPX data has been parsed properly, false otherwise
351 * @see GpxReader#parse
352 */
353 public final boolean isGpxParsedProperly() {
354 return gpxParsedProperly;
355 }
356
357 /**
358 * Downloads notes from the API, given API limit parameters
359 *
360 * @param noteLimit How many notes to download.
361 * @param daysClosed Return notes closed this many days in the past. -1 means all notes, ever. 0 means only unresolved notes.
362 * @param progressMonitor Progress monitor for user feedback
363 * @return List of notes returned by the API
364 * @throws OsmTransferException if any errors happen
365 */
366 public List<Note> parseNotes(int noteLimit, int daysClosed, ProgressMonitor progressMonitor) throws OsmTransferException {
367 return null;
368 }
369
370 /**
371 * Downloads notes from a given raw URL. The URL is assumed to be complete and no API limits are added
372 *
373 * @param progressMonitor progress monitor
374 * @return A list of notes parsed from the URL
375 * @throws OsmTransferException if any error occurs during dialog with OSM API
376 */
377 public List<Note> parseRawNotes(final ProgressMonitor progressMonitor) throws OsmTransferException {
378 return null;
379 }
380
381 /**
382 * Download notes from a URL that contains a bzip2 compressed notes dump file
383 * @param progressMonitor progress monitor
384 * @return A list of notes parsed from the URL
385 * @throws OsmTransferException if any error occurs during dialog with OSM API
386 */
387 public List<Note> parseRawNotesBzip2(final ProgressMonitor progressMonitor) throws OsmTransferException {
388 return null;
389 }
390
391 /**
392 * Returns an attribute from the given DOM node.
393 * @param node DOM node
394 * @param name attribute name
395 * @return attribute value for the given attribute
396 * @since 12510
397 */
398 protected static String getAttribute(Node node, String name) {
399 return node.getAttributes().getNamedItem(name).getNodeValue();
400 }
401
402 /**
403 * DOM document parser.
404 * @param <R> resulting type
405 * @since 12510
406 */
407 @FunctionalInterface
408 protected interface DomParser<R> {
409 /**
410 * Parses a given DOM document.
411 * @param doc DOM document
412 * @return parsed data
413 * @throws XmlParsingException if an XML parsing error occurs
414 */
415 R parse(Document doc) throws XmlParsingException;
416 }
417
418 /**
419 * Fetches generic data from the DOM document resulting an API call.
420 * @param api the OSM API call
421 * @param subtask the subtask translated message
422 * @param parser the parser converting the DOM document (OSM API result)
423 * @param <T> data type
424 * @param monitor The progress monitor
425 * @param reason The reason to show on console. Can be {@code null} if no reason is given
426 * @return The converted data
427 * @throws OsmTransferException if something goes wrong
428 * @since 12510
429 */
430 public <T> T fetchData(String api, String subtask, DomParser<T> parser, ProgressMonitor monitor, String reason)
431 throws OsmTransferException {
432 try {
433 monitor.beginTask("");
434 monitor.indeterminateSubTask(subtask);
435 try (InputStream in = getInputStream(api, monitor.createSubTaskMonitor(1, true), reason)) {
436 return parser.parse(Utils.parseSafeDOM(in));
437 }
438 } catch (OsmTransferException e) {
439 throw e;
440 } catch (IOException | ParserConfigurationException | SAXException e) {
441 throw new OsmTransferException(e);
442 } finally {
443 monitor.finishTask();
444 }
445 }
446}
Note: See TracBrowser for help on using the repository browser.