source: josm/trunk/src/org/openstreetmap/josm/io/OsmServerReader.java@ 12806

Last change on this file since 12806 was 12711, checked in by Don-vip, 7 years ago

see #15141, see #15167 - use correct message ("Downloading data" instead of "Uploading data") when downloading data from Overpass API / POST
  • Property svn:eol-style set to native
File size: 18.0 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.io;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.IOException;
7import java.io.InputStream;
8import java.net.HttpURLConnection;
9import java.net.MalformedURLException;
10import java.net.URL;
11import java.util.List;
12
13import javax.xml.parsers.ParserConfigurationException;
14
15import org.openstreetmap.josm.Main;
16import org.openstreetmap.josm.data.gpx.GpxData;
17import org.openstreetmap.josm.data.notes.Note;
18import org.openstreetmap.josm.data.osm.DataSet;
19import org.openstreetmap.josm.gui.progress.ProgressMonitor;
20import org.openstreetmap.josm.io.auth.CredentialsAgentException;
21import org.openstreetmap.josm.io.auth.CredentialsManager;
22import org.openstreetmap.josm.tools.HttpClient;
23import org.openstreetmap.josm.tools.Logging;
24import org.openstreetmap.josm.tools.Utils;
25import org.openstreetmap.josm.tools.XmlParsingException;
26import org.w3c.dom.Document;
27import org.w3c.dom.Node;
28import org.xml.sax.SAXException;
29
30/**
31 * This DataReader reads directly from the REST API of the osm server.
32 *
33 * It supports plain text transfer as well as gzip or deflate encoded transfers;
34 * if compressed transfers are unwanted, set property osm-server.use-compression
35 * to false.
36 *
37 * @author imi
38 */
39public abstract class OsmServerReader extends OsmConnection {
40 private final OsmApi api = OsmApi.getOsmApi();
41 private boolean doAuthenticate;
42 protected boolean gpxParsedProperly;
43
44 /**
45 * Constructs a new {@code OsmServerReader}.
46 */
47 public OsmServerReader() {
48 try {
49 doAuthenticate = OsmApi.isUsingOAuth() && CredentialsManager.getInstance().lookupOAuthAccessToken() != null;
50 } catch (CredentialsAgentException e) {
51 Logging.warn(e);
52 }
53 }
54
55 /**
56 * Open a connection to the given url and return a reader on the input stream
57 * from that connection. In case of user cancel, return <code>null</code>.
58 * Relative URL's are directed to API base URL.
59 * @param urlStr The url to connect to.
60 * @param progressMonitor progress monitoring and abort handler
61 * @return A reader reading the input stream (servers answer) or <code>null</code>.
62 * @throws OsmTransferException if data transfer errors occur
63 */
64 protected InputStream getInputStream(String urlStr, ProgressMonitor progressMonitor) throws OsmTransferException {
65 return getInputStream(urlStr, progressMonitor, null);
66 }
67
68 /**
69 * Open a connection to the given url and return a reader on the input stream
70 * from that connection. In case of user cancel, return <code>null</code>.
71 * Relative URL's are directed to API base URL.
72 * @param urlStr The url to connect to.
73 * @param progressMonitor progress monitoring and abort handler
74 * @param reason The reason to show on console. Can be {@code null} if no reason is given
75 * @return A reader reading the input stream (servers answer) or <code>null</code>.
76 * @throws OsmTransferException if data transfer errors occur
77 */
78 protected InputStream getInputStream(String urlStr, ProgressMonitor progressMonitor, String reason) throws OsmTransferException {
79 try {
80 api.initialize(progressMonitor);
81 String url = urlStr.startsWith("http") ? urlStr : (getBaseUrl() + urlStr);
82 return getInputStreamRaw(url, progressMonitor, reason);
83 } finally {
84 progressMonitor.invalidate();
85 }
86 }
87
88 /**
89 * Return the base URL for relative URL requests
90 * @return base url of API
91 */
92 protected String getBaseUrl() {
93 return api.getBaseUrl();
94 }
95
96 /**
97 * Open a connection to the given url and return a reader on the input stream
98 * from that connection. In case of user cancel, return <code>null</code>.
99 * @param urlStr The exact url to connect to.
100 * @param progressMonitor progress monitoring and abort handler
101 * @return An reader reading the input stream (servers answer) or <code>null</code>.
102 * @throws OsmTransferException if data transfer errors occur
103 */
104 protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor) throws OsmTransferException {
105 return getInputStreamRaw(urlStr, progressMonitor, null);
106 }
107
108 /**
109 * Open a connection to the given url and return a reader on the input stream
110 * from that connection. In case of user cancel, return <code>null</code>.
111 * @param urlStr The exact url to connect to.
112 * @param progressMonitor progress monitoring and abort handler
113 * @param reason The reason to show on console. Can be {@code null} if no reason is given
114 * @return An reader reading the input stream (servers answer) or <code>null</code>.
115 * @throws OsmTransferException if data transfer errors occur
116 */
117 protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason) throws OsmTransferException {
118 return getInputStreamRaw(urlStr, progressMonitor, reason, false);
119 }
120
121 /**
122 * Open a connection to the given url (if HTTP, trough a GET request) and return a reader on the input stream
123 * from that connection. In case of user cancel, return <code>null</code>.
124 * @param urlStr The exact url to connect to.
125 * @param progressMonitor progress monitoring and abort handler
126 * @param reason The reason to show on console. Can be {@code null} if no reason is given
127 * @param uncompressAccordingToContentDisposition Whether to inspect the HTTP header {@code Content-Disposition}
128 * for {@code filename} and uncompress a gzip/bzip2 stream.
129 * @return An reader reading the input stream (servers answer) or <code>null</code>.
130 * @throws OsmTransferException if data transfer errors occur
131 */
132 protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason,
133 boolean uncompressAccordingToContentDisposition) throws OsmTransferException {
134 return getInputStreamRaw(urlStr, progressMonitor, reason, uncompressAccordingToContentDisposition, "GET", null);
135 }
136
137 /**
138 * Open a connection to the given url (if HTTP, with the specified method) and return a reader on the input stream
139 * from that connection. In case of user cancel, return <code>null</code>.
140 * @param urlStr The exact url to connect to.
141 * @param progressMonitor progress monitoring and abort handler
142 * @param reason The reason to show on console. Can be {@code null} if no reason is given
143 * @param uncompressAccordingToContentDisposition Whether to inspect the HTTP header {@code Content-Disposition}
144 * for {@code filename} and uncompress a gzip/bzip2 stream.
145 * @param httpMethod HTTP method ("GET", "POST" or "PUT")
146 * @param requestBody HTTP request body (for "POST" and "PUT" methods only). Must be null for "GET" method.
147 * @return An reader reading the input stream (servers answer) or <code>null</code>.
148 * @throws OsmTransferException if data transfer errors occur
149 * @since 12596
150 */
151 @SuppressWarnings("resource")
152 protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason,
153 boolean uncompressAccordingToContentDisposition, String httpMethod, byte[] requestBody) throws OsmTransferException {
154 try {
155 OnlineResource.JOSM_WEBSITE.checkOfflineAccess(urlStr, Main.getJOSMWebsite());
156 OnlineResource.OSM_API.checkOfflineAccess(urlStr, OsmApi.getOsmApi().getServerUrl());
157
158 URL url = null;
159 try {
160 url = new URL(urlStr.replace(" ", "%20"));
161 } catch (MalformedURLException e) {
162 throw new OsmTransferException(e);
163 }
164
165 if ("file".equals(url.getProtocol())) {
166 try {
167 return url.openStream();
168 } catch (IOException e) {
169 throw new OsmTransferException(e);
170 }
171 }
172
173 final HttpClient client = HttpClient.create(url, httpMethod)
174 .setFinishOnCloseOutput(false)
175 .setReasonForRequest(reason)
176 .setOutputMessage(tr("Downloading data..."))
177 .setRequestBody(requestBody);
178 activeConnection = client;
179 adaptRequest(client);
180 if (doAuthenticate) {
181 addAuth(client);
182 }
183 if (cancel)
184 throw new OsmTransferCanceledException("Operation canceled");
185
186 final HttpClient.Response response;
187 try {
188 response = client.connect(progressMonitor);
189 } catch (IOException e) {
190 Logging.error(e);
191 OsmTransferException ote = new OsmTransferException(
192 tr("Could not connect to the OSM server. Please check your internet connection."), e);
193 ote.setUrl(url.toString());
194 throw ote;
195 }
196 try {
197 if (response.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED)
198 throw new OsmApiException(HttpURLConnection.HTTP_UNAUTHORIZED, null, null);
199
200 if (response.getResponseCode() == HttpURLConnection.HTTP_PROXY_AUTH)
201 throw new OsmTransferCanceledException("Proxy Authentication Required");
202
203 if (response.getResponseCode() != HttpURLConnection.HTTP_OK) {
204 String errorHeader = response.getHeaderField("Error");
205 String errorBody = fetchResponseText(response);
206 throw new OsmApiException(response.getResponseCode(), errorHeader, errorBody, url.toString());
207 }
208
209 response.uncompressAccordingToContentDisposition(uncompressAccordingToContentDisposition);
210 return response.getContent();
211 } catch (OsmTransferException e) {
212 throw e;
213 } catch (IOException e) {
214 throw new OsmTransferException(e);
215 }
216 } finally {
217 progressMonitor.invalidate();
218 }
219 }
220
221 private static String fetchResponseText(final HttpClient.Response response) {
222 try {
223 return response.fetchContent();
224 } catch (IOException e) {
225 Logging.error(e);
226 return tr("Reading error text failed.");
227 }
228 }
229
230 /**
231 * Allows subclasses to modify the request.
232 * @param request the prepared request
233 * @since 9308
234 */
235 protected void adaptRequest(HttpClient request) {
236 }
237
238 /**
239 * Download OSM files from somewhere
240 * @param progressMonitor The progress monitor
241 * @return The corresponding dataset
242 * @throws OsmTransferException if any error occurs
243 */
244 public abstract DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException;
245
246 /**
247 * Download OSM Change files from somewhere
248 * @param progressMonitor The progress monitor
249 * @return The corresponding dataset
250 * @throws OsmTransferException if any error occurs
251 */
252 public DataSet parseOsmChange(final ProgressMonitor progressMonitor) throws OsmTransferException {
253 return null;
254 }
255
256 /**
257 * Download BZip2-compressed OSM Change files from somewhere
258 * @param progressMonitor The progress monitor
259 * @return The corresponding dataset
260 * @throws OsmTransferException if any error occurs
261 */
262 public DataSet parseOsmChangeBzip2(final ProgressMonitor progressMonitor) throws OsmTransferException {
263 return null;
264 }
265
266 /**
267 * Download GZip-compressed OSM Change files from somewhere
268 * @param progressMonitor The progress monitor
269 * @return The corresponding dataset
270 * @throws OsmTransferException if any error occurs
271 */
272 public DataSet parseOsmChangeGzip(final ProgressMonitor progressMonitor) throws OsmTransferException {
273 return null;
274 }
275
276 /**
277 * Retrieve raw gps waypoints from the server API.
278 * @param progressMonitor The progress monitor
279 * @return The corresponding GPX tracks
280 * @throws OsmTransferException if any error occurs
281 */
282 public GpxData parseRawGps(final ProgressMonitor progressMonitor) throws OsmTransferException {
283 return null;
284 }
285
286 /**
287 * Retrieve BZip2-compressed GPX files from somewhere.
288 * @param progressMonitor The progress monitor
289 * @return The corresponding GPX tracks
290 * @throws OsmTransferException if any error occurs
291 * @since 6244
292 */
293 public GpxData parseRawGpsBzip2(final ProgressMonitor progressMonitor) throws OsmTransferException {
294 return null;
295 }
296
297 /**
298 * Download BZip2-compressed OSM files from somewhere
299 * @param progressMonitor The progress monitor
300 * @return The corresponding dataset
301 * @throws OsmTransferException if any error occurs
302 */
303 public DataSet parseOsmBzip2(final ProgressMonitor progressMonitor) throws OsmTransferException {
304 return null;
305 }
306
307 /**
308 * Download GZip-compressed OSM files from somewhere
309 * @param progressMonitor The progress monitor
310 * @return The corresponding dataset
311 * @throws OsmTransferException if any error occurs
312 */
313 public DataSet parseOsmGzip(final ProgressMonitor progressMonitor) throws OsmTransferException {
314 return null;
315 }
316
317 /**
318 * Download Zip-compressed OSM files from somewhere
319 * @param progressMonitor The progress monitor
320 * @return The corresponding dataset
321 * @throws OsmTransferException if any error occurs
322 * @since 6882
323 */
324 public DataSet parseOsmZip(final ProgressMonitor progressMonitor) throws OsmTransferException {
325 return null;
326 }
327
328 /**
329 * Returns true if this reader is adding authentication credentials to the read
330 * request sent to the server.
331 *
332 * @return true if this reader is adding authentication credentials to the read
333 * request sent to the server
334 */
335 public boolean isDoAuthenticate() {
336 return doAuthenticate;
337 }
338
339 /**
340 * Sets whether this reader adds authentication credentials to the read
341 * request sent to the server.
342 *
343 * @param doAuthenticate true if this reader adds authentication credentials to the read
344 * request sent to the server
345 */
346 public void setDoAuthenticate(boolean doAuthenticate) {
347 this.doAuthenticate = doAuthenticate;
348 }
349
350 /**
351 * Determines if the GPX data has been parsed properly.
352 * @return true if the GPX data has been parsed properly, false otherwise
353 * @see GpxReader#parse
354 */
355 public final boolean isGpxParsedProperly() {
356 return gpxParsedProperly;
357 }
358
359 /**
360 * Downloads notes from the API, given API limit parameters
361 *
362 * @param noteLimit How many notes to download.
363 * @param daysClosed Return notes closed this many days in the past. -1 means all notes, ever. 0 means only unresolved notes.
364 * @param progressMonitor Progress monitor for user feedback
365 * @return List of notes returned by the API
366 * @throws OsmTransferException if any errors happen
367 */
368 public List<Note> parseNotes(int noteLimit, int daysClosed, ProgressMonitor progressMonitor) throws OsmTransferException {
369 return null;
370 }
371
372 /**
373 * Downloads notes from a given raw URL. The URL is assumed to be complete and no API limits are added
374 *
375 * @param progressMonitor progress monitor
376 * @return A list of notes parsed from the URL
377 * @throws OsmTransferException if any error occurs during dialog with OSM API
378 */
379 public List<Note> parseRawNotes(final ProgressMonitor progressMonitor) throws OsmTransferException {
380 return null;
381 }
382
383 /**
384 * Download notes from a URL that contains a bzip2 compressed notes dump file
385 * @param progressMonitor progress monitor
386 * @return A list of notes parsed from the URL
387 * @throws OsmTransferException if any error occurs during dialog with OSM API
388 */
389 public List<Note> parseRawNotesBzip2(final ProgressMonitor progressMonitor) throws OsmTransferException {
390 return null;
391 }
392
393 /**
394 * Returns an attribute from the given DOM node.
395 * @param node DOM node
396 * @param name attribute name
397 * @return attribute value for the given attribute
398 * @since 12510
399 */
400 protected static String getAttribute(Node node, String name) {
401 return node.getAttributes().getNamedItem(name).getNodeValue();
402 }
403
404 /**
405 * DOM document parser.
406 * @param <R> resulting type
407 * @since 12510
408 */
409 @FunctionalInterface
410 protected interface DomParser<R> {
411 /**
412 * Parses a given DOM document.
413 * @param doc DOM document
414 * @return parsed data
415 * @throws XmlParsingException if an XML parsing error occurs
416 */
417 R parse(Document doc) throws XmlParsingException;
418 }
419
420 /**
421 * Fetches generic data from the DOM document resulting an API call.
422 * @param api the OSM API call
423 * @param subtask the subtask translated message
424 * @param parser the parser converting the DOM document (OSM API result)
425 * @param <T> data type
426 * @param monitor The progress monitor
427 * @param reason The reason to show on console. Can be {@code null} if no reason is given
428 * @return The converted data
429 * @throws OsmTransferException if something goes wrong
430 * @since 12510
431 */
432 public <T> T fetchData(String api, String subtask, DomParser<T> parser, ProgressMonitor monitor, String reason)
433 throws OsmTransferException {
434 try {
435 monitor.beginTask("");
436 monitor.indeterminateSubTask(subtask);
437 try (InputStream in = getInputStream(api, monitor.createSubTaskMonitor(1, true), reason)) {
438 return parser.parse(Utils.parseSafeDOM(in));
439 }
440 } catch (OsmTransferException e) {
441 throw e;
442 } catch (IOException | ParserConfigurationException | SAXException e) {
443 throw new OsmTransferException(e);
444 } finally {
445 monitor.finishTask();
446 }
447 }
448}
Note: See TracBrowser for help on using the repository browser.