source: josm/trunk/src/org/openstreetmap/josm/io/OverpassDownloadReader.java

Last change on this file was 18211, checked in by Don-vip, 3 years ago

global use of !Utils.isEmpty/isBlank
  • Property svn:eol-style set to native
File size: 18.2 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.io;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.IOException;
7import java.io.InputStream;
8import java.nio.charset.StandardCharsets;
9import java.time.Duration;
10import java.time.LocalDateTime;
11import java.time.Period;
12import java.time.ZoneId;
13import java.time.ZoneOffset;
14import java.time.format.DateTimeParseException;
15import java.util.Arrays;
16import java.util.EnumMap;
17import java.util.List;
18import java.util.Locale;
19import java.util.Map;
20import java.util.NoSuchElementException;
21import java.util.Objects;
22import java.util.concurrent.ConcurrentHashMap;
23import java.util.concurrent.TimeUnit;
24import java.util.regex.Matcher;
25import java.util.regex.Pattern;
26
27import javax.xml.stream.XMLStreamConstants;
28import javax.xml.stream.XMLStreamException;
29
30import org.openstreetmap.josm.data.Bounds;
31import org.openstreetmap.josm.data.DataSource;
32import org.openstreetmap.josm.data.coor.LatLon;
33import org.openstreetmap.josm.data.osm.BBox;
34import org.openstreetmap.josm.data.osm.DataSet;
35import org.openstreetmap.josm.data.osm.DataSetMerger;
36import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
37import org.openstreetmap.josm.data.osm.PrimitiveId;
38import org.openstreetmap.josm.data.preferences.BooleanProperty;
39import org.openstreetmap.josm.data.preferences.ListProperty;
40import org.openstreetmap.josm.data.preferences.StringProperty;
41import org.openstreetmap.josm.gui.download.OverpassDownloadSource;
42import org.openstreetmap.josm.gui.progress.ProgressMonitor;
43import org.openstreetmap.josm.io.NameFinder.SearchResult;
44import org.openstreetmap.josm.tools.HttpClient;
45import org.openstreetmap.josm.tools.Logging;
46import org.openstreetmap.josm.tools.UncheckedParseException;
47import org.openstreetmap.josm.tools.Utils;
48import org.openstreetmap.josm.tools.date.DateUtils;
49
50/**
51 * Read content from an Overpass server.
52 *
53 * @since 8744
54 */
55public class OverpassDownloadReader extends BoundingBoxDownloader {
56
57 /**
58 * Property for current Overpass server.
59 * @since 12816
60 */
61 public static final StringProperty OVERPASS_SERVER = new StringProperty("download.overpass.server",
62 "https://overpass-api.de/api/");
63 /**
64 * Property for list of known Overpass servers.
65 * @since 12816
66 */
67 public static final ListProperty OVERPASS_SERVER_HISTORY = new ListProperty("download.overpass.servers",
68 Arrays.asList("https://overpass-api.de/api/", "http://overpass.openstreetmap.ru/cgi/"));
69 /**
70 * Property to determine if Overpass API should be used for multi-fetch download.
71 * @since 12816
72 */
73 public static final BooleanProperty FOR_MULTI_FETCH = new BooleanProperty("download.overpass.for-multi-fetch", false);
74
75 private static final String DATA_PREFIX = "?data=";
76
77 static final class OverpassOsmReader extends OsmReader {
78 @Override
79 protected void parseUnknown(boolean printWarning) throws XMLStreamException {
80 if ("remark".equals(parser.getLocalName()) && parser.getEventType() == XMLStreamConstants.START_ELEMENT) {
81 final String text = parser.getElementText();
82 if (text.contains("runtime error")) {
83 throw new XMLStreamException(text);
84 }
85 }
86 super.parseUnknown(printWarning);
87 }
88 }
89
90 static final class OverpassOsmJsonReader extends OsmJsonReader {
91
92 }
93
94 /**
95 * Possible Overpass API output format, with the {@code [out:<directive>]} statement.
96 * @since 11916
97 */
98 public enum OverpassOutputFormat {
99 /** Default output format: plain OSM XML */
100 OSM_XML("xml"),
101 /** OSM JSON format (not GeoJson) */
102 OSM_JSON("json"),
103 /** CSV, see https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#Output_Format_.28out.29 */
104 CSV("csv"),
105 /** Custom, see https://overpass-api.de/output_formats.html#custom */
106 CUSTOM("custom"),
107 /** Popup, see https://overpass-api.de/output_formats.html#popup */
108 POPUP("popup"),
109 /** PBF, see https://josm.openstreetmap.de/ticket/14653 */
110 PBF("pbf");
111
112 private final String directive;
113
114 OverpassOutputFormat(String directive) {
115 this.directive = directive;
116 }
117
118 /**
119 * Returns the directive used in {@code [out:<directive>]} statement.
120 * @return the directive used in {@code [out:<directive>]} statement
121 */
122 public String getDirective() {
123 return directive;
124 }
125
126 /**
127 * Returns the {@code OverpassOutputFormat} matching the given directive.
128 * @param directive directive used in {@code [out:<directive>]} statement
129 * @return {@code OverpassOutputFormat} matching the given directive
130 * @throws IllegalArgumentException in case of invalid directive
131 */
132 static OverpassOutputFormat from(String directive) {
133 for (OverpassOutputFormat oof : values()) {
134 if (oof.directive.equals(directive)) {
135 return oof;
136 }
137 }
138 throw new IllegalArgumentException(directive);
139 }
140 }
141
142 static final Pattern OUTPUT_FORMAT_STATEMENT = Pattern.compile(".*\\[out:([a-z]{3,})\\].*", Pattern.DOTALL);
143
144 static final Map<OverpassOutputFormat, Class<? extends AbstractReader>> outputFormatReaders = new ConcurrentHashMap<>();
145
146 final String overpassServer;
147 final String overpassQuery;
148
149 /**
150 * Constructs a new {@code OverpassDownloadReader}.
151 *
152 * @param downloadArea The area to download
153 * @param overpassServer The Overpass server to use
154 * @param overpassQuery The Overpass query
155 */
156 public OverpassDownloadReader(Bounds downloadArea, String overpassServer, String overpassQuery) {
157 super(downloadArea);
158 setDoAuthenticate(false);
159 this.overpassServer = overpassServer;
160 this.overpassQuery = overpassQuery.trim();
161 }
162
163 /**
164 * Registers an OSM reader for the given Overpass output format.
165 * @param format Overpass output format
166 * @param readerClass OSM reader class
167 * @return the previous value associated with {@code format}, or {@code null} if there was no mapping
168 */
169 public static final Class<? extends AbstractReader> registerOverpassOutputFormatReader(
170 OverpassOutputFormat format, Class<? extends AbstractReader> readerClass) {
171 return outputFormatReaders.put(Objects.requireNonNull(format), Objects.requireNonNull(readerClass));
172 }
173
174 static {
175 registerOverpassOutputFormatReader(OverpassOutputFormat.OSM_XML, OverpassOsmReader.class);
176 registerOverpassOutputFormatReader(OverpassOutputFormat.OSM_JSON, OverpassOsmJsonReader.class);
177 }
178
179 @Override
180 protected String getBaseUrl() {
181 return overpassServer;
182 }
183
184 @Override
185 protected String getRequestForBbox(double lon1, double lat1, double lon2, double lat2) {
186 if (overpassQuery.isEmpty())
187 return super.getRequestForBbox(lon1, lat1, lon2, lat2);
188 else {
189 final String query = this.overpassQuery
190 .replace("{{bbox}}", bbox(lon1, lat1, lon2, lat2))
191 .replace("{{center}}", center(lon1, lat1, lon2, lat2));
192 final String expandedOverpassQuery = expandExtendedQueries(query);
193 return "interpreter" + DATA_PREFIX + Utils.encodeUrl(expandedOverpassQuery);
194 }
195 }
196
197 /**
198 * Evaluates some features of overpass turbo extended query syntax.
199 * See https://wiki.openstreetmap.org/wiki/Overpass_turbo/Extended_Overpass_Turbo_Queries
200 * @param query unexpanded query
201 * @return expanded query
202 */
203 static String expandExtendedQueries(String query) {
204 final StringBuffer sb = new StringBuffer();
205 final Matcher matcher = Pattern.compile("\\{\\{(date|geocodeArea|geocodeBbox|geocodeCoords|geocodeId):([^}]+)\\}\\}").matcher(query);
206 while (matcher.find()) {
207 try {
208 switch (matcher.group(1)) {
209 case "date":
210 matcher.appendReplacement(sb, date(matcher.group(2), LocalDateTime.now(ZoneId.systemDefault())));
211 break;
212 case "geocodeArea":
213 matcher.appendReplacement(sb, geocodeArea(matcher.group(2)));
214 break;
215 case "geocodeBbox":
216 matcher.appendReplacement(sb, geocodeBbox(matcher.group(2)));
217 break;
218 case "geocodeCoords":
219 matcher.appendReplacement(sb, geocodeCoords(matcher.group(2)));
220 break;
221 case "geocodeId":
222 matcher.appendReplacement(sb, geocodeId(matcher.group(2)));
223 break;
224 default:
225 Logging.warn("Unsupported syntax: " + matcher.group(1));
226 }
227 } catch (UncheckedParseException | DateTimeParseException | IOException | NoSuchElementException | IndexOutOfBoundsException ex) {
228 final String msg = tr("Failed to evaluate {0}", matcher.group());
229 Logging.log(Logging.LEVEL_WARN, msg, ex);
230 matcher.appendReplacement(sb, "// " + msg + "\n");
231 }
232 }
233 matcher.appendTail(sb);
234 return sb.toString();
235 }
236
237 static String bbox(double lon1, double lat1, double lon2, double lat2) {
238 return lat1 + "," + lon1 + "," + lat2 + "," + lon2;
239 }
240
241 static String center(double lon1, double lat1, double lon2, double lat2) {
242 LatLon c = new BBox(lon1, lat1, lon2, lat2).getCenter();
243 return c.lat()+ "," + c.lon();
244 }
245
246 static String date(String dateOrHumanDuration, LocalDateTime from) {
247 try {
248 return DateUtils.parseInstant(dateOrHumanDuration).toString();
249 } catch (UncheckedParseException e) {
250 Logging.trace(e);
251 }
252 return duration(dateOrHumanDuration, from);
253 }
254
255 static String duration(String humanDuration, LocalDateTime from) {
256 // Convert to ISO 8601. Replace months by X temporarily to avoid conflict with minutes
257 String duration = humanDuration.toLowerCase(Locale.ENGLISH).replace(" ", "")
258 .replaceAll("years?", "Y").replaceAll("months?", "X").replaceAll("weeks?", "W")
259 .replaceAll("days?", "D").replaceAll("hours?", "H").replaceAll("minutes?", "M").replaceAll("seconds?", "S");
260 Matcher matcher = Pattern.compile(
261 "((?:[0-9]+Y)?(?:[0-9]+X)?(?:[0-9]+W)?)"+
262 "((?:[0-9]+D)?)" +
263 "((?:[0-9]+H)?(?:[0-9]+M)?(?:[0-9]+(?:[.,][0-9]{0,9})?S)?)?").matcher(duration);
264 boolean javaPer = false;
265 boolean javaDur = false;
266 if (matcher.matches()) {
267 javaPer = !Utils.isEmpty(matcher.group(1));
268 javaDur = !Utils.isEmpty(matcher.group(3));
269 duration = 'P' + matcher.group(1).replace('X', 'M') + matcher.group(2);
270 if (javaDur) {
271 duration += 'T' + matcher.group(3);
272 }
273 }
274
275 // Duration is now a full ISO 8601 duration string. Unfortunately Java does not allow to parse it entirely.
276 // We must split the "period" (years, months, weeks, days) from the "duration" (days, hours, minutes, seconds).
277 Period p = null;
278 Duration d = null;
279 int idx = duration.indexOf('T');
280 if (javaPer) {
281 p = Period.parse(javaDur ? duration.substring(0, idx) : duration);
282 }
283 if (javaDur) {
284 d = Duration.parse(javaPer ? 'P' + duration.substring(idx) : duration);
285 } else if (!javaPer) {
286 d = Duration.parse(duration);
287 }
288
289 // Now that period and duration are known, compute the correct date/time
290 LocalDateTime dt = from;
291 if (p != null) {
292 dt = dt.minus(p);
293 }
294 if (d != null) {
295 dt = dt.minus(d);
296 }
297
298 // Returns the date/time formatted in ISO 8601
299 return dt.toInstant(ZoneOffset.UTC).toString();
300 }
301
302 private static SearchResult searchName(String area) throws IOException {
303 return searchName(NameFinder.queryNominatim(area));
304 }
305
306 static SearchResult searchName(List<SearchResult> results) {
307 return results.stream().filter(
308 x -> OsmPrimitiveType.NODE != x.getOsmId().getType()).iterator().next();
309 }
310
311 static String geocodeArea(String area) throws IOException {
312 // Offsets defined in https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#By_element_id
313 final EnumMap<OsmPrimitiveType, Long> idOffset = new EnumMap<>(OsmPrimitiveType.class);
314 idOffset.put(OsmPrimitiveType.NODE, 0L);
315 idOffset.put(OsmPrimitiveType.WAY, 2_400_000_000L);
316 idOffset.put(OsmPrimitiveType.RELATION, 3_600_000_000L);
317 final PrimitiveId osmId = searchName(area).getOsmId();
318 Logging.debug("Area ''{0}'' resolved to {1}", area, osmId);
319 return String.format(Locale.ENGLISH, "area(%d)", osmId.getUniqueId() + idOffset.get(osmId.getType()));
320 }
321
322 static String geocodeBbox(String area) throws IOException {
323 Bounds bounds = searchName(area).getBounds();
324 return bounds.getMinLat() + "," + bounds.getMinLon() + "," + bounds.getMaxLat() + "," + bounds.getMaxLon();
325 }
326
327 static String geocodeCoords(String area) throws IOException {
328 SearchResult result = searchName(area);
329 return result.getLat() + "," + result.getLon();
330 }
331
332 static String geocodeId(String area) throws IOException {
333 PrimitiveId osmId = searchName(area).getOsmId();
334 return String.format(Locale.ENGLISH, "%s(%d)", osmId.getType().getAPIName(), osmId.getUniqueId());
335 }
336
337 @Override
338 protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason,
339 boolean uncompressAccordingToContentDisposition) throws OsmTransferException {
340 try {
341 int index = urlStr.indexOf(DATA_PREFIX);
342 // Make an HTTP POST request instead of a simple GET, allows more complex queries
343 return super.getInputStreamRaw(urlStr.substring(0, index),
344 progressMonitor, reason, uncompressAccordingToContentDisposition,
345 "POST", Utils.decodeUrl(urlStr.substring(index + DATA_PREFIX.length())).getBytes(StandardCharsets.UTF_8));
346 } catch (OsmApiException ex) {
347 final String errorIndicator = "Error</strong>: ";
348 if (ex.getMessage() != null && ex.getMessage().contains(errorIndicator)) {
349 final String errorPlusRest = ex.getMessage().split(errorIndicator, -1)[1];
350 if (errorPlusRest != null) {
351 ex.setErrorHeader(errorPlusRest.split("</", -1)[0].replaceAll(".*::request_read_and_idx::", ""));
352 }
353 }
354 throw ex;
355 }
356 }
357
358 @Override
359 protected void adaptRequest(HttpClient request) {
360 // see https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#timeout
361 final Matcher timeoutMatcher = Pattern.compile("\\[timeout:(\\d+)\\]").matcher(overpassQuery);
362 final int timeout;
363 if (timeoutMatcher.find()) {
364 timeout = (int) TimeUnit.SECONDS.toMillis(Integer.parseInt(timeoutMatcher.group(1)));
365 } else {
366 timeout = (int) TimeUnit.MINUTES.toMillis(3);
367 }
368 request.setConnectTimeout(timeout);
369 request.setReadTimeout(timeout);
370 }
371
372 @Override
373 protected String getTaskName() {
374 return tr("Contacting Server...");
375 }
376
377 @Override
378 protected DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
379 AbstractReader reader = null;
380 Matcher m = OUTPUT_FORMAT_STATEMENT.matcher(overpassQuery);
381 if (m.matches()) {
382 Class<? extends AbstractReader> readerClass = outputFormatReaders.get(OverpassOutputFormat.from(m.group(1)));
383 if (readerClass != null) {
384 try {
385 reader = readerClass.getDeclaredConstructor().newInstance();
386 } catch (ReflectiveOperationException | IllegalArgumentException | SecurityException e) {
387 Logging.error(e);
388 }
389 }
390 }
391 if (reader == null) {
392 reader = new OverpassOsmReader();
393 }
394 return reader.doParseDataSet(source, progressMonitor);
395 }
396
397 @Override
398 public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException {
399
400 DataSet ds = super.parseOsm(progressMonitor);
401 if (!considerAsFullDownload()) {
402 DataSet noBounds = new DataSet();
403 DataSetMerger dsm = new DataSetMerger(noBounds, ds);
404 dsm.merge(null, false);
405 return dsm.getTargetDataSet();
406 } else {
407 // add bounds if necessary (note that Overpass API does not return bounds in the response XML)
408 if (ds != null && ds.getDataSources().isEmpty() && overpassQuery.contains("{{bbox}}")) {
409 if (crosses180th) {
410 Bounds bounds = new Bounds(lat1, lon1, lat2, 180.0);
411 DataSource src = new DataSource(bounds, getBaseUrl());
412 ds.addDataSource(src);
413
414 bounds = new Bounds(lat1, -180.0, lat2, lon2);
415 src = new DataSource(bounds, getBaseUrl());
416 ds.addDataSource(src);
417 } else {
418 Bounds bounds = new Bounds(lat1, lon1, lat2, lon2);
419 DataSource src = new DataSource(bounds, getBaseUrl());
420 ds.addDataSource(src);
421 }
422 }
423 return ds;
424 }
425 }
426
427 /**
428 * Fixes Overpass API query to make sure it will be accepted by JOSM.
429 * @param query Overpass query to check
430 * @return fixed query
431 * @since 13335
432 */
433 public static String fixQuery(String query) {
434 return query == null ? query : query
435 .replaceAll("out( body| skel| ids)?( id| qt)?;", "out meta$2;")
436 .replaceAll("(?s)\\[out:(csv)[^\\]]*\\]", "[out:xml]");
437 }
438
439 @Override
440 public boolean considerAsFullDownload() {
441 return overpassQuery.equals(OverpassDownloadSource.FULL_DOWNLOAD_QUERY);
442 }
443}
Note: See TracBrowser for help on using the repository browser.