source: josm/trunk/src/org/openstreetmap/josm/io/OverpassDownloadReader.java@ 17982

Last change on this file since 17982 was 17982, checked in by Don-vip, 3 years ago

fix #21021 - see #18164 - OverpassTurboQueryWizard: fix "newer" statements
  • Property svn:eol-style set to native
File size: 17.9 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.io;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.IOException;
7import java.io.InputStream;
8import java.nio.charset.StandardCharsets;
9import java.time.Duration;
10import java.time.LocalDateTime;
11import java.time.Period;
12import java.time.ZoneId;
13import java.time.ZoneOffset;
14import java.time.format.DateTimeParseException;
15import java.util.Arrays;
16import java.util.EnumMap;
17import java.util.List;
18import java.util.Locale;
19import java.util.Map;
20import java.util.NoSuchElementException;
21import java.util.Objects;
22import java.util.concurrent.ConcurrentHashMap;
23import java.util.concurrent.TimeUnit;
24import java.util.regex.Matcher;
25import java.util.regex.Pattern;
26
27import javax.xml.stream.XMLStreamConstants;
28import javax.xml.stream.XMLStreamException;
29
30import org.openstreetmap.josm.data.Bounds;
31import org.openstreetmap.josm.data.DataSource;
32import org.openstreetmap.josm.data.coor.LatLon;
33import org.openstreetmap.josm.data.osm.BBox;
34import org.openstreetmap.josm.data.osm.DataSet;
35import org.openstreetmap.josm.data.osm.DataSetMerger;
36import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
37import org.openstreetmap.josm.data.osm.PrimitiveId;
38import org.openstreetmap.josm.data.preferences.BooleanProperty;
39import org.openstreetmap.josm.data.preferences.ListProperty;
40import org.openstreetmap.josm.data.preferences.StringProperty;
41import org.openstreetmap.josm.gui.download.OverpassDownloadSource;
42import org.openstreetmap.josm.gui.progress.ProgressMonitor;
43import org.openstreetmap.josm.io.NameFinder.SearchResult;
44import org.openstreetmap.josm.tools.HttpClient;
45import org.openstreetmap.josm.tools.Logging;
46import org.openstreetmap.josm.tools.UncheckedParseException;
47import org.openstreetmap.josm.tools.Utils;
48
49/**
50 * Read content from an Overpass server.
51 *
52 * @since 8744
53 */
54public class OverpassDownloadReader extends BoundingBoxDownloader {
55
56 /**
57 * Property for current Overpass server.
58 * @since 12816
59 */
60 public static final StringProperty OVERPASS_SERVER = new StringProperty("download.overpass.server",
61 "https://overpass-api.de/api/");
62 /**
63 * Property for list of known Overpass servers.
64 * @since 12816
65 */
66 public static final ListProperty OVERPASS_SERVER_HISTORY = new ListProperty("download.overpass.servers",
67 Arrays.asList("https://overpass-api.de/api/", "http://overpass.openstreetmap.ru/cgi/"));
68 /**
69 * Property to determine if Overpass API should be used for multi-fetch download.
70 * @since 12816
71 */
72 public static final BooleanProperty FOR_MULTI_FETCH = new BooleanProperty("download.overpass.for-multi-fetch", false);
73
74 private static final String DATA_PREFIX = "?data=";
75
76 static final class OverpassOsmReader extends OsmReader {
77 @Override
78 protected void parseUnknown(boolean printWarning) throws XMLStreamException {
79 if ("remark".equals(parser.getLocalName()) && parser.getEventType() == XMLStreamConstants.START_ELEMENT) {
80 final String text = parser.getElementText();
81 if (text.contains("runtime error")) {
82 throw new XMLStreamException(text);
83 }
84 }
85 super.parseUnknown(printWarning);
86 }
87 }
88
89 static final class OverpassOsmJsonReader extends OsmJsonReader {
90
91 }
92
93 /**
94 * Possible Overpass API output format, with the {@code [out:<directive>]} statement.
95 * @since 11916
96 */
97 public enum OverpassOutputFormat {
98 /** Default output format: plain OSM XML */
99 OSM_XML("xml"),
100 /** OSM JSON format (not GeoJson) */
101 OSM_JSON("json"),
102 /** CSV, see https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#Output_Format_.28out.29 */
103 CSV("csv"),
104 /** Custom, see https://overpass-api.de/output_formats.html#custom */
105 CUSTOM("custom"),
106 /** Popup, see https://overpass-api.de/output_formats.html#popup */
107 POPUP("popup"),
108 /** PBF, see https://josm.openstreetmap.de/ticket/14653 */
109 PBF("pbf");
110
111 private final String directive;
112
113 OverpassOutputFormat(String directive) {
114 this.directive = directive;
115 }
116
117 /**
118 * Returns the directive used in {@code [out:<directive>]} statement.
119 * @return the directive used in {@code [out:<directive>]} statement
120 */
121 public String getDirective() {
122 return directive;
123 }
124
125 /**
126 * Returns the {@code OverpassOutputFormat} matching the given directive.
127 * @param directive directive used in {@code [out:<directive>]} statement
128 * @return {@code OverpassOutputFormat} matching the given directive
129 * @throws IllegalArgumentException in case of invalid directive
130 */
131 static OverpassOutputFormat from(String directive) {
132 for (OverpassOutputFormat oof : values()) {
133 if (oof.directive.equals(directive)) {
134 return oof;
135 }
136 }
137 throw new IllegalArgumentException(directive);
138 }
139 }
140
141 static final Pattern OUTPUT_FORMAT_STATEMENT = Pattern.compile(".*\\[out:([a-z]{3,})\\].*", Pattern.DOTALL);
142
143 static final Map<OverpassOutputFormat, Class<? extends AbstractReader>> outputFormatReaders = new ConcurrentHashMap<>();
144
145 final String overpassServer;
146 final String overpassQuery;
147
148 /**
149 * Constructs a new {@code OverpassDownloadReader}.
150 *
151 * @param downloadArea The area to download
152 * @param overpassServer The Overpass server to use
153 * @param overpassQuery The Overpass query
154 */
155 public OverpassDownloadReader(Bounds downloadArea, String overpassServer, String overpassQuery) {
156 super(downloadArea);
157 setDoAuthenticate(false);
158 this.overpassServer = overpassServer;
159 this.overpassQuery = overpassQuery.trim();
160 }
161
162 /**
163 * Registers an OSM reader for the given Overpass output format.
164 * @param format Overpass output format
165 * @param readerClass OSM reader class
166 * @return the previous value associated with {@code format}, or {@code null} if there was no mapping
167 */
168 public static final Class<? extends AbstractReader> registerOverpassOutputFormatReader(
169 OverpassOutputFormat format, Class<? extends AbstractReader> readerClass) {
170 return outputFormatReaders.put(Objects.requireNonNull(format), Objects.requireNonNull(readerClass));
171 }
172
173 static {
174 registerOverpassOutputFormatReader(OverpassOutputFormat.OSM_XML, OverpassOsmReader.class);
175 registerOverpassOutputFormatReader(OverpassOutputFormat.OSM_JSON, OverpassOsmJsonReader.class);
176 }
177
178 @Override
179 protected String getBaseUrl() {
180 return overpassServer;
181 }
182
183 @Override
184 protected String getRequestForBbox(double lon1, double lat1, double lon2, double lat2) {
185 if (overpassQuery.isEmpty())
186 return super.getRequestForBbox(lon1, lat1, lon2, lat2);
187 else {
188 final String query = this.overpassQuery
189 .replace("{{bbox}}", bbox(lon1, lat1, lon2, lat2))
190 .replace("{{center}}", center(lon1, lat1, lon2, lat2));
191 final String expandedOverpassQuery = expandExtendedQueries(query);
192 return "interpreter" + DATA_PREFIX + Utils.encodeUrl(expandedOverpassQuery);
193 }
194 }
195
196 /**
197 * Evaluates some features of overpass turbo extended query syntax.
198 * See https://wiki.openstreetmap.org/wiki/Overpass_turbo/Extended_Overpass_Turbo_Queries
199 * @param query unexpanded query
200 * @return expanded query
201 */
202 static String expandExtendedQueries(String query) {
203 final StringBuffer sb = new StringBuffer();
204 final Matcher matcher = Pattern.compile("\\{\\{(date|geocodeArea|geocodeBbox|geocodeCoords|geocodeId):([^}]+)\\}\\}").matcher(query);
205 while (matcher.find()) {
206 try {
207 switch (matcher.group(1)) {
208 case "date":
209 matcher.appendReplacement(sb, date(matcher.group(2), LocalDateTime.now(ZoneId.systemDefault())));
210 break;
211 case "geocodeArea":
212 matcher.appendReplacement(sb, geocodeArea(matcher.group(2)));
213 break;
214 case "geocodeBbox":
215 matcher.appendReplacement(sb, geocodeBbox(matcher.group(2)));
216 break;
217 case "geocodeCoords":
218 matcher.appendReplacement(sb, geocodeCoords(matcher.group(2)));
219 break;
220 case "geocodeId":
221 matcher.appendReplacement(sb, geocodeId(matcher.group(2)));
222 break;
223 default:
224 Logging.warn("Unsupported syntax: " + matcher.group(1));
225 }
226 } catch (UncheckedParseException | DateTimeParseException | IOException | NoSuchElementException | IndexOutOfBoundsException ex) {
227 final String msg = tr("Failed to evaluate {0}", matcher.group());
228 Logging.log(Logging.LEVEL_WARN, msg, ex);
229 matcher.appendReplacement(sb, "// " + msg + "\n");
230 }
231 }
232 matcher.appendTail(sb);
233 return sb.toString();
234 }
235
236 static String bbox(double lon1, double lat1, double lon2, double lat2) {
237 return lat1 + "," + lon1 + "," + lat2 + "," + lon2;
238 }
239
240 static String center(double lon1, double lat1, double lon2, double lat2) {
241 LatLon c = new BBox(lon1, lat1, lon2, lat2).getCenter();
242 return c.lat()+ "," + c.lon();
243 }
244
245 static String date(String humanDuration, LocalDateTime from) {
246 // Convert to ISO 8601. Replace months by X temporarily to avoid conflict with minutes
247 String duration = humanDuration.toLowerCase(Locale.ENGLISH).replace(" ", "")
248 .replaceAll("years?", "Y").replaceAll("months?", "X").replaceAll("weeks?", "W")
249 .replaceAll("days?", "D").replaceAll("hours?", "H").replaceAll("minutes?", "M").replaceAll("seconds?", "S");
250 Matcher matcher = Pattern.compile(
251 "((?:[0-9]+Y)?(?:[0-9]+X)?(?:[0-9]+W)?)"+
252 "((?:[0-9]+D)?)" +
253 "((?:[0-9]+H)?(?:[0-9]+M)?(?:[0-9]+(?:[.,][0-9]{0,9})?S)?)?").matcher(duration);
254 boolean javaPer = false;
255 boolean javaDur = false;
256 if (matcher.matches()) {
257 javaPer = matcher.group(1) != null && !matcher.group(1).isEmpty();
258 javaDur = matcher.group(3) != null && !matcher.group(3).isEmpty();
259 duration = 'P' + matcher.group(1).replace('X', 'M') + matcher.group(2);
260 if (javaDur) {
261 duration += 'T' + matcher.group(3);
262 }
263 }
264
265 // Duration is now a full ISO 8601 duration string. Unfortunately Java does not allow to parse it entirely.
266 // We must split the "period" (years, months, weeks, days) from the "duration" (days, hours, minutes, seconds).
267 Period p = null;
268 Duration d = null;
269 int idx = duration.indexOf('T');
270 if (javaPer) {
271 p = Period.parse(javaDur ? duration.substring(0, idx) : duration);
272 }
273 if (javaDur) {
274 d = Duration.parse(javaPer ? 'P' + duration.substring(idx) : duration);
275 } else if (!javaPer) {
276 d = Duration.parse(duration);
277 }
278
279 // Now that period and duration are known, compute the correct date/time
280 LocalDateTime dt = from;
281 if (p != null) {
282 dt = dt.minus(p);
283 }
284 if (d != null) {
285 dt = dt.minus(d);
286 }
287
288 // Returns the date/time formatted in ISO 8601
289 return dt.toInstant(ZoneOffset.UTC).toString();
290 }
291
292 private static SearchResult searchName(String area) throws IOException {
293 return searchName(NameFinder.queryNominatim(area));
294 }
295
296 static SearchResult searchName(List<SearchResult> results) {
297 return results.stream().filter(
298 x -> OsmPrimitiveType.NODE != x.getOsmId().getType()).iterator().next();
299 }
300
301 static String geocodeArea(String area) throws IOException {
302 // Offsets defined in https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#By_element_id
303 final EnumMap<OsmPrimitiveType, Long> idOffset = new EnumMap<>(OsmPrimitiveType.class);
304 idOffset.put(OsmPrimitiveType.NODE, 0L);
305 idOffset.put(OsmPrimitiveType.WAY, 2_400_000_000L);
306 idOffset.put(OsmPrimitiveType.RELATION, 3_600_000_000L);
307 final PrimitiveId osmId = searchName(area).getOsmId();
308 Logging.debug("Area ''{0}'' resolved to {1}", area, osmId);
309 return String.format(Locale.ENGLISH, "area(%d)", osmId.getUniqueId() + idOffset.get(osmId.getType()));
310 }
311
312 static String geocodeBbox(String area) throws IOException {
313 Bounds bounds = searchName(area).getBounds();
314 return bounds.getMinLat() + "," + bounds.getMinLon() + "," + bounds.getMaxLat() + "," + bounds.getMaxLon();
315 }
316
317 static String geocodeCoords(String area) throws IOException {
318 SearchResult result = searchName(area);
319 return result.getLat() + "," + result.getLon();
320 }
321
322 static String geocodeId(String area) throws IOException {
323 PrimitiveId osmId = searchName(area).getOsmId();
324 return String.format(Locale.ENGLISH, "%s(%d)", osmId.getType().getAPIName(), osmId.getUniqueId());
325 }
326
327 @Override
328 protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason,
329 boolean uncompressAccordingToContentDisposition) throws OsmTransferException {
330 try {
331 int index = urlStr.indexOf(DATA_PREFIX);
332 // Make an HTTP POST request instead of a simple GET, allows more complex queries
333 return super.getInputStreamRaw(urlStr.substring(0, index),
334 progressMonitor, reason, uncompressAccordingToContentDisposition,
335 "POST", Utils.decodeUrl(urlStr.substring(index + DATA_PREFIX.length())).getBytes(StandardCharsets.UTF_8));
336 } catch (OsmApiException ex) {
337 final String errorIndicator = "Error</strong>: ";
338 if (ex.getMessage() != null && ex.getMessage().contains(errorIndicator)) {
339 final String errorPlusRest = ex.getMessage().split(errorIndicator, -1)[1];
340 if (errorPlusRest != null) {
341 ex.setErrorHeader(errorPlusRest.split("</", -1)[0].replaceAll(".*::request_read_and_idx::", ""));
342 }
343 }
344 throw ex;
345 }
346 }
347
348 @Override
349 protected void adaptRequest(HttpClient request) {
350 // see https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#timeout
351 final Matcher timeoutMatcher = Pattern.compile("\\[timeout:(\\d+)\\]").matcher(overpassQuery);
352 final int timeout;
353 if (timeoutMatcher.find()) {
354 timeout = (int) TimeUnit.SECONDS.toMillis(Integer.parseInt(timeoutMatcher.group(1)));
355 } else {
356 timeout = (int) TimeUnit.MINUTES.toMillis(3);
357 }
358 request.setConnectTimeout(timeout);
359 request.setReadTimeout(timeout);
360 }
361
362 @Override
363 protected String getTaskName() {
364 return tr("Contacting Server...");
365 }
366
367 @Override
368 protected DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
369 AbstractReader reader = null;
370 Matcher m = OUTPUT_FORMAT_STATEMENT.matcher(overpassQuery);
371 if (m.matches()) {
372 Class<? extends AbstractReader> readerClass = outputFormatReaders.get(OverpassOutputFormat.from(m.group(1)));
373 if (readerClass != null) {
374 try {
375 reader = readerClass.getDeclaredConstructor().newInstance();
376 } catch (ReflectiveOperationException | IllegalArgumentException | SecurityException e) {
377 Logging.error(e);
378 }
379 }
380 }
381 if (reader == null) {
382 reader = new OverpassOsmReader();
383 }
384 return reader.doParseDataSet(source, progressMonitor);
385 }
386
387 @Override
388 public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException {
389
390 DataSet ds = super.parseOsm(progressMonitor);
391 if (!considerAsFullDownload()) {
392 DataSet noBounds = new DataSet();
393 DataSetMerger dsm = new DataSetMerger(noBounds, ds);
394 dsm.merge(null, false);
395 return dsm.getTargetDataSet();
396 } else {
397 // add bounds if necessary (note that Overpass API does not return bounds in the response XML)
398 if (ds != null && ds.getDataSources().isEmpty() && overpassQuery.contains("{{bbox}}")) {
399 if (crosses180th) {
400 Bounds bounds = new Bounds(lat1, lon1, lat2, 180.0);
401 DataSource src = new DataSource(bounds, getBaseUrl());
402 ds.addDataSource(src);
403
404 bounds = new Bounds(lat1, -180.0, lat2, lon2);
405 src = new DataSource(bounds, getBaseUrl());
406 ds.addDataSource(src);
407 } else {
408 Bounds bounds = new Bounds(lat1, lon1, lat2, lon2);
409 DataSource src = new DataSource(bounds, getBaseUrl());
410 ds.addDataSource(src);
411 }
412 }
413 return ds;
414 }
415 }
416
417 /**
418 * Fixes Overpass API query to make sure it will be accepted by JOSM.
419 * @param query Overpass query to check
420 * @return fixed query
421 * @since 13335
422 */
423 public static String fixQuery(String query) {
424 return query == null ? query : query
425 .replaceAll("out( body| skel| ids)?( id| qt)?;", "out meta$2;")
426 .replaceAll("(?s)\\[out:(csv)[^\\]]*\\]", "[out:xml]");
427 }
428
429 @Override
430 public boolean considerAsFullDownload() {
431 return overpassQuery.equals(OverpassDownloadSource.FULL_DOWNLOAD_QUERY);
432 }
433}
Note: See TracBrowser for help on using the repository browser.