source: josm/trunk/src/org/openstreetmap/josm/io/OverpassDownloadReader.java@ 17379

Last change on this file since 17379 was 17333, checked in by Don-vip, 3 years ago

see #20129 - Fix typos and misspellings in the code (patch by gaben)
  • Property svn:eol-style set to native
File size: 17.8 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.io;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.IOException;
7import java.io.InputStream;
8import java.nio.charset.StandardCharsets;
9import java.time.Duration;
10import java.time.LocalDateTime;
11import java.time.Period;
12import java.time.ZoneId;
13import java.time.ZoneOffset;
14import java.util.Arrays;
15import java.util.EnumMap;
16import java.util.List;
17import java.util.Locale;
18import java.util.Map;
19import java.util.NoSuchElementException;
20import java.util.Objects;
21import java.util.concurrent.ConcurrentHashMap;
22import java.util.concurrent.TimeUnit;
23import java.util.regex.Matcher;
24import java.util.regex.Pattern;
25
26import javax.xml.stream.XMLStreamConstants;
27import javax.xml.stream.XMLStreamException;
28
29import org.openstreetmap.josm.data.Bounds;
30import org.openstreetmap.josm.data.DataSource;
31import org.openstreetmap.josm.data.coor.LatLon;
32import org.openstreetmap.josm.data.osm.BBox;
33import org.openstreetmap.josm.data.osm.DataSet;
34import org.openstreetmap.josm.data.osm.DataSetMerger;
35import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
36import org.openstreetmap.josm.data.osm.PrimitiveId;
37import org.openstreetmap.josm.data.preferences.BooleanProperty;
38import org.openstreetmap.josm.data.preferences.ListProperty;
39import org.openstreetmap.josm.data.preferences.StringProperty;
40import org.openstreetmap.josm.gui.download.OverpassDownloadSource;
41import org.openstreetmap.josm.gui.progress.ProgressMonitor;
42import org.openstreetmap.josm.io.NameFinder.SearchResult;
43import org.openstreetmap.josm.tools.HttpClient;
44import org.openstreetmap.josm.tools.Logging;
45import org.openstreetmap.josm.tools.UncheckedParseException;
46import org.openstreetmap.josm.tools.Utils;
47
48/**
49 * Read content from an Overpass server.
50 *
51 * @since 8744
52 */
53public class OverpassDownloadReader extends BoundingBoxDownloader {
54
55 /**
56 * Property for current Overpass server.
57 * @since 12816
58 */
59 public static final StringProperty OVERPASS_SERVER = new StringProperty("download.overpass.server",
60 "https://overpass-api.de/api/");
61 /**
62 * Property for list of known Overpass servers.
63 * @since 12816
64 */
65 public static final ListProperty OVERPASS_SERVER_HISTORY = new ListProperty("download.overpass.servers",
66 Arrays.asList("https://overpass-api.de/api/", "http://overpass.openstreetmap.ru/cgi/"));
67 /**
68 * Property to determine if Overpass API should be used for multi-fetch download.
69 * @since 12816
70 */
71 public static final BooleanProperty FOR_MULTI_FETCH = new BooleanProperty("download.overpass.for-multi-fetch", false);
72
73 private static final String DATA_PREFIX = "?data=";
74
75 static final class OverpassOsmReader extends OsmReader {
76 @Override
77 protected void parseUnknown(boolean printWarning) throws XMLStreamException {
78 if ("remark".equals(parser.getLocalName()) && parser.getEventType() == XMLStreamConstants.START_ELEMENT) {
79 final String text = parser.getElementText();
80 if (text.contains("runtime error")) {
81 throw new XMLStreamException(text);
82 }
83 }
84 super.parseUnknown(printWarning);
85 }
86 }
87
88 static final class OverpassOsmJsonReader extends OsmJsonReader {
89
90 }
91
92 /**
93 * Possible Overpass API output format, with the {@code [out:<directive>]} statement.
94 * @since 11916
95 */
96 public enum OverpassOutputFormat {
97 /** Default output format: plain OSM XML */
98 OSM_XML("xml"),
99 /** OSM JSON format (not GeoJson) */
100 OSM_JSON("json"),
101 /** CSV, see https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#Output_Format_.28out.29 */
102 CSV("csv"),
103 /** Custom, see https://overpass-api.de/output_formats.html#custom */
104 CUSTOM("custom"),
105 /** Popup, see https://overpass-api.de/output_formats.html#popup */
106 POPUP("popup"),
107 /** PBF, see https://josm.openstreetmap.de/ticket/14653 */
108 PBF("pbf");
109
110 private final String directive;
111
112 OverpassOutputFormat(String directive) {
113 this.directive = directive;
114 }
115
116 /**
117 * Returns the directive used in {@code [out:<directive>]} statement.
118 * @return the directive used in {@code [out:<directive>]} statement
119 */
120 public String getDirective() {
121 return directive;
122 }
123
124 /**
125 * Returns the {@code OverpassOutputFormat} matching the given directive.
126 * @param directive directive used in {@code [out:<directive>]} statement
127 * @return {@code OverpassOutputFormat} matching the given directive
128 * @throws IllegalArgumentException in case of invalid directive
129 */
130 static OverpassOutputFormat from(String directive) {
131 for (OverpassOutputFormat oof : values()) {
132 if (oof.directive.equals(directive)) {
133 return oof;
134 }
135 }
136 throw new IllegalArgumentException(directive);
137 }
138 }
139
140 static final Pattern OUTPUT_FORMAT_STATEMENT = Pattern.compile(".*\\[out:([a-z]{3,})\\].*", Pattern.DOTALL);
141
142 static final Map<OverpassOutputFormat, Class<? extends AbstractReader>> outputFormatReaders = new ConcurrentHashMap<>();
143
144 final String overpassServer;
145 final String overpassQuery;
146
147 /**
148 * Constructs a new {@code OverpassDownloadReader}.
149 *
150 * @param downloadArea The area to download
151 * @param overpassServer The Overpass server to use
152 * @param overpassQuery The Overpass query
153 */
154 public OverpassDownloadReader(Bounds downloadArea, String overpassServer, String overpassQuery) {
155 super(downloadArea);
156 setDoAuthenticate(false);
157 this.overpassServer = overpassServer;
158 this.overpassQuery = overpassQuery.trim();
159 }
160
161 /**
162 * Registers an OSM reader for the given Overpass output format.
163 * @param format Overpass output format
164 * @param readerClass OSM reader class
165 * @return the previous value associated with {@code format}, or {@code null} if there was no mapping
166 */
167 public static final Class<? extends AbstractReader> registerOverpassOutputFormatReader(
168 OverpassOutputFormat format, Class<? extends AbstractReader> readerClass) {
169 return outputFormatReaders.put(Objects.requireNonNull(format), Objects.requireNonNull(readerClass));
170 }
171
172 static {
173 registerOverpassOutputFormatReader(OverpassOutputFormat.OSM_XML, OverpassOsmReader.class);
174 registerOverpassOutputFormatReader(OverpassOutputFormat.OSM_JSON, OverpassOsmJsonReader.class);
175 }
176
177 @Override
178 protected String getBaseUrl() {
179 return overpassServer;
180 }
181
182 @Override
183 protected String getRequestForBbox(double lon1, double lat1, double lon2, double lat2) {
184 if (overpassQuery.isEmpty())
185 return super.getRequestForBbox(lon1, lat1, lon2, lat2);
186 else {
187 final String query = this.overpassQuery
188 .replace("{{bbox}}", bbox(lon1, lat1, lon2, lat2))
189 .replace("{{center}}", center(lon1, lat1, lon2, lat2));
190 final String expandedOverpassQuery = expandExtendedQueries(query);
191 return "interpreter" + DATA_PREFIX + Utils.encodeUrl(expandedOverpassQuery);
192 }
193 }
194
195 /**
196 * Evaluates some features of overpass turbo extended query syntax.
197 * See https://wiki.openstreetmap.org/wiki/Overpass_turbo/Extended_Overpass_Turbo_Queries
198 * @param query unexpanded query
199 * @return expanded query
200 */
201 static String expandExtendedQueries(String query) {
202 final StringBuffer sb = new StringBuffer();
203 final Matcher matcher = Pattern.compile("\\{\\{(date|geocodeArea|geocodeBbox|geocodeCoords|geocodeId):([^}]+)\\}\\}").matcher(query);
204 while (matcher.find()) {
205 try {
206 switch (matcher.group(1)) {
207 case "date":
208 matcher.appendReplacement(sb, date(matcher.group(2), LocalDateTime.now(ZoneId.systemDefault())));
209 break;
210 case "geocodeArea":
211 matcher.appendReplacement(sb, geocodeArea(matcher.group(2)));
212 break;
213 case "geocodeBbox":
214 matcher.appendReplacement(sb, geocodeBbox(matcher.group(2)));
215 break;
216 case "geocodeCoords":
217 matcher.appendReplacement(sb, geocodeCoords(matcher.group(2)));
218 break;
219 case "geocodeId":
220 matcher.appendReplacement(sb, geocodeId(matcher.group(2)));
221 break;
222 default:
223 Logging.warn("Unsupported syntax: " + matcher.group(1));
224 }
225 } catch (UncheckedParseException | IOException | NoSuchElementException | IndexOutOfBoundsException ex) {
226 final String msg = tr("Failed to evaluate {0}", matcher.group());
227 Logging.log(Logging.LEVEL_WARN, msg, ex);
228 matcher.appendReplacement(sb, "// " + msg + "\n");
229 }
230 }
231 matcher.appendTail(sb);
232 return sb.toString();
233 }
234
235 static String bbox(double lon1, double lat1, double lon2, double lat2) {
236 return lat1 + "," + lon1 + "," + lat2 + "," + lon2;
237 }
238
239 static String center(double lon1, double lat1, double lon2, double lat2) {
240 LatLon c = new BBox(lon1, lat1, lon2, lat2).getCenter();
241 return c.lat()+ "," + c.lon();
242 }
243
244 static String date(String humanDuration, LocalDateTime from) {
245 // Convert to ISO 8601. Replace months by X temporarily to avoid conflict with minutes
246 String duration = humanDuration.toLowerCase(Locale.ENGLISH).replace(" ", "")
247 .replaceAll("years?", "Y").replaceAll("months?", "X").replaceAll("weeks?", "W")
248 .replaceAll("days?", "D").replaceAll("hours?", "H").replaceAll("minutes?", "M").replaceAll("seconds?", "S");
249 Matcher matcher = Pattern.compile(
250 "((?:[0-9]+Y)?(?:[0-9]+X)?(?:[0-9]+W)?)"+
251 "((?:[0-9]+D)?)" +
252 "((?:[0-9]+H)?(?:[0-9]+M)?(?:[0-9]+(?:[.,][0-9]{0,9})?S)?)?").matcher(duration);
253 boolean javaPer = false;
254 boolean javaDur = false;
255 if (matcher.matches()) {
256 javaPer = matcher.group(1) != null && !matcher.group(1).isEmpty();
257 javaDur = matcher.group(3) != null && !matcher.group(3).isEmpty();
258 duration = 'P' + matcher.group(1).replace('X', 'M') + matcher.group(2);
259 if (javaDur) {
260 duration += 'T' + matcher.group(3);
261 }
262 }
263
264 // Duration is now a full ISO 8601 duration string. Unfortunately Java does not allow to parse it entirely.
265 // We must split the "period" (years, months, weeks, days) from the "duration" (days, hours, minutes, seconds).
266 Period p = null;
267 Duration d = null;
268 int idx = duration.indexOf('T');
269 if (javaPer) {
270 p = Period.parse(javaDur ? duration.substring(0, idx) : duration);
271 }
272 if (javaDur) {
273 d = Duration.parse(javaPer ? 'P' + duration.substring(idx, duration.length()) : duration);
274 } else if (!javaPer) {
275 d = Duration.parse(duration);
276 }
277
278 // Now that period and duration are known, compute the correct date/time
279 LocalDateTime dt = from;
280 if (p != null) {
281 dt = dt.minus(p);
282 }
283 if (d != null) {
284 dt = dt.minus(d);
285 }
286
287 // Returns the date/time formatted in ISO 8601
288 return dt.toInstant(ZoneOffset.UTC).toString();
289 }
290
291 private static SearchResult searchName(String area) throws IOException {
292 return searchName(NameFinder.queryNominatim(area));
293 }
294
295 static SearchResult searchName(List<SearchResult> results) {
296 return results.stream().filter(
297 x -> OsmPrimitiveType.NODE != x.getOsmId().getType()).iterator().next();
298 }
299
300 static String geocodeArea(String area) throws IOException {
301 // Offsets defined in https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#By_element_id
302 final EnumMap<OsmPrimitiveType, Long> idOffset = new EnumMap<>(OsmPrimitiveType.class);
303 idOffset.put(OsmPrimitiveType.NODE, 0L);
304 idOffset.put(OsmPrimitiveType.WAY, 2_400_000_000L);
305 idOffset.put(OsmPrimitiveType.RELATION, 3_600_000_000L);
306 final PrimitiveId osmId = searchName(area).getOsmId();
307 Logging.debug("Area ''{0}'' resolved to {1}", area, osmId);
308 return String.format("area(%d)", osmId.getUniqueId() + idOffset.get(osmId.getType()));
309 }
310
311 static String geocodeBbox(String area) throws IOException {
312 Bounds bounds = searchName(area).getBounds();
313 return bounds.getMinLat() + "," + bounds.getMinLon() + "," + bounds.getMaxLat() + "," + bounds.getMaxLon();
314 }
315
316 static String geocodeCoords(String area) throws IOException {
317 SearchResult result = searchName(area);
318 return result.getLat() + "," + result.getLon();
319 }
320
321 static String geocodeId(String area) throws IOException {
322 PrimitiveId osmId = searchName(area).getOsmId();
323 return String.format("%s(%d)", osmId.getType().getAPIName(), osmId.getUniqueId());
324 }
325
326 @Override
327 protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason,
328 boolean uncompressAccordingToContentDisposition) throws OsmTransferException {
329 try {
330 int index = urlStr.indexOf(DATA_PREFIX);
331 // Make an HTTP POST request instead of a simple GET, allows more complex queries
332 return super.getInputStreamRaw(urlStr.substring(0, index),
333 progressMonitor, reason, uncompressAccordingToContentDisposition,
334 "POST", Utils.decodeUrl(urlStr.substring(index + DATA_PREFIX.length())).getBytes(StandardCharsets.UTF_8));
335 } catch (OsmApiException ex) {
336 final String errorIndicator = "Error</strong>: ";
337 if (ex.getMessage() != null && ex.getMessage().contains(errorIndicator)) {
338 final String errorPlusRest = ex.getMessage().split(errorIndicator, -1)[1];
339 if (errorPlusRest != null) {
340 ex.setErrorHeader(errorPlusRest.split("</", -1)[0].replaceAll(".*::request_read_and_idx::", ""));
341 }
342 }
343 throw ex;
344 }
345 }
346
347 @Override
348 protected void adaptRequest(HttpClient request) {
349 // see https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#timeout
350 final Matcher timeoutMatcher = Pattern.compile("\\[timeout:(\\d+)\\]").matcher(overpassQuery);
351 final int timeout;
352 if (timeoutMatcher.find()) {
353 timeout = (int) TimeUnit.SECONDS.toMillis(Integer.parseInt(timeoutMatcher.group(1)));
354 } else {
355 timeout = (int) TimeUnit.MINUTES.toMillis(3);
356 }
357 request.setConnectTimeout(timeout);
358 request.setReadTimeout(timeout);
359 }
360
361 @Override
362 protected String getTaskName() {
363 return tr("Contacting Server...");
364 }
365
366 @Override
367 protected DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
368 AbstractReader reader = null;
369 Matcher m = OUTPUT_FORMAT_STATEMENT.matcher(overpassQuery);
370 if (m.matches()) {
371 Class<? extends AbstractReader> readerClass = outputFormatReaders.get(OverpassOutputFormat.from(m.group(1)));
372 if (readerClass != null) {
373 try {
374 reader = readerClass.getDeclaredConstructor().newInstance();
375 } catch (ReflectiveOperationException | IllegalArgumentException | SecurityException e) {
376 Logging.error(e);
377 }
378 }
379 }
380 if (reader == null) {
381 reader = new OverpassOsmReader();
382 }
383 return reader.doParseDataSet(source, progressMonitor);
384 }
385
386 @Override
387 public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException {
388
389 DataSet ds = super.parseOsm(progressMonitor);
390 if (!considerAsFullDownload()) {
391 DataSet noBounds = new DataSet();
392 DataSetMerger dsm = new DataSetMerger(noBounds, ds);
393 dsm.merge(null, false);
394 return dsm.getTargetDataSet();
395 } else {
396 // add bounds if necessary (note that Overpass API does not return bounds in the response XML)
397 if (ds != null && ds.getDataSources().isEmpty() && overpassQuery.contains("{{bbox}}")) {
398 if (crosses180th) {
399 Bounds bounds = new Bounds(lat1, lon1, lat2, 180.0);
400 DataSource src = new DataSource(bounds, getBaseUrl());
401 ds.addDataSource(src);
402
403 bounds = new Bounds(lat1, -180.0, lat2, lon2);
404 src = new DataSource(bounds, getBaseUrl());
405 ds.addDataSource(src);
406 } else {
407 Bounds bounds = new Bounds(lat1, lon1, lat2, lon2);
408 DataSource src = new DataSource(bounds, getBaseUrl());
409 ds.addDataSource(src);
410 }
411 }
412 return ds;
413 }
414 }
415
416 /**
417 * Fixes Overpass API query to make sure it will be accepted by JOSM.
418 * @param query Overpass query to check
419 * @return fixed query
420 * @since 13335
421 */
422 public static String fixQuery(String query) {
423 return query == null ? query : query
424 .replaceAll("out( body| skel| ids)?( id| qt)?;", "out meta$2;")
425 .replaceAll("(?s)\\[out:(csv)[^\\]]*\\]", "[out:xml]");
426 }
427
428 @Override
429 public boolean considerAsFullDownload() {
430 return overpassQuery.equals(OverpassDownloadSource.FULL_DOWNLOAD_QUERY);
431 }
432}
Note: See TracBrowser for help on using the repository browser.