source: josm/trunk/src/org/openstreetmap/josm/io/OverpassDownloadReader.java@ 14062

Last change on this file since 14062 was 14015, checked in by Don-vip, 6 years ago

add unit tests on nominatim area name resolution
  • Property svn:eol-style set to native
File size: 17.0 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.io;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.IOException;
7import java.io.InputStream;
8import java.nio.charset.StandardCharsets;
9import java.time.Duration;
10import java.time.LocalDateTime;
11import java.time.Period;
12import java.time.ZoneOffset;
13import java.util.Arrays;
14import java.util.EnumMap;
15import java.util.List;
16import java.util.Locale;
17import java.util.Map;
18import java.util.NoSuchElementException;
19import java.util.Objects;
20import java.util.concurrent.ConcurrentHashMap;
21import java.util.concurrent.TimeUnit;
22import java.util.regex.Matcher;
23import java.util.regex.Pattern;
24
25import javax.xml.stream.XMLStreamConstants;
26import javax.xml.stream.XMLStreamException;
27
28import org.openstreetmap.josm.data.Bounds;
29import org.openstreetmap.josm.data.DataSource;
30import org.openstreetmap.josm.data.coor.LatLon;
31import org.openstreetmap.josm.data.osm.BBox;
32import org.openstreetmap.josm.data.osm.DataSet;
33import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
34import org.openstreetmap.josm.data.osm.PrimitiveId;
35import org.openstreetmap.josm.data.preferences.BooleanProperty;
36import org.openstreetmap.josm.data.preferences.ListProperty;
37import org.openstreetmap.josm.data.preferences.StringProperty;
38import org.openstreetmap.josm.gui.progress.ProgressMonitor;
39import org.openstreetmap.josm.io.NameFinder.SearchResult;
40import org.openstreetmap.josm.tools.HttpClient;
41import org.openstreetmap.josm.tools.Logging;
42import org.openstreetmap.josm.tools.UncheckedParseException;
43import org.openstreetmap.josm.tools.Utils;
44
45/**
46 * Read content from an Overpass server.
47 *
48 * @since 8744
49 */
50public class OverpassDownloadReader extends BoundingBoxDownloader {
51
52 /**
53 * Property for current Overpass server.
54 * @since 12816
55 */
56 public static final StringProperty OVERPASS_SERVER = new StringProperty("download.overpass.server",
57 "https://overpass-api.de/api/");
58 /**
59 * Property for list of known Overpass servers.
60 * @since 12816
61 */
62 public static final ListProperty OVERPASS_SERVER_HISTORY = new ListProperty("download.overpass.servers",
63 Arrays.asList("https://overpass-api.de/api/", "http://overpass.openstreetmap.ru/cgi/"));
64 /**
65 * Property to determine if Overpass API should be used for multi-fetch download.
66 * @since 12816
67 */
68 public static final BooleanProperty FOR_MULTI_FETCH = new BooleanProperty("download.overpass.for-multi-fetch", false);
69
70 private static final String DATA_PREFIX = "?data=";
71
72 static final class OverpassOsmReader extends OsmReader {
73 @Override
74 protected void parseUnknown(boolean printWarning) throws XMLStreamException {
75 if ("remark".equals(parser.getLocalName()) && parser.getEventType() == XMLStreamConstants.START_ELEMENT) {
76 final String text = parser.getElementText();
77 if (text.contains("runtime error")) {
78 throw new XMLStreamException(text);
79 }
80 }
81 super.parseUnknown(printWarning);
82 }
83 }
84
85 /**
86 * Possible Overpass API output format, with the {@code [out:<directive>]} statement.
87 * @since 11916
88 */
89 public enum OverpassOutpoutFormat {
90 /** Default output format: plain OSM XML */
91 OSM_XML("xml"),
92 /** OSM JSON format (not GeoJson) */
93 OSM_JSON("json"),
94 /** CSV, see https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#Output_Format_.28out.29 */
95 CSV("csv"),
96 /** Custom, see https://overpass-api.de/output_formats.html#custom */
97 CUSTOM("custom"),
98 /** Popup, see https://overpass-api.de/output_formats.html#popup */
99 POPUP("popup"),
100 /** PBF, see https://josm.openstreetmap.de/ticket/14653 */
101 PBF("pbf");
102
103 private final String directive;
104
105 OverpassOutpoutFormat(String directive) {
106 this.directive = directive;
107 }
108
109 /**
110 * Returns the directive used in {@code [out:<directive>]} statement.
111 * @return the directive used in {@code [out:<directive>]} statement
112 */
113 public String getDirective() {
114 return directive;
115 }
116
117 /**
118 * Returns the {@code OverpassOutpoutFormat} matching the given directive.
119 * @param directive directive used in {@code [out:<directive>]} statement
120 * @return {@code OverpassOutpoutFormat} matching the given directive
121 * @throws IllegalArgumentException in case of invalid directive
122 */
123 static OverpassOutpoutFormat from(String directive) {
124 for (OverpassOutpoutFormat oof : values()) {
125 if (oof.directive.equals(directive)) {
126 return oof;
127 }
128 }
129 throw new IllegalArgumentException(directive);
130 }
131 }
132
133 static final Pattern OUTPUT_FORMAT_STATEMENT = Pattern.compile(".*\\[out:([a-z]{3,})\\].*", Pattern.DOTALL);
134
135 static final Map<OverpassOutpoutFormat, Class<? extends AbstractReader>> outputFormatReaders = new ConcurrentHashMap<>();
136
137 final String overpassServer;
138 final String overpassQuery;
139
140 /**
141 * Constructs a new {@code OverpassDownloadReader}.
142 *
143 * @param downloadArea The area to download
144 * @param overpassServer The Overpass server to use
145 * @param overpassQuery The Overpass query
146 */
147 public OverpassDownloadReader(Bounds downloadArea, String overpassServer, String overpassQuery) {
148 super(downloadArea);
149 setDoAuthenticate(false);
150 this.overpassServer = overpassServer;
151 this.overpassQuery = overpassQuery.trim();
152 }
153
154 /**
155 * Registers an OSM reader for the given Overpass output format.
156 * @param format Overpass output format
157 * @param readerClass OSM reader class
158 * @return the previous value associated with {@code format}, or {@code null} if there was no mapping
159 */
160 public static final Class<? extends AbstractReader> registerOverpassOutpoutFormatReader(
161 OverpassOutpoutFormat format, Class<? extends AbstractReader> readerClass) {
162 return outputFormatReaders.put(Objects.requireNonNull(format), Objects.requireNonNull(readerClass));
163 }
164
165 static {
166 registerOverpassOutpoutFormatReader(OverpassOutpoutFormat.OSM_XML, OverpassOsmReader.class);
167 }
168
169 @Override
170 protected String getBaseUrl() {
171 return overpassServer;
172 }
173
174 @Override
175 protected String getRequestForBbox(double lon1, double lat1, double lon2, double lat2) {
176 if (overpassQuery.isEmpty())
177 return super.getRequestForBbox(lon1, lat1, lon2, lat2);
178 else {
179 final String query = this.overpassQuery
180 .replace("{{bbox}}", bbox(lon1, lat1, lon2, lat2))
181 .replace("{{center}}", center(lon1, lat1, lon2, lat2));
182 final String expandedOverpassQuery = expandExtendedQueries(query);
183 return "interpreter" + DATA_PREFIX + Utils.encodeUrl(expandedOverpassQuery);
184 }
185 }
186
187 /**
188 * Evaluates some features of overpass turbo extended query syntax.
189 * See https://wiki.openstreetmap.org/wiki/Overpass_turbo/Extended_Overpass_Turbo_Queries
190 * @param query unexpanded query
191 * @return expanded query
192 */
193 static String expandExtendedQueries(String query) {
194 final StringBuffer sb = new StringBuffer();
195 final Matcher matcher = Pattern.compile("\\{\\{(date|geocodeArea|geocodeBbox|geocodeCoords|geocodeId):([^}]+)\\}\\}").matcher(query);
196 while (matcher.find()) {
197 try {
198 switch (matcher.group(1)) {
199 case "date":
200 matcher.appendReplacement(sb, date(matcher.group(2), LocalDateTime.now()));
201 break;
202 case "geocodeArea":
203 matcher.appendReplacement(sb, geocodeArea(matcher.group(2)));
204 break;
205 case "geocodeBbox":
206 matcher.appendReplacement(sb, geocodeBbox(matcher.group(2)));
207 break;
208 case "geocodeCoords":
209 matcher.appendReplacement(sb, geocodeCoords(matcher.group(2)));
210 break;
211 case "geocodeId":
212 matcher.appendReplacement(sb, geocodeId(matcher.group(2)));
213 break;
214 default:
215 Logging.warn("Unsupported syntax: " + matcher.group(1));
216 }
217 } catch (UncheckedParseException | IOException | NoSuchElementException | IndexOutOfBoundsException ex) {
218 final String msg = tr("Failed to evaluate {0}", matcher.group());
219 Logging.log(Logging.LEVEL_WARN, msg, ex);
220 matcher.appendReplacement(sb, "// " + msg + "\n");
221 }
222 }
223 matcher.appendTail(sb);
224 return sb.toString();
225 }
226
227 static String bbox(double lon1, double lat1, double lon2, double lat2) {
228 return lat1 + "," + lon1 + "," + lat2 + "," + lon2;
229 }
230
231 static String center(double lon1, double lat1, double lon2, double lat2) {
232 LatLon c = new BBox(lon1, lat1, lon2, lat2).getCenter();
233 return c.lat()+ "," + c.lon();
234 }
235
236 static String date(String humanDuration, LocalDateTime from) {
237 // Convert to ISO 8601. Replace months by X temporarily to avoid conflict with minutes
238 String duration = humanDuration.toLowerCase(Locale.ENGLISH).replace(" ", "")
239 .replaceAll("years?", "Y").replaceAll("months?", "X").replaceAll("weeks?", "W")
240 .replaceAll("days?", "D").replaceAll("hours?", "H").replaceAll("minutes?", "M").replaceAll("seconds?", "S");
241 Matcher matcher = Pattern.compile(
242 "((?:[0-9]+Y)?(?:[0-9]+X)?(?:[0-9]+W)?)"+
243 "((?:[0-9]+D)?)" +
244 "((?:[0-9]+H)?(?:[0-9]+M)?(?:[0-9]+(?:[.,][0-9]{0,9})?S)?)?").matcher(duration);
245 boolean javaPer = false;
246 boolean javaDur = false;
247 if (matcher.matches()) {
248 javaPer = matcher.group(1) != null && !matcher.group(1).isEmpty();
249 javaDur = matcher.group(3) != null && !matcher.group(3).isEmpty();
250 duration = 'P' + matcher.group(1).replace('X', 'M') + matcher.group(2);
251 if (javaDur) {
252 duration += 'T' + matcher.group(3);
253 }
254 }
255
256 // Duration is now a full ISO 8601 duration string. Unfortunately Java does not allow to parse it entirely.
257 // We must split the "period" (years, months, weeks, days) from the "duration" (days, hours, minutes, seconds).
258 Period p = null;
259 Duration d = null;
260 int idx = duration.indexOf('T');
261 if (javaPer) {
262 p = Period.parse(javaDur ? duration.substring(0, idx) : duration);
263 }
264 if (javaDur) {
265 d = Duration.parse(javaPer ? 'P' + duration.substring(idx, duration.length()) : duration);
266 } else if (!javaPer) {
267 d = Duration.parse(duration);
268 }
269
270 // Now that period and duration are known, compute the correct date/time
271 LocalDateTime dt = from;
272 if (p != null) {
273 dt = dt.minus(p);
274 }
275 if (d != null) {
276 dt = dt.minus(d);
277 }
278
279 // Returns the date/time formatted in ISO 8601
280 return dt.toInstant(ZoneOffset.UTC).toString();
281 }
282
283 private static SearchResult searchName(String area) throws IOException {
284 return searchName(NameFinder.queryNominatim(area));
285 }
286
287 static SearchResult searchName(List<SearchResult> results) {
288 return results.stream().filter(
289 x -> !OsmPrimitiveType.NODE.equals(x.getOsmId().getType())).iterator().next();
290 }
291
292 static String geocodeArea(String area) throws IOException {
293 // Offsets defined in https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#By_element_id
294 final EnumMap<OsmPrimitiveType, Long> idOffset = new EnumMap<>(OsmPrimitiveType.class);
295 idOffset.put(OsmPrimitiveType.NODE, 0L);
296 idOffset.put(OsmPrimitiveType.WAY, 2_400_000_000L);
297 idOffset.put(OsmPrimitiveType.RELATION, 3_600_000_000L);
298 final PrimitiveId osmId = searchName(area).getOsmId();
299 Logging.debug("Area '{0}' resolved to {1}", area, osmId);
300 return String.format("area(%d)", osmId.getUniqueId() + idOffset.get(osmId.getType()));
301 }
302
303 static String geocodeBbox(String area) throws IOException {
304 Bounds bounds = searchName(area).getBounds();
305 return bounds.getMinLat() + "," + bounds.getMinLon() + "," + bounds.getMaxLat() + "," + bounds.getMaxLon();
306 }
307
308 static String geocodeCoords(String area) throws IOException {
309 SearchResult result = searchName(area);
310 return result.getLat() + "," + result.getLon();
311 }
312
313 static String geocodeId(String area) throws IOException {
314 PrimitiveId osmId = searchName(area).getOsmId();
315 return String.format("%s(%d)", osmId.getType().getAPIName(), osmId.getUniqueId());
316 }
317
318 @Override
319 protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason,
320 boolean uncompressAccordingToContentDisposition) throws OsmTransferException {
321 try {
322 int index = urlStr.indexOf(DATA_PREFIX);
323 // Make an HTTP POST request instead of a simple GET, allows more complex queries
324 return super.getInputStreamRaw(urlStr.substring(0, index),
325 progressMonitor, reason, uncompressAccordingToContentDisposition,
326 "POST", Utils.decodeUrl(urlStr.substring(index + DATA_PREFIX.length())).getBytes(StandardCharsets.UTF_8));
327 } catch (OsmApiException ex) {
328 final String errorIndicator = "Error</strong>: ";
329 if (ex.getMessage() != null && ex.getMessage().contains(errorIndicator)) {
330 final String errorPlusRest = ex.getMessage().split(errorIndicator)[1];
331 if (errorPlusRest != null) {
332 ex.setErrorHeader(errorPlusRest.split("</")[0].replaceAll(".*::request_read_and_idx::", ""));
333 }
334 }
335 throw ex;
336 }
337 }
338
339 @Override
340 protected void adaptRequest(HttpClient request) {
341 // see https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#timeout
342 final Matcher timeoutMatcher = Pattern.compile("\\[timeout:(\\d+)\\]").matcher(overpassQuery);
343 final int timeout;
344 if (timeoutMatcher.find()) {
345 timeout = (int) TimeUnit.SECONDS.toMillis(Integer.parseInt(timeoutMatcher.group(1)));
346 } else {
347 timeout = (int) TimeUnit.MINUTES.toMillis(3);
348 }
349 request.setConnectTimeout(timeout);
350 request.setReadTimeout(timeout);
351 }
352
353 @Override
354 protected String getTaskName() {
355 return tr("Contacting Server...");
356 }
357
358 @Override
359 protected DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
360 AbstractReader reader = null;
361 Matcher m = OUTPUT_FORMAT_STATEMENT.matcher(overpassQuery);
362 if (m.matches()) {
363 Class<? extends AbstractReader> readerClass = outputFormatReaders.get(OverpassOutpoutFormat.from(m.group(1)));
364 if (readerClass != null) {
365 try {
366 reader = readerClass.getDeclaredConstructor().newInstance();
367 } catch (ReflectiveOperationException | IllegalArgumentException | SecurityException e) {
368 Logging.error(e);
369 }
370 }
371 }
372 if (reader == null) {
373 reader = new OverpassOsmReader();
374 }
375 return reader.doParseDataSet(source, progressMonitor);
376 }
377
378 @Override
379 public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException {
380
381 DataSet ds = super.parseOsm(progressMonitor);
382
383 // add bounds if necessary (note that Overpass API does not return bounds in the response XML)
384 if (ds != null && ds.getDataSources().isEmpty() && overpassQuery.contains("{{bbox}}")) {
385 if (crosses180th) {
386 Bounds bounds = new Bounds(lat1, lon1, lat2, 180.0);
387 DataSource src = new DataSource(bounds, getBaseUrl());
388 ds.addDataSource(src);
389
390 bounds = new Bounds(lat1, -180.0, lat2, lon2);
391 src = new DataSource(bounds, getBaseUrl());
392 ds.addDataSource(src);
393 } else {
394 Bounds bounds = new Bounds(lat1, lon1, lat2, lon2);
395 DataSource src = new DataSource(bounds, getBaseUrl());
396 ds.addDataSource(src);
397 }
398 }
399
400 return ds;
401 }
402
403 /**
404 * Fixes Overpass API query to make sure it will be accepted by JOSM.
405 * @param query Overpass query to check
406 * @return fixed query
407 * @since 13335
408 */
409 public static String fixQuery(String query) {
410 return query == null ? query : query
411 .replaceAll("out( body| skel| ids)?( id| qt)?;", "out meta$2;")
412 .replaceAll("(?s)\\[out:(json|csv)[^\\]]*\\]", "[out:xml]");
413 }
414}
Note: See TracBrowser for help on using the repository browser.