source: josm/trunk/src/org/openstreetmap/josm/io/OverpassDownloadReader.java@ 13402

Last change on this file since 13402 was 13359, checked in by Don-vip, 6 years ago

fix #15800 - mistake in replacements
  • Property svn:eol-style set to native
File size: 16.8 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.io;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.IOException;
7import java.io.InputStream;
8import java.nio.charset.StandardCharsets;
9import java.time.Duration;
10import java.time.LocalDateTime;
11import java.time.Period;
12import java.time.ZoneOffset;
13import java.util.Arrays;
14import java.util.EnumMap;
15import java.util.Locale;
16import java.util.Map;
17import java.util.NoSuchElementException;
18import java.util.Objects;
19import java.util.concurrent.ConcurrentHashMap;
20import java.util.concurrent.TimeUnit;
21import java.util.regex.Matcher;
22import java.util.regex.Pattern;
23
24import javax.xml.stream.XMLStreamConstants;
25import javax.xml.stream.XMLStreamException;
26
27import org.openstreetmap.josm.data.Bounds;
28import org.openstreetmap.josm.data.DataSource;
29import org.openstreetmap.josm.data.coor.LatLon;
30import org.openstreetmap.josm.data.osm.BBox;
31import org.openstreetmap.josm.data.osm.DataSet;
32import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
33import org.openstreetmap.josm.data.osm.PrimitiveId;
34import org.openstreetmap.josm.data.preferences.BooleanProperty;
35import org.openstreetmap.josm.data.preferences.ListProperty;
36import org.openstreetmap.josm.data.preferences.StringProperty;
37import org.openstreetmap.josm.gui.progress.ProgressMonitor;
38import org.openstreetmap.josm.io.NameFinder.SearchResult;
39import org.openstreetmap.josm.tools.HttpClient;
40import org.openstreetmap.josm.tools.Logging;
41import org.openstreetmap.josm.tools.UncheckedParseException;
42import org.openstreetmap.josm.tools.Utils;
43
44/**
45 * Read content from an Overpass server.
46 *
47 * @since 8744
48 */
49public class OverpassDownloadReader extends BoundingBoxDownloader {
50
51 /**
52 * Property for current Overpass server.
53 * @since 12816
54 */
55 public static final StringProperty OVERPASS_SERVER = new StringProperty("download.overpass.server",
56 "https://overpass-api.de/api/");
57 /**
58 * Property for list of known Overpass servers.
59 * @since 12816
60 */
61 public static final ListProperty OVERPASS_SERVER_HISTORY = new ListProperty("download.overpass.servers",
62 Arrays.asList("https://overpass-api.de/api/", "http://overpass.osm.rambler.ru/cgi/"));
63 /**
64 * Property to determine if Overpass API should be used for multi-fetch download.
65 * @since 12816
66 */
67 public static final BooleanProperty FOR_MULTI_FETCH = new BooleanProperty("download.overpass.for-multi-fetch", false);
68
69 private static final String DATA_PREFIX = "?data=";
70
71 static final class OverpassOsmReader extends OsmReader {
72 @Override
73 protected void parseUnknown(boolean printWarning) throws XMLStreamException {
74 if ("remark".equals(parser.getLocalName()) && parser.getEventType() == XMLStreamConstants.START_ELEMENT) {
75 final String text = parser.getElementText();
76 if (text.contains("runtime error")) {
77 throw new XMLStreamException(text);
78 }
79 }
80 super.parseUnknown(printWarning);
81 }
82 }
83
84 /**
85 * Possible Overpass API output format, with the {@code [out:<directive>]} statement.
86 * @since 11916
87 */
88 public enum OverpassOutpoutFormat {
89 /** Default output format: plain OSM XML */
90 OSM_XML("xml"),
91 /** OSM JSON format (not GeoJson) */
92 OSM_JSON("json"),
93 /** CSV, see https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#Output_Format_.28out.29 */
94 CSV("csv"),
95 /** Custom, see https://overpass-api.de/output_formats.html#custom */
96 CUSTOM("custom"),
97 /** Popup, see https://overpass-api.de/output_formats.html#popup */
98 POPUP("popup"),
99 /** PBF, see https://josm.openstreetmap.de/ticket/14653 */
100 PBF("pbf");
101
102 private final String directive;
103
104 OverpassOutpoutFormat(String directive) {
105 this.directive = directive;
106 }
107
108 /**
109 * Returns the directive used in {@code [out:<directive>]} statement.
110 * @return the directive used in {@code [out:<directive>]} statement
111 */
112 public String getDirective() {
113 return directive;
114 }
115
116 /**
117 * Returns the {@code OverpassOutpoutFormat} matching the given directive.
118 * @param directive directive used in {@code [out:<directive>]} statement
119 * @return {@code OverpassOutpoutFormat} matching the given directive
120 * @throws IllegalArgumentException in case of invalid directive
121 */
122 static OverpassOutpoutFormat from(String directive) {
123 for (OverpassOutpoutFormat oof : values()) {
124 if (oof.directive.equals(directive)) {
125 return oof;
126 }
127 }
128 throw new IllegalArgumentException(directive);
129 }
130 }
131
132 static final Pattern OUTPUT_FORMAT_STATEMENT = Pattern.compile(".*\\[out:([a-z]{3,})\\].*", Pattern.DOTALL);
133
134 static final Map<OverpassOutpoutFormat, Class<? extends AbstractReader>> outputFormatReaders = new ConcurrentHashMap<>();
135
136 final String overpassServer;
137 final String overpassQuery;
138
139 /**
140 * Constructs a new {@code OverpassDownloadReader}.
141 *
142 * @param downloadArea The area to download
143 * @param overpassServer The Overpass server to use
144 * @param overpassQuery The Overpass query
145 */
146 public OverpassDownloadReader(Bounds downloadArea, String overpassServer, String overpassQuery) {
147 super(downloadArea);
148 setDoAuthenticate(false);
149 this.overpassServer = overpassServer;
150 this.overpassQuery = overpassQuery.trim();
151 }
152
153 /**
154 * Registers an OSM reader for the given Overpass output format.
155 * @param format Overpass output format
156 * @param readerClass OSM reader class
157 * @return the previous value associated with {@code format}, or {@code null} if there was no mapping
158 */
159 public static final Class<? extends AbstractReader> registerOverpassOutpoutFormatReader(
160 OverpassOutpoutFormat format, Class<? extends AbstractReader> readerClass) {
161 return outputFormatReaders.put(Objects.requireNonNull(format), Objects.requireNonNull(readerClass));
162 }
163
164 static {
165 registerOverpassOutpoutFormatReader(OverpassOutpoutFormat.OSM_XML, OverpassOsmReader.class);
166 }
167
168 @Override
169 protected String getBaseUrl() {
170 return overpassServer;
171 }
172
173 @Override
174 protected String getRequestForBbox(double lon1, double lat1, double lon2, double lat2) {
175 if (overpassQuery.isEmpty())
176 return super.getRequestForBbox(lon1, lat1, lon2, lat2);
177 else {
178 final String query = this.overpassQuery
179 .replace("{{bbox}}", bbox(lon1, lat1, lon2, lat2))
180 .replace("{{center}}", center(lon1, lat1, lon2, lat2));
181 final String expandedOverpassQuery = expandExtendedQueries(query);
182 return "interpreter" + DATA_PREFIX + Utils.encodeUrl(expandedOverpassQuery);
183 }
184 }
185
186 /**
187 * Evaluates some features of overpass turbo extended query syntax.
188 * See https://wiki.openstreetmap.org/wiki/Overpass_turbo/Extended_Overpass_Turbo_Queries
189 * @param query unexpanded query
190 * @return expanded query
191 */
192 static String expandExtendedQueries(String query) {
193 final StringBuffer sb = new StringBuffer();
194 final Matcher matcher = Pattern.compile("\\{\\{(date|geocodeArea|geocodeBbox|geocodeCoords|geocodeId):([^}]+)\\}\\}").matcher(query);
195 while (matcher.find()) {
196 try {
197 switch (matcher.group(1)) {
198 case "date":
199 matcher.appendReplacement(sb, date(matcher.group(2), LocalDateTime.now()));
200 break;
201 case "geocodeArea":
202 matcher.appendReplacement(sb, geocodeArea(matcher.group(2)));
203 break;
204 case "geocodeBbox":
205 matcher.appendReplacement(sb, geocodeBbox(matcher.group(2)));
206 break;
207 case "geocodeCoords":
208 matcher.appendReplacement(sb, geocodeCoords(matcher.group(2)));
209 break;
210 case "geocodeId":
211 matcher.appendReplacement(sb, geocodeId(matcher.group(2)));
212 break;
213 default:
214 Logging.warn("Unsupported syntax: " + matcher.group(1));
215 }
216 } catch (UncheckedParseException | IOException | NoSuchElementException | IndexOutOfBoundsException ex) {
217 final String msg = tr("Failed to evaluate {0}", matcher.group());
218 Logging.log(Logging.LEVEL_WARN, msg, ex);
219 matcher.appendReplacement(sb, "// " + msg + "\n");
220 }
221 }
222 matcher.appendTail(sb);
223 return sb.toString();
224 }
225
226 static String bbox(double lon1, double lat1, double lon2, double lat2) {
227 return lat1 + "," + lon1 + "," + lat2 + "," + lon2;
228 }
229
230 static String center(double lon1, double lat1, double lon2, double lat2) {
231 LatLon c = new BBox(lon1, lat1, lon2, lat2).getCenter();
232 return c.lat()+ "," + c.lon();
233 }
234
235 static String date(String humanDuration, LocalDateTime from) {
236 // Convert to ISO 8601. Replace months by X temporarily to avoid conflict with minutes
237 String duration = humanDuration.toLowerCase(Locale.ENGLISH).replace(" ", "")
238 .replaceAll("years?", "Y").replaceAll("months?", "X").replaceAll("weeks?", "W")
239 .replaceAll("days?", "D").replaceAll("hours?", "H").replaceAll("minutes?", "M").replaceAll("seconds?", "S");
240 Matcher matcher = Pattern.compile(
241 "((?:[0-9]+Y)?(?:[0-9]+X)?(?:[0-9]+W)?)"+
242 "((?:[0-9]+D)?)" +
243 "((?:[0-9]+H)?(?:[0-9]+M)?(?:[0-9]+(?:[.,][0-9]{0,9})?S)?)?").matcher(duration);
244 boolean javaPer = false;
245 boolean javaDur = false;
246 if (matcher.matches()) {
247 javaPer = matcher.group(1) != null && !matcher.group(1).isEmpty();
248 javaDur = matcher.group(3) != null && !matcher.group(3).isEmpty();
249 duration = 'P' + matcher.group(1).replace('X', 'M') + matcher.group(2);
250 if (javaDur) {
251 duration += 'T' + matcher.group(3);
252 }
253 }
254
255 // Duration is now a full ISO 8601 duration string. Unfortunately Java does not allow to parse it entirely.
256 // We must split the "period" (years, months, weeks, days) from the "duration" (days, hours, minutes, seconds).
257 Period p = null;
258 Duration d = null;
259 int idx = duration.indexOf('T');
260 if (javaPer) {
261 p = Period.parse(javaDur ? duration.substring(0, idx) : duration);
262 }
263 if (javaDur) {
264 d = Duration.parse(javaPer ? 'P' + duration.substring(idx, duration.length()) : duration);
265 } else if (!javaPer) {
266 d = Duration.parse(duration);
267 }
268
269 // Now that period and duration are known, compute the correct date/time
270 LocalDateTime dt = from;
271 if (p != null) {
272 dt = dt.minus(p);
273 }
274 if (d != null) {
275 dt = dt.minus(d);
276 }
277
278 // Returns the date/time formatted in ISO 8601
279 return dt.toInstant(ZoneOffset.UTC).toString();
280 }
281
282 private static SearchResult searchName(String area) throws IOException {
283 return NameFinder.queryNominatim(area).stream().filter(
284 x -> !OsmPrimitiveType.NODE.equals(x.getOsmId().getType())).iterator().next();
285 }
286
287 static String geocodeArea(String area) throws IOException {
288 // Offsets defined in https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#By_element_id
289 final EnumMap<OsmPrimitiveType, Long> idOffset = new EnumMap<>(OsmPrimitiveType.class);
290 idOffset.put(OsmPrimitiveType.NODE, 0L);
291 idOffset.put(OsmPrimitiveType.WAY, 2_400_000_000L);
292 idOffset.put(OsmPrimitiveType.RELATION, 3_600_000_000L);
293 final PrimitiveId osmId = searchName(area).getOsmId();
294 return String.format("area(%d)", osmId.getUniqueId() + idOffset.get(osmId.getType()));
295 }
296
297 static String geocodeBbox(String area) throws IOException {
298 Bounds bounds = searchName(area).getBounds();
299 return bounds.getMinLat() + "," + bounds.getMinLon() + "," + bounds.getMaxLat() + "," + bounds.getMaxLon();
300 }
301
302 static String geocodeCoords(String area) throws IOException {
303 SearchResult result = searchName(area);
304 return result.getLat() + "," + result.getLon();
305 }
306
307 static String geocodeId(String area) throws IOException {
308 PrimitiveId osmId = searchName(area).getOsmId();
309 return String.format("%s(%d)", osmId.getType().getAPIName(), osmId.getUniqueId());
310 }
311
312 @Override
313 protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason,
314 boolean uncompressAccordingToContentDisposition) throws OsmTransferException {
315 try {
316 int index = urlStr.indexOf(DATA_PREFIX);
317 // Make an HTTP POST request instead of a simple GET, allows more complex queries
318 return super.getInputStreamRaw(urlStr.substring(0, index),
319 progressMonitor, reason, uncompressAccordingToContentDisposition,
320 "POST", Utils.decodeUrl(urlStr.substring(index + DATA_PREFIX.length())).getBytes(StandardCharsets.UTF_8));
321 } catch (OsmApiException ex) {
322 final String errorIndicator = "Error</strong>: ";
323 if (ex.getMessage() != null && ex.getMessage().contains(errorIndicator)) {
324 final String errorPlusRest = ex.getMessage().split(errorIndicator)[1];
325 if (errorPlusRest != null) {
326 ex.setErrorHeader(errorPlusRest.split("</")[0].replaceAll(".*::request_read_and_idx::", ""));
327 }
328 }
329 throw ex;
330 }
331 }
332
333 @Override
334 protected void adaptRequest(HttpClient request) {
335 // see https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#timeout
336 final Matcher timeoutMatcher = Pattern.compile("\\[timeout:(\\d+)\\]").matcher(overpassQuery);
337 final int timeout;
338 if (timeoutMatcher.find()) {
339 timeout = (int) TimeUnit.SECONDS.toMillis(Integer.parseInt(timeoutMatcher.group(1)));
340 } else {
341 timeout = (int) TimeUnit.MINUTES.toMillis(3);
342 }
343 request.setConnectTimeout(timeout);
344 request.setReadTimeout(timeout);
345 }
346
347 @Override
348 protected String getTaskName() {
349 return tr("Contacting Server...");
350 }
351
352 @Override
353 protected DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
354 AbstractReader reader = null;
355 Matcher m = OUTPUT_FORMAT_STATEMENT.matcher(overpassQuery);
356 if (m.matches()) {
357 Class<? extends AbstractReader> readerClass = outputFormatReaders.get(OverpassOutpoutFormat.from(m.group(1)));
358 if (readerClass != null) {
359 try {
360 reader = readerClass.getDeclaredConstructor().newInstance();
361 } catch (ReflectiveOperationException | IllegalArgumentException | SecurityException e) {
362 Logging.error(e);
363 }
364 }
365 }
366 if (reader == null) {
367 reader = new OverpassOsmReader();
368 }
369 return reader.doParseDataSet(source, progressMonitor);
370 }
371
372 @Override
373 public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException {
374
375 DataSet ds = super.parseOsm(progressMonitor);
376
377 // add bounds if necessary (note that Overpass API does not return bounds in the response XML)
378 if (ds != null && ds.getDataSources().isEmpty() && overpassQuery.contains("{{bbox}}")) {
379 if (crosses180th) {
380 Bounds bounds = new Bounds(lat1, lon1, lat2, 180.0);
381 DataSource src = new DataSource(bounds, getBaseUrl());
382 ds.addDataSource(src);
383
384 bounds = new Bounds(lat1, -180.0, lat2, lon2);
385 src = new DataSource(bounds, getBaseUrl());
386 ds.addDataSource(src);
387 } else {
388 Bounds bounds = new Bounds(lat1, lon1, lat2, lon2);
389 DataSource src = new DataSource(bounds, getBaseUrl());
390 ds.addDataSource(src);
391 }
392 }
393
394 return ds;
395 }
396
397 /**
398 * Fixes Overpass API query to make sure it will be accepted by JOSM.
399 * @param query Overpass query to check
400 * @return fixed query
401 * @since 13335
402 */
403 public static String fixQuery(String query) {
404 return query == null ? query : query
405 .replaceAll("out( body| skel| ids)?( id| qt)?;", "out meta$2;")
406 .replaceAll("(?s)\\[out:(json|csv)[^\\]]*\\]", "[out:xml]");
407 }
408}
Note: See TracBrowser for help on using the repository browser.