1 | // License: GPL. For details, see LICENSE file.
|
---|
2 | package org.openstreetmap.josm.io;
|
---|
3 |
|
---|
4 | import static org.openstreetmap.josm.tools.I18n.tr;
|
---|
5 |
|
---|
6 | import java.io.IOException;
|
---|
7 | import java.io.InputStream;
|
---|
8 | import java.nio.charset.StandardCharsets;
|
---|
9 | import java.time.Duration;
|
---|
10 | import java.time.LocalDateTime;
|
---|
11 | import java.time.Period;
|
---|
12 | import java.time.ZoneOffset;
|
---|
13 | import java.util.Arrays;
|
---|
14 | import java.util.EnumMap;
|
---|
15 | import java.util.Locale;
|
---|
16 | import java.util.Map;
|
---|
17 | import java.util.NoSuchElementException;
|
---|
18 | import java.util.Objects;
|
---|
19 | import java.util.concurrent.ConcurrentHashMap;
|
---|
20 | import java.util.concurrent.TimeUnit;
|
---|
21 | import java.util.regex.Matcher;
|
---|
22 | import java.util.regex.Pattern;
|
---|
23 |
|
---|
24 | import javax.xml.stream.XMLStreamConstants;
|
---|
25 | import javax.xml.stream.XMLStreamException;
|
---|
26 |
|
---|
27 | import org.openstreetmap.josm.data.Bounds;
|
---|
28 | import org.openstreetmap.josm.data.DataSource;
|
---|
29 | import org.openstreetmap.josm.data.coor.LatLon;
|
---|
30 | import org.openstreetmap.josm.data.osm.BBox;
|
---|
31 | import org.openstreetmap.josm.data.osm.DataSet;
|
---|
32 | import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
|
---|
33 | import org.openstreetmap.josm.data.osm.PrimitiveId;
|
---|
34 | import org.openstreetmap.josm.data.preferences.BooleanProperty;
|
---|
35 | import org.openstreetmap.josm.data.preferences.ListProperty;
|
---|
36 | import org.openstreetmap.josm.data.preferences.StringProperty;
|
---|
37 | import org.openstreetmap.josm.gui.progress.ProgressMonitor;
|
---|
38 | import org.openstreetmap.josm.io.NameFinder.SearchResult;
|
---|
39 | import org.openstreetmap.josm.tools.HttpClient;
|
---|
40 | import org.openstreetmap.josm.tools.Logging;
|
---|
41 | import org.openstreetmap.josm.tools.UncheckedParseException;
|
---|
42 | import org.openstreetmap.josm.tools.Utils;
|
---|
43 |
|
---|
44 | /**
|
---|
45 | * Read content from an Overpass server.
|
---|
46 | *
|
---|
47 | * @since 8744
|
---|
48 | */
|
---|
49 | public class OverpassDownloadReader extends BoundingBoxDownloader {
|
---|
50 |
|
---|
51 | /**
|
---|
52 | * Property for current Overpass server.
|
---|
53 | * @since 12816
|
---|
54 | */
|
---|
55 | public static final StringProperty OVERPASS_SERVER = new StringProperty("download.overpass.server",
|
---|
56 | "https://overpass-api.de/api/");
|
---|
57 | /**
|
---|
58 | * Property for list of known Overpass servers.
|
---|
59 | * @since 12816
|
---|
60 | */
|
---|
61 | public static final ListProperty OVERPASS_SERVER_HISTORY = new ListProperty("download.overpass.servers",
|
---|
62 | Arrays.asList("https://overpass-api.de/api/", "http://overpass.osm.rambler.ru/cgi/"));
|
---|
63 | /**
|
---|
64 | * Property to determine if Overpass API should be used for multi-fetch download.
|
---|
65 | * @since 12816
|
---|
66 | */
|
---|
67 | public static final BooleanProperty FOR_MULTI_FETCH = new BooleanProperty("download.overpass.for-multi-fetch", false);
|
---|
68 |
|
---|
69 | private static final String DATA_PREFIX = "?data=";
|
---|
70 |
|
---|
71 | static final class OverpassOsmReader extends OsmReader {
|
---|
72 | @Override
|
---|
73 | protected void parseUnknown(boolean printWarning) throws XMLStreamException {
|
---|
74 | if ("remark".equals(parser.getLocalName()) && parser.getEventType() == XMLStreamConstants.START_ELEMENT) {
|
---|
75 | final String text = parser.getElementText();
|
---|
76 | if (text.contains("runtime error")) {
|
---|
77 | throw new XMLStreamException(text);
|
---|
78 | }
|
---|
79 | }
|
---|
80 | super.parseUnknown(printWarning);
|
---|
81 | }
|
---|
82 | }
|
---|
83 |
|
---|
84 | /**
|
---|
85 | * Possible Overpass API output format, with the {@code [out:<directive>]} statement.
|
---|
86 | * @since 11916
|
---|
87 | */
|
---|
88 | public enum OverpassOutpoutFormat {
|
---|
89 | /** Default output format: plain OSM XML */
|
---|
90 | OSM_XML("xml"),
|
---|
91 | /** OSM JSON format (not GeoJson) */
|
---|
92 | OSM_JSON("json"),
|
---|
93 | /** CSV, see https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#Output_Format_.28out.29 */
|
---|
94 | CSV("csv"),
|
---|
95 | /** Custom, see https://overpass-api.de/output_formats.html#custom */
|
---|
96 | CUSTOM("custom"),
|
---|
97 | /** Popup, see https://overpass-api.de/output_formats.html#popup */
|
---|
98 | POPUP("popup"),
|
---|
99 | /** PBF, see https://josm.openstreetmap.de/ticket/14653 */
|
---|
100 | PBF("pbf");
|
---|
101 |
|
---|
102 | private final String directive;
|
---|
103 |
|
---|
104 | OverpassOutpoutFormat(String directive) {
|
---|
105 | this.directive = directive;
|
---|
106 | }
|
---|
107 |
|
---|
108 | /**
|
---|
109 | * Returns the directive used in {@code [out:<directive>]} statement.
|
---|
110 | * @return the directive used in {@code [out:<directive>]} statement
|
---|
111 | */
|
---|
112 | public String getDirective() {
|
---|
113 | return directive;
|
---|
114 | }
|
---|
115 |
|
---|
116 | /**
|
---|
117 | * Returns the {@code OverpassOutpoutFormat} matching the given directive.
|
---|
118 | * @param directive directive used in {@code [out:<directive>]} statement
|
---|
119 | * @return {@code OverpassOutpoutFormat} matching the given directive
|
---|
120 | * @throws IllegalArgumentException in case of invalid directive
|
---|
121 | */
|
---|
122 | static OverpassOutpoutFormat from(String directive) {
|
---|
123 | for (OverpassOutpoutFormat oof : values()) {
|
---|
124 | if (oof.directive.equals(directive)) {
|
---|
125 | return oof;
|
---|
126 | }
|
---|
127 | }
|
---|
128 | throw new IllegalArgumentException(directive);
|
---|
129 | }
|
---|
130 | }
|
---|
131 |
|
---|
132 | static final Pattern OUTPUT_FORMAT_STATEMENT = Pattern.compile(".*\\[out:([a-z]{3,})\\].*", Pattern.DOTALL);
|
---|
133 |
|
---|
134 | static final Map<OverpassOutpoutFormat, Class<? extends AbstractReader>> outputFormatReaders = new ConcurrentHashMap<>();
|
---|
135 |
|
---|
136 | final String overpassServer;
|
---|
137 | final String overpassQuery;
|
---|
138 |
|
---|
139 | /**
|
---|
140 | * Constructs a new {@code OverpassDownloadReader}.
|
---|
141 | *
|
---|
142 | * @param downloadArea The area to download
|
---|
143 | * @param overpassServer The Overpass server to use
|
---|
144 | * @param overpassQuery The Overpass query
|
---|
145 | */
|
---|
146 | public OverpassDownloadReader(Bounds downloadArea, String overpassServer, String overpassQuery) {
|
---|
147 | super(downloadArea);
|
---|
148 | setDoAuthenticate(false);
|
---|
149 | this.overpassServer = overpassServer;
|
---|
150 | this.overpassQuery = overpassQuery.trim();
|
---|
151 | }
|
---|
152 |
|
---|
153 | /**
|
---|
154 | * Registers an OSM reader for the given Overpass output format.
|
---|
155 | * @param format Overpass output format
|
---|
156 | * @param readerClass OSM reader class
|
---|
157 | * @return the previous value associated with {@code format}, or {@code null} if there was no mapping
|
---|
158 | */
|
---|
159 | public static final Class<? extends AbstractReader> registerOverpassOutpoutFormatReader(
|
---|
160 | OverpassOutpoutFormat format, Class<? extends AbstractReader> readerClass) {
|
---|
161 | return outputFormatReaders.put(Objects.requireNonNull(format), Objects.requireNonNull(readerClass));
|
---|
162 | }
|
---|
163 |
|
---|
164 | static {
|
---|
165 | registerOverpassOutpoutFormatReader(OverpassOutpoutFormat.OSM_XML, OverpassOsmReader.class);
|
---|
166 | }
|
---|
167 |
|
---|
168 | @Override
|
---|
169 | protected String getBaseUrl() {
|
---|
170 | return overpassServer;
|
---|
171 | }
|
---|
172 |
|
---|
173 | @Override
|
---|
174 | protected String getRequestForBbox(double lon1, double lat1, double lon2, double lat2) {
|
---|
175 | if (overpassQuery.isEmpty())
|
---|
176 | return super.getRequestForBbox(lon1, lat1, lon2, lat2);
|
---|
177 | else {
|
---|
178 | final String query = this.overpassQuery
|
---|
179 | .replace("{{bbox}}", bbox(lon1, lat1, lon2, lat2))
|
---|
180 | .replace("{{center}}", center(lon1, lat1, lon2, lat2));
|
---|
181 | final String expandedOverpassQuery = expandExtendedQueries(query);
|
---|
182 | return "interpreter" + DATA_PREFIX + Utils.encodeUrl(expandedOverpassQuery);
|
---|
183 | }
|
---|
184 | }
|
---|
185 |
|
---|
186 | /**
|
---|
187 | * Evaluates some features of overpass turbo extended query syntax.
|
---|
188 | * See https://wiki.openstreetmap.org/wiki/Overpass_turbo/Extended_Overpass_Turbo_Queries
|
---|
189 | * @param query unexpanded query
|
---|
190 | * @return expanded query
|
---|
191 | */
|
---|
192 | static String expandExtendedQueries(String query) {
|
---|
193 | final StringBuffer sb = new StringBuffer();
|
---|
194 | final Matcher matcher = Pattern.compile("\\{\\{(date|geocodeArea|geocodeBbox|geocodeCoords|geocodeId):([^}]+)\\}\\}").matcher(query);
|
---|
195 | while (matcher.find()) {
|
---|
196 | try {
|
---|
197 | switch (matcher.group(1)) {
|
---|
198 | case "date":
|
---|
199 | matcher.appendReplacement(sb, date(matcher.group(2), LocalDateTime.now()));
|
---|
200 | break;
|
---|
201 | case "geocodeArea":
|
---|
202 | matcher.appendReplacement(sb, geocodeArea(matcher.group(2)));
|
---|
203 | break;
|
---|
204 | case "geocodeBbox":
|
---|
205 | matcher.appendReplacement(sb, geocodeBbox(matcher.group(2)));
|
---|
206 | break;
|
---|
207 | case "geocodeCoords":
|
---|
208 | matcher.appendReplacement(sb, geocodeCoords(matcher.group(2)));
|
---|
209 | break;
|
---|
210 | case "geocodeId":
|
---|
211 | matcher.appendReplacement(sb, geocodeId(matcher.group(2)));
|
---|
212 | break;
|
---|
213 | default:
|
---|
214 | Logging.warn("Unsupported syntax: " + matcher.group(1));
|
---|
215 | }
|
---|
216 | } catch (UncheckedParseException | IOException | NoSuchElementException | IndexOutOfBoundsException ex) {
|
---|
217 | final String msg = tr("Failed to evaluate {0}", matcher.group());
|
---|
218 | Logging.log(Logging.LEVEL_WARN, msg, ex);
|
---|
219 | matcher.appendReplacement(sb, "// " + msg + "\n");
|
---|
220 | }
|
---|
221 | }
|
---|
222 | matcher.appendTail(sb);
|
---|
223 | return sb.toString();
|
---|
224 | }
|
---|
225 |
|
---|
226 | static String bbox(double lon1, double lat1, double lon2, double lat2) {
|
---|
227 | return lat1 + "," + lon1 + "," + lat2 + "," + lon2;
|
---|
228 | }
|
---|
229 |
|
---|
230 | static String center(double lon1, double lat1, double lon2, double lat2) {
|
---|
231 | LatLon c = new BBox(lon1, lat1, lon2, lat2).getCenter();
|
---|
232 | return c.lat()+ "," + c.lon();
|
---|
233 | }
|
---|
234 |
|
---|
235 | static String date(String humanDuration, LocalDateTime from) {
|
---|
236 | // Convert to ISO 8601. Replace months by X temporarily to avoid conflict with minutes
|
---|
237 | String duration = humanDuration.toLowerCase(Locale.ENGLISH).replace(" ", "")
|
---|
238 | .replaceAll("years?", "Y").replaceAll("months?", "X").replaceAll("weeks?", "W")
|
---|
239 | .replaceAll("days?", "D").replaceAll("hours?", "H").replaceAll("minutes?", "M").replaceAll("seconds?", "S");
|
---|
240 | Matcher matcher = Pattern.compile(
|
---|
241 | "((?:[0-9]+Y)?(?:[0-9]+X)?(?:[0-9]+W)?)"+
|
---|
242 | "((?:[0-9]+D)?)" +
|
---|
243 | "((?:[0-9]+H)?(?:[0-9]+M)?(?:[0-9]+(?:[.,][0-9]{0,9})?S)?)?").matcher(duration);
|
---|
244 | boolean javaPer = false;
|
---|
245 | boolean javaDur = false;
|
---|
246 | if (matcher.matches()) {
|
---|
247 | javaPer = matcher.group(1) != null && !matcher.group(1).isEmpty();
|
---|
248 | javaDur = matcher.group(3) != null && !matcher.group(3).isEmpty();
|
---|
249 | duration = 'P' + matcher.group(1).replace('X', 'M') + matcher.group(2);
|
---|
250 | if (javaDur) {
|
---|
251 | duration += 'T' + matcher.group(3);
|
---|
252 | }
|
---|
253 | }
|
---|
254 |
|
---|
255 | // Duration is now a full ISO 8601 duration string. Unfortunately Java does not allow to parse it entirely.
|
---|
256 | // We must split the "period" (years, months, weeks, days) from the "duration" (days, hours, minutes, seconds).
|
---|
257 | Period p = null;
|
---|
258 | Duration d = null;
|
---|
259 | int idx = duration.indexOf('T');
|
---|
260 | if (javaPer) {
|
---|
261 | p = Period.parse(javaDur ? duration.substring(0, idx) : duration);
|
---|
262 | }
|
---|
263 | if (javaDur) {
|
---|
264 | d = Duration.parse(javaPer ? 'P' + duration.substring(idx, duration.length()) : duration);
|
---|
265 | } else if (!javaPer) {
|
---|
266 | d = Duration.parse(duration);
|
---|
267 | }
|
---|
268 |
|
---|
269 | // Now that period and duration are known, compute the correct date/time
|
---|
270 | LocalDateTime dt = from;
|
---|
271 | if (p != null) {
|
---|
272 | dt = dt.minus(p);
|
---|
273 | }
|
---|
274 | if (d != null) {
|
---|
275 | dt = dt.minus(d);
|
---|
276 | }
|
---|
277 |
|
---|
278 | // Returns the date/time formatted in ISO 8601
|
---|
279 | return dt.toInstant(ZoneOffset.UTC).toString();
|
---|
280 | }
|
---|
281 |
|
---|
282 | private static SearchResult searchName(String area) throws IOException {
|
---|
283 | return NameFinder.queryNominatim(area).stream().filter(
|
---|
284 | x -> !OsmPrimitiveType.NODE.equals(x.getOsmId().getType())).iterator().next();
|
---|
285 | }
|
---|
286 |
|
---|
287 | static String geocodeArea(String area) throws IOException {
|
---|
288 | // Offsets defined in https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#By_element_id
|
---|
289 | final EnumMap<OsmPrimitiveType, Long> idOffset = new EnumMap<>(OsmPrimitiveType.class);
|
---|
290 | idOffset.put(OsmPrimitiveType.NODE, 0L);
|
---|
291 | idOffset.put(OsmPrimitiveType.WAY, 2_400_000_000L);
|
---|
292 | idOffset.put(OsmPrimitiveType.RELATION, 3_600_000_000L);
|
---|
293 | final PrimitiveId osmId = searchName(area).getOsmId();
|
---|
294 | return String.format("area(%d)", osmId.getUniqueId() + idOffset.get(osmId.getType()));
|
---|
295 | }
|
---|
296 |
|
---|
297 | static String geocodeBbox(String area) throws IOException {
|
---|
298 | Bounds bounds = searchName(area).getBounds();
|
---|
299 | return bounds.getMinLat() + "," + bounds.getMinLon() + "," + bounds.getMaxLat() + "," + bounds.getMaxLon();
|
---|
300 | }
|
---|
301 |
|
---|
302 | static String geocodeCoords(String area) throws IOException {
|
---|
303 | SearchResult result = searchName(area);
|
---|
304 | return result.getLat() + "," + result.getLon();
|
---|
305 | }
|
---|
306 |
|
---|
307 | static String geocodeId(String area) throws IOException {
|
---|
308 | PrimitiveId osmId = searchName(area).getOsmId();
|
---|
309 | return String.format("%s(%d)", osmId.getType().getAPIName(), osmId.getUniqueId());
|
---|
310 | }
|
---|
311 |
|
---|
312 | @Override
|
---|
313 | protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason,
|
---|
314 | boolean uncompressAccordingToContentDisposition) throws OsmTransferException {
|
---|
315 | try {
|
---|
316 | int index = urlStr.indexOf(DATA_PREFIX);
|
---|
317 | // Make an HTTP POST request instead of a simple GET, allows more complex queries
|
---|
318 | return super.getInputStreamRaw(urlStr.substring(0, index),
|
---|
319 | progressMonitor, reason, uncompressAccordingToContentDisposition,
|
---|
320 | "POST", Utils.decodeUrl(urlStr.substring(index + DATA_PREFIX.length())).getBytes(StandardCharsets.UTF_8));
|
---|
321 | } catch (OsmApiException ex) {
|
---|
322 | final String errorIndicator = "Error</strong>: ";
|
---|
323 | if (ex.getMessage() != null && ex.getMessage().contains(errorIndicator)) {
|
---|
324 | final String errorPlusRest = ex.getMessage().split(errorIndicator)[1];
|
---|
325 | if (errorPlusRest != null) {
|
---|
326 | ex.setErrorHeader(errorPlusRest.split("</")[0].replaceAll(".*::request_read_and_idx::", ""));
|
---|
327 | }
|
---|
328 | }
|
---|
329 | throw ex;
|
---|
330 | }
|
---|
331 | }
|
---|
332 |
|
---|
333 | @Override
|
---|
334 | protected void adaptRequest(HttpClient request) {
|
---|
335 | // see https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#timeout
|
---|
336 | final Matcher timeoutMatcher = Pattern.compile("\\[timeout:(\\d+)\\]").matcher(overpassQuery);
|
---|
337 | final int timeout;
|
---|
338 | if (timeoutMatcher.find()) {
|
---|
339 | timeout = (int) TimeUnit.SECONDS.toMillis(Integer.parseInt(timeoutMatcher.group(1)));
|
---|
340 | } else {
|
---|
341 | timeout = (int) TimeUnit.MINUTES.toMillis(3);
|
---|
342 | }
|
---|
343 | request.setConnectTimeout(timeout);
|
---|
344 | request.setReadTimeout(timeout);
|
---|
345 | }
|
---|
346 |
|
---|
347 | @Override
|
---|
348 | protected String getTaskName() {
|
---|
349 | return tr("Contacting Server...");
|
---|
350 | }
|
---|
351 |
|
---|
352 | @Override
|
---|
353 | protected DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
|
---|
354 | AbstractReader reader = null;
|
---|
355 | Matcher m = OUTPUT_FORMAT_STATEMENT.matcher(overpassQuery);
|
---|
356 | if (m.matches()) {
|
---|
357 | Class<? extends AbstractReader> readerClass = outputFormatReaders.get(OverpassOutpoutFormat.from(m.group(1)));
|
---|
358 | if (readerClass != null) {
|
---|
359 | try {
|
---|
360 | reader = readerClass.getDeclaredConstructor().newInstance();
|
---|
361 | } catch (ReflectiveOperationException | IllegalArgumentException | SecurityException e) {
|
---|
362 | Logging.error(e);
|
---|
363 | }
|
---|
364 | }
|
---|
365 | }
|
---|
366 | if (reader == null) {
|
---|
367 | reader = new OverpassOsmReader();
|
---|
368 | }
|
---|
369 | return reader.doParseDataSet(source, progressMonitor);
|
---|
370 | }
|
---|
371 |
|
---|
372 | @Override
|
---|
373 | public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException {
|
---|
374 |
|
---|
375 | DataSet ds = super.parseOsm(progressMonitor);
|
---|
376 |
|
---|
377 | // add bounds if necessary (note that Overpass API does not return bounds in the response XML)
|
---|
378 | if (ds != null && ds.getDataSources().isEmpty() && overpassQuery.contains("{{bbox}}")) {
|
---|
379 | if (crosses180th) {
|
---|
380 | Bounds bounds = new Bounds(lat1, lon1, lat2, 180.0);
|
---|
381 | DataSource src = new DataSource(bounds, getBaseUrl());
|
---|
382 | ds.addDataSource(src);
|
---|
383 |
|
---|
384 | bounds = new Bounds(lat1, -180.0, lat2, lon2);
|
---|
385 | src = new DataSource(bounds, getBaseUrl());
|
---|
386 | ds.addDataSource(src);
|
---|
387 | } else {
|
---|
388 | Bounds bounds = new Bounds(lat1, lon1, lat2, lon2);
|
---|
389 | DataSource src = new DataSource(bounds, getBaseUrl());
|
---|
390 | ds.addDataSource(src);
|
---|
391 | }
|
---|
392 | }
|
---|
393 |
|
---|
394 | return ds;
|
---|
395 | }
|
---|
396 |
|
---|
397 | /**
|
---|
398 | * Fixes Overpass API query to make sure it will be accepted by JOSM.
|
---|
399 | * @param query Overpass query to check
|
---|
400 | * @return fixed query
|
---|
401 | * @since 13335
|
---|
402 | */
|
---|
403 | public static String fixQuery(String query) {
|
---|
404 | return query == null ? query : query
|
---|
405 | .replaceAll("out( body| skel| ids)?( id| qt)?;", "out meta$2;")
|
---|
406 | .replaceAll("(?s)\\[out:(json|csv)[^\\]]*\\]", "[out:xml]");
|
---|
407 | }
|
---|
408 | }
|
---|