[8764] | 1 | // License: GPL. For details, see LICENSE file.
|
---|
[8744] | 2 | package org.openstreetmap.josm.io;
|
---|
| 3 |
|
---|
[8787] | 4 | import static org.openstreetmap.josm.tools.I18n.tr;
|
---|
| 5 |
|
---|
[11003] | 6 | import java.io.IOException;
|
---|
[8782] | 7 | import java.io.InputStream;
|
---|
[12596] | 8 | import java.nio.charset.StandardCharsets;
|
---|
[12714] | 9 | import java.time.Duration;
|
---|
| 10 | import java.time.LocalDateTime;
|
---|
| 11 | import java.time.Period;
|
---|
| 12 | import java.time.ZoneOffset;
|
---|
[11003] | 13 | import java.util.EnumMap;
|
---|
[12714] | 14 | import java.util.Locale;
|
---|
[11916] | 15 | import java.util.Map;
|
---|
[11009] | 16 | import java.util.NoSuchElementException;
|
---|
[11916] | 17 | import java.util.Objects;
|
---|
| 18 | import java.util.concurrent.ConcurrentHashMap;
|
---|
[11288] | 19 | import java.util.concurrent.TimeUnit;
|
---|
[9308] | 20 | import java.util.regex.Matcher;
|
---|
| 21 | import java.util.regex.Pattern;
|
---|
[8782] | 22 |
|
---|
[8788] | 23 | import javax.xml.stream.XMLStreamConstants;
|
---|
| 24 | import javax.xml.stream.XMLStreamException;
|
---|
| 25 |
|
---|
[8744] | 26 | import org.openstreetmap.josm.data.Bounds;
|
---|
| 27 | import org.openstreetmap.josm.data.DataSource;
|
---|
[12714] | 28 | import org.openstreetmap.josm.data.coor.LatLon;
|
---|
| 29 | import org.openstreetmap.josm.data.osm.BBox;
|
---|
[8744] | 30 | import org.openstreetmap.josm.data.osm.DataSet;
|
---|
[11003] | 31 | import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
|
---|
| 32 | import org.openstreetmap.josm.data.osm.PrimitiveId;
|
---|
[8744] | 33 | import org.openstreetmap.josm.gui.progress.ProgressMonitor;
|
---|
[12714] | 34 | import org.openstreetmap.josm.io.NameFinder.SearchResult;
|
---|
[9308] | 35 | import org.openstreetmap.josm.tools.HttpClient;
|
---|
[12620] | 36 | import org.openstreetmap.josm.tools.Logging;
|
---|
[11003] | 37 | import org.openstreetmap.josm.tools.UncheckedParseException;
|
---|
[8744] | 38 | import org.openstreetmap.josm.tools.Utils;
|
---|
| 39 |
|
---|
| 40 | /**
|
---|
| 41 | * Read content from an Overpass server.
|
---|
| 42 | *
|
---|
| 43 | * @since 8744
|
---|
| 44 | */
|
---|
| 45 | public class OverpassDownloadReader extends BoundingBoxDownloader {
|
---|
| 46 |
|
---|
[12596] | 47 | private static final String DATA_PREFIX = "?data=";
|
---|
| 48 |
|
---|
[11366] | 49 | static final class OverpassOsmReader extends OsmReader {
|
---|
| 50 | @Override
|
---|
| 51 | protected void parseUnknown(boolean printWarning) throws XMLStreamException {
|
---|
| 52 | if ("remark".equals(parser.getLocalName()) && parser.getEventType() == XMLStreamConstants.START_ELEMENT) {
|
---|
| 53 | final String text = parser.getElementText();
|
---|
| 54 | if (text.contains("runtime error")) {
|
---|
| 55 | throw new XMLStreamException(text);
|
---|
| 56 | }
|
---|
| 57 | }
|
---|
| 58 | super.parseUnknown(printWarning);
|
---|
| 59 | }
|
---|
| 60 | }
|
---|
| 61 |
|
---|
[11916] | 62 | /**
|
---|
| 63 | * Possible Overpass API output format, with the {@code [out:<directive>]} statement.
|
---|
| 64 | * @since 11916
|
---|
| 65 | */
|
---|
[11918] | 66 | public enum OverpassOutpoutFormat {
|
---|
[11916] | 67 | /** Default output format: plain OSM XML */
|
---|
| 68 | OSM_XML("xml"),
|
---|
| 69 | /** OSM JSON format (not GeoJson) */
|
---|
| 70 | OSM_JSON("json"),
|
---|
| 71 | /** CSV, see https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#Output_Format_.28out.29 */
|
---|
| 72 | CSV("csv"),
|
---|
| 73 | /** Custom, see https://overpass-api.de/output_formats.html#custom */
|
---|
| 74 | CUSTOM("custom"),
|
---|
| 75 | /** Popup, see https://overpass-api.de/output_formats.html#popup */
|
---|
| 76 | POPUP("popup"),
|
---|
| 77 | /** PBF, see https://josm.openstreetmap.de/ticket/14653 */
|
---|
| 78 | PBF("pbf");
|
---|
| 79 |
|
---|
| 80 | private final String directive;
|
---|
| 81 |
|
---|
| 82 | OverpassOutpoutFormat(String directive) {
|
---|
| 83 | this.directive = directive;
|
---|
| 84 | }
|
---|
| 85 |
|
---|
| 86 | /**
|
---|
| 87 | * Returns the directive used in {@code [out:<directive>]} statement.
|
---|
| 88 | * @return the directive used in {@code [out:<directive>]} statement
|
---|
| 89 | */
|
---|
| 90 | public String getDirective() {
|
---|
| 91 | return directive;
|
---|
| 92 | }
|
---|
| 93 |
|
---|
| 94 | /**
|
---|
| 95 | * Returns the {@code OverpassOutpoutFormat} matching the given directive.
|
---|
| 96 | * @param directive directive used in {@code [out:<directive>]} statement
|
---|
| 97 | * @return {@code OverpassOutpoutFormat} matching the given directive
|
---|
| 98 | * @throws IllegalArgumentException in case of invalid directive
|
---|
| 99 | */
|
---|
| 100 | static OverpassOutpoutFormat from(String directive) {
|
---|
| 101 | for (OverpassOutpoutFormat oof : values()) {
|
---|
| 102 | if (oof.directive.equals(directive)) {
|
---|
| 103 | return oof;
|
---|
| 104 | }
|
---|
| 105 | }
|
---|
| 106 | throw new IllegalArgumentException(directive);
|
---|
| 107 | }
|
---|
| 108 | }
|
---|
| 109 |
|
---|
| 110 | static final Pattern OUTPUT_FORMAT_STATEMENT = Pattern.compile(".*\\[out:([a-z]{3,})\\].*", Pattern.DOTALL);
|
---|
| 111 |
|
---|
[12542] | 112 | static final Map<OverpassOutpoutFormat, Class<? extends AbstractReader>> outputFormatReaders = new ConcurrentHashMap<>();
|
---|
[11916] | 113 |
|
---|
[8744] | 114 | final String overpassServer;
|
---|
| 115 | final String overpassQuery;
|
---|
| 116 |
|
---|
| 117 | /**
|
---|
| 118 | * Constructs a new {@code OverpassDownloadReader}.
|
---|
| 119 | *
|
---|
| 120 | * @param downloadArea The area to download
|
---|
| 121 | * @param overpassServer The Overpass server to use
|
---|
| 122 | * @param overpassQuery The Overpass query
|
---|
| 123 | */
|
---|
| 124 | public OverpassDownloadReader(Bounds downloadArea, String overpassServer, String overpassQuery) {
|
---|
| 125 | super(downloadArea);
|
---|
[12598] | 126 | setDoAuthenticate(false);
|
---|
[8744] | 127 | this.overpassServer = overpassServer;
|
---|
| 128 | this.overpassQuery = overpassQuery.trim();
|
---|
| 129 | }
|
---|
| 130 |
|
---|
[11916] | 131 | /**
|
---|
| 132 | * Registers an OSM reader for the given Overpass output format.
|
---|
| 133 | * @param format Overpass output format
|
---|
| 134 | * @param readerClass OSM reader class
|
---|
| 135 | * @return the previous value associated with {@code format}, or {@code null} if there was no mapping
|
---|
| 136 | */
|
---|
[11919] | 137 | public static final Class<? extends AbstractReader> registerOverpassOutpoutFormatReader(
|
---|
| 138 | OverpassOutpoutFormat format, Class<? extends AbstractReader> readerClass) {
|
---|
[12542] | 139 | return outputFormatReaders.put(Objects.requireNonNull(format), Objects.requireNonNull(readerClass));
|
---|
[11916] | 140 | }
|
---|
| 141 |
|
---|
| 142 | static {
|
---|
| 143 | registerOverpassOutpoutFormatReader(OverpassOutpoutFormat.OSM_XML, OverpassOsmReader.class);
|
---|
| 144 | }
|
---|
| 145 |
|
---|
[8744] | 146 | @Override
|
---|
| 147 | protected String getBaseUrl() {
|
---|
| 148 | return overpassServer;
|
---|
| 149 | }
|
---|
| 150 |
|
---|
| 151 | @Override
|
---|
| 152 | protected String getRequestForBbox(double lon1, double lat1, double lon2, double lat2) {
|
---|
| 153 | if (overpassQuery.isEmpty())
|
---|
| 154 | return super.getRequestForBbox(lon1, lat1, lon2, lat2);
|
---|
| 155 | else {
|
---|
[12714] | 156 | final String query = this.overpassQuery
|
---|
| 157 | .replace("{{bbox}}", bbox(lon1, lat1, lon2, lat2))
|
---|
| 158 | .replace("{{center}}", center(lon1, lat1, lon2, lat2));
|
---|
[11009] | 159 | final String expandedOverpassQuery = expandExtendedQueries(query);
|
---|
[12596] | 160 | return "interpreter" + DATA_PREFIX + Utils.encodeUrl(expandedOverpassQuery);
|
---|
[8744] | 161 | }
|
---|
| 162 | }
|
---|
| 163 |
|
---|
[11003] | 164 | /**
|
---|
| 165 | * Evaluates some features of overpass turbo extended query syntax.
|
---|
| 166 | * See https://wiki.openstreetmap.org/wiki/Overpass_turbo/Extended_Overpass_Turbo_Queries
|
---|
[11009] | 167 | * @param query unexpanded query
|
---|
| 168 | * @return expanded query
|
---|
[11003] | 169 | */
|
---|
| 170 | static String expandExtendedQueries(String query) {
|
---|
| 171 | final StringBuffer sb = new StringBuffer();
|
---|
[12714] | 172 | final Matcher matcher = Pattern.compile("\\{\\{(date|geocodeArea|geocodeBbox|geocodeCoords|geocodeId):([^}]+)\\}\\}").matcher(query);
|
---|
[11003] | 173 | while (matcher.find()) {
|
---|
| 174 | try {
|
---|
| 175 | switch (matcher.group(1)) {
|
---|
[12714] | 176 | case "date":
|
---|
| 177 | matcher.appendReplacement(sb, date(matcher.group(2), LocalDateTime.now()));
|
---|
| 178 | break;
|
---|
[11003] | 179 | case "geocodeArea":
|
---|
| 180 | matcher.appendReplacement(sb, geocodeArea(matcher.group(2)));
|
---|
[11048] | 181 | break;
|
---|
[12714] | 182 | case "geocodeBbox":
|
---|
| 183 | matcher.appendReplacement(sb, geocodeBbox(matcher.group(2)));
|
---|
| 184 | break;
|
---|
| 185 | case "geocodeCoords":
|
---|
| 186 | matcher.appendReplacement(sb, geocodeCoords(matcher.group(2)));
|
---|
| 187 | break;
|
---|
| 188 | case "geocodeId":
|
---|
| 189 | matcher.appendReplacement(sb, geocodeId(matcher.group(2)));
|
---|
| 190 | break;
|
---|
[11048] | 191 | default:
|
---|
[12620] | 192 | Logging.warn("Unsupported syntax: " + matcher.group(1));
|
---|
[11003] | 193 | }
|
---|
[12714] | 194 | } catch (UncheckedParseException | IOException | NoSuchElementException | IndexOutOfBoundsException ex) {
|
---|
[11003] | 195 | final String msg = tr("Failed to evaluate {0}", matcher.group());
|
---|
[12620] | 196 | Logging.log(Logging.LEVEL_WARN, msg, ex);
|
---|
[11003] | 197 | matcher.appendReplacement(sb, "// " + msg + "\n");
|
---|
[8744] | 198 | }
|
---|
| 199 | }
|
---|
[11003] | 200 | matcher.appendTail(sb);
|
---|
| 201 | return sb.toString();
|
---|
| 202 | }
|
---|
[8744] | 203 |
|
---|
[12714] | 204 | static String bbox(double lon1, double lat1, double lon2, double lat2) {
|
---|
| 205 | return lat1 + "," + lon1 + "," + lat2 + "," + lon2;
|
---|
| 206 | }
|
---|
| 207 |
|
---|
| 208 | static String center(double lon1, double lat1, double lon2, double lat2) {
|
---|
| 209 | LatLon c = new BBox(lon1, lat1, lon2, lat2).getCenter();
|
---|
| 210 | return c.lat()+ "," + c.lon();
|
---|
| 211 | }
|
---|
| 212 |
|
---|
| 213 | static String date(String humanDuration, LocalDateTime from) {
|
---|
| 214 | // Convert to ISO 8601. Replace months by X temporarily to avoid conflict with minutes
|
---|
| 215 | String duration = humanDuration.toLowerCase(Locale.ENGLISH).replace(" ", "")
|
---|
| 216 | .replaceAll("years?", "Y").replaceAll("months?", "X").replaceAll("weeks?", "W")
|
---|
| 217 | .replaceAll("days?", "D").replaceAll("hours?", "H").replaceAll("minutes?", "M").replaceAll("seconds?", "S");
|
---|
| 218 | Matcher matcher = Pattern.compile(
|
---|
| 219 | "((?:[0-9]+Y)?(?:[0-9]+X)?(?:[0-9]+W)?)"+
|
---|
| 220 | "((?:[0-9]+D)?)" +
|
---|
| 221 | "((?:[0-9]+H)?(?:[0-9]+M)?(?:[0-9]+(?:[.,][0-9]{0,9})?S)?)?").matcher(duration);
|
---|
| 222 | boolean javaPer = false;
|
---|
| 223 | boolean javaDur = false;
|
---|
| 224 | if (matcher.matches()) {
|
---|
| 225 | javaPer = matcher.group(1) != null && !matcher.group(1).isEmpty();
|
---|
| 226 | javaDur = matcher.group(3) != null && !matcher.group(3).isEmpty();
|
---|
| 227 | duration = 'P' + matcher.group(1).replace('X', 'M') + matcher.group(2);
|
---|
| 228 | if (javaDur) {
|
---|
| 229 | duration += 'T' + matcher.group(3);
|
---|
| 230 | }
|
---|
| 231 | }
|
---|
| 232 |
|
---|
| 233 | // Duration is now a full ISO 8601 duration string. Unfortunately Java does not allow to parse it entirely.
|
---|
| 234 | // We must split the "period" (years, months, weeks, days) from the "duration" (days, hours, minutes, seconds).
|
---|
| 235 | Period p = null;
|
---|
| 236 | Duration d = null;
|
---|
| 237 | int idx = duration.indexOf('T');
|
---|
| 238 | if (javaPer) {
|
---|
| 239 | p = Period.parse(javaDur ? duration.substring(0, idx) : duration);
|
---|
| 240 | }
|
---|
| 241 | if (javaDur) {
|
---|
| 242 | d = Duration.parse(javaPer ? 'P' + duration.substring(idx, duration.length()) : duration);
|
---|
| 243 | } else if (!javaPer) {
|
---|
| 244 | d = Duration.parse(duration);
|
---|
| 245 | }
|
---|
| 246 |
|
---|
| 247 | // Now that period and duration are known, compute the correct date/time
|
---|
| 248 | LocalDateTime dt = from;
|
---|
| 249 | if (p != null) {
|
---|
| 250 | dt = dt.minus(p);
|
---|
| 251 | }
|
---|
| 252 | if (d != null) {
|
---|
| 253 | dt = dt.minus(d);
|
---|
| 254 | }
|
---|
| 255 |
|
---|
| 256 | // Returns the date/time formatted in ISO 8601
|
---|
| 257 | return dt.toInstant(ZoneOffset.UTC).toString();
|
---|
| 258 | }
|
---|
| 259 |
|
---|
| 260 | private static SearchResult searchName(String area) throws IOException {
|
---|
| 261 | return NameFinder.queryNominatim(area).stream().filter(
|
---|
| 262 | x -> !OsmPrimitiveType.NODE.equals(x.getOsmId().getType())).iterator().next();
|
---|
| 263 | }
|
---|
| 264 |
|
---|
| 265 | static String geocodeArea(String area) throws IOException {
|
---|
[11003] | 266 | // Offsets defined in https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#By_element_id
|
---|
| 267 | final EnumMap<OsmPrimitiveType, Long> idOffset = new EnumMap<>(OsmPrimitiveType.class);
|
---|
| 268 | idOffset.put(OsmPrimitiveType.NODE, 0L);
|
---|
[11009] | 269 | idOffset.put(OsmPrimitiveType.WAY, 2_400_000_000L);
|
---|
| 270 | idOffset.put(OsmPrimitiveType.RELATION, 3_600_000_000L);
|
---|
[12714] | 271 | final PrimitiveId osmId = searchName(area).getOsmId();
|
---|
| 272 | return String.format("area(%d)", osmId.getUniqueId() + idOffset.get(osmId.getType()));
|
---|
[8744] | 273 | }
|
---|
| 274 |
|
---|
[12714] | 275 | static String geocodeBbox(String area) throws IOException {
|
---|
| 276 | Bounds bounds = searchName(area).getBounds();
|
---|
| 277 | return bounds.getMinLat() + "," + bounds.getMinLon() + "," + bounds.getMaxLat() + "," + bounds.getMaxLon();
|
---|
| 278 | }
|
---|
| 279 |
|
---|
| 280 | static String geocodeCoords(String area) throws IOException {
|
---|
| 281 | SearchResult result = searchName(area);
|
---|
| 282 | return result.getLat() + "," + result.getLon();
|
---|
| 283 | }
|
---|
| 284 |
|
---|
| 285 | static String geocodeId(String area) throws IOException {
|
---|
| 286 | PrimitiveId osmId = searchName(area).getOsmId();
|
---|
| 287 | return String.format("%s(%d)", osmId.getType().getAPIName(), osmId.getUniqueId());
|
---|
| 288 | }
|
---|
| 289 |
|
---|
[8744] | 290 | @Override
|
---|
[8782] | 291 | protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason,
|
---|
| 292 | boolean uncompressAccordingToContentDisposition) throws OsmTransferException {
|
---|
| 293 | try {
|
---|
[12596] | 294 | int index = urlStr.indexOf(DATA_PREFIX);
|
---|
| 295 | // Make an HTTP POST request instead of a simple GET, allows more complex queries
|
---|
| 296 | return super.getInputStreamRaw(urlStr.substring(0, index),
|
---|
| 297 | progressMonitor, reason, uncompressAccordingToContentDisposition,
|
---|
| 298 | "POST", Utils.decodeUrl(urlStr.substring(index + DATA_PREFIX.length())).getBytes(StandardCharsets.UTF_8));
|
---|
[8782] | 299 | } catch (OsmApiException ex) {
|
---|
| 300 | final String errorIndicator = "Error</strong>: ";
|
---|
| 301 | if (ex.getMessage() != null && ex.getMessage().contains(errorIndicator)) {
|
---|
| 302 | final String errorPlusRest = ex.getMessage().split(errorIndicator)[1];
|
---|
| 303 | if (errorPlusRest != null) {
|
---|
[11987] | 304 | ex.setErrorHeader(errorPlusRest.split("</")[0].replaceAll(".*::request_read_and_idx::", ""));
|
---|
[8782] | 305 | }
|
---|
| 306 | }
|
---|
| 307 | throw ex;
|
---|
| 308 | }
|
---|
| 309 | }
|
---|
| 310 |
|
---|
| 311 | @Override
|
---|
[9308] | 312 | protected void adaptRequest(HttpClient request) {
|
---|
| 313 | // see https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#timeout
|
---|
| 314 | final Matcher timeoutMatcher = Pattern.compile("\\[timeout:(\\d+)\\]").matcher(overpassQuery);
|
---|
[9318] | 315 | final int timeout;
|
---|
[9308] | 316 | if (timeoutMatcher.find()) {
|
---|
[11288] | 317 | timeout = (int) TimeUnit.SECONDS.toMillis(Integer.parseInt(timeoutMatcher.group(1)));
|
---|
[9318] | 318 | } else {
|
---|
[11288] | 319 | timeout = (int) TimeUnit.MINUTES.toMillis(3);
|
---|
[9308] | 320 | }
|
---|
[9318] | 321 | request.setConnectTimeout(timeout);
|
---|
| 322 | request.setReadTimeout(timeout);
|
---|
[9308] | 323 | }
|
---|
| 324 |
|
---|
| 325 | @Override
|
---|
[8787] | 326 | protected String getTaskName() {
|
---|
| 327 | return tr("Contacting Server...");
|
---|
| 328 | }
|
---|
| 329 |
|
---|
| 330 | @Override
|
---|
[8788] | 331 | protected DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
|
---|
[11919] | 332 | AbstractReader reader = null;
|
---|
[11916] | 333 | Matcher m = OUTPUT_FORMAT_STATEMENT.matcher(overpassQuery);
|
---|
| 334 | if (m.matches()) {
|
---|
[12542] | 335 | Class<? extends AbstractReader> readerClass = outputFormatReaders.get(OverpassOutpoutFormat.from(m.group(1)));
|
---|
[11916] | 336 | if (readerClass != null) {
|
---|
| 337 | try {
|
---|
[11924] | 338 | reader = readerClass.getDeclaredConstructor().newInstance();
|
---|
[11916] | 339 | } catch (ReflectiveOperationException | IllegalArgumentException | SecurityException e) {
|
---|
[12620] | 340 | Logging.error(e);
|
---|
[11916] | 341 | }
|
---|
| 342 | }
|
---|
| 343 | }
|
---|
| 344 | if (reader == null) {
|
---|
| 345 | reader = new OverpassOsmReader();
|
---|
| 346 | }
|
---|
| 347 | return reader.doParseDataSet(source, progressMonitor);
|
---|
[8788] | 348 | }
|
---|
| 349 |
|
---|
| 350 | @Override
|
---|
[8744] | 351 | public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException {
|
---|
| 352 |
|
---|
| 353 | DataSet ds = super.parseOsm(progressMonitor);
|
---|
| 354 |
|
---|
| 355 | // add bounds if necessary (note that Overpass API does not return bounds in the response XML)
|
---|
[11627] | 356 | if (ds != null && ds.getDataSources().isEmpty() && overpassQuery.contains("{{bbox}}")) {
|
---|
[8744] | 357 | if (crosses180th) {
|
---|
| 358 | Bounds bounds = new Bounds(lat1, lon1, lat2, 180.0);
|
---|
| 359 | DataSource src = new DataSource(bounds, getBaseUrl());
|
---|
[11627] | 360 | ds.addDataSource(src);
|
---|
[8744] | 361 |
|
---|
| 362 | bounds = new Bounds(lat1, -180.0, lat2, lon2);
|
---|
| 363 | src = new DataSource(bounds, getBaseUrl());
|
---|
[11627] | 364 | ds.addDataSource(src);
|
---|
[8744] | 365 | } else {
|
---|
| 366 | Bounds bounds = new Bounds(lat1, lon1, lat2, lon2);
|
---|
| 367 | DataSource src = new DataSource(bounds, getBaseUrl());
|
---|
[11627] | 368 | ds.addDataSource(src);
|
---|
[8744] | 369 | }
|
---|
| 370 | }
|
---|
| 371 |
|
---|
| 372 | return ds;
|
---|
| 373 | }
|
---|
| 374 | }
|
---|