source: josm/trunk/src/org/openstreetmap/josm/io/OverpassDownloadReader.java@ 12596

Last change on this file since 12596 was 12596, checked in by Don-vip, 7 years ago

fix #15141 - Make HTTP POST requests to Overpass API - allows longer/more complex queries
  • Property svn:eol-style set to native
File size: 11.2 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.io;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.IOException;
7import java.io.InputStream;
8import java.nio.charset.StandardCharsets;
9import java.util.EnumMap;
10import java.util.Map;
11import java.util.NoSuchElementException;
12import java.util.Objects;
13import java.util.concurrent.ConcurrentHashMap;
14import java.util.concurrent.TimeUnit;
15import java.util.regex.Matcher;
16import java.util.regex.Pattern;
17
18import javax.xml.stream.XMLStreamConstants;
19import javax.xml.stream.XMLStreamException;
20
21import org.openstreetmap.josm.Main;
22import org.openstreetmap.josm.data.Bounds;
23import org.openstreetmap.josm.data.DataSource;
24import org.openstreetmap.josm.data.osm.DataSet;
25import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
26import org.openstreetmap.josm.data.osm.PrimitiveId;
27import org.openstreetmap.josm.gui.progress.ProgressMonitor;
28import org.openstreetmap.josm.tools.HttpClient;
29import org.openstreetmap.josm.tools.UncheckedParseException;
30import org.openstreetmap.josm.tools.Utils;
31
32/**
33 * Read content from an Overpass server.
34 *
35 * @since 8744
36 */
37public class OverpassDownloadReader extends BoundingBoxDownloader {
38
39 private static final String DATA_PREFIX = "?data=";
40
41 static final class OverpassOsmReader extends OsmReader {
42 @Override
43 protected void parseUnknown(boolean printWarning) throws XMLStreamException {
44 if ("remark".equals(parser.getLocalName()) && parser.getEventType() == XMLStreamConstants.START_ELEMENT) {
45 final String text = parser.getElementText();
46 if (text.contains("runtime error")) {
47 throw new XMLStreamException(text);
48 }
49 }
50 super.parseUnknown(printWarning);
51 }
52 }
53
54 /**
55 * Possible Overpass API output format, with the {@code [out:<directive>]} statement.
56 * @since 11916
57 */
58 public enum OverpassOutpoutFormat {
59 /** Default output format: plain OSM XML */
60 OSM_XML("xml"),
61 /** OSM JSON format (not GeoJson) */
62 OSM_JSON("json"),
63 /** CSV, see https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#Output_Format_.28out.29 */
64 CSV("csv"),
65 /** Custom, see https://overpass-api.de/output_formats.html#custom */
66 CUSTOM("custom"),
67 /** Popup, see https://overpass-api.de/output_formats.html#popup */
68 POPUP("popup"),
69 /** PBF, see https://josm.openstreetmap.de/ticket/14653 */
70 PBF("pbf");
71
72 private final String directive;
73
74 OverpassOutpoutFormat(String directive) {
75 this.directive = directive;
76 }
77
78 /**
79 * Returns the directive used in {@code [out:<directive>]} statement.
80 * @return the directive used in {@code [out:<directive>]} statement
81 */
82 public String getDirective() {
83 return directive;
84 }
85
86 /**
87 * Returns the {@code OverpassOutpoutFormat} matching the given directive.
88 * @param directive directive used in {@code [out:<directive>]} statement
89 * @return {@code OverpassOutpoutFormat} matching the given directive
90 * @throws IllegalArgumentException in case of invalid directive
91 */
92 static OverpassOutpoutFormat from(String directive) {
93 for (OverpassOutpoutFormat oof : values()) {
94 if (oof.directive.equals(directive)) {
95 return oof;
96 }
97 }
98 throw new IllegalArgumentException(directive);
99 }
100 }
101
102 static final Pattern OUTPUT_FORMAT_STATEMENT = Pattern.compile(".*\\[out:([a-z]{3,})\\].*", Pattern.DOTALL);
103
104 static final Map<OverpassOutpoutFormat, Class<? extends AbstractReader>> outputFormatReaders = new ConcurrentHashMap<>();
105
106 final String overpassServer;
107 final String overpassQuery;
108
109 /**
110 * Constructs a new {@code OverpassDownloadReader}.
111 *
112 * @param downloadArea The area to download
113 * @param overpassServer The Overpass server to use
114 * @param overpassQuery The Overpass query
115 */
116 public OverpassDownloadReader(Bounds downloadArea, String overpassServer, String overpassQuery) {
117 super(downloadArea);
118 this.overpassServer = overpassServer;
119 this.overpassQuery = overpassQuery.trim();
120 }
121
122 /**
123 * Registers an OSM reader for the given Overpass output format.
124 * @param format Overpass output format
125 * @param readerClass OSM reader class
126 * @return the previous value associated with {@code format}, or {@code null} if there was no mapping
127 */
128 public static final Class<? extends AbstractReader> registerOverpassOutpoutFormatReader(
129 OverpassOutpoutFormat format, Class<? extends AbstractReader> readerClass) {
130 return outputFormatReaders.put(Objects.requireNonNull(format), Objects.requireNonNull(readerClass));
131 }
132
133 static {
134 registerOverpassOutpoutFormatReader(OverpassOutpoutFormat.OSM_XML, OverpassOsmReader.class);
135 }
136
137 @Override
138 protected String getBaseUrl() {
139 return overpassServer;
140 }
141
142 @Override
143 protected String getRequestForBbox(double lon1, double lat1, double lon2, double lat2) {
144 if (overpassQuery.isEmpty())
145 return super.getRequestForBbox(lon1, lat1, lon2, lat2);
146 else {
147 final String query = this.overpassQuery.replace("{{bbox}}", lat1 + "," + lon1 + "," + lat2 + "," + lon2);
148 final String expandedOverpassQuery = expandExtendedQueries(query);
149 return "interpreter" + DATA_PREFIX + Utils.encodeUrl(expandedOverpassQuery);
150 }
151 }
152
153 /**
154 * Evaluates some features of overpass turbo extended query syntax.
155 * See https://wiki.openstreetmap.org/wiki/Overpass_turbo/Extended_Overpass_Turbo_Queries
156 * @param query unexpanded query
157 * @return expanded query
158 */
159 static String expandExtendedQueries(String query) {
160 final StringBuffer sb = new StringBuffer();
161 final Matcher matcher = Pattern.compile("\\{\\{(geocodeArea):([^}]+)\\}\\}").matcher(query);
162 while (matcher.find()) {
163 try {
164 switch (matcher.group(1)) {
165 case "geocodeArea":
166 matcher.appendReplacement(sb, geocodeArea(matcher.group(2)));
167 break;
168 default:
169 Main.warn("Unsupported syntax: " + matcher.group(1));
170 }
171 } catch (UncheckedParseException ex) {
172 final String msg = tr("Failed to evaluate {0}", matcher.group());
173 Main.warn(ex, msg);
174 matcher.appendReplacement(sb, "// " + msg + "\n");
175 }
176 }
177 matcher.appendTail(sb);
178 return sb.toString();
179 }
180
181 private static String geocodeArea(String area) {
182 // Offsets defined in https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#By_element_id
183 final EnumMap<OsmPrimitiveType, Long> idOffset = new EnumMap<>(OsmPrimitiveType.class);
184 idOffset.put(OsmPrimitiveType.NODE, 0L);
185 idOffset.put(OsmPrimitiveType.WAY, 2_400_000_000L);
186 idOffset.put(OsmPrimitiveType.RELATION, 3_600_000_000L);
187 try {
188 final PrimitiveId osmId = NameFinder.queryNominatim(area).stream().filter(
189 x -> !OsmPrimitiveType.NODE.equals(x.getOsmId().getType())).iterator().next().getOsmId();
190 return String.format("area(%d)", osmId.getUniqueId() + idOffset.get(osmId.getType()));
191 } catch (IOException | NoSuchElementException | IndexOutOfBoundsException ex) {
192 throw new UncheckedParseException(ex);
193 }
194 }
195
196 @Override
197 protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason,
198 boolean uncompressAccordingToContentDisposition) throws OsmTransferException {
199 try {
200 int index = urlStr.indexOf(DATA_PREFIX);
201 // Make an HTTP POST request instead of a simple GET, allows more complex queries
202 return super.getInputStreamRaw(urlStr.substring(0, index),
203 progressMonitor, reason, uncompressAccordingToContentDisposition,
204 "POST", Utils.decodeUrl(urlStr.substring(index + DATA_PREFIX.length())).getBytes(StandardCharsets.UTF_8));
205 } catch (OsmApiException ex) {
206 final String errorIndicator = "Error</strong>: ";
207 if (ex.getMessage() != null && ex.getMessage().contains(errorIndicator)) {
208 final String errorPlusRest = ex.getMessage().split(errorIndicator)[1];
209 if (errorPlusRest != null) {
210 ex.setErrorHeader(errorPlusRest.split("</")[0].replaceAll(".*::request_read_and_idx::", ""));
211 }
212 }
213 throw ex;
214 }
215 }
216
217 @Override
218 protected void adaptRequest(HttpClient request) {
219 // see https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#timeout
220 final Matcher timeoutMatcher = Pattern.compile("\\[timeout:(\\d+)\\]").matcher(overpassQuery);
221 final int timeout;
222 if (timeoutMatcher.find()) {
223 timeout = (int) TimeUnit.SECONDS.toMillis(Integer.parseInt(timeoutMatcher.group(1)));
224 } else {
225 timeout = (int) TimeUnit.MINUTES.toMillis(3);
226 }
227 request.setConnectTimeout(timeout);
228 request.setReadTimeout(timeout);
229 }
230
231 @Override
232 protected String getTaskName() {
233 return tr("Contacting Server...");
234 }
235
236 @Override
237 protected DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
238 AbstractReader reader = null;
239 Matcher m = OUTPUT_FORMAT_STATEMENT.matcher(overpassQuery);
240 if (m.matches()) {
241 Class<? extends AbstractReader> readerClass = outputFormatReaders.get(OverpassOutpoutFormat.from(m.group(1)));
242 if (readerClass != null) {
243 try {
244 reader = readerClass.getDeclaredConstructor().newInstance();
245 } catch (ReflectiveOperationException | IllegalArgumentException | SecurityException e) {
246 Main.error(e);
247 }
248 }
249 }
250 if (reader == null) {
251 reader = new OverpassOsmReader();
252 }
253 return reader.doParseDataSet(source, progressMonitor);
254 }
255
256 @Override
257 public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException {
258
259 DataSet ds = super.parseOsm(progressMonitor);
260
261 // add bounds if necessary (note that Overpass API does not return bounds in the response XML)
262 if (ds != null && ds.getDataSources().isEmpty() && overpassQuery.contains("{{bbox}}")) {
263 if (crosses180th) {
264 Bounds bounds = new Bounds(lat1, lon1, lat2, 180.0);
265 DataSource src = new DataSource(bounds, getBaseUrl());
266 ds.addDataSource(src);
267
268 bounds = new Bounds(lat1, -180.0, lat2, lon2);
269 src = new DataSource(bounds, getBaseUrl());
270 ds.addDataSource(src);
271 } else {
272 Bounds bounds = new Bounds(lat1, lon1, lat2, lon2);
273 DataSource src = new DataSource(bounds, getBaseUrl());
274 ds.addDataSource(src);
275 }
276 }
277
278 return ds;
279 }
280}
Note: See TracBrowser for help on using the repository browser.