source: josm/trunk/src/org/openstreetmap/josm/io/OverpassDownloadReader.java@ 12279

Last change on this file since 12279 was 11987, checked in by Don-vip, 7 years ago

checkstyle
  • Property svn:eol-style set to native
File size: 10.8 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.io;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.IOException;
7import java.io.InputStream;
8import java.util.EnumMap;
9import java.util.Map;
10import java.util.NoSuchElementException;
11import java.util.Objects;
12import java.util.concurrent.ConcurrentHashMap;
13import java.util.concurrent.TimeUnit;
14import java.util.regex.Matcher;
15import java.util.regex.Pattern;
16
17import javax.xml.stream.XMLStreamConstants;
18import javax.xml.stream.XMLStreamException;
19
20import org.openstreetmap.josm.Main;
21import org.openstreetmap.josm.data.Bounds;
22import org.openstreetmap.josm.data.DataSource;
23import org.openstreetmap.josm.data.osm.DataSet;
24import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
25import org.openstreetmap.josm.data.osm.PrimitiveId;
26import org.openstreetmap.josm.gui.progress.ProgressMonitor;
27import org.openstreetmap.josm.tools.HttpClient;
28import org.openstreetmap.josm.tools.UncheckedParseException;
29import org.openstreetmap.josm.tools.Utils;
30
31/**
32 * Read content from an Overpass server.
33 *
34 * @since 8744
35 */
36public class OverpassDownloadReader extends BoundingBoxDownloader {
37
38 static final class OverpassOsmReader extends OsmReader {
39 @Override
40 protected void parseUnknown(boolean printWarning) throws XMLStreamException {
41 if ("remark".equals(parser.getLocalName()) && parser.getEventType() == XMLStreamConstants.START_ELEMENT) {
42 final String text = parser.getElementText();
43 if (text.contains("runtime error")) {
44 throw new XMLStreamException(text);
45 }
46 }
47 super.parseUnknown(printWarning);
48 }
49 }
50
51 /**
52 * Possible Overpass API output format, with the {@code [out:<directive>]} statement.
53 * @since 11916
54 */
55 public enum OverpassOutpoutFormat {
56 /** Default output format: plain OSM XML */
57 OSM_XML("xml"),
58 /** OSM JSON format (not GeoJson) */
59 OSM_JSON("json"),
60 /** CSV, see https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#Output_Format_.28out.29 */
61 CSV("csv"),
62 /** Custom, see https://overpass-api.de/output_formats.html#custom */
63 CUSTOM("custom"),
64 /** Popup, see https://overpass-api.de/output_formats.html#popup */
65 POPUP("popup"),
66 /** PBF, see https://josm.openstreetmap.de/ticket/14653 */
67 PBF("pbf");
68
69 private final String directive;
70
71 OverpassOutpoutFormat(String directive) {
72 this.directive = directive;
73 }
74
75 /**
76 * Returns the directive used in {@code [out:<directive>]} statement.
77 * @return the directive used in {@code [out:<directive>]} statement
78 */
79 public String getDirective() {
80 return directive;
81 }
82
83 /**
84 * Returns the {@code OverpassOutpoutFormat} matching the given directive.
85 * @param directive directive used in {@code [out:<directive>]} statement
86 * @return {@code OverpassOutpoutFormat} matching the given directive
87 * @throws IllegalArgumentException in case of invalid directive
88 */
89 static OverpassOutpoutFormat from(String directive) {
90 for (OverpassOutpoutFormat oof : values()) {
91 if (oof.directive.equals(directive)) {
92 return oof;
93 }
94 }
95 throw new IllegalArgumentException(directive);
96 }
97 }
98
99 static final Pattern OUTPUT_FORMAT_STATEMENT = Pattern.compile(".*\\[out:([a-z]{3,})\\].*", Pattern.DOTALL);
100
101 static final Map<OverpassOutpoutFormat, Class<? extends AbstractReader>> outputFormatReaders = new ConcurrentHashMap<>();
102
103 final String overpassServer;
104 final String overpassQuery;
105
106 /**
107 * Constructs a new {@code OverpassDownloadReader}.
108 *
109 * @param downloadArea The area to download
110 * @param overpassServer The Overpass server to use
111 * @param overpassQuery The Overpass query
112 */
113 public OverpassDownloadReader(Bounds downloadArea, String overpassServer, String overpassQuery) {
114 super(downloadArea);
115 this.overpassServer = overpassServer;
116 this.overpassQuery = overpassQuery.trim();
117 }
118
119 /**
120 * Registers an OSM reader for the given Overpass output format.
121 * @param format Overpass output format
122 * @param readerClass OSM reader class
123 * @return the previous value associated with {@code format}, or {@code null} if there was no mapping
124 */
125 public static final Class<? extends AbstractReader> registerOverpassOutpoutFormatReader(
126 OverpassOutpoutFormat format, Class<? extends AbstractReader> readerClass) {
127 return outputFormatReaders.put(Objects.requireNonNull(format), Objects.requireNonNull(readerClass));
128 }
129
130 static {
131 registerOverpassOutpoutFormatReader(OverpassOutpoutFormat.OSM_XML, OverpassOsmReader.class);
132 }
133
134 @Override
135 protected String getBaseUrl() {
136 return overpassServer;
137 }
138
139 @Override
140 protected String getRequestForBbox(double lon1, double lat1, double lon2, double lat2) {
141 if (overpassQuery.isEmpty())
142 return super.getRequestForBbox(lon1, lat1, lon2, lat2);
143 else {
144 final String query = this.overpassQuery.replace("{{bbox}}", lat1 + "," + lon1 + "," + lat2 + "," + lon2);
145 final String expandedOverpassQuery = expandExtendedQueries(query);
146 return "interpreter?data=" + Utils.encodeUrl(expandedOverpassQuery);
147 }
148 }
149
150 /**
151 * Evaluates some features of overpass turbo extended query syntax.
152 * See https://wiki.openstreetmap.org/wiki/Overpass_turbo/Extended_Overpass_Turbo_Queries
153 * @param query unexpanded query
154 * @return expanded query
155 */
156 static String expandExtendedQueries(String query) {
157 final StringBuffer sb = new StringBuffer();
158 final Matcher matcher = Pattern.compile("\\{\\{(geocodeArea):([^}]+)\\}\\}").matcher(query);
159 while (matcher.find()) {
160 try {
161 switch (matcher.group(1)) {
162 case "geocodeArea":
163 matcher.appendReplacement(sb, geocodeArea(matcher.group(2)));
164 break;
165 default:
166 Main.warn("Unsupported syntax: " + matcher.group(1));
167 }
168 } catch (UncheckedParseException ex) {
169 final String msg = tr("Failed to evaluate {0}", matcher.group());
170 Main.warn(ex, msg);
171 matcher.appendReplacement(sb, "// " + msg + "\n");
172 }
173 }
174 matcher.appendTail(sb);
175 return sb.toString();
176 }
177
178 private static String geocodeArea(String area) {
179 // Offsets defined in https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#By_element_id
180 final EnumMap<OsmPrimitiveType, Long> idOffset = new EnumMap<>(OsmPrimitiveType.class);
181 idOffset.put(OsmPrimitiveType.NODE, 0L);
182 idOffset.put(OsmPrimitiveType.WAY, 2_400_000_000L);
183 idOffset.put(OsmPrimitiveType.RELATION, 3_600_000_000L);
184 try {
185 final PrimitiveId osmId = NameFinder.queryNominatim(area).stream().filter(
186 x -> !OsmPrimitiveType.NODE.equals(x.getOsmId().getType())).iterator().next().getOsmId();
187 return String.format("area(%d)", osmId.getUniqueId() + idOffset.get(osmId.getType()));
188 } catch (IOException | NoSuchElementException | IndexOutOfBoundsException ex) {
189 throw new UncheckedParseException(ex);
190 }
191 }
192
193 @Override
194 protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason,
195 boolean uncompressAccordingToContentDisposition) throws OsmTransferException {
196 try {
197 return super.getInputStreamRaw(urlStr, progressMonitor, reason, uncompressAccordingToContentDisposition);
198 } catch (OsmApiException ex) {
199 final String errorIndicator = "Error</strong>: ";
200 if (ex.getMessage() != null && ex.getMessage().contains(errorIndicator)) {
201 final String errorPlusRest = ex.getMessage().split(errorIndicator)[1];
202 if (errorPlusRest != null) {
203 ex.setErrorHeader(errorPlusRest.split("</")[0].replaceAll(".*::request_read_and_idx::", ""));
204 }
205 }
206 throw ex;
207 }
208 }
209
210 @Override
211 protected void adaptRequest(HttpClient request) {
212 // see https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#timeout
213 final Matcher timeoutMatcher = Pattern.compile("\\[timeout:(\\d+)\\]").matcher(overpassQuery);
214 final int timeout;
215 if (timeoutMatcher.find()) {
216 timeout = (int) TimeUnit.SECONDS.toMillis(Integer.parseInt(timeoutMatcher.group(1)));
217 } else {
218 timeout = (int) TimeUnit.MINUTES.toMillis(3);
219 }
220 request.setConnectTimeout(timeout);
221 request.setReadTimeout(timeout);
222 }
223
224 @Override
225 protected String getTaskName() {
226 return tr("Contacting Server...");
227 }
228
229 @Override
230 protected DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
231 AbstractReader reader = null;
232 Matcher m = OUTPUT_FORMAT_STATEMENT.matcher(overpassQuery);
233 if (m.matches()) {
234 Class<? extends AbstractReader> readerClass = outputFormatReaders.get(OverpassOutpoutFormat.from(m.group(1)));
235 if (readerClass != null) {
236 try {
237 reader = readerClass.getDeclaredConstructor().newInstance();
238 } catch (ReflectiveOperationException | IllegalArgumentException | SecurityException e) {
239 Main.error(e);
240 }
241 }
242 }
243 if (reader == null) {
244 reader = new OverpassOsmReader();
245 }
246 return reader.doParseDataSet(source, progressMonitor);
247 }
248
249 @Override
250 public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException {
251
252 DataSet ds = super.parseOsm(progressMonitor);
253
254 // add bounds if necessary (note that Overpass API does not return bounds in the response XML)
255 if (ds != null && ds.getDataSources().isEmpty() && overpassQuery.contains("{{bbox}}")) {
256 if (crosses180th) {
257 Bounds bounds = new Bounds(lat1, lon1, lat2, 180.0);
258 DataSource src = new DataSource(bounds, getBaseUrl());
259 ds.addDataSource(src);
260
261 bounds = new Bounds(lat1, -180.0, lat2, lon2);
262 src = new DataSource(bounds, getBaseUrl());
263 ds.addDataSource(src);
264 } else {
265 Bounds bounds = new Bounds(lat1, lon1, lat2, lon2);
266 DataSource src = new DataSource(bounds, getBaseUrl());
267 ds.addDataSource(src);
268 }
269 }
270
271 return ds;
272 }
273}
Note: See TracBrowser for help on using the repository browser.