source: josm/trunk/src/org/openstreetmap/josm/io/OsmReader.java@ 17379

Last change on this file since 17379 was 16641, checked in by simon04, 4 years ago

fix #18258 - OsmReader: Allow end user to know what the original id of a feature was (patch by taylor.smock, modified)
  • Property svn:eol-style set to native
File size: 19.9 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.io;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.InputStream;
7import java.util.Arrays;
8import java.util.Collection;
9import java.util.Collections;
10import java.util.Objects;
11import java.util.Set;
12import java.util.TreeSet;
13import java.util.regex.Matcher;
14import java.util.regex.Pattern;
15
16import javax.xml.stream.Location;
17import javax.xml.stream.XMLStreamConstants;
18import javax.xml.stream.XMLStreamException;
19import javax.xml.stream.XMLStreamReader;
20
21import org.openstreetmap.josm.data.osm.Changeset;
22import org.openstreetmap.josm.data.osm.DataSet;
23import org.openstreetmap.josm.data.osm.Node;
24import org.openstreetmap.josm.data.osm.PrimitiveData;
25import org.openstreetmap.josm.data.osm.Relation;
26import org.openstreetmap.josm.data.osm.RelationMemberData;
27import org.openstreetmap.josm.data.osm.Tagged;
28import org.openstreetmap.josm.data.osm.Way;
29import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
30import org.openstreetmap.josm.gui.progress.ProgressMonitor;
31import org.openstreetmap.josm.tools.Logging;
32import org.openstreetmap.josm.tools.UncheckedParseException;
33import org.openstreetmap.josm.tools.XmlUtils;
34
35/**
36 * Parser for the Osm API (XML output). Read from an input stream and construct a dataset out of it.
37 *
38 * For each xml element, there is a dedicated method.
39 * The XMLStreamReader cursor points to the start of the element, when the method is
40 * entered, and it must point to the end of the same element, when it is exited.
41 */
42public class OsmReader extends AbstractReader {
43
44 /**
45 * Options are used to change how the xml data is parsed.
46 * For example, {@link Options#CONVERT_UNKNOWN_TO_TAGS} is used to convert unknown XML attributes to a tag for the object.
47 * @since 16641
48 */
49 public enum Options {
50 /**
51 * Convert unknown XML attributes to tags
52 */
53 CONVERT_UNKNOWN_TO_TAGS,
54 /**
55 * Save the original id of an object (currently stored in `current_id`)
56 */
57 SAVE_ORIGINAL_ID
58 }
59
60 protected XMLStreamReader parser;
61
62 /** The {@link OsmReader.Options} to use when parsing the xml data */
63 protected final Collection<Options> options;
64
65 private static final Set<String> COMMON_XML_ATTRIBUTES = new TreeSet<>();
66
67 static {
68 COMMON_XML_ATTRIBUTES.add("id");
69 COMMON_XML_ATTRIBUTES.add("timestamp");
70 COMMON_XML_ATTRIBUTES.add("user");
71 COMMON_XML_ATTRIBUTES.add("uid");
72 COMMON_XML_ATTRIBUTES.add("visible");
73 COMMON_XML_ATTRIBUTES.add("version");
74 COMMON_XML_ATTRIBUTES.add("action");
75 COMMON_XML_ATTRIBUTES.add("changeset");
76 COMMON_XML_ATTRIBUTES.add("lat");
77 COMMON_XML_ATTRIBUTES.add("lon");
78 }
79
80 /**
81 * constructor (for private and subclasses use only)
82 *
83 * @see #parseDataSet(InputStream, ProgressMonitor)
84 */
85 protected OsmReader() {
86 this((Options) null);
87 }
88
89 /**
90 * constructor (for private and subclasses use only)
91 * @param options The options to use when reading data
92 *
93 * @see #parseDataSet(InputStream, ProgressMonitor)
94 * @since 16641
95 */
96 protected OsmReader(Options... options) {
97 // Restricts visibility
98 this.options = options == null ? Collections.emptyList() : Arrays.asList(options);
99 }
100
101 protected void setParser(XMLStreamReader parser) {
102 this.parser = parser;
103 }
104
105 protected void throwException(Throwable th) throws XMLStreamException {
106 throw new XmlStreamParsingException(th.getMessage(), parser.getLocation(), th);
107 }
108
109 protected void throwException(String msg, Throwable th) throws XMLStreamException {
110 throw new XmlStreamParsingException(msg, parser.getLocation(), th);
111 }
112
113 protected void throwException(String msg) throws XMLStreamException {
114 throw new XmlStreamParsingException(msg, parser.getLocation());
115 }
116
117 protected void parse() throws XMLStreamException {
118 int event = parser.getEventType();
119 while (true) {
120 if (event == XMLStreamConstants.START_ELEMENT) {
121 parseRoot();
122 } else if (event == XMLStreamConstants.END_ELEMENT)
123 return;
124 if (parser.hasNext()) {
125 event = parser.next();
126 } else {
127 break;
128 }
129 }
130 parser.close();
131 }
132
133 protected void parseRoot() throws XMLStreamException {
134 if ("osm".equals(parser.getLocalName())) {
135 parseOsm();
136 } else {
137 parseUnknown();
138 }
139 }
140
141 private void parseOsm() throws XMLStreamException {
142 try {
143 parseVersion(parser.getAttributeValue(null, "version"));
144 parseDownloadPolicy("download", parser.getAttributeValue(null, "download"));
145 parseUploadPolicy("upload", parser.getAttributeValue(null, "upload"));
146 parseLocked(parser.getAttributeValue(null, "locked"));
147 } catch (IllegalDataException e) {
148 throwException(e);
149 }
150 String generator = parser.getAttributeValue(null, "generator");
151 Long uploadChangesetId = null;
152 if (parser.getAttributeValue(null, "upload-changeset") != null) {
153 uploadChangesetId = getLong("upload-changeset");
154 }
155 while (parser.hasNext()) {
156 int event = parser.next();
157
158 if (cancel) {
159 cancel = false;
160 throw new OsmParsingCanceledException(tr("Reading was canceled"), parser.getLocation());
161 }
162
163 if (event == XMLStreamConstants.START_ELEMENT) {
164 switch (parser.getLocalName()) {
165 case "bounds":
166 parseBounds(generator);
167 break;
168 case "node":
169 parseNode();
170 break;
171 case "way":
172 parseWay();
173 break;
174 case "relation":
175 parseRelation();
176 break;
177 case "changeset":
178 parseChangeset(uploadChangesetId);
179 break;
180 case "remark": // Used by Overpass API
181 parseRemark();
182 break;
183 default:
184 parseUnknown();
185 }
186 } else if (event == XMLStreamConstants.END_ELEMENT) {
187 return;
188 }
189 }
190 }
191
192 private void handleIllegalDataException(IllegalDataException e) throws XMLStreamException {
193 Throwable cause = e.getCause();
194 if (cause instanceof XMLStreamException) {
195 throw (XMLStreamException) cause;
196 } else {
197 throwException(e);
198 }
199 }
200
201 private void parseRemark() throws XMLStreamException {
202 while (parser.hasNext()) {
203 int event = parser.next();
204 if (event == XMLStreamConstants.CHARACTERS) {
205 ds.setRemark(parser.getText());
206 } else if (event == XMLStreamConstants.END_ELEMENT) {
207 return;
208 }
209 }
210 }
211
212 private void parseBounds(String generator) throws XMLStreamException {
213 String minlon = parser.getAttributeValue(null, "minlon");
214 String minlat = parser.getAttributeValue(null, "minlat");
215 String maxlon = parser.getAttributeValue(null, "maxlon");
216 String maxlat = parser.getAttributeValue(null, "maxlat");
217 String origin = parser.getAttributeValue(null, "origin");
218 try {
219 parseBounds(generator, minlon, minlat, maxlon, maxlat, origin);
220 } catch (IllegalDataException e) {
221 handleIllegalDataException(e);
222 }
223 jumpToEnd();
224 }
225
226 protected Node parseNode() throws XMLStreamException {
227 String lat = parser.getAttributeValue(null, "lat");
228 String lon = parser.getAttributeValue(null, "lon");
229 try {
230 return parseNode(lat, lon, this::readCommon, this::parseNodeTags);
231 } catch (IllegalDataException e) {
232 handleIllegalDataException(e);
233 }
234 return null;
235 }
236
237 private void parseNodeTags(Node n) throws IllegalDataException {
238 try {
239 while (parser.hasNext()) {
240 int event = parser.next();
241 if (event == XMLStreamConstants.START_ELEMENT) {
242 if ("tag".equals(parser.getLocalName())) {
243 parseTag(n);
244 } else {
245 parseUnknown();
246 }
247 } else if (event == XMLStreamConstants.END_ELEMENT) {
248 return;
249 }
250 }
251 } catch (XMLStreamException e) {
252 throw new IllegalDataException(e);
253 }
254 }
255
256 protected Way parseWay() throws XMLStreamException {
257 try {
258 return parseWay(this::readCommon, this::parseWayNodesAndTags);
259 } catch (IllegalDataException e) {
260 handleIllegalDataException(e);
261 }
262 return null;
263 }
264
265 private void parseWayNodesAndTags(Way w, Collection<Long> nodeIds) throws IllegalDataException {
266 try {
267 while (parser.hasNext()) {
268 int event = parser.next();
269 if (event == XMLStreamConstants.START_ELEMENT) {
270 switch (parser.getLocalName()) {
271 case "nd":
272 nodeIds.add(parseWayNode(w));
273 break;
274 case "tag":
275 parseTag(w);
276 break;
277 default:
278 parseUnknown();
279 }
280 } else if (event == XMLStreamConstants.END_ELEMENT) {
281 break;
282 }
283 }
284 } catch (XMLStreamException e) {
285 throw new IllegalDataException(e);
286 }
287 }
288
289 private long parseWayNode(Way w) throws XMLStreamException {
290 if (parser.getAttributeValue(null, "ref") == null) {
291 throwException(
292 tr("Missing mandatory attribute ''{0}'' on <nd> of way {1}.", "ref", Long.toString(w.getUniqueId()))
293 );
294 }
295 long id = getLong("ref");
296 if (id == 0) {
297 throwException(
298 tr("Illegal value of attribute ''ref'' of element <nd>. Got {0}.", Long.toString(id))
299 );
300 }
301 jumpToEnd();
302 return id;
303 }
304
305 protected Relation parseRelation() throws XMLStreamException {
306 try {
307 return parseRelation(this::readCommon, this::parseRelationMembersAndTags);
308 } catch (IllegalDataException e) {
309 handleIllegalDataException(e);
310 }
311 return null;
312 }
313
314 private void parseRelationMembersAndTags(Relation r, Collection<RelationMemberData> members) throws IllegalDataException {
315 try {
316 while (parser.hasNext()) {
317 int event = parser.next();
318 if (event == XMLStreamConstants.START_ELEMENT) {
319 switch (parser.getLocalName()) {
320 case "member":
321 members.add(parseRelationMember(r));
322 break;
323 case "tag":
324 parseTag(r);
325 break;
326 default:
327 parseUnknown();
328 }
329 } else if (event == XMLStreamConstants.END_ELEMENT) {
330 break;
331 }
332 }
333 } catch (XMLStreamException e) {
334 throw new IllegalDataException(e);
335 }
336 }
337
338 private RelationMemberData parseRelationMember(Relation r) throws XMLStreamException {
339 RelationMemberData result = null;
340 try {
341 String ref = parser.getAttributeValue(null, "ref");
342 String type = parser.getAttributeValue(null, "type");
343 String role = parser.getAttributeValue(null, "role");
344 result = parseRelationMember(r, ref, type, role);
345 jumpToEnd();
346 } catch (IllegalDataException e) {
347 handleIllegalDataException(e);
348 }
349 return result;
350 }
351
352 private void parseChangeset(Long uploadChangesetId) throws XMLStreamException {
353
354 Long id = null;
355 if (parser.getAttributeValue(null, "id") != null) {
356 id = getLong("id");
357 }
358 // Read changeset info if neither upload-changeset nor id are set, or if they are both set to the same value
359 if (Objects.equals(id, uploadChangesetId)) {
360 uploadChangeset = new Changeset(id != null ? id.intValue() : 0);
361 while (true) {
362 int event = parser.next();
363 if (event == XMLStreamConstants.START_ELEMENT) {
364 if ("tag".equals(parser.getLocalName())) {
365 parseTag(uploadChangeset);
366 } else {
367 parseUnknown();
368 }
369 } else if (event == XMLStreamConstants.END_ELEMENT)
370 return;
371 }
372 } else {
373 jumpToEnd(false);
374 }
375 }
376
377 private void parseTag(Tagged t) throws XMLStreamException {
378 String key = parser.getAttributeValue(null, "k");
379 String value = parser.getAttributeValue(null, "v");
380 try {
381 parseTag(t, key, value);
382 } catch (IllegalDataException e) {
383 throwException(e);
384 }
385 jumpToEnd();
386 }
387
388 protected void parseUnknown(boolean printWarning) throws XMLStreamException {
389 final String element = parser.getLocalName();
390 if (printWarning && ("note".equals(element) || "meta".equals(element))) {
391 // we know that Overpass API returns those elements
392 Logging.debug(tr("Undefined element ''{0}'' found in input stream. Skipping.", element));
393 } else if (printWarning) {
394 Logging.info(tr("Undefined element ''{0}'' found in input stream. Skipping.", element));
395 }
396 while (true) {
397 int event = parser.next();
398 if (event == XMLStreamConstants.START_ELEMENT) {
399 parseUnknown(false); /* no more warning for inner elements */
400 } else if (event == XMLStreamConstants.END_ELEMENT)
401 return;
402 }
403 }
404
405 protected void parseUnknown() throws XMLStreamException {
406 parseUnknown(true);
407 }
408
409 /**
410 * When cursor is at the start of an element, moves it to the end tag of that element.
411 * Nested content is skipped.
412 *
413 * This is basically the same code as parseUnknown(), except for the warnings, which
414 * are displayed for inner elements and not at top level.
415 * @param printWarning if {@code true}, a warning message will be printed if an unknown element is met
416 * @throws XMLStreamException if there is an error processing the underlying XML source
417 */
418 protected final void jumpToEnd(boolean printWarning) throws XMLStreamException {
419 while (true) {
420 int event = parser.next();
421 if (event == XMLStreamConstants.START_ELEMENT) {
422 parseUnknown(printWarning);
423 } else if (event == XMLStreamConstants.END_ELEMENT)
424 return;
425 }
426 }
427
428 protected final void jumpToEnd() throws XMLStreamException {
429 jumpToEnd(true);
430 }
431
432 /**
433 * Read out the common attributes and put them into current OsmPrimitive.
434 * @param current primitive to update
435 * @throws IllegalDataException if there is an error processing the underlying XML source
436 */
437 private void readCommon(PrimitiveData current) throws IllegalDataException {
438 try {
439 parseId(current, getLong("id"));
440 parseTimestamp(current, parser.getAttributeValue(null, "timestamp"));
441 parseUser(current, parser.getAttributeValue(null, "user"), parser.getAttributeValue(null, "uid"));
442 parseVisible(current, parser.getAttributeValue(null, "visible"));
443 parseVersion(current, parser.getAttributeValue(null, "version"));
444 parseAction(current, parser.getAttributeValue(null, "action"));
445 parseChangeset(current, parser.getAttributeValue(null, "changeset"));
446
447 if (options.contains(Options.SAVE_ORIGINAL_ID)) {
448 parseTag(current, "current_id", Long.toString(getLong("id")));
449 }
450 if (options.contains(Options.CONVERT_UNKNOWN_TO_TAGS)) {
451 for (int i = 0; i < parser.getAttributeCount(); i++) {
452 if (!COMMON_XML_ATTRIBUTES.contains(parser.getAttributeLocalName(i))) {
453 parseTag(current, parser.getAttributeLocalName(i), parser.getAttributeValue(i));
454 }
455 }
456 }
457 } catch (UncheckedParseException | XMLStreamException e) {
458 throw new IllegalDataException(e);
459 }
460 }
461
462 private long getLong(String name) throws XMLStreamException {
463 String value = parser.getAttributeValue(null, name);
464 try {
465 return getLong(name, value);
466 } catch (IllegalDataException e) {
467 throwException(e);
468 }
469 return 0; // should not happen
470 }
471
472 /**
473 * Exception thrown after user cancelation.
474 */
475 private static final class OsmParsingCanceledException extends XmlStreamParsingException implements ImportCancelException {
476 /**
477 * Constructs a new {@code OsmParsingCanceledException}.
478 * @param msg The error message
479 * @param location The parser location
480 */
481 OsmParsingCanceledException(String msg, Location location) {
482 super(msg, location);
483 }
484 }
485
486 @Override
487 protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
488 return doParseDataSet(source, progressMonitor, ir -> {
489 try {
490 setParser(XmlUtils.newSafeXMLInputFactory().createXMLStreamReader(ir));
491 parse();
492 } catch (XmlStreamParsingException | UncheckedParseException e) {
493 throw new IllegalDataException(e.getMessage(), e);
494 } catch (XMLStreamException e) {
495 String msg = e.getMessage();
496 Pattern p = Pattern.compile("Message: (.+)");
497 Matcher m = p.matcher(msg);
498 if (m.find()) {
499 msg = m.group(1);
500 }
501 if (e.getLocation() != null)
502 throw new IllegalDataException(tr("Line {0} column {1}: ",
503 e.getLocation().getLineNumber(), e.getLocation().getColumnNumber()) + msg, e);
504 else
505 throw new IllegalDataException(msg, e);
506 }
507 });
508 }
509
510 /**
511 * Parse the given input source and return the dataset.
512 *
513 * @param source the source input stream. Must not be null.
514 * @param progressMonitor the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed
515 *
516 * @return the dataset with the parsed data
517 * @throws IllegalDataException if an error was found while parsing the data from the source
518 * @throws IllegalArgumentException if source is null
519 */
520 public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
521 return parseDataSet(source, progressMonitor, (Options) null);
522 }
523
524 /**
525 * Parse the given input source and return the dataset.
526 *
527 * @param source the source input stream. Must not be null.
528 * @param progressMonitor the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed
529 * @param options The options to use when parsing the dataset
530 *
531 * @return the dataset with the parsed data
532 * @throws IllegalDataException if an error was found while parsing the data from the source
533 * @throws IllegalArgumentException if source is null
534 * @since 16641
535 */
536 public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor, Options... options)
537 throws IllegalDataException {
538 return new OsmReader(options).doParseDataSet(source, progressMonitor);
539 }
540}
Note: See TracBrowser for help on using the repository browser.