source: josm/trunk/src/org/openstreetmap/josm/io/OsmReader.java@ 9917

Last change on this file since 9917 was 9326, checked in by simon04, 8 years ago

OsmReader: do not inform about undefined elements send from Overpass API

This gets rid of:

Undefined element 'node' found in input stream. Skipping.
Undefined element 'meta' found in input stream. Skipping.
  • Property svn:eol-style set to native
File size: 26.0 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.io;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.InputStream;
7import java.io.InputStreamReader;
8import java.text.MessageFormat;
9import java.util.ArrayList;
10import java.util.Collection;
11import java.util.List;
12import java.util.regex.Matcher;
13import java.util.regex.Pattern;
14
15import javax.xml.stream.Location;
16import javax.xml.stream.XMLInputFactory;
17import javax.xml.stream.XMLStreamConstants;
18import javax.xml.stream.XMLStreamException;
19import javax.xml.stream.XMLStreamReader;
20
21import org.openstreetmap.josm.Main;
22import org.openstreetmap.josm.data.Bounds;
23import org.openstreetmap.josm.data.DataSource;
24import org.openstreetmap.josm.data.coor.LatLon;
25import org.openstreetmap.josm.data.osm.Changeset;
26import org.openstreetmap.josm.data.osm.DataSet;
27import org.openstreetmap.josm.data.osm.Node;
28import org.openstreetmap.josm.data.osm.NodeData;
29import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
30import org.openstreetmap.josm.data.osm.PrimitiveData;
31import org.openstreetmap.josm.data.osm.Relation;
32import org.openstreetmap.josm.data.osm.RelationData;
33import org.openstreetmap.josm.data.osm.RelationMemberData;
34import org.openstreetmap.josm.data.osm.Tagged;
35import org.openstreetmap.josm.data.osm.User;
36import org.openstreetmap.josm.data.osm.Way;
37import org.openstreetmap.josm.data.osm.WayData;
38import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
39import org.openstreetmap.josm.gui.progress.ProgressMonitor;
40import org.openstreetmap.josm.tools.CheckParameterUtil;
41import org.openstreetmap.josm.tools.date.DateUtils;
42
43/**
44 * Parser for the Osm Api. Read from an input stream and construct a dataset out of it.
45 *
46 * For each xml element, there is a dedicated method.
47 * The XMLStreamReader cursor points to the start of the element, when the method is
48 * entered, and it must point to the end of the same element, when it is exited.
49 */
50public class OsmReader extends AbstractReader {
51
52 protected XMLStreamReader parser;
53
54 protected boolean cancel;
55
56 /** Used by plugins to register themselves as data postprocessors. */
57 private static volatile List<OsmServerReadPostprocessor> postprocessors;
58
59 /** Register a new postprocessor.
60 * @param pp postprocessor
61 * @see #deregisterPostprocessor
62 */
63 public static void registerPostprocessor(OsmServerReadPostprocessor pp) {
64 if (postprocessors == null) {
65 postprocessors = new ArrayList<>();
66 }
67 postprocessors.add(pp);
68 }
69
70 /**
71 * Deregister a postprocessor previously registered with {@link #registerPostprocessor}.
72 * @param pp postprocessor
73 * @see #registerPostprocessor
74 */
75 public static void deregisterPostprocessor(OsmServerReadPostprocessor pp) {
76 if (postprocessors != null) {
77 postprocessors.remove(pp);
78 }
79 }
80
81 /**
82 * constructor (for private and subclasses use only)
83 *
84 * @see #parseDataSet(InputStream, ProgressMonitor)
85 */
86 protected OsmReader() {
87 // Restricts visibility
88 }
89
90 protected void setParser(XMLStreamReader parser) {
91 this.parser = parser;
92 }
93
94 protected void throwException(String msg, Throwable th) throws XMLStreamException {
95 throw new OsmParsingException(msg, parser.getLocation(), th);
96 }
97
98 protected void throwException(String msg) throws XMLStreamException {
99 throw new OsmParsingException(msg, parser.getLocation());
100 }
101
102 protected void parse() throws XMLStreamException {
103 int event = parser.getEventType();
104 while (true) {
105 if (event == XMLStreamConstants.START_ELEMENT) {
106 parseRoot();
107 } else if (event == XMLStreamConstants.END_ELEMENT)
108 return;
109 if (parser.hasNext()) {
110 event = parser.next();
111 } else {
112 break;
113 }
114 }
115 parser.close();
116 }
117
118 protected void parseRoot() throws XMLStreamException {
119 if ("osm".equals(parser.getLocalName())) {
120 parseOsm();
121 } else {
122 parseUnknown();
123 }
124 }
125
126 private void parseOsm() throws XMLStreamException {
127 String v = parser.getAttributeValue(null, "version");
128 if (v == null) {
129 throwException(tr("Missing mandatory attribute ''{0}''.", "version"));
130 }
131 if (!"0.6".equals(v)) {
132 throwException(tr("Unsupported version: {0}", v));
133 }
134 ds.setVersion(v);
135 String upload = parser.getAttributeValue(null, "upload");
136 if (upload != null) {
137 ds.setUploadDiscouraged(!Boolean.parseBoolean(upload));
138 }
139 String generator = parser.getAttributeValue(null, "generator");
140 Long uploadChangesetId = null;
141 if (parser.getAttributeValue(null, "upload-changeset") != null) {
142 uploadChangesetId = getLong("upload-changeset");
143 }
144 while (true) {
145 int event = parser.next();
146
147 if (cancel) {
148 cancel = false;
149 throw new OsmParsingCanceledException(tr("Reading was canceled"), parser.getLocation());
150 }
151
152 if (event == XMLStreamConstants.START_ELEMENT) {
153 switch (parser.getLocalName()) {
154 case "bounds":
155 parseBounds(generator);
156 break;
157 case "node":
158 parseNode();
159 break;
160 case "way":
161 parseWay();
162 break;
163 case "relation":
164 parseRelation();
165 break;
166 case "changeset":
167 parseChangeset(uploadChangesetId);
168 break;
169 default:
170 parseUnknown();
171 }
172 } else if (event == XMLStreamConstants.END_ELEMENT)
173 return;
174 }
175 }
176
177 private void parseBounds(String generator) throws XMLStreamException {
178 String minlon = parser.getAttributeValue(null, "minlon");
179 String minlat = parser.getAttributeValue(null, "minlat");
180 String maxlon = parser.getAttributeValue(null, "maxlon");
181 String maxlat = parser.getAttributeValue(null, "maxlat");
182 String origin = parser.getAttributeValue(null, "origin");
183 if (minlon != null && maxlon != null && minlat != null && maxlat != null) {
184 if (origin == null) {
185 origin = generator;
186 }
187 Bounds bounds = new Bounds(
188 Double.parseDouble(minlat), Double.parseDouble(minlon),
189 Double.parseDouble(maxlat), Double.parseDouble(maxlon));
190 if (bounds.isOutOfTheWorld()) {
191 Bounds copy = new Bounds(bounds);
192 bounds.normalize();
193 Main.info("Bbox " + copy + " is out of the world, normalized to " + bounds);
194 }
195 DataSource src = new DataSource(bounds, origin);
196 ds.dataSources.add(src);
197 } else {
198 throwException(tr("Missing mandatory attributes on element ''bounds''. " +
199 "Got minlon=''{0}'',minlat=''{1}'',maxlon=''{3}'',maxlat=''{4}'', origin=''{5}''.",
200 minlon, minlat, maxlon, maxlat, origin
201 ));
202 }
203 jumpToEnd();
204 }
205
206 protected Node parseNode() throws XMLStreamException {
207 NodeData nd = new NodeData();
208 String lat = parser.getAttributeValue(null, "lat");
209 String lon = parser.getAttributeValue(null, "lon");
210 if (lat != null && lon != null) {
211 nd.setCoor(new LatLon(Double.parseDouble(lat), Double.parseDouble(lon)));
212 }
213 readCommon(nd);
214 Node n = new Node(nd.getId(), nd.getVersion());
215 n.setVisible(nd.isVisible());
216 n.load(nd);
217 externalIdMap.put(nd.getPrimitiveId(), n);
218 while (true) {
219 int event = parser.next();
220 if (event == XMLStreamConstants.START_ELEMENT) {
221 if ("tag".equals(parser.getLocalName())) {
222 parseTag(n);
223 } else {
224 parseUnknown();
225 }
226 } else if (event == XMLStreamConstants.END_ELEMENT)
227 return n;
228 }
229 }
230
231 protected Way parseWay() throws XMLStreamException {
232 WayData wd = new WayData();
233 readCommon(wd);
234 Way w = new Way(wd.getId(), wd.getVersion());
235 w.setVisible(wd.isVisible());
236 w.load(wd);
237 externalIdMap.put(wd.getPrimitiveId(), w);
238
239 Collection<Long> nodeIds = new ArrayList<>();
240 while (true) {
241 int event = parser.next();
242 if (event == XMLStreamConstants.START_ELEMENT) {
243 switch (parser.getLocalName()) {
244 case "nd":
245 nodeIds.add(parseWayNode(w));
246 break;
247 case "tag":
248 parseTag(w);
249 break;
250 default:
251 parseUnknown();
252 }
253 } else if (event == XMLStreamConstants.END_ELEMENT) {
254 break;
255 }
256 }
257 if (w.isDeleted() && !nodeIds.isEmpty()) {
258 Main.info(tr("Deleted way {0} contains nodes", w.getUniqueId()));
259 nodeIds = new ArrayList<>();
260 }
261 ways.put(wd.getUniqueId(), nodeIds);
262 return w;
263 }
264
265 private long parseWayNode(Way w) throws XMLStreamException {
266 if (parser.getAttributeValue(null, "ref") == null) {
267 throwException(
268 tr("Missing mandatory attribute ''{0}'' on <nd> of way {1}.", "ref", w.getUniqueId())
269 );
270 }
271 long id = getLong("ref");
272 if (id == 0) {
273 throwException(
274 tr("Illegal value of attribute ''ref'' of element <nd>. Got {0}.", id)
275 );
276 }
277 jumpToEnd();
278 return id;
279 }
280
281 protected Relation parseRelation() throws XMLStreamException {
282 RelationData rd = new RelationData();
283 readCommon(rd);
284 Relation r = new Relation(rd.getId(), rd.getVersion());
285 r.setVisible(rd.isVisible());
286 r.load(rd);
287 externalIdMap.put(rd.getPrimitiveId(), r);
288
289 Collection<RelationMemberData> members = new ArrayList<>();
290 while (true) {
291 int event = parser.next();
292 if (event == XMLStreamConstants.START_ELEMENT) {
293 switch (parser.getLocalName()) {
294 case "member":
295 members.add(parseRelationMember(r));
296 break;
297 case "tag":
298 parseTag(r);
299 break;
300 default:
301 parseUnknown();
302 }
303 } else if (event == XMLStreamConstants.END_ELEMENT) {
304 break;
305 }
306 }
307 if (r.isDeleted() && !members.isEmpty()) {
308 Main.info(tr("Deleted relation {0} contains members", r.getUniqueId()));
309 members = new ArrayList<>();
310 }
311 relations.put(rd.getUniqueId(), members);
312 return r;
313 }
314
315 private RelationMemberData parseRelationMember(Relation r) throws XMLStreamException {
316 String role = null;
317 OsmPrimitiveType type = null;
318 long id = 0;
319 String value = parser.getAttributeValue(null, "ref");
320 if (value == null) {
321 throwException(tr("Missing attribute ''ref'' on member in relation {0}.", r.getUniqueId()));
322 }
323 try {
324 id = Long.parseLong(value);
325 } catch (NumberFormatException e) {
326 throwException(tr("Illegal value for attribute ''ref'' on member in relation {0}. Got {1}", Long.toString(r.getUniqueId()),
327 value), e);
328 }
329 value = parser.getAttributeValue(null, "type");
330 if (value == null) {
331 throwException(tr("Missing attribute ''type'' on member {0} in relation {1}.", Long.toString(id), Long.toString(r.getUniqueId())));
332 }
333 try {
334 type = OsmPrimitiveType.fromApiTypeName(value);
335 } catch (IllegalArgumentException e) {
336 throwException(tr("Illegal value for attribute ''type'' on member {0} in relation {1}. Got {2}.",
337 Long.toString(id), Long.toString(r.getUniqueId()), value), e);
338 }
339 value = parser.getAttributeValue(null, "role");
340 role = value;
341
342 if (id == 0) {
343 throwException(tr("Incomplete <member> specification with ref=0"));
344 }
345 jumpToEnd();
346 return new RelationMemberData(role, type, id);
347 }
348
349 private void parseChangeset(Long uploadChangesetId) throws XMLStreamException {
350
351 Long id = null;
352 if (parser.getAttributeValue(null, "id") != null) {
353 id = getLong("id");
354 }
355 // Read changeset info if neither upload-changeset nor id are set, or if they are both set to the same value
356 if (id == uploadChangesetId || (id != null && id.equals(uploadChangesetId))) {
357 uploadChangeset = new Changeset(id != null ? id.intValue() : 0);
358 while (true) {
359 int event = parser.next();
360 if (event == XMLStreamConstants.START_ELEMENT) {
361 if ("tag".equals(parser.getLocalName())) {
362 parseTag(uploadChangeset);
363 } else {
364 parseUnknown();
365 }
366 } else if (event == XMLStreamConstants.END_ELEMENT)
367 return;
368 }
369 } else {
370 jumpToEnd(false);
371 }
372 }
373
374 private void parseTag(Tagged t) throws XMLStreamException {
375 String key = parser.getAttributeValue(null, "k");
376 String value = parser.getAttributeValue(null, "v");
377 if (key == null || value == null) {
378 throwException(tr("Missing key or value attribute in tag."));
379 } else {
380 t.put(key.intern(), value.intern());
381 }
382 jumpToEnd();
383 }
384
385 protected void parseUnknown(boolean printWarning) throws XMLStreamException {
386 final String element = parser.getLocalName();
387 if (printWarning && ("note".equals(element) || "meta".equals(element))) {
388 // we know that Overpass API returns those elements
389 Main.debug(tr("Undefined element ''{0}'' found in input stream. Skipping.", element));
390 } else if (printWarning) {
391 Main.info(tr("Undefined element ''{0}'' found in input stream. Skipping.", element));
392 }
393 while (true) {
394 int event = parser.next();
395 if (event == XMLStreamConstants.START_ELEMENT) {
396 parseUnknown(false); /* no more warning for inner elements */
397 } else if (event == XMLStreamConstants.END_ELEMENT)
398 return;
399 }
400 }
401
402 protected void parseUnknown() throws XMLStreamException {
403 parseUnknown(true);
404 }
405
406 /**
407 * When cursor is at the start of an element, moves it to the end tag of that element.
408 * Nested content is skipped.
409 *
410 * This is basically the same code as parseUnknown(), except for the warnings, which
411 * are displayed for inner elements and not at top level.
412 * @param printWarning if {@code true}, a warning message will be printed if an unknown element is met
413 * @throws XMLStreamException if there is an error processing the underlying XML source
414 */
415 private void jumpToEnd(boolean printWarning) throws XMLStreamException {
416 while (true) {
417 int event = parser.next();
418 if (event == XMLStreamConstants.START_ELEMENT) {
419 parseUnknown(printWarning);
420 } else if (event == XMLStreamConstants.END_ELEMENT)
421 return;
422 }
423 }
424
425 private void jumpToEnd() throws XMLStreamException {
426 jumpToEnd(true);
427 }
428
429 private User createUser(String uid, String name) throws XMLStreamException {
430 if (uid == null) {
431 if (name == null)
432 return null;
433 return User.createLocalUser(name);
434 }
435 try {
436 long id = Long.parseLong(uid);
437 return User.createOsmUser(id, name);
438 } catch (NumberFormatException e) {
439 throwException(MessageFormat.format("Illegal value for attribute ''uid''. Got ''{0}''.", uid), e);
440 }
441 return null;
442 }
443
444 /**
445 * Read out the common attributes and put them into current OsmPrimitive.
446 * @param current primitive to update
447 * @throws XMLStreamException if there is an error processing the underlying XML source
448 */
449 private void readCommon(PrimitiveData current) throws XMLStreamException {
450 current.setId(getLong("id"));
451 if (current.getUniqueId() == 0) {
452 throwException(tr("Illegal object with ID=0."));
453 }
454
455 String time = parser.getAttributeValue(null, "timestamp");
456 if (time != null && !time.isEmpty()) {
457 current.setRawTimestamp((int) (DateUtils.tsFromString(time)/1000));
458 }
459
460 String user = parser.getAttributeValue(null, "user");
461 String uid = parser.getAttributeValue(null, "uid");
462 current.setUser(createUser(uid, user));
463
464 String visible = parser.getAttributeValue(null, "visible");
465 if (visible != null) {
466 current.setVisible(Boolean.parseBoolean(visible));
467 }
468
469 String versionString = parser.getAttributeValue(null, "version");
470 int version = 0;
471 if (versionString != null) {
472 try {
473 version = Integer.parseInt(versionString);
474 } catch (NumberFormatException e) {
475 throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.",
476 Long.toString(current.getUniqueId()), versionString), e);
477 }
478 switch (ds.getVersion()) {
479 case "0.6":
480 if (version <= 0 && !current.isNew()) {
481 throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.",
482 Long.toString(current.getUniqueId()), versionString));
483 } else if (version < 0 && current.isNew()) {
484 Main.warn(tr("Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.",
485 current.getUniqueId(), version, 0, "0.6"));
486 version = 0;
487 }
488 break;
489 default:
490 // should not happen. API version has been checked before
491 throwException(tr("Unknown or unsupported API version. Got {0}.", ds.getVersion()));
492 }
493 } else {
494 // version expected for OSM primitives with an id assigned by the server (id > 0), since API 0.6
495 if (!current.isNew() && ds.getVersion() != null && "0.6".equals(ds.getVersion())) {
496 throwException(tr("Missing attribute ''version'' on OSM primitive with ID {0}.", Long.toString(current.getUniqueId())));
497 }
498 }
499 current.setVersion(version);
500
501 String action = parser.getAttributeValue(null, "action");
502 if (action == null) {
503 // do nothing
504 } else if ("delete".equals(action)) {
505 current.setDeleted(true);
506 current.setModified(current.isVisible());
507 } else if ("modify".equals(action)) {
508 current.setModified(true);
509 }
510
511 String v = parser.getAttributeValue(null, "changeset");
512 if (v == null) {
513 current.setChangesetId(0);
514 } else {
515 try {
516 current.setChangesetId(Integer.parseInt(v));
517 } catch (IllegalArgumentException e) {
518 Main.debug(e.getMessage());
519 if (current.isNew()) {
520 // for a new primitive we just log a warning
521 Main.info(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.",
522 v, current.getUniqueId()));
523 current.setChangesetId(0);
524 } else {
525 // for an existing primitive this is a problem
526 throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v), e);
527 }
528 } catch (IllegalStateException e) {
529 // thrown for positive changeset id on new primitives
530 Main.info(e.getMessage());
531 current.setChangesetId(0);
532 }
533 if (current.getChangesetId() <= 0) {
534 if (current.isNew()) {
535 // for a new primitive we just log a warning
536 Main.info(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.",
537 v, current.getUniqueId()));
538 current.setChangesetId(0);
539 } else {
540 // for an existing primitive this is a problem
541 throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v));
542 }
543 }
544 }
545 }
546
547 private long getLong(String name) throws XMLStreamException {
548 String value = parser.getAttributeValue(null, name);
549 if (value == null) {
550 throwException(tr("Missing required attribute ''{0}''.", name));
551 }
552 try {
553 return Long.parseLong(value);
554 } catch (NumberFormatException e) {
555 throwException(tr("Illegal long value for attribute ''{0}''. Got ''{1}''.", name, value), e);
556 }
557 return 0; // should not happen
558 }
559
560 private static class OsmParsingException extends XMLStreamException {
561
562 OsmParsingException(String msg, Location location) {
563 super(msg); /* cannot use super(msg, location) because it messes with the message preventing localization */
564 this.location = location;
565 }
566
567 OsmParsingException(String msg, Location location, Throwable th) {
568 super(msg, th);
569 this.location = location;
570 }
571
572 @Override
573 public String getMessage() {
574 String msg = super.getMessage();
575 if (msg == null) {
576 msg = getClass().getName();
577 }
578 if (getLocation() == null)
579 return msg;
580 msg += ' ' + tr("(at line {0}, column {1})", getLocation().getLineNumber(), getLocation().getColumnNumber());
581 int offset = getLocation().getCharacterOffset();
582 if (offset > -1) {
583 msg += ". "+ tr("{0} bytes have been read", offset);
584 }
585 return msg;
586 }
587 }
588
589 /**
590 * Exception thrown after user cancelation.
591 */
592 private static final class OsmParsingCanceledException extends OsmParsingException implements ImportCancelException {
593 /**
594 * Constructs a new {@code OsmParsingCanceledException}.
595 * @param msg The error message
596 * @param location The parser location
597 */
598 OsmParsingCanceledException(String msg, Location location) {
599 super(msg, location);
600 }
601 }
602
603 protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
604 if (progressMonitor == null) {
605 progressMonitor = NullProgressMonitor.INSTANCE;
606 }
607 ProgressMonitor.CancelListener cancelListener = new ProgressMonitor.CancelListener() {
608 @Override public void operationCanceled() {
609 cancel = true;
610 }
611 };
612 progressMonitor.addCancelListener(cancelListener);
613 CheckParameterUtil.ensureParameterNotNull(source, "source");
614 try {
615 progressMonitor.beginTask(tr("Prepare OSM data...", 2));
616 progressMonitor.indeterminateSubTask(tr("Parsing OSM data..."));
617
618 try (InputStreamReader ir = UTFInputStreamReader.create(source)) {
619 XMLStreamReader parser = XMLInputFactory.newInstance().createXMLStreamReader(ir);
620 setParser(parser);
621 parse();
622 }
623 progressMonitor.worked(1);
624
625 progressMonitor.indeterminateSubTask(tr("Preparing data set..."));
626 prepareDataSet();
627 progressMonitor.worked(1);
628
629 // iterate over registered postprocessors and give them each a chance
630 // to modify the dataset we have just loaded.
631 if (postprocessors != null) {
632 for (OsmServerReadPostprocessor pp : postprocessors) {
633 pp.postprocessDataSet(getDataSet(), progressMonitor);
634 }
635 }
636 return getDataSet();
637 } catch (IllegalDataException e) {
638 throw e;
639 } catch (OsmParsingException e) {
640 throw new IllegalDataException(e.getMessage(), e);
641 } catch (XMLStreamException e) {
642 String msg = e.getMessage();
643 Pattern p = Pattern.compile("Message: (.+)");
644 Matcher m = p.matcher(msg);
645 if (m.find()) {
646 msg = m.group(1);
647 }
648 if (e.getLocation() != null)
649 throw new IllegalDataException(tr("Line {0} column {1}: ",
650 e.getLocation().getLineNumber(), e.getLocation().getColumnNumber()) + msg, e);
651 else
652 throw new IllegalDataException(msg, e);
653 } catch (Exception e) {
654 throw new IllegalDataException(e);
655 } finally {
656 progressMonitor.finishTask();
657 progressMonitor.removeCancelListener(cancelListener);
658 }
659 }
660
661 /**
662 * Parse the given input source and return the dataset.
663 *
664 * @param source the source input stream. Must not be null.
665 * @param progressMonitor the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed
666 *
667 * @return the dataset with the parsed data
668 * @throws IllegalDataException if an error was found while parsing the data from the source
669 * @throws IllegalArgumentException if source is null
670 */
671 public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
672 return new OsmReader().doParseDataSet(source, progressMonitor);
673 }
674}
Note: See TracBrowser for help on using the repository browser.