source: josm/trunk/src/org/openstreetmap/josm/io/OsmReader.java@ 11435

Last change on this file since 11435 was 11435, checked in by Don-vip, 7 years ago

fix #14199 - JOSM drops empty tags on loading and thus prevents correcting them
  • Property svn:eol-style set to native
File size: 25.5 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.io;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.IOException;
7import java.io.InputStream;
8import java.io.InputStreamReader;
9import java.text.MessageFormat;
10import java.util.ArrayList;
11import java.util.Collection;
12import java.util.List;
13import java.util.Objects;
14import java.util.regex.Matcher;
15import java.util.regex.Pattern;
16
17import javax.xml.stream.Location;
18import javax.xml.stream.XMLInputFactory;
19import javax.xml.stream.XMLStreamConstants;
20import javax.xml.stream.XMLStreamException;
21import javax.xml.stream.XMLStreamReader;
22
23import org.openstreetmap.josm.Main;
24import org.openstreetmap.josm.data.Bounds;
25import org.openstreetmap.josm.data.DataSource;
26import org.openstreetmap.josm.data.coor.LatLon;
27import org.openstreetmap.josm.data.osm.AbstractPrimitive;
28import org.openstreetmap.josm.data.osm.Changeset;
29import org.openstreetmap.josm.data.osm.DataSet;
30import org.openstreetmap.josm.data.osm.Node;
31import org.openstreetmap.josm.data.osm.NodeData;
32import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
33import org.openstreetmap.josm.data.osm.PrimitiveData;
34import org.openstreetmap.josm.data.osm.Relation;
35import org.openstreetmap.josm.data.osm.RelationData;
36import org.openstreetmap.josm.data.osm.RelationMemberData;
37import org.openstreetmap.josm.data.osm.Tagged;
38import org.openstreetmap.josm.data.osm.User;
39import org.openstreetmap.josm.data.osm.Way;
40import org.openstreetmap.josm.data.osm.WayData;
41import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
42import org.openstreetmap.josm.gui.progress.ProgressMonitor;
43import org.openstreetmap.josm.tools.CheckParameterUtil;
44import org.openstreetmap.josm.tools.Utils;
45import org.openstreetmap.josm.tools.date.DateUtils;
46
47/**
48 * Parser for the Osm Api. Read from an input stream and construct a dataset out of it.
49 *
50 * For each xml element, there is a dedicated method.
51 * The XMLStreamReader cursor points to the start of the element, when the method is
52 * entered, and it must point to the end of the same element, when it is exited.
53 */
54public class OsmReader extends AbstractReader {
55
56 protected XMLStreamReader parser;
57
58 protected boolean cancel;
59
60 /** Used by plugins to register themselves as data postprocessors. */
61 private static volatile List<OsmServerReadPostprocessor> postprocessors;
62
63 /** Register a new postprocessor.
64 * @param pp postprocessor
65 * @see #deregisterPostprocessor
66 */
67 public static void registerPostprocessor(OsmServerReadPostprocessor pp) {
68 if (postprocessors == null) {
69 postprocessors = new ArrayList<>();
70 }
71 postprocessors.add(pp);
72 }
73
74 /**
75 * Deregister a postprocessor previously registered with {@link #registerPostprocessor}.
76 * @param pp postprocessor
77 * @see #registerPostprocessor
78 */
79 public static void deregisterPostprocessor(OsmServerReadPostprocessor pp) {
80 if (postprocessors != null) {
81 postprocessors.remove(pp);
82 }
83 }
84
85 /**
86 * constructor (for private and subclasses use only)
87 *
88 * @see #parseDataSet(InputStream, ProgressMonitor)
89 */
90 protected OsmReader() {
91 // Restricts visibility
92 }
93
94 protected void setParser(XMLStreamReader parser) {
95 this.parser = parser;
96 }
97
98 protected void throwException(String msg, Throwable th) throws XMLStreamException {
99 throw new XmlStreamParsingException(msg, parser.getLocation(), th);
100 }
101
102 protected void throwException(String msg) throws XMLStreamException {
103 throw new XmlStreamParsingException(msg, parser.getLocation());
104 }
105
106 protected void parse() throws XMLStreamException {
107 int event = parser.getEventType();
108 while (true) {
109 if (event == XMLStreamConstants.START_ELEMENT) {
110 parseRoot();
111 } else if (event == XMLStreamConstants.END_ELEMENT)
112 return;
113 if (parser.hasNext()) {
114 event = parser.next();
115 } else {
116 break;
117 }
118 }
119 parser.close();
120 }
121
122 protected void parseRoot() throws XMLStreamException {
123 if ("osm".equals(parser.getLocalName())) {
124 parseOsm();
125 } else {
126 parseUnknown();
127 }
128 }
129
130 private void parseOsm() throws XMLStreamException {
131 String v = parser.getAttributeValue(null, "version");
132 if (v == null) {
133 throwException(tr("Missing mandatory attribute ''{0}''.", "version"));
134 }
135 if (!"0.6".equals(v)) {
136 throwException(tr("Unsupported version: {0}", v));
137 }
138 ds.setVersion(v);
139 String upload = parser.getAttributeValue(null, "upload");
140 if (upload != null) {
141 ds.setUploadDiscouraged(!Boolean.parseBoolean(upload));
142 }
143 String generator = parser.getAttributeValue(null, "generator");
144 Long uploadChangesetId = null;
145 if (parser.getAttributeValue(null, "upload-changeset") != null) {
146 uploadChangesetId = getLong("upload-changeset");
147 }
148 while (true) {
149 int event = parser.next();
150
151 if (cancel) {
152 cancel = false;
153 throw new OsmParsingCanceledException(tr("Reading was canceled"), parser.getLocation());
154 }
155
156 if (event == XMLStreamConstants.START_ELEMENT) {
157 switch (parser.getLocalName()) {
158 case "bounds":
159 parseBounds(generator);
160 break;
161 case "node":
162 parseNode();
163 break;
164 case "way":
165 parseWay();
166 break;
167 case "relation":
168 parseRelation();
169 break;
170 case "changeset":
171 parseChangeset(uploadChangesetId);
172 break;
173 default:
174 parseUnknown();
175 }
176 } else if (event == XMLStreamConstants.END_ELEMENT)
177 return;
178 }
179 }
180
181 private void parseBounds(String generator) throws XMLStreamException {
182 String minlon = parser.getAttributeValue(null, "minlon");
183 String minlat = parser.getAttributeValue(null, "minlat");
184 String maxlon = parser.getAttributeValue(null, "maxlon");
185 String maxlat = parser.getAttributeValue(null, "maxlat");
186 String origin = parser.getAttributeValue(null, "origin");
187 if (minlon != null && maxlon != null && minlat != null && maxlat != null) {
188 if (origin == null) {
189 origin = generator;
190 }
191 Bounds bounds = new Bounds(
192 Double.parseDouble(minlat), Double.parseDouble(minlon),
193 Double.parseDouble(maxlat), Double.parseDouble(maxlon));
194 if (bounds.isOutOfTheWorld()) {
195 Bounds copy = new Bounds(bounds);
196 bounds.normalize();
197 Main.info("Bbox " + copy + " is out of the world, normalized to " + bounds);
198 }
199 DataSource src = new DataSource(bounds, origin);
200 ds.dataSources.add(src);
201 } else {
202 throwException(tr("Missing mandatory attributes on element ''bounds''. " +
203 "Got minlon=''{0}'',minlat=''{1}'',maxlon=''{3}'',maxlat=''{4}'', origin=''{5}''.",
204 minlon, minlat, maxlon, maxlat, origin
205 ));
206 }
207 jumpToEnd();
208 }
209
210 protected Node parseNode() throws XMLStreamException {
211 NodeData nd = new NodeData();
212 String lat = parser.getAttributeValue(null, "lat");
213 String lon = parser.getAttributeValue(null, "lon");
214 if (lat != null && lon != null) {
215 nd.setCoor(new LatLon(Double.parseDouble(lat), Double.parseDouble(lon)));
216 }
217 readCommon(nd);
218 Node n = new Node(nd.getId(), nd.getVersion());
219 n.setVisible(nd.isVisible());
220 n.load(nd);
221 externalIdMap.put(nd.getPrimitiveId(), n);
222 while (true) {
223 int event = parser.next();
224 if (event == XMLStreamConstants.START_ELEMENT) {
225 if ("tag".equals(parser.getLocalName())) {
226 parseTag(n);
227 } else {
228 parseUnknown();
229 }
230 } else if (event == XMLStreamConstants.END_ELEMENT)
231 return n;
232 }
233 }
234
235 protected Way parseWay() throws XMLStreamException {
236 WayData wd = new WayData();
237 readCommon(wd);
238 Way w = new Way(wd.getId(), wd.getVersion());
239 w.setVisible(wd.isVisible());
240 w.load(wd);
241 externalIdMap.put(wd.getPrimitiveId(), w);
242
243 Collection<Long> nodeIds = new ArrayList<>();
244 while (true) {
245 int event = parser.next();
246 if (event == XMLStreamConstants.START_ELEMENT) {
247 switch (parser.getLocalName()) {
248 case "nd":
249 nodeIds.add(parseWayNode(w));
250 break;
251 case "tag":
252 parseTag(w);
253 break;
254 default:
255 parseUnknown();
256 }
257 } else if (event == XMLStreamConstants.END_ELEMENT) {
258 break;
259 }
260 }
261 if (w.isDeleted() && !nodeIds.isEmpty()) {
262 Main.info(tr("Deleted way {0} contains nodes", w.getUniqueId()));
263 nodeIds = new ArrayList<>();
264 }
265 ways.put(wd.getUniqueId(), nodeIds);
266 return w;
267 }
268
269 private long parseWayNode(Way w) throws XMLStreamException {
270 if (parser.getAttributeValue(null, "ref") == null) {
271 throwException(
272 tr("Missing mandatory attribute ''{0}'' on <nd> of way {1}.", "ref", w.getUniqueId())
273 );
274 }
275 long id = getLong("ref");
276 if (id == 0) {
277 throwException(
278 tr("Illegal value of attribute ''ref'' of element <nd>. Got {0}.", id)
279 );
280 }
281 jumpToEnd();
282 return id;
283 }
284
285 protected Relation parseRelation() throws XMLStreamException {
286 RelationData rd = new RelationData();
287 readCommon(rd);
288 Relation r = new Relation(rd.getId(), rd.getVersion());
289 r.setVisible(rd.isVisible());
290 r.load(rd);
291 externalIdMap.put(rd.getPrimitiveId(), r);
292
293 Collection<RelationMemberData> members = new ArrayList<>();
294 while (true) {
295 int event = parser.next();
296 if (event == XMLStreamConstants.START_ELEMENT) {
297 switch (parser.getLocalName()) {
298 case "member":
299 members.add(parseRelationMember(r));
300 break;
301 case "tag":
302 parseTag(r);
303 break;
304 default:
305 parseUnknown();
306 }
307 } else if (event == XMLStreamConstants.END_ELEMENT) {
308 break;
309 }
310 }
311 if (r.isDeleted() && !members.isEmpty()) {
312 Main.info(tr("Deleted relation {0} contains members", r.getUniqueId()));
313 members = new ArrayList<>();
314 }
315 relations.put(rd.getUniqueId(), members);
316 return r;
317 }
318
319 private RelationMemberData parseRelationMember(Relation r) throws XMLStreamException {
320 OsmPrimitiveType type = null;
321 long id = 0;
322 String value = parser.getAttributeValue(null, "ref");
323 if (value == null) {
324 throwException(tr("Missing attribute ''ref'' on member in relation {0}.", r.getUniqueId()));
325 }
326 try {
327 id = Long.parseLong(value);
328 } catch (NumberFormatException e) {
329 throwException(tr("Illegal value for attribute ''ref'' on member in relation {0}. Got {1}", Long.toString(r.getUniqueId()),
330 value), e);
331 }
332 value = parser.getAttributeValue(null, "type");
333 if (value == null) {
334 throwException(tr("Missing attribute ''type'' on member {0} in relation {1}.", Long.toString(id), Long.toString(r.getUniqueId())));
335 }
336 try {
337 type = OsmPrimitiveType.fromApiTypeName(value);
338 } catch (IllegalArgumentException e) {
339 throwException(tr("Illegal value for attribute ''type'' on member {0} in relation {1}. Got {2}.",
340 Long.toString(id), Long.toString(r.getUniqueId()), value), e);
341 }
342 String role = parser.getAttributeValue(null, "role");
343
344 if (id == 0) {
345 throwException(tr("Incomplete <member> specification with ref=0"));
346 }
347 jumpToEnd();
348 return new RelationMemberData(role, type, id);
349 }
350
351 private void parseChangeset(Long uploadChangesetId) throws XMLStreamException {
352
353 Long id = null;
354 if (parser.getAttributeValue(null, "id") != null) {
355 id = getLong("id");
356 }
357 // Read changeset info if neither upload-changeset nor id are set, or if they are both set to the same value
358 if (Objects.equals(id, uploadChangesetId)) {
359 uploadChangeset = new Changeset(id != null ? id.intValue() : 0);
360 while (true) {
361 int event = parser.next();
362 if (event == XMLStreamConstants.START_ELEMENT) {
363 if ("tag".equals(parser.getLocalName())) {
364 parseTag(uploadChangeset);
365 } else {
366 parseUnknown();
367 }
368 } else if (event == XMLStreamConstants.END_ELEMENT)
369 return;
370 }
371 } else {
372 jumpToEnd(false);
373 }
374 }
375
376 private void parseTag(Tagged t) throws XMLStreamException {
377 String key = parser.getAttributeValue(null, "k");
378 String value = parser.getAttributeValue(null, "v");
379 if (key == null || value == null) {
380 throwException(tr("Missing key or value attribute in tag."));
381 } else if (Utils.isStripEmpty(key) && t instanceof AbstractPrimitive) {
382 // #14199: Empty keys as ignored by AbstractPrimitive#put, but it causes problems to fix existing data
383 // Drop the tag on import, but flag the primitive as modified
384 ((AbstractPrimitive) t).setModified(true);
385 } else {
386 t.put(key.intern(), value.intern());
387 }
388 jumpToEnd();
389 }
390
391 protected void parseUnknown(boolean printWarning) throws XMLStreamException {
392 final String element = parser.getLocalName();
393 if (printWarning && ("note".equals(element) || "meta".equals(element))) {
394 // we know that Overpass API returns those elements
395 Main.debug(tr("Undefined element ''{0}'' found in input stream. Skipping.", element));
396 } else if (printWarning) {
397 Main.info(tr("Undefined element ''{0}'' found in input stream. Skipping.", element));
398 }
399 while (true) {
400 int event = parser.next();
401 if (event == XMLStreamConstants.START_ELEMENT) {
402 parseUnknown(false); /* no more warning for inner elements */
403 } else if (event == XMLStreamConstants.END_ELEMENT)
404 return;
405 }
406 }
407
408 protected void parseUnknown() throws XMLStreamException {
409 parseUnknown(true);
410 }
411
412 /**
413 * When cursor is at the start of an element, moves it to the end tag of that element.
414 * Nested content is skipped.
415 *
416 * This is basically the same code as parseUnknown(), except for the warnings, which
417 * are displayed for inner elements and not at top level.
418 * @param printWarning if {@code true}, a warning message will be printed if an unknown element is met
419 * @throws XMLStreamException if there is an error processing the underlying XML source
420 */
421 private void jumpToEnd(boolean printWarning) throws XMLStreamException {
422 while (true) {
423 int event = parser.next();
424 if (event == XMLStreamConstants.START_ELEMENT) {
425 parseUnknown(printWarning);
426 } else if (event == XMLStreamConstants.END_ELEMENT)
427 return;
428 }
429 }
430
431 private void jumpToEnd() throws XMLStreamException {
432 jumpToEnd(true);
433 }
434
435 private User createUser(String uid, String name) throws XMLStreamException {
436 if (uid == null) {
437 if (name == null)
438 return null;
439 return User.createLocalUser(name);
440 }
441 try {
442 long id = Long.parseLong(uid);
443 return User.createOsmUser(id, name);
444 } catch (NumberFormatException e) {
445 throwException(MessageFormat.format("Illegal value for attribute ''uid''. Got ''{0}''.", uid), e);
446 }
447 return null;
448 }
449
450 /**
451 * Read out the common attributes and put them into current OsmPrimitive.
452 * @param current primitive to update
453 * @throws XMLStreamException if there is an error processing the underlying XML source
454 */
455 private void readCommon(PrimitiveData current) throws XMLStreamException {
456 current.setId(getLong("id"));
457 if (current.getUniqueId() == 0) {
458 throwException(tr("Illegal object with ID=0."));
459 }
460
461 String time = parser.getAttributeValue(null, "timestamp");
462 if (time != null && !time.isEmpty()) {
463 current.setRawTimestamp((int) (DateUtils.tsFromString(time)/1000));
464 }
465
466 String user = parser.getAttributeValue(null, "user");
467 String uid = parser.getAttributeValue(null, "uid");
468 current.setUser(createUser(uid, user));
469
470 String visible = parser.getAttributeValue(null, "visible");
471 if (visible != null) {
472 current.setVisible(Boolean.parseBoolean(visible));
473 }
474
475 String versionString = parser.getAttributeValue(null, "version");
476 int version = 0;
477 if (versionString != null) {
478 try {
479 version = Integer.parseInt(versionString);
480 } catch (NumberFormatException e) {
481 throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.",
482 Long.toString(current.getUniqueId()), versionString), e);
483 }
484 switch (ds.getVersion()) {
485 case "0.6":
486 if (version <= 0 && !current.isNew()) {
487 throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.",
488 Long.toString(current.getUniqueId()), versionString));
489 } else if (version < 0 && current.isNew()) {
490 Main.warn(tr("Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.",
491 current.getUniqueId(), version, 0, "0.6"));
492 version = 0;
493 }
494 break;
495 default:
496 // should not happen. API version has been checked before
497 throwException(tr("Unknown or unsupported API version. Got {0}.", ds.getVersion()));
498 }
499 } else {
500 // version expected for OSM primitives with an id assigned by the server (id > 0), since API 0.6
501 if (!current.isNew() && ds.getVersion() != null && "0.6".equals(ds.getVersion())) {
502 throwException(tr("Missing attribute ''version'' on OSM primitive with ID {0}.", Long.toString(current.getUniqueId())));
503 }
504 }
505 current.setVersion(version);
506
507 String action = parser.getAttributeValue(null, "action");
508 if (action == null) {
509 // do nothing
510 } else if ("delete".equals(action)) {
511 current.setDeleted(true);
512 current.setModified(current.isVisible());
513 } else if ("modify".equals(action)) {
514 current.setModified(true);
515 }
516
517 String v = parser.getAttributeValue(null, "changeset");
518 if (v == null) {
519 current.setChangesetId(0);
520 } else {
521 try {
522 current.setChangesetId(Integer.parseInt(v));
523 } catch (IllegalArgumentException e) {
524 Main.debug(e.getMessage());
525 if (current.isNew()) {
526 // for a new primitive we just log a warning
527 Main.info(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.",
528 v, current.getUniqueId()));
529 current.setChangesetId(0);
530 } else {
531 // for an existing primitive this is a problem
532 throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v), e);
533 }
534 } catch (IllegalStateException e) {
535 // thrown for positive changeset id on new primitives
536 Main.debug(e);
537 Main.info(e.getMessage());
538 current.setChangesetId(0);
539 }
540 if (current.getChangesetId() <= 0) {
541 if (current.isNew()) {
542 // for a new primitive we just log a warning
543 Main.info(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.",
544 v, current.getUniqueId()));
545 current.setChangesetId(0);
546 } else {
547 // for an existing primitive this is a problem
548 throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v));
549 }
550 }
551 }
552 }
553
554 private long getLong(String name) throws XMLStreamException {
555 String value = parser.getAttributeValue(null, name);
556 if (value == null) {
557 throwException(tr("Missing required attribute ''{0}''.", name));
558 }
559 try {
560 return Long.parseLong(value);
561 } catch (NumberFormatException e) {
562 throwException(tr("Illegal long value for attribute ''{0}''. Got ''{1}''.", name, value), e);
563 }
564 return 0; // should not happen
565 }
566
567 /**
568 * Exception thrown after user cancelation.
569 */
570 private static final class OsmParsingCanceledException extends XmlStreamParsingException implements ImportCancelException {
571 /**
572 * Constructs a new {@code OsmParsingCanceledException}.
573 * @param msg The error message
574 * @param location The parser location
575 */
576 OsmParsingCanceledException(String msg, Location location) {
577 super(msg, location);
578 }
579 }
580
581 protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
582 if (progressMonitor == null) {
583 progressMonitor = NullProgressMonitor.INSTANCE;
584 }
585 ProgressMonitor.CancelListener cancelListener = () -> cancel = true;
586 progressMonitor.addCancelListener(cancelListener);
587 CheckParameterUtil.ensureParameterNotNull(source, "source");
588 try {
589 progressMonitor.beginTask(tr("Prepare OSM data...", 2));
590 progressMonitor.indeterminateSubTask(tr("Parsing OSM data..."));
591
592 try (InputStreamReader ir = UTFInputStreamReader.create(source)) {
593 XMLInputFactory factory = XMLInputFactory.newInstance();
594 // do not try to load external entities
595 factory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, Boolean.FALSE);
596 factory.setProperty(XMLInputFactory.SUPPORT_DTD, Boolean.FALSE);
597 setParser(factory.createXMLStreamReader(ir));
598 parse();
599 }
600 progressMonitor.worked(1);
601
602 progressMonitor.indeterminateSubTask(tr("Preparing data set..."));
603 prepareDataSet();
604 progressMonitor.worked(1);
605
606 // iterate over registered postprocessors and give them each a chance
607 // to modify the dataset we have just loaded.
608 if (postprocessors != null) {
609 for (OsmServerReadPostprocessor pp : postprocessors) {
610 pp.postprocessDataSet(getDataSet(), progressMonitor);
611 }
612 }
613 return getDataSet();
614 } catch (IllegalDataException e) {
615 throw e;
616 } catch (XmlStreamParsingException e) {
617 throw new IllegalDataException(e.getMessage(), e);
618 } catch (XMLStreamException e) {
619 String msg = e.getMessage();
620 Pattern p = Pattern.compile("Message: (.+)");
621 Matcher m = p.matcher(msg);
622 if (m.find()) {
623 msg = m.group(1);
624 }
625 if (e.getLocation() != null)
626 throw new IllegalDataException(tr("Line {0} column {1}: ",
627 e.getLocation().getLineNumber(), e.getLocation().getColumnNumber()) + msg, e);
628 else
629 throw new IllegalDataException(msg, e);
630 } catch (IOException e) {
631 throw new IllegalDataException(e);
632 } finally {
633 progressMonitor.finishTask();
634 progressMonitor.removeCancelListener(cancelListener);
635 }
636 }
637
638 /**
639 * Parse the given input source and return the dataset.
640 *
641 * @param source the source input stream. Must not be null.
642 * @param progressMonitor the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed
643 *
644 * @return the dataset with the parsed data
645 * @throws IllegalDataException if an error was found while parsing the data from the source
646 * @throws IllegalArgumentException if source is null
647 */
648 public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
649 return new OsmReader().doParseDataSet(source, progressMonitor);
650 }
651}
Note: See TracBrowser for help on using the repository browser.