source: josm/trunk/src/org/openstreetmap/josm/io/OsmReader.java@ 11746

Last change on this file since 11746 was 11709, checked in by bastiK, 7 years ago

fixed #12731 - Add an option to completely prevent upload of a layer : e.g. "never" to upload=true/false

to set this option, add XML attribute upload='never' to .osm file
  • Property svn:eol-style set to native
File size: 25.7 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.io;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.IOException;
7import java.io.InputStream;
8import java.io.InputStreamReader;
9import java.text.MessageFormat;
10import java.util.ArrayList;
11import java.util.Collection;
12import java.util.List;
13import java.util.Objects;
14import java.util.regex.Matcher;
15import java.util.regex.Pattern;
16
17import javax.xml.stream.Location;
18import javax.xml.stream.XMLInputFactory;
19import javax.xml.stream.XMLStreamConstants;
20import javax.xml.stream.XMLStreamException;
21import javax.xml.stream.XMLStreamReader;
22
23import org.openstreetmap.josm.Main;
24import org.openstreetmap.josm.data.Bounds;
25import org.openstreetmap.josm.data.DataSource;
26import org.openstreetmap.josm.data.coor.LatLon;
27import org.openstreetmap.josm.data.osm.AbstractPrimitive;
28import org.openstreetmap.josm.data.osm.Changeset;
29import org.openstreetmap.josm.data.osm.DataSet;
30import org.openstreetmap.josm.data.osm.DataSet.UploadPolicy;
31import org.openstreetmap.josm.data.osm.Node;
32import org.openstreetmap.josm.data.osm.NodeData;
33import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
34import org.openstreetmap.josm.data.osm.PrimitiveData;
35import org.openstreetmap.josm.data.osm.Relation;
36import org.openstreetmap.josm.data.osm.RelationData;
37import org.openstreetmap.josm.data.osm.RelationMemberData;
38import org.openstreetmap.josm.data.osm.Tagged;
39import org.openstreetmap.josm.data.osm.User;
40import org.openstreetmap.josm.data.osm.Way;
41import org.openstreetmap.josm.data.osm.WayData;
42import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
43import org.openstreetmap.josm.gui.progress.ProgressMonitor;
44import org.openstreetmap.josm.tools.CheckParameterUtil;
45import org.openstreetmap.josm.tools.Utils;
46import org.openstreetmap.josm.tools.date.DateUtils;
47
48/**
49 * Parser for the Osm Api. Read from an input stream and construct a dataset out of it.
50 *
51 * For each xml element, there is a dedicated method.
52 * The XMLStreamReader cursor points to the start of the element, when the method is
53 * entered, and it must point to the end of the same element, when it is exited.
54 */
55public class OsmReader extends AbstractReader {
56
57 protected XMLStreamReader parser;
58
59 protected boolean cancel;
60
61 /** Used by plugins to register themselves as data postprocessors. */
62 private static volatile List<OsmServerReadPostprocessor> postprocessors;
63
64 /** Register a new postprocessor.
65 * @param pp postprocessor
66 * @see #deregisterPostprocessor
67 */
68 public static void registerPostprocessor(OsmServerReadPostprocessor pp) {
69 if (postprocessors == null) {
70 postprocessors = new ArrayList<>();
71 }
72 postprocessors.add(pp);
73 }
74
75 /**
76 * Deregister a postprocessor previously registered with {@link #registerPostprocessor}.
77 * @param pp postprocessor
78 * @see #registerPostprocessor
79 */
80 public static void deregisterPostprocessor(OsmServerReadPostprocessor pp) {
81 if (postprocessors != null) {
82 postprocessors.remove(pp);
83 }
84 }
85
86 /**
87 * constructor (for private and subclasses use only)
88 *
89 * @see #parseDataSet(InputStream, ProgressMonitor)
90 */
91 protected OsmReader() {
92 // Restricts visibility
93 }
94
95 protected void setParser(XMLStreamReader parser) {
96 this.parser = parser;
97 }
98
99 protected void throwException(String msg, Throwable th) throws XMLStreamException {
100 throw new XmlStreamParsingException(msg, parser.getLocation(), th);
101 }
102
103 protected void throwException(String msg) throws XMLStreamException {
104 throw new XmlStreamParsingException(msg, parser.getLocation());
105 }
106
107 protected void parse() throws XMLStreamException {
108 int event = parser.getEventType();
109 while (true) {
110 if (event == XMLStreamConstants.START_ELEMENT) {
111 parseRoot();
112 } else if (event == XMLStreamConstants.END_ELEMENT)
113 return;
114 if (parser.hasNext()) {
115 event = parser.next();
116 } else {
117 break;
118 }
119 }
120 parser.close();
121 }
122
123 protected void parseRoot() throws XMLStreamException {
124 if ("osm".equals(parser.getLocalName())) {
125 parseOsm();
126 } else {
127 parseUnknown();
128 }
129 }
130
131 private void parseOsm() throws XMLStreamException {
132 String v = parser.getAttributeValue(null, "version");
133 if (v == null) {
134 throwException(tr("Missing mandatory attribute ''{0}''.", "version"));
135 }
136 if (!"0.6".equals(v)) {
137 throwException(tr("Unsupported version: {0}", v));
138 }
139 ds.setVersion(v);
140 String upload = parser.getAttributeValue(null, "upload");
141 if (upload != null) {
142 for (UploadPolicy policy : UploadPolicy.values()) {
143 if (policy.getXmlFlag().equalsIgnoreCase(upload)) {
144 ds.setUploadPolicy(policy);
145 break;
146 }
147 }
148 }
149 String generator = parser.getAttributeValue(null, "generator");
150 Long uploadChangesetId = null;
151 if (parser.getAttributeValue(null, "upload-changeset") != null) {
152 uploadChangesetId = getLong("upload-changeset");
153 }
154 while (true) {
155 int event = parser.next();
156
157 if (cancel) {
158 cancel = false;
159 throw new OsmParsingCanceledException(tr("Reading was canceled"), parser.getLocation());
160 }
161
162 if (event == XMLStreamConstants.START_ELEMENT) {
163 switch (parser.getLocalName()) {
164 case "bounds":
165 parseBounds(generator);
166 break;
167 case "node":
168 parseNode();
169 break;
170 case "way":
171 parseWay();
172 break;
173 case "relation":
174 parseRelation();
175 break;
176 case "changeset":
177 parseChangeset(uploadChangesetId);
178 break;
179 default:
180 parseUnknown();
181 }
182 } else if (event == XMLStreamConstants.END_ELEMENT)
183 return;
184 }
185 }
186
187 private void parseBounds(String generator) throws XMLStreamException {
188 String minlon = parser.getAttributeValue(null, "minlon");
189 String minlat = parser.getAttributeValue(null, "minlat");
190 String maxlon = parser.getAttributeValue(null, "maxlon");
191 String maxlat = parser.getAttributeValue(null, "maxlat");
192 String origin = parser.getAttributeValue(null, "origin");
193 if (minlon != null && maxlon != null && minlat != null && maxlat != null) {
194 if (origin == null) {
195 origin = generator;
196 }
197 Bounds bounds = new Bounds(
198 Double.parseDouble(minlat), Double.parseDouble(minlon),
199 Double.parseDouble(maxlat), Double.parseDouble(maxlon));
200 if (bounds.isOutOfTheWorld()) {
201 Bounds copy = new Bounds(bounds);
202 bounds.normalize();
203 Main.info("Bbox " + copy + " is out of the world, normalized to " + bounds);
204 }
205 DataSource src = new DataSource(bounds, origin);
206 ds.addDataSource(src);
207 } else {
208 throwException(tr("Missing mandatory attributes on element ''bounds''. " +
209 "Got minlon=''{0}'',minlat=''{1}'',maxlon=''{3}'',maxlat=''{4}'', origin=''{5}''.",
210 minlon, minlat, maxlon, maxlat, origin
211 ));
212 }
213 jumpToEnd();
214 }
215
216 protected Node parseNode() throws XMLStreamException {
217 NodeData nd = new NodeData();
218 String lat = parser.getAttributeValue(null, "lat");
219 String lon = parser.getAttributeValue(null, "lon");
220 if (lat != null && lon != null) {
221 nd.setCoor(new LatLon(Double.parseDouble(lat), Double.parseDouble(lon)));
222 }
223 readCommon(nd);
224 Node n = new Node(nd.getId(), nd.getVersion());
225 n.setVisible(nd.isVisible());
226 n.load(nd);
227 externalIdMap.put(nd.getPrimitiveId(), n);
228 while (true) {
229 int event = parser.next();
230 if (event == XMLStreamConstants.START_ELEMENT) {
231 if ("tag".equals(parser.getLocalName())) {
232 parseTag(n);
233 } else {
234 parseUnknown();
235 }
236 } else if (event == XMLStreamConstants.END_ELEMENT)
237 return n;
238 }
239 }
240
241 protected Way parseWay() throws XMLStreamException {
242 WayData wd = new WayData();
243 readCommon(wd);
244 Way w = new Way(wd.getId(), wd.getVersion());
245 w.setVisible(wd.isVisible());
246 w.load(wd);
247 externalIdMap.put(wd.getPrimitiveId(), w);
248
249 Collection<Long> nodeIds = new ArrayList<>();
250 while (true) {
251 int event = parser.next();
252 if (event == XMLStreamConstants.START_ELEMENT) {
253 switch (parser.getLocalName()) {
254 case "nd":
255 nodeIds.add(parseWayNode(w));
256 break;
257 case "tag":
258 parseTag(w);
259 break;
260 default:
261 parseUnknown();
262 }
263 } else if (event == XMLStreamConstants.END_ELEMENT) {
264 break;
265 }
266 }
267 if (w.isDeleted() && !nodeIds.isEmpty()) {
268 Main.info(tr("Deleted way {0} contains nodes", w.getUniqueId()));
269 nodeIds = new ArrayList<>();
270 }
271 ways.put(wd.getUniqueId(), nodeIds);
272 return w;
273 }
274
275 private long parseWayNode(Way w) throws XMLStreamException {
276 if (parser.getAttributeValue(null, "ref") == null) {
277 throwException(
278 tr("Missing mandatory attribute ''{0}'' on <nd> of way {1}.", "ref", w.getUniqueId())
279 );
280 }
281 long id = getLong("ref");
282 if (id == 0) {
283 throwException(
284 tr("Illegal value of attribute ''ref'' of element <nd>. Got {0}.", id)
285 );
286 }
287 jumpToEnd();
288 return id;
289 }
290
291 protected Relation parseRelation() throws XMLStreamException {
292 RelationData rd = new RelationData();
293 readCommon(rd);
294 Relation r = new Relation(rd.getId(), rd.getVersion());
295 r.setVisible(rd.isVisible());
296 r.load(rd);
297 externalIdMap.put(rd.getPrimitiveId(), r);
298
299 Collection<RelationMemberData> members = new ArrayList<>();
300 while (true) {
301 int event = parser.next();
302 if (event == XMLStreamConstants.START_ELEMENT) {
303 switch (parser.getLocalName()) {
304 case "member":
305 members.add(parseRelationMember(r));
306 break;
307 case "tag":
308 parseTag(r);
309 break;
310 default:
311 parseUnknown();
312 }
313 } else if (event == XMLStreamConstants.END_ELEMENT) {
314 break;
315 }
316 }
317 if (r.isDeleted() && !members.isEmpty()) {
318 Main.info(tr("Deleted relation {0} contains members", r.getUniqueId()));
319 members = new ArrayList<>();
320 }
321 relations.put(rd.getUniqueId(), members);
322 return r;
323 }
324
325 private RelationMemberData parseRelationMember(Relation r) throws XMLStreamException {
326 OsmPrimitiveType type = null;
327 long id = 0;
328 String value = parser.getAttributeValue(null, "ref");
329 if (value == null) {
330 throwException(tr("Missing attribute ''ref'' on member in relation {0}.", r.getUniqueId()));
331 }
332 try {
333 id = Long.parseLong(value);
334 } catch (NumberFormatException e) {
335 throwException(tr("Illegal value for attribute ''ref'' on member in relation {0}. Got {1}", Long.toString(r.getUniqueId()),
336 value), e);
337 }
338 value = parser.getAttributeValue(null, "type");
339 if (value == null) {
340 throwException(tr("Missing attribute ''type'' on member {0} in relation {1}.", Long.toString(id), Long.toString(r.getUniqueId())));
341 }
342 try {
343 type = OsmPrimitiveType.fromApiTypeName(value);
344 } catch (IllegalArgumentException e) {
345 throwException(tr("Illegal value for attribute ''type'' on member {0} in relation {1}. Got {2}.",
346 Long.toString(id), Long.toString(r.getUniqueId()), value), e);
347 }
348 String role = parser.getAttributeValue(null, "role");
349
350 if (id == 0) {
351 throwException(tr("Incomplete <member> specification with ref=0"));
352 }
353 jumpToEnd();
354 return new RelationMemberData(role, type, id);
355 }
356
357 private void parseChangeset(Long uploadChangesetId) throws XMLStreamException {
358
359 Long id = null;
360 if (parser.getAttributeValue(null, "id") != null) {
361 id = getLong("id");
362 }
363 // Read changeset info if neither upload-changeset nor id are set, or if they are both set to the same value
364 if (Objects.equals(id, uploadChangesetId)) {
365 uploadChangeset = new Changeset(id != null ? id.intValue() : 0);
366 while (true) {
367 int event = parser.next();
368 if (event == XMLStreamConstants.START_ELEMENT) {
369 if ("tag".equals(parser.getLocalName())) {
370 parseTag(uploadChangeset);
371 } else {
372 parseUnknown();
373 }
374 } else if (event == XMLStreamConstants.END_ELEMENT)
375 return;
376 }
377 } else {
378 jumpToEnd(false);
379 }
380 }
381
382 private void parseTag(Tagged t) throws XMLStreamException {
383 String key = parser.getAttributeValue(null, "k");
384 String value = parser.getAttributeValue(null, "v");
385 if (key == null || value == null) {
386 throwException(tr("Missing key or value attribute in tag."));
387 } else if (Utils.isStripEmpty(key) && t instanceof AbstractPrimitive) {
388 // #14199: Empty keys as ignored by AbstractPrimitive#put, but it causes problems to fix existing data
389 // Drop the tag on import, but flag the primitive as modified
390 ((AbstractPrimitive) t).setModified(true);
391 } else {
392 t.put(key.intern(), value.intern());
393 }
394 jumpToEnd();
395 }
396
397 protected void parseUnknown(boolean printWarning) throws XMLStreamException {
398 final String element = parser.getLocalName();
399 if (printWarning && ("note".equals(element) || "meta".equals(element))) {
400 // we know that Overpass API returns those elements
401 Main.debug(tr("Undefined element ''{0}'' found in input stream. Skipping.", element));
402 } else if (printWarning) {
403 Main.info(tr("Undefined element ''{0}'' found in input stream. Skipping.", element));
404 }
405 while (true) {
406 int event = parser.next();
407 if (event == XMLStreamConstants.START_ELEMENT) {
408 parseUnknown(false); /* no more warning for inner elements */
409 } else if (event == XMLStreamConstants.END_ELEMENT)
410 return;
411 }
412 }
413
414 protected void parseUnknown() throws XMLStreamException {
415 parseUnknown(true);
416 }
417
418 /**
419 * When cursor is at the start of an element, moves it to the end tag of that element.
420 * Nested content is skipped.
421 *
422 * This is basically the same code as parseUnknown(), except for the warnings, which
423 * are displayed for inner elements and not at top level.
424 * @param printWarning if {@code true}, a warning message will be printed if an unknown element is met
425 * @throws XMLStreamException if there is an error processing the underlying XML source
426 */
427 private void jumpToEnd(boolean printWarning) throws XMLStreamException {
428 while (true) {
429 int event = parser.next();
430 if (event == XMLStreamConstants.START_ELEMENT) {
431 parseUnknown(printWarning);
432 } else if (event == XMLStreamConstants.END_ELEMENT)
433 return;
434 }
435 }
436
437 private void jumpToEnd() throws XMLStreamException {
438 jumpToEnd(true);
439 }
440
441 private User createUser(String uid, String name) throws XMLStreamException {
442 if (uid == null) {
443 if (name == null)
444 return null;
445 return User.createLocalUser(name);
446 }
447 try {
448 long id = Long.parseLong(uid);
449 return User.createOsmUser(id, name);
450 } catch (NumberFormatException e) {
451 throwException(MessageFormat.format("Illegal value for attribute ''uid''. Got ''{0}''.", uid), e);
452 }
453 return null;
454 }
455
456 /**
457 * Read out the common attributes and put them into current OsmPrimitive.
458 * @param current primitive to update
459 * @throws XMLStreamException if there is an error processing the underlying XML source
460 */
461 private void readCommon(PrimitiveData current) throws XMLStreamException {
462 current.setId(getLong("id"));
463 if (current.getUniqueId() == 0) {
464 throwException(tr("Illegal object with ID=0."));
465 }
466
467 String time = parser.getAttributeValue(null, "timestamp");
468 if (time != null && !time.isEmpty()) {
469 current.setRawTimestamp((int) (DateUtils.tsFromString(time)/1000));
470 }
471
472 String user = parser.getAttributeValue(null, "user");
473 String uid = parser.getAttributeValue(null, "uid");
474 current.setUser(createUser(uid, user));
475
476 String visible = parser.getAttributeValue(null, "visible");
477 if (visible != null) {
478 current.setVisible(Boolean.parseBoolean(visible));
479 }
480
481 String versionString = parser.getAttributeValue(null, "version");
482 int version = 0;
483 if (versionString != null) {
484 try {
485 version = Integer.parseInt(versionString);
486 } catch (NumberFormatException e) {
487 throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.",
488 Long.toString(current.getUniqueId()), versionString), e);
489 }
490 switch (ds.getVersion()) {
491 case "0.6":
492 if (version <= 0 && !current.isNew()) {
493 throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.",
494 Long.toString(current.getUniqueId()), versionString));
495 } else if (version < 0 && current.isNew()) {
496 Main.warn(tr("Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.",
497 current.getUniqueId(), version, 0, "0.6"));
498 version = 0;
499 }
500 break;
501 default:
502 // should not happen. API version has been checked before
503 throwException(tr("Unknown or unsupported API version. Got {0}.", ds.getVersion()));
504 }
505 } else {
506 // version expected for OSM primitives with an id assigned by the server (id > 0), since API 0.6
507 if (!current.isNew() && ds.getVersion() != null && "0.6".equals(ds.getVersion())) {
508 throwException(tr("Missing attribute ''version'' on OSM primitive with ID {0}.", Long.toString(current.getUniqueId())));
509 }
510 }
511 current.setVersion(version);
512
513 String action = parser.getAttributeValue(null, "action");
514 if (action == null) {
515 // do nothing
516 } else if ("delete".equals(action)) {
517 current.setDeleted(true);
518 current.setModified(current.isVisible());
519 } else if ("modify".equals(action)) {
520 current.setModified(true);
521 }
522
523 String v = parser.getAttributeValue(null, "changeset");
524 if (v == null) {
525 current.setChangesetId(0);
526 } else {
527 try {
528 current.setChangesetId(Integer.parseInt(v));
529 } catch (IllegalArgumentException e) {
530 Main.debug(e.getMessage());
531 if (current.isNew()) {
532 // for a new primitive we just log a warning
533 Main.info(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.",
534 v, current.getUniqueId()));
535 current.setChangesetId(0);
536 } else {
537 // for an existing primitive this is a problem
538 throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v), e);
539 }
540 } catch (IllegalStateException e) {
541 // thrown for positive changeset id on new primitives
542 Main.debug(e);
543 Main.info(e.getMessage());
544 current.setChangesetId(0);
545 }
546 if (current.getChangesetId() <= 0) {
547 if (current.isNew()) {
548 // for a new primitive we just log a warning
549 Main.info(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.",
550 v, current.getUniqueId()));
551 current.setChangesetId(0);
552 } else {
553 // for an existing primitive this is a problem
554 throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v));
555 }
556 }
557 }
558 }
559
560 private long getLong(String name) throws XMLStreamException {
561 String value = parser.getAttributeValue(null, name);
562 if (value == null) {
563 throwException(tr("Missing required attribute ''{0}''.", name));
564 }
565 try {
566 return Long.parseLong(value);
567 } catch (NumberFormatException e) {
568 throwException(tr("Illegal long value for attribute ''{0}''. Got ''{1}''.", name, value), e);
569 }
570 return 0; // should not happen
571 }
572
573 /**
574 * Exception thrown after user cancelation.
575 */
576 private static final class OsmParsingCanceledException extends XmlStreamParsingException implements ImportCancelException {
577 /**
578 * Constructs a new {@code OsmParsingCanceledException}.
579 * @param msg The error message
580 * @param location The parser location
581 */
582 OsmParsingCanceledException(String msg, Location location) {
583 super(msg, location);
584 }
585 }
586
587 protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
588 if (progressMonitor == null) {
589 progressMonitor = NullProgressMonitor.INSTANCE;
590 }
591 ProgressMonitor.CancelListener cancelListener = () -> cancel = true;
592 progressMonitor.addCancelListener(cancelListener);
593 CheckParameterUtil.ensureParameterNotNull(source, "source");
594 try {
595 progressMonitor.beginTask(tr("Prepare OSM data...", 2));
596 progressMonitor.indeterminateSubTask(tr("Parsing OSM data..."));
597
598 try (InputStreamReader ir = UTFInputStreamReader.create(source)) {
599 XMLInputFactory factory = XMLInputFactory.newInstance();
600 // do not try to load external entities
601 factory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, Boolean.FALSE);
602 factory.setProperty(XMLInputFactory.SUPPORT_DTD, Boolean.FALSE);
603 setParser(factory.createXMLStreamReader(ir));
604 parse();
605 }
606 progressMonitor.worked(1);
607
608 progressMonitor.indeterminateSubTask(tr("Preparing data set..."));
609 prepareDataSet();
610 progressMonitor.worked(1);
611
612 // iterate over registered postprocessors and give them each a chance
613 // to modify the dataset we have just loaded.
614 if (postprocessors != null) {
615 for (OsmServerReadPostprocessor pp : postprocessors) {
616 pp.postprocessDataSet(getDataSet(), progressMonitor);
617 }
618 }
619 return getDataSet();
620 } catch (IllegalDataException e) {
621 throw e;
622 } catch (XmlStreamParsingException e) {
623 throw new IllegalDataException(e.getMessage(), e);
624 } catch (XMLStreamException e) {
625 String msg = e.getMessage();
626 Pattern p = Pattern.compile("Message: (.+)");
627 Matcher m = p.matcher(msg);
628 if (m.find()) {
629 msg = m.group(1);
630 }
631 if (e.getLocation() != null)
632 throw new IllegalDataException(tr("Line {0} column {1}: ",
633 e.getLocation().getLineNumber(), e.getLocation().getColumnNumber()) + msg, e);
634 else
635 throw new IllegalDataException(msg, e);
636 } catch (IOException e) {
637 throw new IllegalDataException(e);
638 } finally {
639 progressMonitor.finishTask();
640 progressMonitor.removeCancelListener(cancelListener);
641 }
642 }
643
644 /**
645 * Parse the given input source and return the dataset.
646 *
647 * @param source the source input stream. Must not be null.
648 * @param progressMonitor the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed
649 *
650 * @return the dataset with the parsed data
651 * @throws IllegalDataException if an error was found while parsing the data from the source
652 * @throws IllegalArgumentException if source is null
653 */
654 public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
655 return new OsmReader().doParseDataSet(source, progressMonitor);
656 }
657}
Note: See TracBrowser for help on using the repository browser.