source: josm/trunk/src/org/openstreetmap/josm/io/OsmReader.java@ 5996

Last change on this file since 5996 was 5996, checked in by Don-vip, 11 years ago

fix #8771 - <changeset> with no ID rejected
  • Property svn:eol-style set to native
File size: 25.7 KB
Line 
1// License: GPL. See LICENSE file for details.
2package org.openstreetmap.josm.io;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.InputStream;
7import java.io.InputStreamReader;
8import java.text.MessageFormat;
9import java.util.ArrayList;
10import java.util.Collection;
11import java.util.regex.Matcher;
12import java.util.regex.Pattern;
13
14import javax.xml.stream.Location;
15import javax.xml.stream.XMLInputFactory;
16import javax.xml.stream.XMLStreamConstants;
17import javax.xml.stream.XMLStreamException;
18import javax.xml.stream.XMLStreamReader;
19
20import org.openstreetmap.josm.data.Bounds;
21import org.openstreetmap.josm.data.coor.LatLon;
22import org.openstreetmap.josm.data.osm.Changeset;
23import org.openstreetmap.josm.data.osm.DataSet;
24import org.openstreetmap.josm.data.osm.DataSource;
25import org.openstreetmap.josm.data.osm.Node;
26import org.openstreetmap.josm.data.osm.NodeData;
27import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
28import org.openstreetmap.josm.data.osm.PrimitiveData;
29import org.openstreetmap.josm.data.osm.Relation;
30import org.openstreetmap.josm.data.osm.RelationData;
31import org.openstreetmap.josm.data.osm.RelationMemberData;
32import org.openstreetmap.josm.data.osm.Tagged;
33import org.openstreetmap.josm.data.osm.User;
34import org.openstreetmap.josm.data.osm.Way;
35import org.openstreetmap.josm.data.osm.WayData;
36import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
37import org.openstreetmap.josm.gui.progress.ProgressMonitor;
38import org.openstreetmap.josm.tools.CheckParameterUtil;
39import org.openstreetmap.josm.tools.DateUtils;
40
41/**
42 * Parser for the Osm Api. Read from an input stream and construct a dataset out of it.
43 *
44 * For each xml element, there is a dedicated method.
45 * The XMLStreamReader cursor points to the start of the element, when the method is
46 * entered, and it must point to the end of the same element, when it is exited.
47 */
48public class OsmReader extends AbstractReader {
49
50 protected XMLStreamReader parser;
51
52 protected boolean cancel;
53
54 /** Used by plugins to register themselves as data postprocessors. */
55 public static ArrayList<OsmServerReadPostprocessor> postprocessors;
56
57 /** register a new postprocessor */
58 public static void registerPostprocessor(OsmServerReadPostprocessor pp) {
59 if (postprocessors == null) {
60 postprocessors = new ArrayList<OsmServerReadPostprocessor>();
61 }
62 postprocessors.add(pp);
63 }
64
65 /** deregister a postprocessor previously registered with registerPostprocessor */
66 public static void deregisterPostprocessor(OsmServerReadPostprocessor pp) {
67 if (postprocessors != null) {
68 postprocessors.remove(pp);
69 }
70 }
71
72 /**
73 * constructor (for private and subclasses use only)
74 *
75 * @see #parseDataSet(InputStream, ProgressMonitor)
76 */
77 protected OsmReader() {
78 }
79
80 protected void setParser(XMLStreamReader parser) {
81 this.parser = parser;
82 }
83
84 protected void throwException(String msg) throws XMLStreamException {
85 throw new OsmParsingException(msg, parser.getLocation());
86 }
87
88 protected void parse() throws XMLStreamException {
89 int event = parser.getEventType();
90 while (true) {
91 if (event == XMLStreamConstants.START_ELEMENT) {
92 parseRoot();
93 } else if (event == XMLStreamConstants.END_ELEMENT)
94 return;
95 if (parser.hasNext()) {
96 event = parser.next();
97 } else {
98 break;
99 }
100 }
101 parser.close();
102 }
103
104 protected void parseRoot() throws XMLStreamException {
105 if (parser.getLocalName().equals("osm")) {
106 parseOsm();
107 } else {
108 parseUnknown();
109 }
110 }
111
112 private void parseOsm() throws XMLStreamException {
113 String v = parser.getAttributeValue(null, "version");
114 if (v == null) {
115 throwException(tr("Missing mandatory attribute ''{0}''.", "version"));
116 }
117 if (!(v.equals("0.5") || v.equals("0.6"))) {
118 throwException(tr("Unsupported version: {0}", v));
119 }
120 ds.setVersion(v);
121 String upload = parser.getAttributeValue(null, "upload");
122 if (upload != null) {
123 ds.setUploadDiscouraged(!Boolean.parseBoolean(upload));
124 }
125 String generator = parser.getAttributeValue(null, "generator");
126 Long uploadChangesetId = null;
127 if (parser.getAttributeValue(null, "upload-changeset") != null) {
128 uploadChangesetId = getLong("upload-changeset");
129 }
130 while (true) {
131 int event = parser.next();
132
133 if (cancel) {
134 cancel = false;
135 throwException(tr("Reading was canceled"));
136 }
137
138 if (event == XMLStreamConstants.START_ELEMENT) {
139 if (parser.getLocalName().equals("bounds")) {
140 parseBounds(generator);
141 } else if (parser.getLocalName().equals("node")) {
142 parseNode();
143 } else if (parser.getLocalName().equals("way")) {
144 parseWay();
145 } else if (parser.getLocalName().equals("relation")) {
146 parseRelation();
147 } else if (parser.getLocalName().equals("changeset")) {
148 parseChangeset(uploadChangesetId);
149 } else {
150 parseUnknown();
151 }
152 } else if (event == XMLStreamConstants.END_ELEMENT)
153 return;
154 }
155 }
156
157 private void parseBounds(String generator) throws XMLStreamException {
158 String minlon = parser.getAttributeValue(null, "minlon");
159 String minlat = parser.getAttributeValue(null, "minlat");
160 String maxlon = parser.getAttributeValue(null, "maxlon");
161 String maxlat = parser.getAttributeValue(null, "maxlat");
162 String origin = parser.getAttributeValue(null, "origin");
163 if (minlon != null && maxlon != null && minlat != null && maxlat != null) {
164 if (origin == null) {
165 origin = generator;
166 }
167 Bounds bounds = new Bounds(
168 Double.parseDouble(minlat), Double.parseDouble(minlon),
169 Double.parseDouble(maxlat), Double.parseDouble(maxlon));
170 if (bounds.isOutOfTheWorld()) {
171 Bounds copy = new Bounds(bounds);
172 bounds.normalize();
173 System.out.println("Bbox " + copy + " is out of the world, normalized to " + bounds);
174 }
175 DataSource src = new DataSource(bounds, origin);
176 ds.dataSources.add(src);
177 } else {
178 throwException(tr(
179 "Missing mandatory attributes on element ''bounds''. Got minlon=''{0}'',minlat=''{1}'',maxlon=''{3}'',maxlat=''{4}'', origin=''{5}''.",
180 minlon, minlat, maxlon, maxlat, origin
181 ));
182 }
183 jumpToEnd();
184 }
185
186 protected Node parseNode() throws XMLStreamException {
187 NodeData nd = new NodeData();
188 String lat = parser.getAttributeValue(null, "lat");
189 String lon = parser.getAttributeValue(null, "lon");
190 if (lat != null && lon != null) {
191 nd.setCoor(new LatLon(Double.parseDouble(lat), Double.parseDouble(lon)));
192 }
193 readCommon(nd);
194 Node n = new Node(nd.getId(), nd.getVersion());
195 n.setVisible(nd.isVisible());
196 n.load(nd);
197 externalIdMap.put(nd.getPrimitiveId(), n);
198 while (true) {
199 int event = parser.next();
200 if (event == XMLStreamConstants.START_ELEMENT) {
201 if (parser.getLocalName().equals("tag")) {
202 parseTag(n);
203 } else {
204 parseUnknown();
205 }
206 } else if (event == XMLStreamConstants.END_ELEMENT)
207 return n;
208 }
209 }
210
211 protected Way parseWay() throws XMLStreamException {
212 WayData wd = new WayData();
213 readCommon(wd);
214 Way w = new Way(wd.getId(), wd.getVersion());
215 w.setVisible(wd.isVisible());
216 w.load(wd);
217 externalIdMap.put(wd.getPrimitiveId(), w);
218
219 Collection<Long> nodeIds = new ArrayList<Long>();
220 while (true) {
221 int event = parser.next();
222 if (event == XMLStreamConstants.START_ELEMENT) {
223 if (parser.getLocalName().equals("nd")) {
224 nodeIds.add(parseWayNode(w));
225 } else if (parser.getLocalName().equals("tag")) {
226 parseTag(w);
227 } else {
228 parseUnknown();
229 }
230 } else if (event == XMLStreamConstants.END_ELEMENT) {
231 break;
232 }
233 }
234 if (w.isDeleted() && nodeIds.size() > 0) {
235 System.out.println(tr("Deleted way {0} contains nodes", w.getUniqueId()));
236 nodeIds = new ArrayList<Long>();
237 }
238 ways.put(wd.getUniqueId(), nodeIds);
239 return w;
240 }
241
242 private long parseWayNode(Way w) throws XMLStreamException {
243 if (parser.getAttributeValue(null, "ref") == null) {
244 throwException(
245 tr("Missing mandatory attribute ''{0}'' on <nd> of way {1}.", "ref", w.getUniqueId())
246 );
247 }
248 long id = getLong("ref");
249 if (id == 0) {
250 throwException(
251 tr("Illegal value of attribute ''ref'' of element <nd>. Got {0}.", id)
252 );
253 }
254 jumpToEnd();
255 return id;
256 }
257
258 protected Relation parseRelation() throws XMLStreamException {
259 RelationData rd = new RelationData();
260 readCommon(rd);
261 Relation r = new Relation(rd.getId(), rd.getVersion());
262 r.setVisible(rd.isVisible());
263 r.load(rd);
264 externalIdMap.put(rd.getPrimitiveId(), r);
265
266 Collection<RelationMemberData> members = new ArrayList<RelationMemberData>();
267 while (true) {
268 int event = parser.next();
269 if (event == XMLStreamConstants.START_ELEMENT) {
270 if (parser.getLocalName().equals("member")) {
271 members.add(parseRelationMember(r));
272 } else if (parser.getLocalName().equals("tag")) {
273 parseTag(r);
274 } else {
275 parseUnknown();
276 }
277 } else if (event == XMLStreamConstants.END_ELEMENT) {
278 break;
279 }
280 }
281 if (r.isDeleted() && members.size() > 0) {
282 System.out.println(tr("Deleted relation {0} contains members", r.getUniqueId()));
283 members = new ArrayList<RelationMemberData>();
284 }
285 relations.put(rd.getUniqueId(), members);
286 return r;
287 }
288
289 private RelationMemberData parseRelationMember(Relation r) throws XMLStreamException {
290 String role = null;
291 OsmPrimitiveType type = null;
292 long id = 0;
293 String value = parser.getAttributeValue(null, "ref");
294 if (value == null) {
295 throwException(tr("Missing attribute ''ref'' on member in relation {0}.",r.getUniqueId()));
296 }
297 try {
298 id = Long.parseLong(value);
299 } catch(NumberFormatException e) {
300 throwException(tr("Illegal value for attribute ''ref'' on member in relation {0}. Got {1}", Long.toString(r.getUniqueId()),value));
301 }
302 value = parser.getAttributeValue(null, "type");
303 if (value == null) {
304 throwException(tr("Missing attribute ''type'' on member {0} in relation {1}.", Long.toString(id), Long.toString(r.getUniqueId())));
305 }
306 try {
307 type = OsmPrimitiveType.fromApiTypeName(value);
308 } catch(IllegalArgumentException e) {
309 throwException(tr("Illegal value for attribute ''type'' on member {0} in relation {1}. Got {2}.", Long.toString(id), Long.toString(r.getUniqueId()), value));
310 }
311 value = parser.getAttributeValue(null, "role");
312 role = value;
313
314 if (id == 0) {
315 throwException(tr("Incomplete <member> specification with ref=0"));
316 }
317 jumpToEnd();
318 return new RelationMemberData(role, type, id);
319 }
320
321 private void parseChangeset(Long uploadChangesetId) throws XMLStreamException {
322
323 Long id = null;
324 if (parser.getAttributeValue(null, "id") != null) {
325 id = getLong("id");
326 }
327 // Read changeset info if neither upload-changeset nor id are set, or if they are both set to the same value
328 if (id == uploadChangesetId || (id != null && id.equals(uploadChangesetId))) {
329 uploadChangeset = new Changeset(id != null ? id.intValue() : 0);
330 while (true) {
331 int event = parser.next();
332 if (event == XMLStreamConstants.START_ELEMENT) {
333 if (parser.getLocalName().equals("tag")) {
334 parseTag(uploadChangeset);
335 } else {
336 parseUnknown();
337 }
338 } else if (event == XMLStreamConstants.END_ELEMENT)
339 return;
340 }
341 } else {
342 jumpToEnd(false);
343 }
344 }
345
346 private void parseTag(Tagged t) throws XMLStreamException {
347 String key = parser.getAttributeValue(null, "k");
348 String value = parser.getAttributeValue(null, "v");
349 if (key == null || value == null) {
350 throwException(tr("Missing key or value attribute in tag."));
351 }
352 t.put(key.intern(), value.intern());
353 jumpToEnd();
354 }
355
356 protected void parseUnknown(boolean printWarning) throws XMLStreamException {
357 if (printWarning) {
358 System.out.println(tr("Undefined element ''{0}'' found in input stream. Skipping.", parser.getLocalName()));
359 }
360 while (true) {
361 int event = parser.next();
362 if (event == XMLStreamConstants.START_ELEMENT) {
363 parseUnknown(false); /* no more warning for inner elements */
364 } else if (event == XMLStreamConstants.END_ELEMENT)
365 return;
366 }
367 }
368
369 protected void parseUnknown() throws XMLStreamException {
370 parseUnknown(true);
371 }
372
373 /**
374 * When cursor is at the start of an element, moves it to the end tag of that element.
375 * Nested content is skipped.
376 *
377 * This is basically the same code as parseUnknown(), except for the warnings, which
378 * are displayed for inner elements and not at top level.
379 */
380 private void jumpToEnd(boolean printWarning) throws XMLStreamException {
381 while (true) {
382 int event = parser.next();
383 if (event == XMLStreamConstants.START_ELEMENT) {
384 parseUnknown(printWarning);
385 } else if (event == XMLStreamConstants.END_ELEMENT)
386 return;
387 }
388 }
389
390 private void jumpToEnd() throws XMLStreamException {
391 jumpToEnd(true);
392 }
393
394 private User createUser(String uid, String name) throws XMLStreamException {
395 if (uid == null) {
396 if (name == null)
397 return null;
398 return User.createLocalUser(name);
399 }
400 try {
401 long id = Long.parseLong(uid);
402 return User.createOsmUser(id, name);
403 } catch(NumberFormatException e) {
404 throwException(MessageFormat.format("Illegal value for attribute ''uid''. Got ''{0}''.", uid));
405 }
406 return null;
407 }
408
409 /**
410 * Read out the common attributes and put them into current OsmPrimitive.
411 */
412 private void readCommon(PrimitiveData current) throws XMLStreamException {
413 current.setId(getLong("id"));
414 if (current.getUniqueId() == 0) {
415 throwException(tr("Illegal object with ID=0."));
416 }
417
418 String time = parser.getAttributeValue(null, "timestamp");
419 if (time != null && time.length() != 0) {
420 current.setTimestamp(DateUtils.fromString(time));
421 }
422
423 // user attribute added in 0.4 API
424 String user = parser.getAttributeValue(null, "user");
425 // uid attribute added in 0.6 API
426 String uid = parser.getAttributeValue(null, "uid");
427 current.setUser(createUser(uid, user));
428
429 // visible attribute added in 0.4 API
430 String visible = parser.getAttributeValue(null, "visible");
431 if (visible != null) {
432 current.setVisible(Boolean.parseBoolean(visible));
433 }
434
435 String versionString = parser.getAttributeValue(null, "version");
436 int version = 0;
437 if (versionString != null) {
438 try {
439 version = Integer.parseInt(versionString);
440 } catch(NumberFormatException e) {
441 throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.", Long.toString(current.getUniqueId()), versionString));
442 }
443 if (ds.getVersion().equals("0.6")){
444 if (version <= 0 && current.getUniqueId() > 0) {
445 throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.", Long.toString(current.getUniqueId()), versionString));
446 } else if (version < 0 && current.getUniqueId() <= 0) {
447 System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 0, "0.6"));
448 version = 0;
449 }
450 } else if (ds.getVersion().equals("0.5")) {
451 if (version <= 0 && current.getUniqueId() > 0) {
452 System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 1, "0.5"));
453 version = 1;
454 } else if (version < 0 && current.getUniqueId() <= 0) {
455 System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 0, "0.5"));
456 version = 0;
457 }
458 } else {
459 // should not happen. API version has been checked before
460 throwException(tr("Unknown or unsupported API version. Got {0}.", ds.getVersion()));
461 }
462 } else {
463 // version expected for OSM primitives with an id assigned by the server (id > 0), since API 0.6
464 //
465 if (current.getUniqueId() > 0 && ds.getVersion() != null && ds.getVersion().equals("0.6")) {
466 throwException(tr("Missing attribute ''version'' on OSM primitive with ID {0}.", Long.toString(current.getUniqueId())));
467 } else if (current.getUniqueId() > 0 && ds.getVersion() != null && ds.getVersion().equals("0.5")) {
468 // default version in 0.5 files for existing primitives
469 System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 1, "0.5"));
470 version= 1;
471 } else if (current.getUniqueId() <= 0 && ds.getVersion() != null && ds.getVersion().equals("0.5")) {
472 // default version in 0.5 files for new primitives, no warning necessary. This is
473 // (was) legal in API 0.5
474 version= 0;
475 }
476 }
477 current.setVersion(version);
478
479 String action = parser.getAttributeValue(null, "action");
480 if (action == null) {
481 // do nothing
482 } else if (action.equals("delete")) {
483 current.setDeleted(true);
484 current.setModified(current.isVisible());
485 } else if (action.equals("modify")) {
486 current.setModified(true);
487 }
488
489 String v = parser.getAttributeValue(null, "changeset");
490 if (v == null) {
491 current.setChangesetId(0);
492 } else {
493 try {
494 current.setChangesetId(Integer.parseInt(v));
495 } catch(NumberFormatException e) {
496 if (current.getUniqueId() <= 0) {
497 // for a new primitive we just log a warning
498 System.out.println(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", v, current.getUniqueId()));
499 current.setChangesetId(0);
500 } else {
501 // for an existing primitive this is a problem
502 throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v));
503 }
504 }
505 if (current.getChangesetId() <=0) {
506 if (current.getUniqueId() <= 0) {
507 // for a new primitive we just log a warning
508 System.out.println(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", v, current.getUniqueId()));
509 current.setChangesetId(0);
510 } else {
511 // for an existing primitive this is a problem
512 throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v));
513 }
514 }
515 }
516 }
517
518 private long getLong(String name) throws XMLStreamException {
519 String value = parser.getAttributeValue(null, name);
520 if (value == null) {
521 throwException(tr("Missing required attribute ''{0}''.",name));
522 }
523 try {
524 return Long.parseLong(value);
525 } catch(NumberFormatException e) {
526 throwException(tr("Illegal long value for attribute ''{0}''. Got ''{1}''.",name, value));
527 }
528 return 0; // should not happen
529 }
530
531 private static class OsmParsingException extends XMLStreamException {
532 public OsmParsingException() {
533 super();
534 }
535
536 public OsmParsingException(String msg) {
537 super(msg);
538 }
539
540 public OsmParsingException(String msg, Location location) {
541 super(msg); /* cannot use super(msg, location) because it messes with the message preventing localization */
542 this.location = location;
543 }
544
545 public OsmParsingException(String msg, Location location, Throwable th) {
546 super(msg, th);
547 this.location = location;
548 }
549
550 public OsmParsingException(String msg, Throwable th) {
551 super(msg, th);
552 }
553
554 public OsmParsingException(Throwable th) {
555 super(th);
556 }
557
558 @Override
559 public String getMessage() {
560 String msg = super.getMessage();
561 if (msg == null) {
562 msg = getClass().getName();
563 }
564 if (getLocation() == null)
565 return msg;
566 msg = msg + " " + tr("(at line {0}, column {1})", getLocation().getLineNumber(), getLocation().getColumnNumber());
567 return msg;
568 }
569 }
570
571 protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
572 if (progressMonitor == null) {
573 progressMonitor = NullProgressMonitor.INSTANCE;
574 }
575 ProgressMonitor.CancelListener cancelListener = new ProgressMonitor.CancelListener() {
576 @Override public void operationCanceled() {
577 cancel = true;
578 }
579 };
580 progressMonitor.addCancelListener(cancelListener);
581 CheckParameterUtil.ensureParameterNotNull(source, "source");
582 try {
583 progressMonitor.beginTask(tr("Prepare OSM data...", 2));
584 progressMonitor.indeterminateSubTask(tr("Parsing OSM data..."));
585
586 InputStreamReader ir = UTFInputStreamReader.create(source, "UTF-8");
587 XMLStreamReader parser = XMLInputFactory.newInstance().createXMLStreamReader(ir);
588 setParser(parser);
589 parse();
590 progressMonitor.worked(1);
591
592 progressMonitor.indeterminateSubTask(tr("Preparing data set..."));
593 prepareDataSet();
594 progressMonitor.worked(1);
595
596 // iterate over registered postprocessors and give them each a chance
597 // to modify the dataset we have just loaded.
598 if (postprocessors != null) {
599 for (OsmServerReadPostprocessor pp : postprocessors) {
600 pp.postprocessDataSet(getDataSet(), progressMonitor);
601 }
602 }
603 return getDataSet();
604 } catch(IllegalDataException e) {
605 throw e;
606 } catch(OsmParsingException e) {
607 throw new IllegalDataException(e.getMessage(), e);
608 } catch(XMLStreamException e) {
609 String msg = e.getMessage();
610 Pattern p = Pattern.compile("Message: (.+)");
611 Matcher m = p.matcher(msg);
612 if (m.find()) {
613 msg = m.group(1);
614 }
615 if (e.getLocation() != null)
616 throw new IllegalDataException(tr("Line {0} column {1}: ", e.getLocation().getLineNumber(), e.getLocation().getColumnNumber()) + msg, e);
617 else
618 throw new IllegalDataException(msg, e);
619 } catch(Exception e) {
620 throw new IllegalDataException(e);
621 } finally {
622 progressMonitor.finishTask();
623 progressMonitor.removeCancelListener(cancelListener);
624 }
625 }
626
627 /**
628 * Parse the given input source and return the dataset.
629 *
630 * @param source the source input stream. Must not be null.
631 * @param progressMonitor the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed
632 *
633 * @return the dataset with the parsed data
634 * @throws IllegalDataException thrown if the an error was found while parsing the data from the source
635 * @throws IllegalArgumentException thrown if source is null
636 */
637 public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
638 return new OsmReader().doParseDataSet(source, progressMonitor);
639 }
640}
Note: See TracBrowser for help on using the repository browser.