source: josm/trunk/src/org/openstreetmap/josm/io/OsmReader.java@ 5707

Last change on this file since 5707 was 5326, checked in by framm, 12 years ago

handle deleted nodes without lat/lon. apply patches from ticket; closes #7847
  • Property svn:eol-style set to native
File size: 24.9 KB
Line 
1// License: GPL. See LICENSE file for details.
2package org.openstreetmap.josm.io;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.InputStream;
7import java.io.InputStreamReader;
8import java.text.MessageFormat;
9import java.util.ArrayList;
10import java.util.Collection;
11import java.util.regex.Matcher;
12import java.util.regex.Pattern;
13
14import javax.xml.stream.Location;
15import javax.xml.stream.XMLInputFactory;
16import javax.xml.stream.XMLStreamConstants;
17import javax.xml.stream.XMLStreamException;
18import javax.xml.stream.XMLStreamReader;
19
20import org.openstreetmap.josm.data.Bounds;
21import org.openstreetmap.josm.data.coor.LatLon;
22import org.openstreetmap.josm.data.osm.Changeset;
23import org.openstreetmap.josm.data.osm.DataSet;
24import org.openstreetmap.josm.data.osm.DataSource;
25import org.openstreetmap.josm.data.osm.Node;
26import org.openstreetmap.josm.data.osm.NodeData;
27import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
28import org.openstreetmap.josm.data.osm.PrimitiveData;
29import org.openstreetmap.josm.data.osm.Relation;
30import org.openstreetmap.josm.data.osm.RelationData;
31import org.openstreetmap.josm.data.osm.RelationMemberData;
32import org.openstreetmap.josm.data.osm.Tagged;
33import org.openstreetmap.josm.data.osm.User;
34import org.openstreetmap.josm.data.osm.Way;
35import org.openstreetmap.josm.data.osm.WayData;
36import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
37import org.openstreetmap.josm.gui.progress.ProgressMonitor;
38import org.openstreetmap.josm.tools.CheckParameterUtil;
39import org.openstreetmap.josm.tools.DateUtils;
40
41/**
42 * Parser for the Osm Api. Read from an input stream and construct a dataset out of it.
43 *
44 * For each xml element, there is a dedicated method.
45 * The XMLStreamReader cursor points to the start of the element, when the method is
46 * entered, and it must point to the end of the same element, when it is exited.
47 */
48public class OsmReader extends AbstractReader {
49
50 protected XMLStreamReader parser;
51
52 /** Used by plugins to register themselves as data postprocessors. */
53 public static ArrayList<OsmServerReadPostprocessor> postprocessors;
54
55 /** register a new postprocessor */
56 public static void registerPostprocessor(OsmServerReadPostprocessor pp) {
57 if (postprocessors == null) {
58 postprocessors = new ArrayList<OsmServerReadPostprocessor>();
59 }
60 postprocessors.add(pp);
61 }
62
63 /** deregister a postprocessor previously registered with registerPostprocessor */
64 public static void deregisterPostprocessor(OsmServerReadPostprocessor pp) {
65 if (postprocessors != null) {
66 postprocessors.remove(pp);
67 }
68 }
69
70 /**
71 * constructor (for private and subclasses use only)
72 *
73 * @see #parseDataSet(InputStream, DataSet, ProgressMonitor)
74 */
75 protected OsmReader() {
76 }
77
78 protected void setParser(XMLStreamReader parser) {
79 this.parser = parser;
80 }
81
82 protected void throwException(String msg) throws XMLStreamException {
83 throw new OsmParsingException(msg, parser.getLocation());
84 }
85
86 protected void parse() throws XMLStreamException {
87 int event = parser.getEventType();
88 while (true) {
89 if (event == XMLStreamConstants.START_ELEMENT) {
90 parseRoot();
91 } else if (event == XMLStreamConstants.END_ELEMENT)
92 return;
93 if (parser.hasNext()) {
94 event = parser.next();
95 } else {
96 break;
97 }
98 }
99 parser.close();
100 }
101
102 protected void parseRoot() throws XMLStreamException {
103 if (parser.getLocalName().equals("osm")) {
104 parseOsm();
105 } else {
106 parseUnknown();
107 }
108 }
109
110 private void parseOsm() throws XMLStreamException {
111 String v = parser.getAttributeValue(null, "version");
112 if (v == null) {
113 throwException(tr("Missing mandatory attribute ''{0}''.", "version"));
114 }
115 if (!(v.equals("0.5") || v.equals("0.6"))) {
116 throwException(tr("Unsupported version: {0}", v));
117 }
118 ds.setVersion(v);
119 String upload = parser.getAttributeValue(null, "upload");
120 if (upload != null) {
121 ds.setUploadDiscouraged(!Boolean.parseBoolean(upload));
122 }
123 String generator = parser.getAttributeValue(null, "generator");
124 Long uploadChangesetId = null;
125 if (parser.getAttributeValue(null, "upload-changeset") != null) {
126 uploadChangesetId = getLong("upload-changeset");
127 }
128 while (true) {
129 int event = parser.next();
130 if (event == XMLStreamConstants.START_ELEMENT) {
131 if (parser.getLocalName().equals("bounds")) {
132 parseBounds(generator);
133 } else if (parser.getLocalName().equals("node")) {
134 parseNode();
135 } else if (parser.getLocalName().equals("way")) {
136 parseWay();
137 } else if (parser.getLocalName().equals("relation")) {
138 parseRelation();
139 } else if (parser.getLocalName().equals("changeset")) {
140 parseChangeset(uploadChangesetId);
141 } else {
142 parseUnknown();
143 }
144 } else if (event == XMLStreamConstants.END_ELEMENT)
145 return;
146 }
147 }
148
149 private void parseBounds(String generator) throws XMLStreamException {
150 String minlon = parser.getAttributeValue(null, "minlon");
151 String minlat = parser.getAttributeValue(null, "minlat");
152 String maxlon = parser.getAttributeValue(null, "maxlon");
153 String maxlat = parser.getAttributeValue(null, "maxlat");
154 String origin = parser.getAttributeValue(null, "origin");
155 if (minlon != null && maxlon != null && minlat != null && maxlat != null) {
156 if (origin == null) {
157 origin = generator;
158 }
159 Bounds bounds = new Bounds(
160 Double.parseDouble(minlat), Double.parseDouble(minlon),
161 Double.parseDouble(maxlat), Double.parseDouble(maxlon));
162 if (bounds.isOutOfTheWorld()) {
163 Bounds copy = new Bounds(bounds);
164 bounds.normalize();
165 System.out.println("Bbox " + copy + " is out of the world, normalized to " + bounds);
166 }
167 DataSource src = new DataSource(bounds, origin);
168 ds.dataSources.add(src);
169 } else {
170 throwException(tr(
171 "Missing mandatory attributes on element ''bounds''. Got minlon=''{0}'',minlat=''{1}'',maxlon=''{3}'',maxlat=''{4}'', origin=''{5}''.",
172 minlon, minlat, maxlon, maxlat, origin
173 ));
174 }
175 jumpToEnd();
176 }
177
178 protected Node parseNode() throws XMLStreamException {
179 NodeData nd = new NodeData();
180 String lat = parser.getAttributeValue(null, "lat");
181 String lon = parser.getAttributeValue(null, "lon");
182 if (lat != null && lon != null) {
183 nd.setCoor(new LatLon(Double.parseDouble(lat), Double.parseDouble(lon)));
184 }
185 readCommon(nd);
186 Node n = new Node(nd.getId(), nd.getVersion());
187 n.setVisible(nd.isVisible());
188 n.load(nd);
189 externalIdMap.put(nd.getPrimitiveId(), n);
190 while (true) {
191 int event = parser.next();
192 if (event == XMLStreamConstants.START_ELEMENT) {
193 if (parser.getLocalName().equals("tag")) {
194 parseTag(n);
195 } else {
196 parseUnknown();
197 }
198 } else if (event == XMLStreamConstants.END_ELEMENT)
199 return n;
200 }
201 }
202
203 protected Way parseWay() throws XMLStreamException {
204 WayData wd = new WayData();
205 readCommon(wd);
206 Way w = new Way(wd.getId(), wd.getVersion());
207 w.setVisible(wd.isVisible());
208 w.load(wd);
209 externalIdMap.put(wd.getPrimitiveId(), w);
210
211 Collection<Long> nodeIds = new ArrayList<Long>();
212 while (true) {
213 int event = parser.next();
214 if (event == XMLStreamConstants.START_ELEMENT) {
215 if (parser.getLocalName().equals("nd")) {
216 nodeIds.add(parseWayNode(w));
217 } else if (parser.getLocalName().equals("tag")) {
218 parseTag(w);
219 } else {
220 parseUnknown();
221 }
222 } else if (event == XMLStreamConstants.END_ELEMENT) {
223 break;
224 }
225 }
226 if (w.isDeleted() && nodeIds.size() > 0) {
227 System.out.println(tr("Deleted way {0} contains nodes", w.getUniqueId()));
228 nodeIds = new ArrayList<Long>();
229 }
230 ways.put(wd.getUniqueId(), nodeIds);
231 return w;
232 }
233
234 private long parseWayNode(Way w) throws XMLStreamException {
235 if (parser.getAttributeValue(null, "ref") == null) {
236 throwException(
237 tr("Missing mandatory attribute ''{0}'' on <nd> of way {1}.", "ref", w.getUniqueId())
238 );
239 }
240 long id = getLong("ref");
241 if (id == 0) {
242 throwException(
243 tr("Illegal value of attribute ''ref'' of element <nd>. Got {0}.", id)
244 );
245 }
246 jumpToEnd();
247 return id;
248 }
249
250 protected Relation parseRelation() throws XMLStreamException {
251 RelationData rd = new RelationData();
252 readCommon(rd);
253 Relation r = new Relation(rd.getId(), rd.getVersion());
254 r.setVisible(rd.isVisible());
255 r.load(rd);
256 externalIdMap.put(rd.getPrimitiveId(), r);
257
258 Collection<RelationMemberData> members = new ArrayList<RelationMemberData>();
259 while (true) {
260 int event = parser.next();
261 if (event == XMLStreamConstants.START_ELEMENT) {
262 if (parser.getLocalName().equals("member")) {
263 members.add(parseRelationMember(r));
264 } else if (parser.getLocalName().equals("tag")) {
265 parseTag(r);
266 } else {
267 parseUnknown();
268 }
269 } else if (event == XMLStreamConstants.END_ELEMENT) {
270 break;
271 }
272 }
273 if (r.isDeleted() && members.size() > 0) {
274 System.out.println(tr("Deleted relation {0} contains members", r.getUniqueId()));
275 members = new ArrayList<RelationMemberData>();
276 }
277 relations.put(rd.getUniqueId(), members);
278 return r;
279 }
280
281 private RelationMemberData parseRelationMember(Relation r) throws XMLStreamException {
282 String role = null;
283 OsmPrimitiveType type = null;
284 long id = 0;
285 String value = parser.getAttributeValue(null, "ref");
286 if (value == null) {
287 throwException(tr("Missing attribute ''ref'' on member in relation {0}.",r.getUniqueId()));
288 }
289 try {
290 id = Long.parseLong(value);
291 } catch(NumberFormatException e) {
292 throwException(tr("Illegal value for attribute ''ref'' on member in relation {0}. Got {1}", Long.toString(r.getUniqueId()),value));
293 }
294 value = parser.getAttributeValue(null, "type");
295 if (value == null) {
296 throwException(tr("Missing attribute ''type'' on member {0} in relation {1}.", Long.toString(id), Long.toString(r.getUniqueId())));
297 }
298 try {
299 type = OsmPrimitiveType.fromApiTypeName(value);
300 } catch(IllegalArgumentException e) {
301 throwException(tr("Illegal value for attribute ''type'' on member {0} in relation {1}. Got {2}.", Long.toString(id), Long.toString(r.getUniqueId()), value));
302 }
303 value = parser.getAttributeValue(null, "role");
304 role = value;
305
306 if (id == 0) {
307 throwException(tr("Incomplete <member> specification with ref=0"));
308 }
309 jumpToEnd();
310 return new RelationMemberData(role, type, id);
311 }
312
313 private void parseChangeset(Long uploadChangesetId) throws XMLStreamException {
314 long id = getLong("id");
315
316 if (id == uploadChangesetId) {
317 uploadChangeset = new Changeset((int) getLong("id"));
318 while (true) {
319 int event = parser.next();
320 if (event == XMLStreamConstants.START_ELEMENT) {
321 if (parser.getLocalName().equals("tag")) {
322 parseTag(uploadChangeset);
323 } else {
324 parseUnknown();
325 }
326 } else if (event == XMLStreamConstants.END_ELEMENT)
327 return;
328 }
329 } else {
330 jumpToEnd(false);
331 }
332 }
333
334 private void parseTag(Tagged t) throws XMLStreamException {
335 String key = parser.getAttributeValue(null, "k");
336 String value = parser.getAttributeValue(null, "v");
337 if (key == null || value == null) {
338 throwException(tr("Missing key or value attribute in tag."));
339 }
340 t.put(key.intern(), value.intern());
341 jumpToEnd();
342 }
343
344 protected void parseUnknown(boolean printWarning) throws XMLStreamException {
345 if (printWarning) {
346 System.out.println(tr("Undefined element ''{0}'' found in input stream. Skipping.", parser.getLocalName()));
347 }
348 while (true) {
349 int event = parser.next();
350 if (event == XMLStreamConstants.START_ELEMENT) {
351 parseUnknown(false); /* no more warning for inner elements */
352 } else if (event == XMLStreamConstants.END_ELEMENT)
353 return;
354 }
355 }
356
357 protected void parseUnknown() throws XMLStreamException {
358 parseUnknown(true);
359 }
360
361 /**
362 * When cursor is at the start of an element, moves it to the end tag of that element.
363 * Nested content is skipped.
364 *
365 * This is basically the same code as parseUnknown(), except for the warnings, which
366 * are displayed for inner elements and not at top level.
367 */
368 private void jumpToEnd(boolean printWarning) throws XMLStreamException {
369 while (true) {
370 int event = parser.next();
371 if (event == XMLStreamConstants.START_ELEMENT) {
372 parseUnknown(printWarning);
373 } else if (event == XMLStreamConstants.END_ELEMENT)
374 return;
375 }
376 }
377
378 private void jumpToEnd() throws XMLStreamException {
379 jumpToEnd(true);
380 }
381
382 private User createUser(String uid, String name) throws XMLStreamException {
383 if (uid == null) {
384 if (name == null)
385 return null;
386 return User.createLocalUser(name);
387 }
388 try {
389 long id = Long.parseLong(uid);
390 return User.createOsmUser(id, name);
391 } catch(NumberFormatException e) {
392 throwException(MessageFormat.format("Illegal value for attribute ''uid''. Got ''{0}''.", uid));
393 }
394 return null;
395 }
396
397 /**
398 * Read out the common attributes and put them into current OsmPrimitive.
399 */
400 private void readCommon(PrimitiveData current) throws XMLStreamException {
401 current.setId(getLong("id"));
402 if (current.getUniqueId() == 0) {
403 throwException(tr("Illegal object with ID=0."));
404 }
405
406 String time = parser.getAttributeValue(null, "timestamp");
407 if (time != null && time.length() != 0) {
408 current.setTimestamp(DateUtils.fromString(time));
409 }
410
411 // user attribute added in 0.4 API
412 String user = parser.getAttributeValue(null, "user");
413 // uid attribute added in 0.6 API
414 String uid = parser.getAttributeValue(null, "uid");
415 current.setUser(createUser(uid, user));
416
417 // visible attribute added in 0.4 API
418 String visible = parser.getAttributeValue(null, "visible");
419 if (visible != null) {
420 current.setVisible(Boolean.parseBoolean(visible));
421 }
422
423 String versionString = parser.getAttributeValue(null, "version");
424 int version = 0;
425 if (versionString != null) {
426 try {
427 version = Integer.parseInt(versionString);
428 } catch(NumberFormatException e) {
429 throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.", Long.toString(current.getUniqueId()), versionString));
430 }
431 if (ds.getVersion().equals("0.6")){
432 if (version <= 0 && current.getUniqueId() > 0) {
433 throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.", Long.toString(current.getUniqueId()), versionString));
434 } else if (version < 0 && current.getUniqueId() <= 0) {
435 System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 0, "0.6"));
436 version = 0;
437 }
438 } else if (ds.getVersion().equals("0.5")) {
439 if (version <= 0 && current.getUniqueId() > 0) {
440 System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 1, "0.5"));
441 version = 1;
442 } else if (version < 0 && current.getUniqueId() <= 0) {
443 System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 0, "0.5"));
444 version = 0;
445 }
446 } else {
447 // should not happen. API version has been checked before
448 throwException(tr("Unknown or unsupported API version. Got {0}.", ds.getVersion()));
449 }
450 } else {
451 // version expected for OSM primitives with an id assigned by the server (id > 0), since API 0.6
452 //
453 if (current.getUniqueId() > 0 && ds.getVersion() != null && ds.getVersion().equals("0.6")) {
454 throwException(tr("Missing attribute ''version'' on OSM primitive with ID {0}.", Long.toString(current.getUniqueId())));
455 } else if (current.getUniqueId() > 0 && ds.getVersion() != null && ds.getVersion().equals("0.5")) {
456 // default version in 0.5 files for existing primitives
457 System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 1, "0.5"));
458 version= 1;
459 } else if (current.getUniqueId() <= 0 && ds.getVersion() != null && ds.getVersion().equals("0.5")) {
460 // default version in 0.5 files for new primitives, no warning necessary. This is
461 // (was) legal in API 0.5
462 version= 0;
463 }
464 }
465 current.setVersion(version);
466
467 String action = parser.getAttributeValue(null, "action");
468 if (action == null) {
469 // do nothing
470 } else if (action.equals("delete")) {
471 current.setDeleted(true);
472 current.setModified(current.isVisible());
473 } else if (action.equals("modify")) {
474 current.setModified(true);
475 }
476
477 String v = parser.getAttributeValue(null, "changeset");
478 if (v == null) {
479 current.setChangesetId(0);
480 } else {
481 try {
482 current.setChangesetId(Integer.parseInt(v));
483 } catch(NumberFormatException e) {
484 if (current.getUniqueId() <= 0) {
485 // for a new primitive we just log a warning
486 System.out.println(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", v, current.getUniqueId()));
487 current.setChangesetId(0);
488 } else {
489 // for an existing primitive this is a problem
490 throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v));
491 }
492 }
493 if (current.getChangesetId() <=0) {
494 if (current.getUniqueId() <= 0) {
495 // for a new primitive we just log a warning
496 System.out.println(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", v, current.getUniqueId()));
497 current.setChangesetId(0);
498 } else {
499 // for an existing primitive this is a problem
500 throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v));
501 }
502 }
503 }
504 }
505
506 private long getLong(String name) throws XMLStreamException {
507 String value = parser.getAttributeValue(null, name);
508 if (value == null) {
509 throwException(tr("Missing required attribute ''{0}''.",name));
510 }
511 try {
512 return Long.parseLong(value);
513 } catch(NumberFormatException e) {
514 throwException(tr("Illegal long value for attribute ''{0}''. Got ''{1}''.",name, value));
515 }
516 return 0; // should not happen
517 }
518
519 private static class OsmParsingException extends XMLStreamException {
520 public OsmParsingException() {
521 super();
522 }
523
524 public OsmParsingException(String msg) {
525 super(msg);
526 }
527
528 public OsmParsingException(String msg, Location location) {
529 super(msg); /* cannot use super(msg, location) because it messes with the message preventing localization */
530 this.location = location;
531 }
532
533 public OsmParsingException(String msg, Location location, Throwable th) {
534 super(msg, th);
535 this.location = location;
536 }
537
538 public OsmParsingException(String msg, Throwable th) {
539 super(msg, th);
540 }
541
542 public OsmParsingException(Throwable th) {
543 super(th);
544 }
545
546 @Override
547 public String getMessage() {
548 String msg = super.getMessage();
549 if (msg == null) {
550 msg = getClass().getName();
551 }
552 if (getLocation() == null)
553 return msg;
554 msg = msg + " " + tr("(at line {0}, column {1})", getLocation().getLineNumber(), getLocation().getColumnNumber());
555 return msg;
556 }
557 }
558
559 protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
560 if (progressMonitor == null) {
561 progressMonitor = NullProgressMonitor.INSTANCE;
562 }
563 CheckParameterUtil.ensureParameterNotNull(source, "source");
564 try {
565 progressMonitor.beginTask(tr("Prepare OSM data...", 2));
566 progressMonitor.indeterminateSubTask(tr("Parsing OSM data..."));
567
568 InputStreamReader ir = UTFInputStreamReader.create(source, "UTF-8");
569 XMLStreamReader parser = XMLInputFactory.newInstance().createXMLStreamReader(ir);
570 setParser(parser);
571 parse();
572 progressMonitor.worked(1);
573
574 progressMonitor.indeterminateSubTask(tr("Preparing data set..."));
575 prepareDataSet();
576 progressMonitor.worked(1);
577
578 // iterate over registered postprocessors and give them each a chance
579 // to modify the dataset we have just loaded.
580 if (postprocessors != null) {
581 for (OsmServerReadPostprocessor pp : postprocessors) {
582 pp.postprocessDataSet(getDataSet(), progressMonitor);
583 }
584 }
585 return getDataSet();
586 } catch(IllegalDataException e) {
587 throw e;
588 } catch(OsmParsingException e) {
589 throw new IllegalDataException(e.getMessage(), e);
590 } catch(XMLStreamException e) {
591 String msg = e.getMessage();
592 Pattern p = Pattern.compile("Message: (.+)");
593 Matcher m = p.matcher(msg);
594 if (m.find()) {
595 msg = m.group(1);
596 }
597 if (e.getLocation() != null)
598 throw new IllegalDataException(tr("Line {0} column {1}: ", e.getLocation().getLineNumber(), e.getLocation().getColumnNumber()) + msg, e);
599 else
600 throw new IllegalDataException(msg, e);
601 } catch(Exception e) {
602 throw new IllegalDataException(e);
603 } finally {
604 progressMonitor.finishTask();
605 }
606 }
607
608 /**
609 * Parse the given input source and return the dataset.
610 *
611 * @param source the source input stream. Must not be null.
612 * @param progressMonitor the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed
613 *
614 * @return the dataset with the parsed data
615 * @throws IllegalDataException thrown if the an error was found while parsing the data from the source
616 * @throws IllegalArgumentException thrown if source is null
617 */
618 public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
619 return new OsmReader().doParseDataSet(source, progressMonitor);
620 }
621}
Note: See TracBrowser for help on using the repository browser.