source: josm/trunk/src/org/openstreetmap/josm/io/OsmReader.java@ 4532

Last change on this file since 4532 was 4532, checked in by Don-vip, 13 years ago

see #6653 - Import <delete> primitives as deleted primitives
  • Property svn:eol-style set to native
File size: 23.7 KB
Line 
1// License: GPL. See LICENSE file for details.
2package org.openstreetmap.josm.io;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.InputStream;
7import java.io.InputStreamReader;
8import java.text.MessageFormat;
9import java.util.ArrayList;
10import java.util.Collection;
11import java.util.regex.Matcher;
12import java.util.regex.Pattern;
13
14import javax.xml.stream.Location;
15import javax.xml.stream.XMLInputFactory;
16import javax.xml.stream.XMLStreamConstants;
17import javax.xml.stream.XMLStreamException;
18import javax.xml.stream.XMLStreamReader;
19
20import org.openstreetmap.josm.data.Bounds;
21import org.openstreetmap.josm.data.coor.LatLon;
22import org.openstreetmap.josm.data.osm.Changeset;
23import org.openstreetmap.josm.data.osm.DataSet;
24import org.openstreetmap.josm.data.osm.DataSource;
25import org.openstreetmap.josm.data.osm.Node;
26import org.openstreetmap.josm.data.osm.NodeData;
27import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
28import org.openstreetmap.josm.data.osm.PrimitiveData;
29import org.openstreetmap.josm.data.osm.Relation;
30import org.openstreetmap.josm.data.osm.RelationData;
31import org.openstreetmap.josm.data.osm.RelationMemberData;
32import org.openstreetmap.josm.data.osm.Tagged;
33import org.openstreetmap.josm.data.osm.User;
34import org.openstreetmap.josm.data.osm.Way;
35import org.openstreetmap.josm.data.osm.WayData;
36import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
37import org.openstreetmap.josm.gui.progress.ProgressMonitor;
38import org.openstreetmap.josm.tools.CheckParameterUtil;
39import org.openstreetmap.josm.tools.DateUtils;
40
41/**
42 * Parser for the Osm Api. Read from an input stream and construct a dataset out of it.
43 *
44 * For each xml element, there is a dedicated method.
45 * The XMLStreamReader cursor points to the start of the element, when the method is
46 * entered, and it must point to the end of the same element, when it is exited.
47 */
48public class OsmReader extends AbstractReader {
49
50 protected XMLStreamReader parser;
51
52 /**
53 * constructor (for private and subclasses use only)
54 *
55 * @see #parseDataSet(InputStream, DataSet, ProgressMonitor)
56 */
57 protected OsmReader() {
58 }
59
60 protected void setParser(XMLStreamReader parser) {
61 this.parser = parser;
62 }
63
64 protected void throwException(String msg) throws XMLStreamException {
65 throw new OsmParsingException(msg, parser.getLocation());
66 }
67
68 protected void parse() throws XMLStreamException {
69 int event = parser.getEventType();
70 while (true) {
71 if (event == XMLStreamConstants.START_ELEMENT) {
72 parseRoot();
73 } else if (event == XMLStreamConstants.END_ELEMENT) {
74 return;
75 }
76 if (parser.hasNext()) {
77 event = parser.next();
78 } else {
79 break;
80 }
81 }
82 parser.close();
83 }
84
85 protected void parseRoot() throws XMLStreamException {
86 if (parser.getLocalName().equals("osm")) {
87 parseOsm();
88 } else {
89 parseUnknown();
90 }
91 }
92
93 private void parseOsm() throws XMLStreamException {
94 String v = parser.getAttributeValue(null, "version");
95 if (v == null) {
96 throwException(tr("Missing mandatory attribute ''{0}''.", "version"));
97 }
98 if (!(v.equals("0.5") || v.equals("0.6"))) {
99 throwException(tr("Unsupported version: {0}", v));
100 }
101 ds.setVersion(v);
102 String generator = parser.getAttributeValue(null, "generator");
103 Long uploadChangesetId = null;
104 if (parser.getAttributeValue(null, "upload-changeset") != null) {
105 uploadChangesetId = getLong("upload-changeset");
106 }
107 while (true) {
108 int event = parser.next();
109 if (event == XMLStreamConstants.START_ELEMENT) {
110 if (parser.getLocalName().equals("bounds")) {
111 parseBounds(generator);
112 } else if (parser.getLocalName().equals("node")) {
113 parseNode();
114 } else if (parser.getLocalName().equals("way")) {
115 parseWay();
116 } else if (parser.getLocalName().equals("relation")) {
117 parseRelation();
118 } else if (parser.getLocalName().equals("changeset")) {
119 parseChangeset(uploadChangesetId);
120 } else {
121 parseUnknown();
122 }
123 } else if (event == XMLStreamConstants.END_ELEMENT) {
124 return;
125 }
126 }
127 }
128
129 private void parseBounds(String generator) throws XMLStreamException {
130 String minlon = parser.getAttributeValue(null, "minlon");
131 String minlat = parser.getAttributeValue(null, "minlat");
132 String maxlon = parser.getAttributeValue(null, "maxlon");
133 String maxlat = parser.getAttributeValue(null, "maxlat");
134 String origin = parser.getAttributeValue(null, "origin");
135 if (minlon != null && maxlon != null && minlat != null && maxlat != null) {
136 if (origin == null) {
137 origin = generator;
138 }
139 Bounds bounds = new Bounds(
140 Double.parseDouble(minlat), Double.parseDouble(minlon),
141 Double.parseDouble(maxlat), Double.parseDouble(maxlon));
142 if (bounds.isOutOfTheWorld()) {
143 Bounds copy = new Bounds(bounds);
144 bounds.normalize();
145 System.out.println("Bbox " + copy + " is out of the world, normalized to " + bounds);
146 }
147 DataSource src = new DataSource(bounds, origin);
148 ds.dataSources.add(src);
149 } else {
150 throwException(tr(
151 "Missing mandatory attributes on element ''bounds''. Got minlon=''{0}'',minlat=''{1}'',maxlon=''{3}'',maxlat=''{4}'', origin=''{5}''.",
152 minlon, minlat, maxlon, maxlat, origin
153 ));
154 }
155 jumpToEnd();
156 }
157
158 protected Node parseNode() throws XMLStreamException {
159 NodeData nd = new NodeData();
160 nd.setCoor(new LatLon(Double.parseDouble(parser.getAttributeValue(null, "lat")), Double.parseDouble(parser.getAttributeValue(null, "lon"))));
161 readCommon(nd);
162 Node n = new Node(nd.getId(), nd.getVersion());
163 n.setVisible(nd.isVisible());
164 n.load(nd);
165 externalIdMap.put(nd.getPrimitiveId(), n);
166 while (true) {
167 int event = parser.next();
168 if (event == XMLStreamConstants.START_ELEMENT) {
169 if (parser.getLocalName().equals("tag")) {
170 parseTag(n);
171 } else {
172 parseUnknown();
173 }
174 } else if (event == XMLStreamConstants.END_ELEMENT) {
175 return n;
176 }
177 }
178 }
179
180 protected Way parseWay() throws XMLStreamException {
181 WayData wd = new WayData();
182 readCommon(wd);
183 Way w = new Way(wd.getId(), wd.getVersion());
184 w.setVisible(wd.isVisible());
185 w.load(wd);
186 externalIdMap.put(wd.getPrimitiveId(), w);
187
188 Collection<Long> nodeIds = new ArrayList<Long>();
189 while (true) {
190 int event = parser.next();
191 if (event == XMLStreamConstants.START_ELEMENT) {
192 if (parser.getLocalName().equals("nd")) {
193 nodeIds.add(parseWayNode(w));
194 } else if (parser.getLocalName().equals("tag")) {
195 parseTag(w);
196 } else {
197 parseUnknown();
198 }
199 } else if (event == XMLStreamConstants.END_ELEMENT) {
200 break;
201 }
202 }
203 if (w.isDeleted() && nodeIds.size() > 0) {
204 System.out.println(tr("Deleted way {0} contains nodes", w.getUniqueId()));
205 nodeIds = new ArrayList<Long>();
206 }
207 ways.put(wd.getUniqueId(), nodeIds);
208 return w;
209 }
210
211 private long parseWayNode(Way w) throws XMLStreamException {
212 if (parser.getAttributeValue(null, "ref") == null) {
213 throwException(
214 tr("Missing mandatory attribute ''{0}'' on <nd> of way {1}.", "ref", w.getUniqueId())
215 );
216 }
217 long id = getLong("ref");
218 if (id == 0) {
219 throwException(
220 tr("Illegal value of attribute ''ref'' of element <nd>. Got {0}.", id)
221 );
222 }
223 jumpToEnd();
224 return id;
225 }
226
227 protected Relation parseRelation() throws XMLStreamException {
228 RelationData rd = new RelationData();
229 readCommon(rd);
230 Relation r = new Relation(rd.getId(), rd.getVersion());
231 r.setVisible(rd.isVisible());
232 r.load(rd);
233 externalIdMap.put(rd.getPrimitiveId(), r);
234
235 Collection<RelationMemberData> members = new ArrayList<RelationMemberData>();
236 while (true) {
237 int event = parser.next();
238 if (event == XMLStreamConstants.START_ELEMENT) {
239 if (parser.getLocalName().equals("member")) {
240 members.add(parseRelationMember(r));
241 } else if (parser.getLocalName().equals("tag")) {
242 parseTag(r);
243 } else {
244 parseUnknown();
245 }
246 } else if (event == XMLStreamConstants.END_ELEMENT) {
247 break;
248 }
249 }
250 if (r.isDeleted() && members.size() > 0) {
251 System.out.println(tr("Deleted relation {0} contains members", r.getUniqueId()));
252 members = new ArrayList<RelationMemberData>();
253 }
254 relations.put(rd.getUniqueId(), members);
255 return r;
256 }
257
258 private RelationMemberData parseRelationMember(Relation r) throws XMLStreamException {
259 String role = null;
260 OsmPrimitiveType type = null;
261 long id = 0;
262 String value = parser.getAttributeValue(null, "ref");
263 if (value == null) {
264 throwException(tr("Missing attribute ''ref'' on member in relation {0}.",r.getUniqueId()));
265 }
266 try {
267 id = Long.parseLong(value);
268 } catch(NumberFormatException e) {
269 throwException(tr("Illegal value for attribute ''ref'' on member in relation {0}. Got {1}", Long.toString(r.getUniqueId()),value));
270 }
271 value = parser.getAttributeValue(null, "type");
272 if (value == null) {
273 throwException(tr("Missing attribute ''type'' on member {0} in relation {1}.", Long.toString(id), Long.toString(r.getUniqueId())));
274 }
275 try {
276 type = OsmPrimitiveType.fromApiTypeName(value);
277 } catch(IllegalArgumentException e) {
278 throwException(tr("Illegal value for attribute ''type'' on member {0} in relation {1}. Got {2}.", Long.toString(id), Long.toString(r.getUniqueId()), value));
279 }
280 value = parser.getAttributeValue(null, "role");
281 role = value;
282
283 if (id == 0) {
284 throwException(tr("Incomplete <member> specification with ref=0"));
285 }
286 jumpToEnd();
287 return new RelationMemberData(role, type, id);
288 }
289
290 private void parseChangeset(Long uploadChangesetId) throws XMLStreamException {
291 long id = getLong("id");
292
293 if (id == uploadChangesetId) {
294 uploadChangeset = new Changeset((int) getLong("id"));
295 while (true) {
296 int event = parser.next();
297 if (event == XMLStreamConstants.START_ELEMENT) {
298 if (parser.getLocalName().equals("tag")) {
299 parseTag(uploadChangeset);
300 } else {
301 parseUnknown();
302 }
303 } else if (event == XMLStreamConstants.END_ELEMENT) {
304 return;
305 }
306 }
307 } else {
308 jumpToEnd(false);
309 }
310 }
311
312 private void parseTag(Tagged t) throws XMLStreamException {
313 String key = parser.getAttributeValue(null, "k");
314 String value = parser.getAttributeValue(null, "v");
315 if (key == null || value == null) {
316 throwException(tr("Missing key or value attribute in tag."));
317 }
318 t.put(key.intern(), value.intern());
319 jumpToEnd();
320 }
321
322 protected void parseUnknown(boolean printWarning) throws XMLStreamException {
323 if (printWarning) {
324 System.out.println(tr("Undefined element ''{0}'' found in input stream. Skipping.", parser.getLocalName()));
325 }
326 while (true) {
327 int event = parser.next();
328 if (event == XMLStreamConstants.START_ELEMENT) {
329 parseUnknown(false); /* no more warning for inner elements */
330 } else if (event == XMLStreamConstants.END_ELEMENT) {
331 return;
332 }
333 }
334 }
335
336 protected void parseUnknown() throws XMLStreamException {
337 parseUnknown(true);
338 }
339
340 /**
341 * When cursor is at the start of an element, moves it to the end tag of that element.
342 * Nested content is skipped.
343 *
344 * This is basically the same code as parseUnkown(), except for the warnings, which
345 * are displayed for inner elements and not at top level.
346 */
347 private void jumpToEnd(boolean printWarning) throws XMLStreamException {
348 while (true) {
349 int event = parser.next();
350 if (event == XMLStreamConstants.START_ELEMENT) {
351 parseUnknown(printWarning);
352 } else if (event == XMLStreamConstants.END_ELEMENT) {
353 return;
354 }
355 }
356 }
357
358 private void jumpToEnd() throws XMLStreamException {
359 jumpToEnd(true);
360 }
361
362 private User createUser(String uid, String name) throws XMLStreamException {
363 if (uid == null) {
364 if (name == null)
365 return null;
366 return User.createLocalUser(name);
367 }
368 try {
369 long id = Long.parseLong(uid);
370 return User.createOsmUser(id, name);
371 } catch(NumberFormatException e) {
372 throwException(MessageFormat.format("Illegal value for attribute ''uid''. Got ''{0}''.", uid));
373 }
374 return null;
375 }
376
377 /**
378 * Read out the common attributes and put them into current OsmPrimitive.
379 */
380 private void readCommon(PrimitiveData current) throws XMLStreamException {
381 current.setId(getLong("id"));
382 if (current.getUniqueId() == 0) {
383 throwException(tr("Illegal object with ID=0."));
384 }
385
386 String time = parser.getAttributeValue(null, "timestamp");
387 if (time != null && time.length() != 0) {
388 current.setTimestamp(DateUtils.fromString(time));
389 }
390
391 // user attribute added in 0.4 API
392 String user = parser.getAttributeValue(null, "user");
393 // uid attribute added in 0.6 API
394 String uid = parser.getAttributeValue(null, "uid");
395 current.setUser(createUser(uid, user));
396
397 // visible attribute added in 0.4 API
398 String visible = parser.getAttributeValue(null, "visible");
399 if (visible != null) {
400 current.setVisible(Boolean.parseBoolean(visible));
401 }
402
403 String versionString = parser.getAttributeValue(null, "version");
404 int version = 0;
405 if (versionString != null) {
406 try {
407 version = Integer.parseInt(versionString);
408 } catch(NumberFormatException e) {
409 throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.", Long.toString(current.getUniqueId()), versionString));
410 }
411 if (ds.getVersion().equals("0.6")){
412 if (version <= 0 && current.getUniqueId() > 0) {
413 throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.", Long.toString(current.getUniqueId()), versionString));
414 } else if (version < 0 && current.getUniqueId() <= 0) {
415 System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 0, "0.6"));
416 version = 0;
417 }
418 } else if (ds.getVersion().equals("0.5")) {
419 if (version <= 0 && current.getUniqueId() > 0) {
420 System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 1, "0.5"));
421 version = 1;
422 } else if (version < 0 && current.getUniqueId() <= 0) {
423 System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 0, "0.5"));
424 version = 0;
425 }
426 } else {
427 // should not happen. API version has been checked before
428 throwException(tr("Unknown or unsupported API version. Got {0}.", ds.getVersion()));
429 }
430 } else {
431 // version expected for OSM primitives with an id assigned by the server (id > 0), since API 0.6
432 //
433 if (current.getUniqueId() > 0 && ds.getVersion() != null && ds.getVersion().equals("0.6")) {
434 throwException(tr("Missing attribute ''version'' on OSM primitive with ID {0}.", Long.toString(current.getUniqueId())));
435 } else if (current.getUniqueId() > 0 && ds.getVersion() != null && ds.getVersion().equals("0.5")) {
436 // default version in 0.5 files for existing primitives
437 System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 1, "0.5"));
438 version= 1;
439 } else if (current.getUniqueId() <= 0 && ds.getVersion() != null && ds.getVersion().equals("0.5")) {
440 // default version in 0.5 files for new primitives, no warning necessary. This is
441 // (was) legal in API 0.5
442 version= 0;
443 }
444 }
445 current.setVersion(version);
446
447 String action = parser.getAttributeValue(null, "action");
448 if (action == null) {
449 // do nothing
450 } else if (action.equals("delete")) {
451 current.setDeleted(true);
452 current.setModified(current.isVisible());
453 } else if (action.equals("modify")) {
454 current.setModified(true);
455 }
456
457 String v = parser.getAttributeValue(null, "changeset");
458 if (v == null) {
459 current.setChangesetId(0);
460 } else {
461 try {
462 current.setChangesetId(Integer.parseInt(v));
463 } catch(NumberFormatException e) {
464 if (current.getUniqueId() <= 0) {
465 // for a new primitive we just log a warning
466 System.out.println(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", v, current.getUniqueId()));
467 current.setChangesetId(0);
468 } else {
469 // for an existing primitive this is a problem
470 throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v));
471 }
472 }
473 if (current.getChangesetId() <=0) {
474 if (current.getUniqueId() <= 0) {
475 // for a new primitive we just log a warning
476 System.out.println(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", v, current.getUniqueId()));
477 current.setChangesetId(0);
478 } else {
479 // for an existing primitive this is a problem
480 throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v));
481 }
482 }
483 }
484 }
485
486 private long getLong(String name) throws XMLStreamException {
487 String value = parser.getAttributeValue(null, name);
488 if (value == null) {
489 throwException(tr("Missing required attribute ''{0}''.",name));
490 }
491 try {
492 return Long.parseLong(value);
493 } catch(NumberFormatException e) {
494 throwException(tr("Illegal long value for attribute ''{0}''. Got ''{1}''.",name, value));
495 }
496 return 0; // should not happen
497 }
498
499 private static class OsmParsingException extends XMLStreamException {
500 public OsmParsingException() {
501 super();
502 }
503
504 public OsmParsingException(String msg) {
505 super(msg);
506 }
507
508 public OsmParsingException(String msg, Location location) {
509 super(msg); /* cannot use super(msg, location) because it messes with the message preventing localization */
510 this.location = location;
511 }
512
513 public OsmParsingException(String msg, Location location, Throwable th) {
514 super(msg, th);
515 this.location = location;
516 }
517
518 public OsmParsingException(String msg, Throwable th) {
519 super(msg, th);
520 }
521
522 public OsmParsingException(Throwable th) {
523 super(th);
524 }
525
526 @Override
527 public String getMessage() {
528 String msg = super.getMessage();
529 if (msg == null) {
530 msg = getClass().getName();
531 }
532 if (getLocation() == null)
533 return msg;
534 msg = msg + " " + tr("(at line {0}, column {1})", getLocation().getLineNumber(), getLocation().getColumnNumber());
535 return msg;
536 }
537 }
538
539 protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
540 if (progressMonitor == null) {
541 progressMonitor = NullProgressMonitor.INSTANCE;
542 }
543 CheckParameterUtil.ensureParameterNotNull(source, "source");
544 try {
545 progressMonitor.beginTask(tr("Prepare OSM data...", 2));
546 progressMonitor.indeterminateSubTask(tr("Parsing OSM data..."));
547
548 InputStreamReader ir = UTFInputStreamReader.create(source, "UTF-8");
549 XMLStreamReader parser = XMLInputFactory.newInstance().createXMLStreamReader(ir);
550 setParser(parser);
551 parse();
552 progressMonitor.worked(1);
553
554 progressMonitor.indeterminateSubTask(tr("Preparing data set..."));
555 prepareDataSet();
556 progressMonitor.worked(1);
557 return getDataSet();
558 } catch(IllegalDataException e) {
559 throw e;
560 } catch(OsmParsingException e) {
561 throw new IllegalDataException(e.getMessage(), e);
562 } catch(XMLStreamException e) {
563 String msg = e.getMessage();
564 Pattern p = Pattern.compile("Message: (.+)");
565 Matcher m = p.matcher(msg);
566 if (m.find()) {
567 msg = m.group(1);
568 }
569 if (e.getLocation() != null) {
570 throw new IllegalDataException(tr("Line {0} column {1}: ", e.getLocation().getLineNumber(), e.getLocation().getColumnNumber()) + msg, e);
571 } else {
572 throw new IllegalDataException(msg, e);
573 }
574 } catch(Exception e) {
575 throw new IllegalDataException(e);
576 } finally {
577 progressMonitor.finishTask();
578 }
579 }
580
581 /**
582 * Parse the given input source and return the dataset.
583 *
584 * @param source the source input stream. Must not be null.
585 * @param progressMonitor the progress monitor. If null, {@see NullProgressMonitor#INSTANCE} is assumed
586 *
587 * @return the dataset with the parsed data
588 * @throws IllegalDataException thrown if the an error was found while parsing the data from the source
589 * @throws IllegalArgumentException thrown if source is null
590 */
591 public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
592 return new OsmReader().doParseDataSet(source, progressMonitor);
593 }
594}
Note: See TracBrowser for help on using the repository browser.