source: josm/trunk/src/org/openstreetmap/josm/io/OsmReader.java@ 2851

Last change on this file since 2851 was 2751, checked in by Gubaer, 14 years ago

Improved user feedback when downloading relations
  • Property svn:eol-style set to native
File size: 26.7 KB
Line 
1package org.openstreetmap.josm.io;
2
3import static org.openstreetmap.josm.tools.I18n.tr;
4
5import java.io.InputStream;
6import java.io.InputStreamReader;
7import java.util.ArrayList;
8import java.util.Collection;
9import java.util.Date;
10import java.util.HashMap;
11import java.util.LinkedList;
12import java.util.List;
13import java.util.Map;
14import java.util.logging.Level;
15import java.util.logging.Logger;
16
17import javax.xml.parsers.ParserConfigurationException;
18import javax.xml.parsers.SAXParserFactory;
19
20import org.openstreetmap.josm.data.Bounds;
21import org.openstreetmap.josm.data.coor.LatLon;
22import org.openstreetmap.josm.data.osm.DataSet;
23import org.openstreetmap.josm.data.osm.DataSource;
24import org.openstreetmap.josm.data.osm.Node;
25import org.openstreetmap.josm.data.osm.OsmPrimitive;
26import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
27import org.openstreetmap.josm.data.osm.PrimitiveId;
28import org.openstreetmap.josm.data.osm.Relation;
29import org.openstreetmap.josm.data.osm.RelationMember;
30import org.openstreetmap.josm.data.osm.SimplePrimitiveId;
31import org.openstreetmap.josm.data.osm.User;
32import org.openstreetmap.josm.data.osm.Way;
33import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
34import org.openstreetmap.josm.gui.progress.ProgressMonitor;
35import org.openstreetmap.josm.tools.DateUtils;
36import org.xml.sax.Attributes;
37import org.xml.sax.InputSource;
38import org.xml.sax.Locator;
39import org.xml.sax.SAXException;
40import org.xml.sax.helpers.DefaultHandler;
41
42/**
43 * Parser for the Osm Api. Read from an input stream and construct a dataset out of it.
44 *
45 */
46public class OsmReader {
47 static private final Logger logger = Logger.getLogger(OsmReader.class.getName());
48
49 /**
50 * The dataset to add parsed objects to.
51 */
52 private DataSet ds = new DataSet();
53
54 /**
55 * Replies the parsed data set
56 *
57 * @return the parsed data set
58 */
59 public DataSet getDataSet() {
60 return ds;
61 }
62
63 /** the map from external ids to read OsmPrimitives. External ids are
64 * longs too, but in contrast to internal ids negative values are used
65 * to identify primitives unknown to the OSM server
66 *
67 * The keys are strings composed as follows
68 * <ul>
69 * <li>"n" + id for nodes</li>
70 * <li>"w" + id for nodes</li>
71 * <li>"r" + id for nodes</li>
72 * </ul>
73 */
74 private Map<PrimitiveId, OsmPrimitive> externalIdMap = new HashMap<PrimitiveId, OsmPrimitive>();
75
76 /**
77 * constructor (for private use only)
78 *
79 * @see #parseDataSet(InputStream, DataSet, ProgressMonitor)
80 * @see #parseDataSetOsm(InputStream, DataSet, ProgressMonitor)
81 */
82 private OsmReader() {
83 externalIdMap = new HashMap<PrimitiveId, OsmPrimitive>();
84 }
85
86 private static class OsmPrimitiveData {
87 public long id = 0;
88 public boolean modified = false;
89 public boolean deleted = false;
90 public Date timestamp = new Date();
91 public User user = null;
92 public boolean visible = true;
93 public int version = 0;
94 public LatLon latlon = new LatLon(0,0);
95 private OsmPrimitive primitive;
96 private int changesetId;
97
98 public void copyTo(OsmPrimitive osm) {
99 // It's not necessary to call clearOsmId for id < 0. The same thing is done by parameterless constructor
100 if (id > 0) {
101 osm.setOsmId(id, version);
102 }
103 osm.setDeleted(deleted);
104 osm.setModified(modified | deleted);
105 osm.setTimestamp(timestamp);
106 osm.setUser(user);
107 if (!osm.isNew() && changesetId > 0) {
108 osm.setChangesetId(changesetId);
109 } else if (osm.isNew() && changesetId > 0) {
110 System.out.println(tr("Warning: ignoring changeset id {0} for new object with external id {1} and internal id {2}",
111 changesetId,
112 id,
113 osm.getUniqueId()
114 ));
115 }
116 if (! osm.isNew()) {
117 // ignore visible attribute for objects not yet known to the server
118 //
119 osm.setVisible(visible);
120 }
121 osm.mappaintStyle = null;
122 }
123
124 public Node createNode() {
125 Node node = new Node();
126 node.setCoor(latlon);
127 copyTo(node);
128 primitive = node;
129 return node;
130 }
131
132 public Way createWay() {
133 Way way = new Way();
134 copyTo(way);
135 primitive = way;
136 return way;
137 }
138 public Relation createRelation() {
139 Relation relation= new Relation();
140 copyTo(relation);
141 primitive = relation;
142 return relation;
143 }
144
145 public void rememberTag(String key, String value) {
146 primitive.put(key, value);
147 }
148 }
149
150 /**
151 * Used as a temporary storage for relation members, before they
152 * are resolved into pointers to real objects.
153 */
154 private static class RelationMemberData {
155 public String type;
156 public long id;
157 public String role;
158 }
159
160 /**
161 * Data structure for the remaining way objects
162 */
163 private Map<Long, Collection<Long>> ways = new HashMap<Long, Collection<Long>>();
164
165 /**
166 * Data structure for relation objects
167 */
168 private Map<Long, Collection<RelationMemberData>> relations = new HashMap<Long, Collection<RelationMemberData>>();
169
170 private class Parser extends DefaultHandler {
171 private Locator locator;
172
173 @Override
174 public void setDocumentLocator(Locator locator) {
175 this.locator = locator;
176 }
177
178 protected void throwException(String msg) throws OsmDataParsingException{
179 throw new OsmDataParsingException(msg).rememberLocation(locator);
180 }
181 /**
182 * The current osm primitive to be read.
183 */
184 private OsmPrimitiveData current;
185 private String generator;
186
187 @Override public void startElement(String namespaceURI, String localName, String qName, Attributes atts) throws SAXException {
188 if (qName.equals("osm")) {
189 if (atts == null) {
190 throwException(tr("Missing mandatory attribute ''{0}'' of XML element {1}.", "version", "osm"));
191 }
192 String v = atts.getValue("version");
193 if (v == null) {
194 throwException(tr("Missing mandatory attribute ''{0}''.", "version"));
195 }
196 if (!(v.equals("0.5") || v.equals("0.6"))) {
197 throwException(tr("Unsupported version: {0}", v));
198 }
199 // save generator attribute for later use when creating DataSource objects
200 generator = atts.getValue("generator");
201 ds.setVersion(v);
202
203 } else if (qName.equals("bounds")) {
204 // new style bounds.
205 String minlon = atts.getValue("minlon");
206 String minlat = atts.getValue("minlat");
207 String maxlon = atts.getValue("maxlon");
208 String maxlat = atts.getValue("maxlat");
209 String origin = atts.getValue("origin");
210 if (minlon != null && maxlon != null && minlat != null && maxlat != null) {
211 if (origin == null) {
212 origin = generator;
213 }
214 Bounds bounds = new Bounds(
215 new LatLon(Double.parseDouble(minlat), Double.parseDouble(minlon)),
216 new LatLon(Double.parseDouble(maxlat), Double.parseDouble(maxlon)));
217 DataSource src = new DataSource(bounds, origin);
218 ds.dataSources.add(src);
219 } else {
220 throwException(tr(
221 "Missing manadatory attributes on element ''bounds''. Got minlon=''{0}'',minlat=''{1}'',maxlon=''{3}'',maxlat=''{4}'', origin=''{5}''.",
222 minlon, minlat, maxlon, maxlat, origin
223 ));
224 }
225
226 // ---- PARSING NODES AND WAYS ----
227
228 } else if (qName.equals("node")) {
229 current = new OsmPrimitiveData();
230 current.latlon = new LatLon(getDouble(atts, "lat"), getDouble(atts, "lon"));
231 readCommon(atts, current);
232 Node n = current.createNode();
233 externalIdMap.put(new SimplePrimitiveId(current.id, OsmPrimitiveType.NODE), n);
234 } else if (qName.equals("way")) {
235 current = new OsmPrimitiveData();
236 readCommon(atts, current);
237 Way w = current.createWay();
238 externalIdMap.put(new SimplePrimitiveId(current.id, OsmPrimitiveType.WAY), w);
239 ways.put(current.id, new ArrayList<Long>());
240 } else if (qName.equals("nd")) {
241 Collection<Long> list = ways.get(current.id);
242 if (list == null) {
243 throwException(
244 tr("Found XML element <nd> not as direct child of element <way>.")
245 );
246 }
247 if (atts.getValue("ref") == null) {
248 throwException(
249 tr("Missing mandatory attribute ''{0}'' on <nd> of way {1}.", "ref", current.id)
250 );
251 }
252 long id = getLong(atts, "ref");
253 if (id == 0) {
254 throwException(
255 tr("Illegal value of attribute ''ref'' of element <nd>. Got {0}.", id)
256 );
257 }
258 list.add(id);
259
260 // ---- PARSING RELATIONS ----
261
262 } else if (qName.equals("relation")) {
263 current = new OsmPrimitiveData();
264 readCommon(atts, current);
265 Relation r = current.createRelation();
266 externalIdMap.put(new SimplePrimitiveId(current.id, OsmPrimitiveType.RELATION), r);
267 relations.put(current.id, new LinkedList<RelationMemberData>());
268 } else if (qName.equals("member")) {
269 Collection<RelationMemberData> list = relations.get(current.id);
270 if (list == null) {
271 throwException(
272 tr("Found XML element <member> not as direct child of element <relation>.")
273 );
274 }
275 RelationMemberData emd = new RelationMemberData();
276 String value = atts.getValue("ref");
277 if (value == null) {
278 throwException(tr("Missing attribute ''ref'' on member in relation {0}.",current.id));
279 }
280 try {
281 emd.id = Long.parseLong(value);
282 } catch(NumberFormatException e) {
283 throwException(tr("Illegal value for attribute ''ref'' on member in relation {0}. Got {1}", Long.toString(current.id),value));
284 }
285 value = atts.getValue("type");
286 if (value == null) {
287 throwException(tr("Missing attribute ''type'' on member {0} in relation {1}.", Long.toString(emd.id), Long.toString(current.id)));
288 }
289 if (! (value.equals("way") || value.equals("node") || value.equals("relation"))) {
290 throwException(tr("Illegal value for attribute ''type'' on member {0} in relation {1}. Got {2}.", Long.toString(emd.id), Long.toString(current.id), value));
291 }
292 emd.type= value;
293 value = atts.getValue("role");
294 emd.role = value;
295
296 if (emd.id == 0) {
297 throwException(tr("Incomplete <member> specification with ref=0"));
298 }
299
300 list.add(emd);
301
302 // ---- PARSING TAGS (applicable to all objects) ----
303
304 } else if (qName.equals("tag")) {
305 String key = atts.getValue("k");
306 String value = atts.getValue("v");
307 current.rememberTag(key, value);
308 } else {
309 System.out.println(tr("Undefined element ''{0}'' found in input stream. Skipping.", qName));
310 }
311 }
312
313 private double getDouble(Attributes atts, String value) {
314 return Double.parseDouble(atts.getValue(value));
315 }
316
317 private User createUser(String uid, String name) throws SAXException {
318 if (uid == null) {
319 if (name == null)
320 return null;
321 return User.createLocalUser(name);
322 }
323 try {
324 long id = Long.parseLong(uid);
325 return User.createOsmUser(id, name);
326 } catch(NumberFormatException e) {
327 throwException(tr("Illegal value for attribute ''uid''. Got ''{0}''.", uid));
328 }
329 return null;
330 }
331 /**
332 * Read out the common attributes from atts and put them into this.current.
333 */
334 void readCommon(Attributes atts, OsmPrimitiveData current) throws SAXException {
335 current.id = getLong(atts, "id");
336 if (current.id == 0) {
337 throwException(tr("Illegal object with ID=0."));
338 }
339
340 String time = atts.getValue("timestamp");
341 if (time != null && time.length() != 0) {
342 current.timestamp = DateUtils.fromString(time);
343 }
344
345 // user attribute added in 0.4 API
346 String user = atts.getValue("user");
347 // uid attribute added in 0.6 API
348 String uid = atts.getValue("uid");
349 current.user = createUser(uid, user);
350
351 // visible attribute added in 0.4 API
352 String visible = atts.getValue("visible");
353 if (visible != null) {
354 current.visible = Boolean.parseBoolean(visible);
355 }
356
357 String version = atts.getValue("version");
358 current.version = 0;
359 if (version != null) {
360 try {
361 current.version = Integer.parseInt(version);
362 } catch(NumberFormatException e) {
363 throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.", Long.toString(current.id), version));
364 }
365 if (ds.getVersion().equals("0.6")){
366 if (current.version <= 0 && current.id > 0) {
367 throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.", Long.toString(current.id), version));
368 } else if (current.version < 0 && current.id <=0) {
369 System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.id, current.version, 0, "0.6"));
370 current.version = 0;
371 }
372 } else if (ds.getVersion().equals("0.5")) {
373 if (current.version <= 0 && current.id > 0) {
374 System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.id, current.version, 1, "0.5"));
375 current.version = 1;
376 } else if (current.version < 0 && current.id <=0) {
377 System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.id, current.version, 0, "0.5"));
378 current.version = 0;
379 }
380 } else {
381 // should not happen. API version has been checked before
382 throwException(tr("Unknown or unsupported API version. Got {0}.", ds.getVersion()));
383 }
384 } else {
385 // version expected for OSM primitives with an id assigned by the server (id > 0), since API 0.6
386 //
387 if (current.id > 0 && ds.getVersion() != null && ds.getVersion().equals("0.6")) {
388 throwException(tr("Missing attribute ''version'' on OSM primitive with ID {0}.", Long.toString(current.id)));
389 } else if (current.id > 0 && ds.getVersion() != null && ds.getVersion().equals("0.5")) {
390 // default version in 0.5 files for existing primitives
391 System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.id, current.version, 1, "0.5"));
392 current.version= 1;
393 } else if (current.id <= 0 && ds.getVersion() != null && ds.getVersion().equals("0.5")) {
394 // default version in 0.5 files for new primitives, no warning necessary. This is
395 // (was) legal in API 0.5
396 current.version= 0;
397 }
398 }
399
400 String action = atts.getValue("action");
401 if (action == null) {
402 // do nothing
403 } else if (action.equals("delete")) {
404 current.deleted = true;
405 } else if (action.startsWith("modify")) { //FIXME: why startsWith()? why not equals()?
406 current.modified = true;
407 }
408
409 String v = atts.getValue("changeset");
410 if (v == null) {
411 current.changesetId = 0;
412 } else {
413 try {
414 current.changesetId = Integer.parseInt(v);
415 } catch(NumberFormatException e) {
416 if (current.id <= 0) {
417 // for a new primitive we just log a warning
418 System.out.println(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", v, current.id));
419 current.changesetId = 0;
420 } else {
421 // for an existing primitive this is a problem
422 throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v));
423 }
424 }
425 if (current.changesetId <=0) {
426 if (current.id <= 0) {
427 // for a new primitive we just log a warning
428 System.out.println(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", v, current.id));
429 current.changesetId = 0;
430 } else {
431 // for an existing primitive this is a problem
432 throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v));
433 }
434 }
435 }
436 }
437
438 private long getLong(Attributes atts, String name) throws SAXException {
439 String value = atts.getValue(name);
440 if (value == null) {
441 throwException(tr("Missing required attribute ''{0}''.",name));
442 }
443 try {
444 return Long.parseLong(value);
445 } catch(NumberFormatException e) {
446 throwException(tr("Illegal long value for attribute ''{0}''. Got ''{1}''.",name, value));
447 }
448 return 0; // should not happen
449 }
450 }
451
452 /**
453 * Processes the ways after parsing. Rebuilds the list of nodes of each way and
454 * adds the way to the dataset
455 *
456 * @throws IllegalDataException thrown if a data integrity problem is detected
457 */
458 protected void processWaysAfterParsing() throws IllegalDataException{
459 for (Long externalWayId: ways.keySet()) {
460 Way w = (Way)externalIdMap.get(new SimplePrimitiveId(externalWayId, OsmPrimitiveType.WAY));
461 List<Node> wayNodes = new ArrayList<Node>();
462 for (long id : ways.get(externalWayId)) {
463 Node n = (Node)externalIdMap.get(new SimplePrimitiveId(id, OsmPrimitiveType.NODE));
464 if (n == null) {
465 if (id <= 0)
466 throw new IllegalDataException (
467 tr(
468 "Way with external ID ''{0}'' includes missing node with external ID ''{1}''.",
469 externalWayId,
470 id
471 )
472 );
473 // create an incomplete node if necessary
474 //
475 n = (Node)ds.getPrimitiveById(id,OsmPrimitiveType.NODE);
476 if (n == null) {
477 n = new Node(id);
478 ds.addPrimitive(n);
479 }
480 }
481 wayNodes.add(n);
482 }
483 w.setNodes(wayNodes);
484 if (w.hasIncompleteNodes()) {
485 if (logger.isLoggable(Level.FINE)) {
486 logger.fine(tr("Marked way {0} with {1} nodes incomplete because at least one node was missing in the " +
487 "loaded data and is therefore incomplete too.", externalWayId, w.getNodesCount()));
488 }
489 ds.addPrimitive(w);
490 } else {
491 ds.addPrimitive(w);
492 }
493 }
494 }
495
496 /**
497 * Processes the parsed nodes after parsing. Just adds them to
498 * the dataset
499 *
500 */
501 protected void processNodesAfterParsing() {
502 for (OsmPrimitive primitive: externalIdMap.values()) {
503 if (primitive instanceof Node) {
504 this.ds.addPrimitive(primitive);
505 }
506 }
507 }
508
509 /**
510 * Completes the parsed relations with its members.
511 *
512 * @throws IllegalDataException thrown if a data integrity problem is detected, i.e. if a
513 * relation member refers to a local primitive which wasn't available in the data
514 *
515 */
516 private void processRelationsAfterParsing() throws IllegalDataException {
517 for (Long externalRelationId : relations.keySet()) {
518 Relation relation = (Relation) externalIdMap.get(
519 new SimplePrimitiveId(externalRelationId, OsmPrimitiveType.RELATION)
520 );
521 List<RelationMember> relationMembers = new ArrayList<RelationMember>();
522 for (RelationMemberData rm : relations.get(externalRelationId)) {
523 OsmPrimitive primitive = null;
524
525 // lookup the member from the map of already created primitives
526 //
527 try {
528 OsmPrimitiveType type = OsmPrimitiveType.fromApiTypeName(rm.type);
529 primitive = externalIdMap.get(new SimplePrimitiveId(rm.id, type));
530 } catch(IllegalArgumentException e) {
531 throw new IllegalDataException(
532 tr("Unknown relation member type ''{0}'' in relation with external id ''{1}''.", rm.type,externalRelationId)
533 );
534 }
535
536 if (primitive == null) {
537 if (rm.id <= 0)
538 // relation member refers to a primitive with a negative id which was not
539 // found in the data. This is always a data integrity problem and we abort
540 // with an exception
541 //
542 throw new IllegalDataException(
543 tr(
544 "Relation with external id ''{0}'' refers to a missing primitive with external id ''{1}''.",
545 externalRelationId,
546 rm.id
547 )
548 );
549
550 // member refers to OSM primitive which was not present in the parsed data
551 // -> create a new incomplete primitive and add it to the dataset
552 //
553 if (rm.type.equals("node")) {
554 primitive = new Node(rm.id);
555 } else if (rm.type.equals("way")) {
556 primitive = new Way(rm.id);
557 } else if (rm.type.equals("relation")) {
558 primitive = new Relation(rm.id);
559 } else
560 // can't happen, we've been testing for valid member types
561 // at the beginning of this method
562 //
563 throw new AssertionError();
564 ds.addPrimitive(primitive);
565 externalIdMap.put(new SimplePrimitiveId(rm.id, OsmPrimitiveType.fromApiTypeName(rm.type)), primitive);
566 }
567 relationMembers.add(new RelationMember(rm.role, primitive));
568 }
569 relation.setMembers(relationMembers);
570 ds.addPrimitive(relation);
571 }
572 }
573
574 /**
575 * Parse the given input source and return the dataset.
576 *
577 * @param source the source input stream. Must not be null.
578 * @param progressMonitor the progress monitor. If null, {@see NullProgressMonitor#INSTANCE} is assumed
579 *
580 * @return the dataset with the parsed data
581 * @throws IllegalDataException thrown if the an error was found while parsing the data from the source
582 * @throws IllegalArgumentException thrown if source is null
583 */
584 public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
585 if (progressMonitor == null) {
586 progressMonitor = NullProgressMonitor.INSTANCE;
587 }
588 if (source == null)
589 throw new IllegalArgumentException(tr("Parameter ''{0}'' must not be null", "source"));
590 OsmReader reader = new OsmReader();
591 try {
592 progressMonitor.beginTask(tr("Prepare OSM data...", 2));
593 progressMonitor.indeterminateSubTask(tr("Parsing OSM data..."));
594 InputSource inputSource = new InputSource(new InputStreamReader(source, "UTF-8"));
595 SAXParserFactory.newInstance().newSAXParser().parse(inputSource, reader.new Parser());
596 progressMonitor.worked(1);
597
598 progressMonitor.indeterminateSubTask(tr("Preparing data set..."));
599 reader.ds.beginUpdate();
600 try {
601 reader.processNodesAfterParsing();
602 reader.processWaysAfterParsing();
603 reader.processRelationsAfterParsing();
604 } finally {
605 reader.ds.endUpdate();
606 }
607 progressMonitor.worked(1);
608 return reader.getDataSet();
609 } catch(IllegalDataException e) {
610 throw e;
611 } catch(ParserConfigurationException e) {
612 throw new IllegalDataException(e.getMessage(), e);
613 } catch(SAXException e) {
614 throw new IllegalDataException(e.getMessage(), e);
615 } catch(Exception e) {
616 throw new IllegalDataException(e);
617 } finally {
618 progressMonitor.finishTask();
619 }
620 }
621}
Note: See TracBrowser for help on using the repository browser.