source: josm/trunk/src/org/openstreetmap/josm/io/OsmReader.java@ 2946

Last change on this file since 2946 was 2939, checked in by jttt, 14 years ago

Do not add deleted nodes/members to loaded way/relation
  • Property svn:eol-style set to native
File size: 25.3 KB
Line 
1package org.openstreetmap.josm.io;
2
3import static org.openstreetmap.josm.tools.I18n.tr;
4
5import java.io.InputStream;
6import java.io.InputStreamReader;
7import java.text.MessageFormat;
8import java.util.ArrayList;
9import java.util.Collection;
10import java.util.HashMap;
11import java.util.LinkedList;
12import java.util.List;
13import java.util.Map;
14import java.util.logging.Level;
15import java.util.logging.Logger;
16
17import javax.xml.parsers.ParserConfigurationException;
18import javax.xml.parsers.SAXParserFactory;
19
20import org.openstreetmap.josm.data.Bounds;
21import org.openstreetmap.josm.data.coor.LatLon;
22import org.openstreetmap.josm.data.osm.DataSet;
23import org.openstreetmap.josm.data.osm.DataSource;
24import org.openstreetmap.josm.data.osm.Node;
25import org.openstreetmap.josm.data.osm.NodeData;
26import org.openstreetmap.josm.data.osm.OsmPrimitive;
27import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
28import org.openstreetmap.josm.data.osm.PrimitiveData;
29import org.openstreetmap.josm.data.osm.PrimitiveId;
30import org.openstreetmap.josm.data.osm.Relation;
31import org.openstreetmap.josm.data.osm.RelationData;
32import org.openstreetmap.josm.data.osm.RelationMember;
33import org.openstreetmap.josm.data.osm.SimplePrimitiveId;
34import org.openstreetmap.josm.data.osm.User;
35import org.openstreetmap.josm.data.osm.Way;
36import org.openstreetmap.josm.data.osm.WayData;
37import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
38import org.openstreetmap.josm.gui.progress.ProgressMonitor;
39import org.openstreetmap.josm.tools.CheckParameterUtil;
40import org.openstreetmap.josm.tools.DateUtils;
41import org.xml.sax.Attributes;
42import org.xml.sax.InputSource;
43import org.xml.sax.Locator;
44import org.xml.sax.SAXException;
45import org.xml.sax.helpers.DefaultHandler;
46
47/**
48 * Parser for the Osm Api. Read from an input stream and construct a dataset out of it.
49 *
50 */
51public class OsmReader {
52 static private final Logger logger = Logger.getLogger(OsmReader.class.getName());
53
54 /**
55 * The dataset to add parsed objects to.
56 */
57 private DataSet ds = new DataSet();
58
59 /**
60 * Replies the parsed data set
61 *
62 * @return the parsed data set
63 */
64 public DataSet getDataSet() {
65 return ds;
66 }
67
68 /** the map from external ids to read OsmPrimitives. External ids are
69 * longs too, but in contrast to internal ids negative values are used
70 * to identify primitives unknown to the OSM server
71 */
72 private Map<PrimitiveId, OsmPrimitive> externalIdMap = new HashMap<PrimitiveId, OsmPrimitive>();
73
74 /**
75 * constructor (for private use only)
76 *
77 * @see #parseDataSet(InputStream, DataSet, ProgressMonitor)
78 * @see #parseDataSetOsm(InputStream, DataSet, ProgressMonitor)
79 */
80 private OsmReader() {
81 externalIdMap = new HashMap<PrimitiveId, OsmPrimitive>();
82 }
83
84 /**
85 * Used as a temporary storage for relation members, before they
86 * are resolved into pointers to real objects.
87 */
88 private static class RelationMemberData {
89 public OsmPrimitiveType type;
90 public long id;
91 public String role;
92 }
93
94 /**
95 * Data structure for the remaining way objects
96 */
97 private Map<Long, Collection<Long>> ways = new HashMap<Long, Collection<Long>>();
98
99 /**
100 * Data structure for relation objects
101 */
102 private Map<Long, Collection<RelationMemberData>> relations = new HashMap<Long, Collection<RelationMemberData>>();
103
104 private class Parser extends DefaultHandler {
105 private Locator locator;
106
107 @Override
108 public void setDocumentLocator(Locator locator) {
109 this.locator = locator;
110 }
111
112 protected void throwException(String msg) throws OsmDataParsingException{
113 throw new OsmDataParsingException(msg).rememberLocation(locator);
114 }
115 /**
116 * The current osm primitive to be read.
117 */
118 private OsmPrimitive currentPrimitive;
119 private long currentExternalId;
120 private String generator;
121
122 @Override public void startElement(String namespaceURI, String localName, String qName, Attributes atts) throws SAXException {
123 if (qName.equals("osm")) {
124 if (atts == null) {
125 throwException(tr("Missing mandatory attribute ''{0}'' of XML element {1}.", "version", "osm"));
126 }
127 String v = atts.getValue("version");
128 if (v == null) {
129 throwException(tr("Missing mandatory attribute ''{0}''.", "version"));
130 }
131 if (!(v.equals("0.5") || v.equals("0.6"))) {
132 throwException(tr("Unsupported version: {0}", v));
133 }
134 // save generator attribute for later use when creating DataSource objects
135 generator = atts.getValue("generator");
136 ds.setVersion(v);
137
138 } else if (qName.equals("bounds")) {
139 // new style bounds.
140 String minlon = atts.getValue("minlon");
141 String minlat = atts.getValue("minlat");
142 String maxlon = atts.getValue("maxlon");
143 String maxlat = atts.getValue("maxlat");
144 String origin = atts.getValue("origin");
145 if (minlon != null && maxlon != null && minlat != null && maxlat != null) {
146 if (origin == null) {
147 origin = generator;
148 }
149 Bounds bounds = new Bounds(
150 new LatLon(Double.parseDouble(minlat), Double.parseDouble(minlon)),
151 new LatLon(Double.parseDouble(maxlat), Double.parseDouble(maxlon)));
152 DataSource src = new DataSource(bounds, origin);
153 ds.dataSources.add(src);
154 } else {
155 throwException(tr(
156 "Missing manadatory attributes on element ''bounds''. Got minlon=''{0}'',minlat=''{1}'',maxlon=''{3}'',maxlat=''{4}'', origin=''{5}''.",
157 minlon, minlat, maxlon, maxlat, origin
158 ));
159 }
160
161 // ---- PARSING NODES AND WAYS ----
162
163 } else if (qName.equals("node")) {
164 NodeData nd = new NodeData();
165 nd.setCoor(new LatLon(getDouble(atts, "lat"), getDouble(atts, "lon")));
166 readCommon(atts, nd);
167 Node n = new Node(nd.getId(), nd.getVersion());
168 n.load(nd);
169 externalIdMap.put(nd.getPrimitiveId(), n);
170 currentPrimitive = n;
171 currentExternalId = nd.getUniqueId();
172 } else if (qName.equals("way")) {
173 WayData wd = new WayData();
174 readCommon(atts, wd);
175 Way w = new Way(wd.getId(), wd.getVersion());
176 w.load(wd);
177 externalIdMap.put(wd.getPrimitiveId(), w);
178 ways.put(wd.getUniqueId(), new ArrayList<Long>());
179 currentPrimitive = w;
180 currentExternalId = wd.getUniqueId();
181 } else if (qName.equals("nd")) {
182 Collection<Long> list = ways.get(currentExternalId);
183 if (list == null) {
184 throwException(
185 tr("Found XML element <nd> not as direct child of element <way>.")
186 );
187 }
188 if (atts.getValue("ref") == null) {
189 throwException(
190 tr("Missing mandatory attribute ''{0}'' on <nd> of way {1}.", "ref", currentPrimitive.getUniqueId())
191 );
192 }
193 long id = getLong(atts, "ref");
194 if (id == 0) {
195 throwException(
196 tr("Illegal value of attribute ''ref'' of element <nd>. Got {0}.", id)
197 );
198 }
199 if (currentPrimitive.isDeleted()) {
200 logger.info(tr("Deleted way {0} contains nodes", currentPrimitive.getUniqueId()));
201 } else {
202 list.add(id);
203 }
204
205 // ---- PARSING RELATIONS ----
206
207 } else if (qName.equals("relation")) {
208 RelationData rd = new RelationData();
209 readCommon(atts, rd);
210 Relation r = new Relation(rd.getId(), rd.getVersion());
211 r.load(rd);
212 externalIdMap.put(rd.getPrimitiveId(), r);
213 relations.put(rd.getUniqueId(), new LinkedList<RelationMemberData>());
214 currentPrimitive = r;
215 currentExternalId = rd.getUniqueId();
216 } else if (qName.equals("member")) {
217 Collection<RelationMemberData> list = relations.get(currentExternalId);
218 if (list == null) {
219 throwException(
220 tr("Found XML element <member> not as direct child of element <relation>.")
221 );
222 }
223 RelationMemberData emd = new RelationMemberData();
224 String value = atts.getValue("ref");
225 if (value == null) {
226 throwException(tr("Missing attribute ''ref'' on member in relation {0}.",currentPrimitive.getUniqueId()));
227 }
228 try {
229 emd.id = Long.parseLong(value);
230 } catch(NumberFormatException e) {
231 throwException(tr("Illegal value for attribute ''ref'' on member in relation {0}. Got {1}", Long.toString(currentPrimitive.getUniqueId()),value));
232 }
233 value = atts.getValue("type");
234 if (value == null) {
235 throwException(tr("Missing attribute ''type'' on member {0} in relation {1}.", Long.toString(emd.id), Long.toString(currentPrimitive.getUniqueId())));
236 }
237 try {
238 emd.type = OsmPrimitiveType.fromApiTypeName(value);
239 } catch(IllegalArgumentException e) {
240 throwException(tr("Illegal value for attribute ''type'' on member {0} in relation {1}. Got {2}.", Long.toString(emd.id), Long.toString(currentPrimitive.getUniqueId()), value));
241 }
242 value = atts.getValue("role");
243 emd.role = value;
244
245 if (emd.id == 0) {
246 throwException(tr("Incomplete <member> specification with ref=0"));
247 }
248
249 if (currentPrimitive.isDeleted()) {
250 logger.info(tr("Deleted relation {0} contains members", currentPrimitive.getUniqueId()));
251 } else {
252 list.add(emd);
253 }
254
255 // ---- PARSING TAGS (applicable to all objects) ----
256
257 } else if (qName.equals("tag")) {
258 String key = atts.getValue("k");
259 String value = atts.getValue("v");
260 currentPrimitive.put(key, value);
261 } else {
262 System.out.println(tr("Undefined element ''{0}'' found in input stream. Skipping.", qName));
263 }
264 }
265
266 private double getDouble(Attributes atts, String value) {
267 return Double.parseDouble(atts.getValue(value));
268 }
269
270 private User createUser(String uid, String name) throws SAXException {
271 if (uid == null) {
272 if (name == null)
273 return null;
274 return User.createLocalUser(name);
275 }
276 try {
277 long id = Long.parseLong(uid);
278 return User.createOsmUser(id, name);
279 } catch(NumberFormatException e) {
280 throwException(MessageFormat.format("Illegal value for attribute ''uid''. Got ''{0}''.", uid));
281 }
282 return null;
283 }
284 /**
285 * Read out the common attributes from atts and put them into this.current.
286 */
287 void readCommon(Attributes atts, PrimitiveData current) throws SAXException {
288 current.setId(getLong(atts, "id"));
289 if (current.getUniqueId() == 0) {
290 throwException(tr("Illegal object with ID=0."));
291 }
292
293 String time = atts.getValue("timestamp");
294 if (time != null && time.length() != 0) {
295 current.setTimestamp(DateUtils.fromString(time));
296 }
297
298 // user attribute added in 0.4 API
299 String user = atts.getValue("user");
300 // uid attribute added in 0.6 API
301 String uid = atts.getValue("uid");
302 current.setUser(createUser(uid, user));
303
304 // visible attribute added in 0.4 API
305 String visible = atts.getValue("visible");
306 if (visible != null) {
307 current.setVisible(Boolean.parseBoolean(visible));
308 }
309
310 String versionString = atts.getValue("version");
311 int version = 0;
312 if (versionString != null) {
313 try {
314 version = Integer.parseInt(versionString);
315 } catch(NumberFormatException e) {
316 throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.", Long.toString(current.getUniqueId()), versionString));
317 }
318 if (ds.getVersion().equals("0.6")){
319 if (version <= 0 && current.getUniqueId() > 0) {
320 throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.", Long.toString(current.getUniqueId()), versionString));
321 } else if (version < 0 && current.getUniqueId() <= 0) {
322 System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 0, "0.6"));
323 version = 0;
324 }
325 } else if (ds.getVersion().equals("0.5")) {
326 if (version <= 0 && current.getUniqueId() > 0) {
327 System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 1, "0.5"));
328 version = 1;
329 } else if (version < 0 && current.getUniqueId() <= 0) {
330 System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 0, "0.5"));
331 version = 0;
332 }
333 } else {
334 // should not happen. API version has been checked before
335 throwException(tr("Unknown or unsupported API version. Got {0}.", ds.getVersion()));
336 }
337 } else {
338 // version expected for OSM primitives with an id assigned by the server (id > 0), since API 0.6
339 //
340 if (current.getUniqueId() > 0 && ds.getVersion() != null && ds.getVersion().equals("0.6")) {
341 throwException(tr("Missing attribute ''version'' on OSM primitive with ID {0}.", Long.toString(current.getUniqueId())));
342 } else if (current.getUniqueId() > 0 && ds.getVersion() != null && ds.getVersion().equals("0.5")) {
343 // default version in 0.5 files for existing primitives
344 System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 1, "0.5"));
345 version= 1;
346 } else if (current.getUniqueId() <= 0 && ds.getVersion() != null && ds.getVersion().equals("0.5")) {
347 // default version in 0.5 files for new primitives, no warning necessary. This is
348 // (was) legal in API 0.5
349 version= 0;
350 }
351 }
352 current.setVersion(version);
353
354 String action = atts.getValue("action");
355 if (action == null) {
356 // do nothing
357 } else if (action.equals("delete")) {
358 current.setDeleted(true);
359 } else if (action.startsWith("modify")) { //FIXME: why startsWith()? why not equals()?
360 current.setModified(true);
361 }
362
363 String v = atts.getValue("changeset");
364 if (v == null) {
365 current.setChangesetId(0);
366 } else {
367 try {
368 current.setChangesetId(Integer.parseInt(v));
369 } catch(NumberFormatException e) {
370 if (current.getUniqueId() <= 0) {
371 // for a new primitive we just log a warning
372 System.out.println(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", v, current.getUniqueId()));
373 current.setChangesetId(0);
374 } else {
375 // for an existing primitive this is a problem
376 throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v));
377 }
378 }
379 if (current.getChangesetId() <=0) {
380 if (current.getUniqueId() <= 0) {
381 // for a new primitive we just log a warning
382 System.out.println(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", v, current.getUniqueId()));
383 current.setChangesetId(0);
384 } else {
385 // for an existing primitive this is a problem
386 throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v));
387 }
388 }
389 }
390 }
391
392 private long getLong(Attributes atts, String name) throws SAXException {
393 String value = atts.getValue(name);
394 if (value == null) {
395 throwException(tr("Missing required attribute ''{0}''.",name));
396 }
397 try {
398 return Long.parseLong(value);
399 } catch(NumberFormatException e) {
400 throwException(tr("Illegal long value for attribute ''{0}''. Got ''{1}''.",name, value));
401 }
402 return 0; // should not happen
403 }
404 }
405
406 /**
407 * Processes the ways after parsing. Rebuilds the list of nodes of each way and
408 * adds the way to the dataset
409 *
410 * @throws IllegalDataException thrown if a data integrity problem is detected
411 */
412 protected void processWaysAfterParsing() throws IllegalDataException{
413 for (Long externalWayId: ways.keySet()) {
414 Way w = (Way)externalIdMap.get(new SimplePrimitiveId(externalWayId, OsmPrimitiveType.WAY));
415 List<Node> wayNodes = new ArrayList<Node>();
416 for (long id : ways.get(externalWayId)) {
417 Node n = (Node)externalIdMap.get(new SimplePrimitiveId(id, OsmPrimitiveType.NODE));
418 if (n == null) {
419 if (id <= 0)
420 throw new IllegalDataException (
421 tr("Way with external ID ''{0}'' includes missing node with external ID ''{1}''.",
422 externalWayId,
423 id));
424 // create an incomplete node if necessary
425 //
426 n = (Node)ds.getPrimitiveById(id,OsmPrimitiveType.NODE);
427 if (n == null) {
428 n = new Node(id);
429 ds.addPrimitive(n);
430 }
431 }
432 if (n.isDeleted()) {
433 logger.warning(tr("Deleted node {0} is part of way {1}", id, w.getId()));
434 } else {
435 wayNodes.add(n);
436 }
437 }
438 w.setNodes(wayNodes);
439 if (w.hasIncompleteNodes()) {
440 if (logger.isLoggable(Level.FINE)) {
441 logger.fine(tr("Way {0} with {1} nodes has incomplete nodes because at least one node was missing in the loaded data.",
442 externalWayId, w.getNodesCount()));
443 }
444 }
445 ds.addPrimitive(w);
446 }
447 }
448
449 /**
450 * Processes the parsed nodes after parsing. Just adds them to
451 * the dataset
452 *
453 */
454 protected void processNodesAfterParsing() {
455 for (OsmPrimitive primitive: externalIdMap.values()) {
456 if (primitive instanceof Node) {
457 this.ds.addPrimitive(primitive);
458 }
459 }
460 }
461
462 /**
463 * Completes the parsed relations with its members.
464 *
465 * @throws IllegalDataException thrown if a data integrity problem is detected, i.e. if a
466 * relation member refers to a local primitive which wasn't available in the data
467 *
468 */
469 private void processRelationsAfterParsing() throws IllegalDataException {
470 for (Long externalRelationId : relations.keySet()) {
471 Relation relation = (Relation) externalIdMap.get(
472 new SimplePrimitiveId(externalRelationId, OsmPrimitiveType.RELATION)
473 );
474 List<RelationMember> relationMembers = new ArrayList<RelationMember>();
475 for (RelationMemberData rm : relations.get(externalRelationId)) {
476 OsmPrimitive primitive = null;
477
478 // lookup the member from the map of already created primitives
479 primitive = externalIdMap.get(new SimplePrimitiveId(rm.id, rm.type));
480
481 if (primitive == null) {
482 if (rm.id <= 0)
483 // relation member refers to a primitive with a negative id which was not
484 // found in the data. This is always a data integrity problem and we abort
485 // with an exception
486 //
487 throw new IllegalDataException(
488 tr("Relation with external id ''{0}'' refers to a missing primitive with external id ''{1}''.",
489 externalRelationId,
490 rm.id));
491
492 // member refers to OSM primitive which was not present in the parsed data
493 // -> create a new incomplete primitive and add it to the dataset
494 //
495 primitive = ds.getPrimitiveById(rm.id, rm.type);
496 if (primitive == null) {
497 switch (rm.type) {
498 case NODE:
499 primitive = new Node(rm.id); break;
500 case WAY:
501 primitive = new Way(rm.id); break;
502 case RELATION:
503 primitive = new Relation(rm.id); break;
504 default: throw new AssertionError(); // can't happen
505 }
506
507 ds.addPrimitive(primitive);
508 externalIdMap.put(new SimplePrimitiveId(rm.id, rm.type), primitive);
509 }
510 }
511 if (primitive.isDeleted()) {
512 logger.warning(tr("Deleted member {0} is used by relation {1}", primitive.getId(), relation.getId()));
513 } else {
514 relationMembers.add(new RelationMember(rm.role, primitive));
515 }
516 }
517 relation.setMembers(relationMembers);
518 ds.addPrimitive(relation);
519 }
520 }
521
522 /**
523 * Parse the given input source and return the dataset.
524 *
525 * @param source the source input stream. Must not be null.
526 * @param progressMonitor the progress monitor. If null, {@see NullProgressMonitor#INSTANCE} is assumed
527 *
528 * @return the dataset with the parsed data
529 * @throws IllegalDataException thrown if the an error was found while parsing the data from the source
530 * @throws IllegalArgumentException thrown if source is null
531 */
532 public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
533 if (progressMonitor == null) {
534 progressMonitor = NullProgressMonitor.INSTANCE;
535 }
536 CheckParameterUtil.ensureParameterNotNull(source, "source");
537 OsmReader reader = new OsmReader();
538 try {
539 progressMonitor.beginTask(tr("Prepare OSM data...", 2));
540 progressMonitor.indeterminateSubTask(tr("Parsing OSM data..."));
541 InputSource inputSource = new InputSource(new InputStreamReader(source, "UTF-8"));
542 SAXParserFactory.newInstance().newSAXParser().parse(inputSource, reader.new Parser());
543 progressMonitor.worked(1);
544
545 progressMonitor.indeterminateSubTask(tr("Preparing data set..."));
546 reader.ds.beginUpdate();
547 try {
548 reader.processNodesAfterParsing();
549 reader.processWaysAfterParsing();
550 reader.processRelationsAfterParsing();
551 } finally {
552 reader.ds.endUpdate();
553 }
554 progressMonitor.worked(1);
555 return reader.getDataSet();
556 } catch(IllegalDataException e) {
557 throw e;
558 } catch(ParserConfigurationException e) {
559 throw new IllegalDataException(e.getMessage(), e);
560 } catch(SAXException e) {
561 throw new IllegalDataException(e.getMessage(), e);
562 } catch(Exception e) {
563 throw new IllegalDataException(e);
564 } finally {
565 progressMonitor.finishTask();
566 }
567 }
568}
Note: See TracBrowser for help on using the repository browser.