source: josm/trunk/src/org/openstreetmap/josm/io/OsmReader.java@ 3225

Last change on this file since 3225 was 3217, checked in by bastiK, 14 years ago

fixed #4971 NullPointerException at OsmReader.java:586, caused by NullPointerException at Storage.java:317;
fixed #4964 exception undo "node-merge" with nodes children of relations.
  • Property svn:eol-style set to native
File size: 26.3 KB
Line 
1package org.openstreetmap.josm.io;
2
3import static org.openstreetmap.josm.tools.I18n.tr;
4
5import java.io.InputStream;
6import java.io.InputStreamReader;
7import java.text.MessageFormat;
8import java.util.ArrayList;
9import java.util.Collection;
10import java.util.HashMap;
11import java.util.LinkedList;
12import java.util.List;
13import java.util.Map;
14import java.util.logging.Level;
15import java.util.logging.Logger;
16
17import javax.xml.parsers.ParserConfigurationException;
18import javax.xml.parsers.SAXParserFactory;
19
20import org.openstreetmap.josm.data.Bounds;
21import org.openstreetmap.josm.data.coor.LatLon;
22import org.openstreetmap.josm.data.osm.DataSet;
23import org.openstreetmap.josm.data.osm.DataSource;
24import org.openstreetmap.josm.data.osm.Node;
25import org.openstreetmap.josm.data.osm.NodeData;
26import org.openstreetmap.josm.data.osm.OsmPrimitive;
27import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
28import org.openstreetmap.josm.data.osm.PrimitiveData;
29import org.openstreetmap.josm.data.osm.PrimitiveId;
30import org.openstreetmap.josm.data.osm.Relation;
31import org.openstreetmap.josm.data.osm.RelationData;
32import org.openstreetmap.josm.data.osm.RelationMember;
33import org.openstreetmap.josm.data.osm.SimplePrimitiveId;
34import org.openstreetmap.josm.data.osm.Storage;
35import org.openstreetmap.josm.data.osm.User;
36import org.openstreetmap.josm.data.osm.Way;
37import org.openstreetmap.josm.data.osm.WayData;
38import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
39import org.openstreetmap.josm.gui.progress.ProgressMonitor;
40import org.openstreetmap.josm.tools.CheckParameterUtil;
41import org.openstreetmap.josm.tools.DateUtils;
42import org.xml.sax.Attributes;
43import org.xml.sax.InputSource;
44import org.xml.sax.Locator;
45import org.xml.sax.SAXException;
46import org.xml.sax.helpers.DefaultHandler;
47
48/**
49 * Parser for the Osm Api. Read from an input stream and construct a dataset out of it.
50 *
51 */
52public class OsmReader {
53 static private final Logger logger = Logger.getLogger(OsmReader.class.getName());
54
55 /**
56 * The dataset to add parsed objects to.
57 */
58 private DataSet ds = new DataSet();
59
60 /**
61 * Replies the parsed data set
62 *
63 * @return the parsed data set
64 */
65 public DataSet getDataSet() {
66 return ds;
67 }
68
69 /** the map from external ids to read OsmPrimitives. External ids are
70 * longs too, but in contrast to internal ids negative values are used
71 * to identify primitives unknown to the OSM server
72 */
73 private Map<PrimitiveId, OsmPrimitive> externalIdMap = new HashMap<PrimitiveId, OsmPrimitive>();
74
75 /**
76 * constructor (for private use only)
77 *
78 * @see #parseDataSet(InputStream, DataSet, ProgressMonitor)
79 * @see #parseDataSetOsm(InputStream, DataSet, ProgressMonitor)
80 */
81 private OsmReader() {
82 externalIdMap = new HashMap<PrimitiveId, OsmPrimitive>();
83 }
84
85 /**
86 * Used as a temporary storage for relation members, before they
87 * are resolved into pointers to real objects.
88 */
89 private static class RelationMemberData {
90 public OsmPrimitiveType type;
91 public long id;
92 public String role;
93 }
94
95 /**
96 * Data structure for the remaining way objects
97 */
98 private Map<Long, Collection<Long>> ways = new HashMap<Long, Collection<Long>>();
99
100 /**
101 * Data structure for relation objects
102 */
103 private Map<Long, Collection<RelationMemberData>> relations = new HashMap<Long, Collection<RelationMemberData>>();
104
105 private class Parser extends DefaultHandler {
106 private Locator locator;
107
108 @Override
109 public void setDocumentLocator(Locator locator) {
110 this.locator = locator;
111 }
112
113 protected void throwException(String msg) throws OsmDataParsingException{
114 throw new OsmDataParsingException(msg).rememberLocation(locator);
115 }
116 /**
117 * The current osm primitive to be read.
118 */
119 private OsmPrimitive currentPrimitive;
120 private long currentExternalId;
121 private String generator;
122 private Storage<String> internedStrings = new Storage<String>();
123
124 // Memory optimization - see #2312
125 private String intern(String s) {
126 String result = internedStrings.get(s);
127 if (result == null) {
128 internedStrings.put(s);
129 return s;
130 } else
131 return result;
132 }
133
134 @Override public void startElement(String namespaceURI, String localName, String qName, Attributes atts) throws SAXException {
135
136 if (qName.equals("osm")) {
137 if (atts == null) {
138 throwException(tr("Missing mandatory attribute ''{0}'' of XML element {1}.", "version", "osm"));
139 }
140 String v = atts.getValue("version");
141 if (v == null) {
142 throwException(tr("Missing mandatory attribute ''{0}''.", "version"));
143 }
144 if (!(v.equals("0.5") || v.equals("0.6"))) {
145 throwException(tr("Unsupported version: {0}", v));
146 }
147 // save generator attribute for later use when creating DataSource objects
148 generator = atts.getValue("generator");
149 ds.setVersion(v);
150
151 } else if (qName.equals("bounds")) {
152 // new style bounds.
153 String minlon = atts.getValue("minlon");
154 String minlat = atts.getValue("minlat");
155 String maxlon = atts.getValue("maxlon");
156 String maxlat = atts.getValue("maxlat");
157 String origin = atts.getValue("origin");
158 if (minlon != null && maxlon != null && minlat != null && maxlat != null) {
159 if (origin == null) {
160 origin = generator;
161 }
162 Bounds bounds = new Bounds(
163 new LatLon(Double.parseDouble(minlat), Double.parseDouble(minlon)),
164 new LatLon(Double.parseDouble(maxlat), Double.parseDouble(maxlon)));
165 DataSource src = new DataSource(bounds, origin);
166 ds.dataSources.add(src);
167 } else {
168 throwException(tr(
169 "Missing manadatory attributes on element ''bounds''. Got minlon=''{0}'',minlat=''{1}'',maxlon=''{3}'',maxlat=''{4}'', origin=''{5}''.",
170 minlon, minlat, maxlon, maxlat, origin
171 ));
172 }
173
174 // ---- PARSING NODES AND WAYS ----
175
176 } else if (qName.equals("node")) {
177 NodeData nd = new NodeData();
178 nd.setCoor(new LatLon(getDouble(atts, "lat"), getDouble(atts, "lon")));
179 readCommon(atts, nd);
180 Node n = new Node(nd.getId(), nd.getVersion());
181 n.load(nd);
182 externalIdMap.put(nd.getPrimitiveId(), n);
183 currentPrimitive = n;
184 currentExternalId = nd.getUniqueId();
185 } else if (qName.equals("way")) {
186 WayData wd = new WayData();
187 readCommon(atts, wd);
188 Way w = new Way(wd.getId(), wd.getVersion());
189 w.load(wd);
190 externalIdMap.put(wd.getPrimitiveId(), w);
191 ways.put(wd.getUniqueId(), new ArrayList<Long>());
192 currentPrimitive = w;
193 currentExternalId = wd.getUniqueId();
194 } else if (qName.equals("nd")) {
195 Collection<Long> list = ways.get(currentExternalId);
196 if (list == null) {
197 throwException(
198 tr("Found XML element <nd> not as direct child of element <way>.")
199 );
200 }
201 if (atts.getValue("ref") == null) {
202 throwException(
203 tr("Missing mandatory attribute ''{0}'' on <nd> of way {1}.", "ref", currentPrimitive.getUniqueId())
204 );
205 }
206 long id = getLong(atts, "ref");
207 if (id == 0) {
208 throwException(
209 tr("Illegal value of attribute ''ref'' of element <nd>. Got {0}.", id)
210 );
211 }
212 if (currentPrimitive.isDeleted()) {
213 logger.info(tr("Deleted way {0} contains nodes", currentPrimitive.getUniqueId()));
214 } else {
215 list.add(id);
216 }
217
218 // ---- PARSING RELATIONS ----
219
220 } else if (qName.equals("relation")) {
221 RelationData rd = new RelationData();
222 readCommon(atts, rd);
223 Relation r = new Relation(rd.getId(), rd.getVersion());
224 r.load(rd);
225 externalIdMap.put(rd.getPrimitiveId(), r);
226 relations.put(rd.getUniqueId(), new LinkedList<RelationMemberData>());
227 currentPrimitive = r;
228 currentExternalId = rd.getUniqueId();
229 } else if (qName.equals("member")) {
230 Collection<RelationMemberData> list = relations.get(currentExternalId);
231 if (list == null) {
232 throwException(
233 tr("Found XML element <member> not as direct child of element <relation>.")
234 );
235 }
236 RelationMemberData emd = new RelationMemberData();
237 String value = atts.getValue("ref");
238 if (value == null) {
239 throwException(tr("Missing attribute ''ref'' on member in relation {0}.",currentPrimitive.getUniqueId()));
240 }
241 try {
242 emd.id = Long.parseLong(value);
243 } catch(NumberFormatException e) {
244 throwException(tr("Illegal value for attribute ''ref'' on member in relation {0}. Got {1}", Long.toString(currentPrimitive.getUniqueId()),value));
245 }
246 value = atts.getValue("type");
247 if (value == null) {
248 throwException(tr("Missing attribute ''type'' on member {0} in relation {1}.", Long.toString(emd.id), Long.toString(currentPrimitive.getUniqueId())));
249 }
250 try {
251 emd.type = OsmPrimitiveType.fromApiTypeName(value);
252 } catch(IllegalArgumentException e) {
253 throwException(tr("Illegal value for attribute ''type'' on member {0} in relation {1}. Got {2}.", Long.toString(emd.id), Long.toString(currentPrimitive.getUniqueId()), value));
254 }
255 value = atts.getValue("role");
256 emd.role = value;
257
258 if (emd.id == 0) {
259 throwException(tr("Incomplete <member> specification with ref=0"));
260 }
261
262 if (currentPrimitive.isDeleted()) {
263 logger.info(tr("Deleted relation {0} contains members", currentPrimitive.getUniqueId()));
264 } else {
265 list.add(emd);
266 }
267
268 // ---- PARSING TAGS (applicable to all objects) ----
269
270 } else if (qName.equals("tag")) {
271 String key = atts.getValue("k");
272 String value = atts.getValue("v");
273 if (key == null || value == null) {
274 throwException(tr("Missing key or value attribute in tag."));
275 }
276 currentPrimitive.put(intern(key), intern(value));
277
278 } else {
279 System.out.println(tr("Undefined element ''{0}'' found in input stream. Skipping.", qName));
280 }
281 }
282
283 private double getDouble(Attributes atts, String value) {
284 return Double.parseDouble(atts.getValue(value));
285 }
286
287 private User createUser(String uid, String name) throws SAXException {
288 if (uid == null) {
289 if (name == null)
290 return null;
291 return User.createLocalUser(name);
292 }
293 try {
294 long id = Long.parseLong(uid);
295 return User.createOsmUser(id, name);
296 } catch(NumberFormatException e) {
297 throwException(MessageFormat.format("Illegal value for attribute ''uid''. Got ''{0}''.", uid));
298 }
299 return null;
300 }
301 /**
302 * Read out the common attributes from atts and put them into this.current.
303 */
304 void readCommon(Attributes atts, PrimitiveData current) throws SAXException {
305 current.setId(getLong(atts, "id"));
306 if (current.getUniqueId() == 0) {
307 throwException(tr("Illegal object with ID=0."));
308 }
309
310 String time = atts.getValue("timestamp");
311 if (time != null && time.length() != 0) {
312 current.setTimestamp(DateUtils.fromString(time));
313 }
314
315 // user attribute added in 0.4 API
316 String user = atts.getValue("user");
317 // uid attribute added in 0.6 API
318 String uid = atts.getValue("uid");
319 current.setUser(createUser(uid, user));
320
321 // visible attribute added in 0.4 API
322 String visible = atts.getValue("visible");
323 if (visible != null) {
324 current.setVisible(Boolean.parseBoolean(visible));
325 }
326
327 String versionString = atts.getValue("version");
328 int version = 0;
329 if (versionString != null) {
330 try {
331 version = Integer.parseInt(versionString);
332 } catch(NumberFormatException e) {
333 throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.", Long.toString(current.getUniqueId()), versionString));
334 }
335 if (ds.getVersion().equals("0.6")){
336 if (version <= 0 && current.getUniqueId() > 0) {
337 throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.", Long.toString(current.getUniqueId()), versionString));
338 } else if (version < 0 && current.getUniqueId() <= 0) {
339 System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 0, "0.6"));
340 version = 0;
341 }
342 } else if (ds.getVersion().equals("0.5")) {
343 if (version <= 0 && current.getUniqueId() > 0) {
344 System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 1, "0.5"));
345 version = 1;
346 } else if (version < 0 && current.getUniqueId() <= 0) {
347 System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 0, "0.5"));
348 version = 0;
349 }
350 } else {
351 // should not happen. API version has been checked before
352 throwException(tr("Unknown or unsupported API version. Got {0}.", ds.getVersion()));
353 }
354 } else {
355 // version expected for OSM primitives with an id assigned by the server (id > 0), since API 0.6
356 //
357 if (current.getUniqueId() > 0 && ds.getVersion() != null && ds.getVersion().equals("0.6")) {
358 throwException(tr("Missing attribute ''version'' on OSM primitive with ID {0}.", Long.toString(current.getUniqueId())));
359 } else if (current.getUniqueId() > 0 && ds.getVersion() != null && ds.getVersion().equals("0.5")) {
360 // default version in 0.5 files for existing primitives
361 System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 1, "0.5"));
362 version= 1;
363 } else if (current.getUniqueId() <= 0 && ds.getVersion() != null && ds.getVersion().equals("0.5")) {
364 // default version in 0.5 files for new primitives, no warning necessary. This is
365 // (was) legal in API 0.5
366 version= 0;
367 }
368 }
369 current.setVersion(version);
370
371 String action = atts.getValue("action");
372 if (action == null) {
373 // do nothing
374 } else if (action.equals("delete")) {
375 current.setDeleted(true);
376 current.setModified(true);
377 } else if (action.equals("modify")) {
378 current.setModified(true);
379 }
380
381 String v = atts.getValue("changeset");
382 if (v == null) {
383 current.setChangesetId(0);
384 } else {
385 try {
386 current.setChangesetId(Integer.parseInt(v));
387 } catch(NumberFormatException e) {
388 if (current.getUniqueId() <= 0) {
389 // for a new primitive we just log a warning
390 System.out.println(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", v, current.getUniqueId()));
391 current.setChangesetId(0);
392 } else {
393 // for an existing primitive this is a problem
394 throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v));
395 }
396 }
397 if (current.getChangesetId() <=0) {
398 if (current.getUniqueId() <= 0) {
399 // for a new primitive we just log a warning
400 System.out.println(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", v, current.getUniqueId()));
401 current.setChangesetId(0);
402 } else {
403 // for an existing primitive this is a problem
404 throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v));
405 }
406 }
407 }
408 }
409
410 private long getLong(Attributes atts, String name) throws SAXException {
411 String value = atts.getValue(name);
412 if (value == null) {
413 throwException(tr("Missing required attribute ''{0}''.",name));
414 }
415 try {
416 return Long.parseLong(value);
417 } catch(NumberFormatException e) {
418 throwException(tr("Illegal long value for attribute ''{0}''. Got ''{1}''.",name, value));
419 }
420 return 0; // should not happen
421 }
422 }
423
424 /**
425 * Processes the ways after parsing. Rebuilds the list of nodes of each way and
426 * adds the way to the dataset
427 *
428 * @throws IllegalDataException thrown if a data integrity problem is detected
429 */
430 protected void processWaysAfterParsing() throws IllegalDataException{
431 for (Long externalWayId: ways.keySet()) {
432 Way w = (Way)externalIdMap.get(new SimplePrimitiveId(externalWayId, OsmPrimitiveType.WAY));
433 List<Node> wayNodes = new ArrayList<Node>();
434 for (long id : ways.get(externalWayId)) {
435 Node n = (Node)externalIdMap.get(new SimplePrimitiveId(id, OsmPrimitiveType.NODE));
436 if (n == null) {
437 if (id <= 0)
438 throw new IllegalDataException (
439 tr("Way with external ID ''{0}'' includes missing node with external ID ''{1}''.",
440 externalWayId,
441 id));
442 // create an incomplete node if necessary
443 //
444 n = (Node)ds.getPrimitiveById(id,OsmPrimitiveType.NODE);
445 if (n == null) {
446 n = new Node(id);
447 ds.addPrimitive(n);
448 }
449 }
450 if (n.isDeleted()) {
451 logger.warning(tr("Deleted node {0} is part of way {1}", id, w.getId()));
452 } else {
453 wayNodes.add(n);
454 }
455 }
456 w.setNodes(wayNodes);
457 if (w.hasIncompleteNodes()) {
458 if (logger.isLoggable(Level.FINE)) {
459 logger.fine(tr("Way {0} with {1} nodes has incomplete nodes because at least one node was missing in the loaded data.",
460 externalWayId, w.getNodesCount()));
461 }
462 }
463 ds.addPrimitive(w);
464 }
465 }
466
467 /**
468 * Processes the parsed nodes after parsing. Just adds them to
469 * the dataset
470 *
471 */
472 protected void processNodesAfterParsing() {
473 for (OsmPrimitive primitive: externalIdMap.values()) {
474 if (primitive instanceof Node) {
475 this.ds.addPrimitive(primitive);
476 }
477 }
478 }
479
480 /**
481 * Completes the parsed relations with its members.
482 *
483 * @throws IllegalDataException thrown if a data integrity problem is detected, i.e. if a
484 * relation member refers to a local primitive which wasn't available in the data
485 *
486 */
487 private void processRelationsAfterParsing() throws IllegalDataException {
488
489 // First add all relations to make sure that when relation reference other relation, the referenced will be already in dataset
490 for (Long externalRelationId : relations.keySet()) {
491 Relation relation = (Relation) externalIdMap.get(
492 new SimplePrimitiveId(externalRelationId, OsmPrimitiveType.RELATION)
493 );
494 ds.addPrimitive(relation);
495 }
496
497 for (Long externalRelationId : relations.keySet()) {
498 Relation relation = (Relation) externalIdMap.get(
499 new SimplePrimitiveId(externalRelationId, OsmPrimitiveType.RELATION)
500 );
501 List<RelationMember> relationMembers = new ArrayList<RelationMember>();
502 for (RelationMemberData rm : relations.get(externalRelationId)) {
503 OsmPrimitive primitive = null;
504
505 // lookup the member from the map of already created primitives
506 primitive = externalIdMap.get(new SimplePrimitiveId(rm.id, rm.type));
507
508 if (primitive == null) {
509 if (rm.id <= 0)
510 // relation member refers to a primitive with a negative id which was not
511 // found in the data. This is always a data integrity problem and we abort
512 // with an exception
513 //
514 throw new IllegalDataException(
515 tr("Relation with external id ''{0}'' refers to a missing primitive with external id ''{1}''.",
516 externalRelationId,
517 rm.id));
518
519 // member refers to OSM primitive which was not present in the parsed data
520 // -> create a new incomplete primitive and add it to the dataset
521 //
522 primitive = ds.getPrimitiveById(rm.id, rm.type);
523 if (primitive == null) {
524 switch (rm.type) {
525 case NODE:
526 primitive = new Node(rm.id); break;
527 case WAY:
528 primitive = new Way(rm.id); break;
529 case RELATION:
530 primitive = new Relation(rm.id); break;
531 default: throw new AssertionError(); // can't happen
532 }
533
534 ds.addPrimitive(primitive);
535 externalIdMap.put(new SimplePrimitiveId(rm.id, rm.type), primitive);
536 }
537 }
538 if (primitive.isDeleted()) {
539 logger.warning(tr("Deleted member {0} is used by relation {1}", primitive.getId(), relation.getId()));
540 } else {
541 relationMembers.add(new RelationMember(rm.role, primitive));
542 }
543 }
544 relation.setMembers(relationMembers);
545 }
546 }
547
548 /**
549 * Parse the given input source and return the dataset.
550 *
551 * @param source the source input stream. Must not be null.
552 * @param progressMonitor the progress monitor. If null, {@see NullProgressMonitor#INSTANCE} is assumed
553 *
554 * @return the dataset with the parsed data
555 * @throws IllegalDataException thrown if the an error was found while parsing the data from the source
556 * @throws IllegalArgumentException thrown if source is null
557 */
558 public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
559 if (progressMonitor == null) {
560 progressMonitor = NullProgressMonitor.INSTANCE;
561 }
562 CheckParameterUtil.ensureParameterNotNull(source, "source");
563 OsmReader reader = new OsmReader();
564 try {
565 progressMonitor.beginTask(tr("Prepare OSM data...", 2));
566 progressMonitor.indeterminateSubTask(tr("Parsing OSM data..."));
567 InputSource inputSource = new InputSource(new InputStreamReader(source, "UTF-8"));
568 SAXParserFactory.newInstance().newSAXParser().parse(inputSource, reader.new Parser());
569 progressMonitor.worked(1);
570
571 progressMonitor.indeterminateSubTask(tr("Preparing data set..."));
572 reader.ds.beginUpdate();
573 try {
574 reader.processNodesAfterParsing();
575 reader.processWaysAfterParsing();
576 reader.processRelationsAfterParsing();
577 } finally {
578 reader.ds.endUpdate();
579 }
580 progressMonitor.worked(1);
581 return reader.getDataSet();
582 } catch(IllegalDataException e) {
583 throw e;
584 } catch(ParserConfigurationException e) {
585 throw new IllegalDataException(e.getMessage(), e);
586 } catch(SAXException e) {
587 throw new IllegalDataException(e.getMessage(), e);
588 } catch(Exception e) {
589 throw new IllegalDataException(e);
590 } finally {
591 progressMonitor.finishTask();
592 }
593 }
594}
Note: See TracBrowser for help on using the repository browser.