source: josm/trunk/src/org/openstreetmap/josm/io/OsmReader.java@ 3321

Last change on this file since 3321 was 3226, checked in by jttt, 14 years ago

Show line number in error message when loading of osm file fails. See #4487
  • Property svn:eol-style set to native
File size: 27.1 KB
Line 
1package org.openstreetmap.josm.io;
2
3import static org.openstreetmap.josm.tools.I18n.tr;
4
5import java.io.InputStream;
6import java.io.InputStreamReader;
7import java.text.MessageFormat;
8import java.util.ArrayList;
9import java.util.Collection;
10import java.util.HashMap;
11import java.util.LinkedList;
12import java.util.List;
13import java.util.Map;
14import java.util.logging.Level;
15import java.util.logging.Logger;
16
17import javax.xml.parsers.ParserConfigurationException;
18import javax.xml.parsers.SAXParserFactory;
19
20import org.openstreetmap.josm.data.Bounds;
21import org.openstreetmap.josm.data.coor.LatLon;
22import org.openstreetmap.josm.data.osm.DataSet;
23import org.openstreetmap.josm.data.osm.DataSource;
24import org.openstreetmap.josm.data.osm.Node;
25import org.openstreetmap.josm.data.osm.NodeData;
26import org.openstreetmap.josm.data.osm.OsmPrimitive;
27import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
28import org.openstreetmap.josm.data.osm.PrimitiveData;
29import org.openstreetmap.josm.data.osm.PrimitiveId;
30import org.openstreetmap.josm.data.osm.Relation;
31import org.openstreetmap.josm.data.osm.RelationData;
32import org.openstreetmap.josm.data.osm.RelationMember;
33import org.openstreetmap.josm.data.osm.SimplePrimitiveId;
34import org.openstreetmap.josm.data.osm.Storage;
35import org.openstreetmap.josm.data.osm.User;
36import org.openstreetmap.josm.data.osm.Way;
37import org.openstreetmap.josm.data.osm.WayData;
38import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
39import org.openstreetmap.josm.gui.progress.ProgressMonitor;
40import org.openstreetmap.josm.tools.CheckParameterUtil;
41import org.openstreetmap.josm.tools.DateUtils;
42import org.xml.sax.Attributes;
43import org.xml.sax.InputSource;
44import org.xml.sax.Locator;
45import org.xml.sax.SAXException;
46import org.xml.sax.SAXParseException;
47import org.xml.sax.helpers.DefaultHandler;
48
49/**
50 * Parser for the Osm Api. Read from an input stream and construct a dataset out of it.
51 *
52 */
53public class OsmReader {
54 static private final Logger logger = Logger.getLogger(OsmReader.class.getName());
55
56 /**
57 * The dataset to add parsed objects to.
58 */
59 private DataSet ds = new DataSet();
60
61 /**
62 * Replies the parsed data set
63 *
64 * @return the parsed data set
65 */
66 public DataSet getDataSet() {
67 return ds;
68 }
69
70 /** the map from external ids to read OsmPrimitives. External ids are
71 * longs too, but in contrast to internal ids negative values are used
72 * to identify primitives unknown to the OSM server
73 */
74 private Map<PrimitiveId, OsmPrimitive> externalIdMap = new HashMap<PrimitiveId, OsmPrimitive>();
75
76 /**
77 * constructor (for private use only)
78 *
79 * @see #parseDataSet(InputStream, DataSet, ProgressMonitor)
80 * @see #parseDataSetOsm(InputStream, DataSet, ProgressMonitor)
81 */
82 private OsmReader() {
83 externalIdMap = new HashMap<PrimitiveId, OsmPrimitive>();
84 }
85
86 /**
87 * Used as a temporary storage for relation members, before they
88 * are resolved into pointers to real objects.
89 */
90 private static class RelationMemberData {
91 public OsmPrimitiveType type;
92 public long id;
93 public String role;
94 }
95
96 /**
97 * Data structure for the remaining way objects
98 */
99 private Map<Long, Collection<Long>> ways = new HashMap<Long, Collection<Long>>();
100
101 /**
102 * Data structure for relation objects
103 */
104 private Map<Long, Collection<RelationMemberData>> relations = new HashMap<Long, Collection<RelationMemberData>>();
105
106 private class Parser extends DefaultHandler {
107 private Locator locator;
108
109 @Override
110 public void setDocumentLocator(Locator locator) {
111 this.locator = locator;
112 }
113
114 protected void throwException(String msg) throws OsmDataParsingException{
115 throw new OsmDataParsingException(msg).rememberLocation(locator);
116 }
117 /**
118 * The current osm primitive to be read.
119 */
120 private OsmPrimitive currentPrimitive;
121 private long currentExternalId;
122 private String generator;
123 private Storage<String> internedStrings = new Storage<String>();
124
125 // Memory optimization - see #2312
126 private String intern(String s) {
127 String result = internedStrings.get(s);
128 if (result == null) {
129 internedStrings.put(s);
130 return s;
131 } else
132 return result;
133 }
134
135 @Override public void startElement(String namespaceURI, String localName, String qName, Attributes atts) throws SAXException {
136
137 try {
138 if (qName.equals("osm")) {
139 if (atts == null) {
140 throwException(tr("Missing mandatory attribute ''{0}'' of XML element {1}.", "version", "osm"));
141 }
142 String v = atts.getValue("version");
143 if (v == null) {
144 throwException(tr("Missing mandatory attribute ''{0}''.", "version"));
145 }
146 if (!(v.equals("0.5") || v.equals("0.6"))) {
147 throwException(tr("Unsupported version: {0}", v));
148 }
149 // save generator attribute for later use when creating DataSource objects
150 generator = atts.getValue("generator");
151 ds.setVersion(v);
152
153 } else if (qName.equals("bounds")) {
154 // new style bounds.
155 String minlon = atts.getValue("minlon");
156 String minlat = atts.getValue("minlat");
157 String maxlon = atts.getValue("maxlon");
158 String maxlat = atts.getValue("maxlat");
159 String origin = atts.getValue("origin");
160 if (minlon != null && maxlon != null && minlat != null && maxlat != null) {
161 if (origin == null) {
162 origin = generator;
163 }
164 Bounds bounds = new Bounds(
165 new LatLon(Double.parseDouble(minlat), Double.parseDouble(minlon)),
166 new LatLon(Double.parseDouble(maxlat), Double.parseDouble(maxlon)));
167 DataSource src = new DataSource(bounds, origin);
168 ds.dataSources.add(src);
169 } else {
170 throwException(tr(
171 "Missing manadatory attributes on element ''bounds''. Got minlon=''{0}'',minlat=''{1}'',maxlon=''{3}'',maxlat=''{4}'', origin=''{5}''.",
172 minlon, minlat, maxlon, maxlat, origin
173 ));
174 }
175
176 // ---- PARSING NODES AND WAYS ----
177
178 } else if (qName.equals("node")) {
179 NodeData nd = new NodeData();
180 nd.setCoor(new LatLon(getDouble(atts, "lat"), getDouble(atts, "lon")));
181 readCommon(atts, nd);
182 Node n = new Node(nd.getId(), nd.getVersion());
183 n.load(nd);
184 externalIdMap.put(nd.getPrimitiveId(), n);
185 currentPrimitive = n;
186 currentExternalId = nd.getUniqueId();
187 } else if (qName.equals("way")) {
188 WayData wd = new WayData();
189 readCommon(atts, wd);
190 Way w = new Way(wd.getId(), wd.getVersion());
191 w.load(wd);
192 externalIdMap.put(wd.getPrimitiveId(), w);
193 ways.put(wd.getUniqueId(), new ArrayList<Long>());
194 currentPrimitive = w;
195 currentExternalId = wd.getUniqueId();
196 } else if (qName.equals("nd")) {
197 Collection<Long> list = ways.get(currentExternalId);
198 if (list == null) {
199 throwException(
200 tr("Found XML element <nd> not as direct child of element <way>.")
201 );
202 }
203 if (atts.getValue("ref") == null) {
204 throwException(
205 tr("Missing mandatory attribute ''{0}'' on <nd> of way {1}.", "ref", currentPrimitive.getUniqueId())
206 );
207 }
208 long id = getLong(atts, "ref");
209 if (id == 0) {
210 throwException(
211 tr("Illegal value of attribute ''ref'' of element <nd>. Got {0}.", id)
212 );
213 }
214 if (currentPrimitive.isDeleted()) {
215 logger.info(tr("Deleted way {0} contains nodes", currentPrimitive.getUniqueId()));
216 } else {
217 list.add(id);
218 }
219
220 // ---- PARSING RELATIONS ----
221
222 } else if (qName.equals("relation")) {
223 RelationData rd = new RelationData();
224 readCommon(atts, rd);
225 Relation r = new Relation(rd.getId(), rd.getVersion());
226 r.load(rd);
227 externalIdMap.put(rd.getPrimitiveId(), r);
228 relations.put(rd.getUniqueId(), new LinkedList<RelationMemberData>());
229 currentPrimitive = r;
230 currentExternalId = rd.getUniqueId();
231 } else if (qName.equals("member")) {
232 Collection<RelationMemberData> list = relations.get(currentExternalId);
233 if (list == null) {
234 throwException(
235 tr("Found XML element <member> not as direct child of element <relation>.")
236 );
237 }
238 RelationMemberData emd = new RelationMemberData();
239 String value = atts.getValue("ref");
240 if (value == null) {
241 throwException(tr("Missing attribute ''ref'' on member in relation {0}.",currentPrimitive.getUniqueId()));
242 }
243 try {
244 emd.id = Long.parseLong(value);
245 } catch(NumberFormatException e) {
246 throwException(tr("Illegal value for attribute ''ref'' on member in relation {0}. Got {1}", Long.toString(currentPrimitive.getUniqueId()),value));
247 }
248 value = atts.getValue("type");
249 if (value == null) {
250 throwException(tr("Missing attribute ''type'' on member {0} in relation {1}.", Long.toString(emd.id), Long.toString(currentPrimitive.getUniqueId())));
251 }
252 try {
253 emd.type = OsmPrimitiveType.fromApiTypeName(value);
254 } catch(IllegalArgumentException e) {
255 throwException(tr("Illegal value for attribute ''type'' on member {0} in relation {1}. Got {2}.", Long.toString(emd.id), Long.toString(currentPrimitive.getUniqueId()), value));
256 }
257 value = atts.getValue("role");
258 emd.role = value;
259
260 if (emd.id == 0) {
261 throwException(tr("Incomplete <member> specification with ref=0"));
262 }
263
264 if (currentPrimitive.isDeleted()) {
265 logger.info(tr("Deleted relation {0} contains members", currentPrimitive.getUniqueId()));
266 } else {
267 list.add(emd);
268 }
269
270 // ---- PARSING TAGS (applicable to all objects) ----
271
272 } else if (qName.equals("tag")) {
273 String key = atts.getValue("k");
274 String value = atts.getValue("v");
275 if (key == null || value == null) {
276 throwException(tr("Missing key or value attribute in tag."));
277 }
278 currentPrimitive.put(intern(key), intern(value));
279
280 } else {
281 System.out.println(tr("Undefined element ''{0}'' found in input stream. Skipping.", qName));
282 }
283 } catch (Exception e) {
284 throw new SAXParseException(e.getMessage(), locator, e);
285 }
286 }
287
288 private double getDouble(Attributes atts, String value) {
289 return Double.parseDouble(atts.getValue(value));
290 }
291
292 private User createUser(String uid, String name) throws SAXException {
293 if (uid == null) {
294 if (name == null)
295 return null;
296 return User.createLocalUser(name);
297 }
298 try {
299 long id = Long.parseLong(uid);
300 return User.createOsmUser(id, name);
301 } catch(NumberFormatException e) {
302 throwException(MessageFormat.format("Illegal value for attribute ''uid''. Got ''{0}''.", uid));
303 }
304 return null;
305 }
306 /**
307 * Read out the common attributes from atts and put them into this.current.
308 */
309 void readCommon(Attributes atts, PrimitiveData current) throws SAXException {
310 current.setId(getLong(atts, "id"));
311 if (current.getUniqueId() == 0) {
312 throwException(tr("Illegal object with ID=0."));
313 }
314
315 String time = atts.getValue("timestamp");
316 if (time != null && time.length() != 0) {
317 current.setTimestamp(DateUtils.fromString(time));
318 }
319
320 // user attribute added in 0.4 API
321 String user = atts.getValue("user");
322 // uid attribute added in 0.6 API
323 String uid = atts.getValue("uid");
324 current.setUser(createUser(uid, user));
325
326 // visible attribute added in 0.4 API
327 String visible = atts.getValue("visible");
328 if (visible != null) {
329 current.setVisible(Boolean.parseBoolean(visible));
330 }
331
332 String versionString = atts.getValue("version");
333 int version = 0;
334 if (versionString != null) {
335 try {
336 version = Integer.parseInt(versionString);
337 } catch(NumberFormatException e) {
338 throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.", Long.toString(current.getUniqueId()), versionString));
339 }
340 if (ds.getVersion().equals("0.6")){
341 if (version <= 0 && current.getUniqueId() > 0) {
342 throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.", Long.toString(current.getUniqueId()), versionString));
343 } else if (version < 0 && current.getUniqueId() <= 0) {
344 System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 0, "0.6"));
345 version = 0;
346 }
347 } else if (ds.getVersion().equals("0.5")) {
348 if (version <= 0 && current.getUniqueId() > 0) {
349 System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 1, "0.5"));
350 version = 1;
351 } else if (version < 0 && current.getUniqueId() <= 0) {
352 System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 0, "0.5"));
353 version = 0;
354 }
355 } else {
356 // should not happen. API version has been checked before
357 throwException(tr("Unknown or unsupported API version. Got {0}.", ds.getVersion()));
358 }
359 } else {
360 // version expected for OSM primitives with an id assigned by the server (id > 0), since API 0.6
361 //
362 if (current.getUniqueId() > 0 && ds.getVersion() != null && ds.getVersion().equals("0.6")) {
363 throwException(tr("Missing attribute ''version'' on OSM primitive with ID {0}.", Long.toString(current.getUniqueId())));
364 } else if (current.getUniqueId() > 0 && ds.getVersion() != null && ds.getVersion().equals("0.5")) {
365 // default version in 0.5 files for existing primitives
366 System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 1, "0.5"));
367 version= 1;
368 } else if (current.getUniqueId() <= 0 && ds.getVersion() != null && ds.getVersion().equals("0.5")) {
369 // default version in 0.5 files for new primitives, no warning necessary. This is
370 // (was) legal in API 0.5
371 version= 0;
372 }
373 }
374 current.setVersion(version);
375
376 String action = atts.getValue("action");
377 if (action == null) {
378 // do nothing
379 } else if (action.equals("delete")) {
380 current.setDeleted(true);
381 current.setModified(true);
382 } else if (action.equals("modify")) {
383 current.setModified(true);
384 }
385
386 String v = atts.getValue("changeset");
387 if (v == null) {
388 current.setChangesetId(0);
389 } else {
390 try {
391 current.setChangesetId(Integer.parseInt(v));
392 } catch(NumberFormatException e) {
393 if (current.getUniqueId() <= 0) {
394 // for a new primitive we just log a warning
395 System.out.println(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", v, current.getUniqueId()));
396 current.setChangesetId(0);
397 } else {
398 // for an existing primitive this is a problem
399 throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v));
400 }
401 }
402 if (current.getChangesetId() <=0) {
403 if (current.getUniqueId() <= 0) {
404 // for a new primitive we just log a warning
405 System.out.println(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", v, current.getUniqueId()));
406 current.setChangesetId(0);
407 } else {
408 // for an existing primitive this is a problem
409 throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v));
410 }
411 }
412 }
413 }
414
415 private long getLong(Attributes atts, String name) throws SAXException {
416 String value = atts.getValue(name);
417 if (value == null) {
418 throwException(tr("Missing required attribute ''{0}''.",name));
419 }
420 try {
421 return Long.parseLong(value);
422 } catch(NumberFormatException e) {
423 throwException(tr("Illegal long value for attribute ''{0}''. Got ''{1}''.",name, value));
424 }
425 return 0; // should not happen
426 }
427 }
428
429 /**
430 * Processes the ways after parsing. Rebuilds the list of nodes of each way and
431 * adds the way to the dataset
432 *
433 * @throws IllegalDataException thrown if a data integrity problem is detected
434 */
435 protected void processWaysAfterParsing() throws IllegalDataException{
436 for (Long externalWayId: ways.keySet()) {
437 Way w = (Way)externalIdMap.get(new SimplePrimitiveId(externalWayId, OsmPrimitiveType.WAY));
438 List<Node> wayNodes = new ArrayList<Node>();
439 for (long id : ways.get(externalWayId)) {
440 Node n = (Node)externalIdMap.get(new SimplePrimitiveId(id, OsmPrimitiveType.NODE));
441 if (n == null) {
442 if (id <= 0)
443 throw new IllegalDataException (
444 tr("Way with external ID ''{0}'' includes missing node with external ID ''{1}''.",
445 externalWayId,
446 id));
447 // create an incomplete node if necessary
448 //
449 n = (Node)ds.getPrimitiveById(id,OsmPrimitiveType.NODE);
450 if (n == null) {
451 n = new Node(id);
452 ds.addPrimitive(n);
453 }
454 }
455 if (n.isDeleted()) {
456 logger.warning(tr("Deleted node {0} is part of way {1}", id, w.getId()));
457 } else {
458 wayNodes.add(n);
459 }
460 }
461 w.setNodes(wayNodes);
462 if (w.hasIncompleteNodes()) {
463 if (logger.isLoggable(Level.FINE)) {
464 logger.fine(tr("Way {0} with {1} nodes has incomplete nodes because at least one node was missing in the loaded data.",
465 externalWayId, w.getNodesCount()));
466 }
467 }
468 ds.addPrimitive(w);
469 }
470 }
471
472 /**
473 * Processes the parsed nodes after parsing. Just adds them to
474 * the dataset
475 *
476 */
477 protected void processNodesAfterParsing() {
478 for (OsmPrimitive primitive: externalIdMap.values()) {
479 if (primitive instanceof Node) {
480 this.ds.addPrimitive(primitive);
481 }
482 }
483 }
484
485 /**
486 * Completes the parsed relations with its members.
487 *
488 * @throws IllegalDataException thrown if a data integrity problem is detected, i.e. if a
489 * relation member refers to a local primitive which wasn't available in the data
490 *
491 */
492 private void processRelationsAfterParsing() throws IllegalDataException {
493
494 // First add all relations to make sure that when relation reference other relation, the referenced will be already in dataset
495 for (Long externalRelationId : relations.keySet()) {
496 Relation relation = (Relation) externalIdMap.get(
497 new SimplePrimitiveId(externalRelationId, OsmPrimitiveType.RELATION)
498 );
499 ds.addPrimitive(relation);
500 }
501
502 for (Long externalRelationId : relations.keySet()) {
503 Relation relation = (Relation) externalIdMap.get(
504 new SimplePrimitiveId(externalRelationId, OsmPrimitiveType.RELATION)
505 );
506 List<RelationMember> relationMembers = new ArrayList<RelationMember>();
507 for (RelationMemberData rm : relations.get(externalRelationId)) {
508 OsmPrimitive primitive = null;
509
510 // lookup the member from the map of already created primitives
511 primitive = externalIdMap.get(new SimplePrimitiveId(rm.id, rm.type));
512
513 if (primitive == null) {
514 if (rm.id <= 0)
515 // relation member refers to a primitive with a negative id which was not
516 // found in the data. This is always a data integrity problem and we abort
517 // with an exception
518 //
519 throw new IllegalDataException(
520 tr("Relation with external id ''{0}'' refers to a missing primitive with external id ''{1}''.",
521 externalRelationId,
522 rm.id));
523
524 // member refers to OSM primitive which was not present in the parsed data
525 // -> create a new incomplete primitive and add it to the dataset
526 //
527 primitive = ds.getPrimitiveById(rm.id, rm.type);
528 if (primitive == null) {
529 switch (rm.type) {
530 case NODE:
531 primitive = new Node(rm.id); break;
532 case WAY:
533 primitive = new Way(rm.id); break;
534 case RELATION:
535 primitive = new Relation(rm.id); break;
536 default: throw new AssertionError(); // can't happen
537 }
538
539 ds.addPrimitive(primitive);
540 externalIdMap.put(new SimplePrimitiveId(rm.id, rm.type), primitive);
541 }
542 }
543 if (primitive.isDeleted()) {
544 logger.warning(tr("Deleted member {0} is used by relation {1}", primitive.getId(), relation.getId()));
545 } else {
546 relationMembers.add(new RelationMember(rm.role, primitive));
547 }
548 }
549 relation.setMembers(relationMembers);
550 }
551 }
552
553 /**
554 * Parse the given input source and return the dataset.
555 *
556 * @param source the source input stream. Must not be null.
557 * @param progressMonitor the progress monitor. If null, {@see NullProgressMonitor#INSTANCE} is assumed
558 *
559 * @return the dataset with the parsed data
560 * @throws IllegalDataException thrown if the an error was found while parsing the data from the source
561 * @throws IllegalArgumentException thrown if source is null
562 */
563 public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
564 if (progressMonitor == null) {
565 progressMonitor = NullProgressMonitor.INSTANCE;
566 }
567 CheckParameterUtil.ensureParameterNotNull(source, "source");
568 OsmReader reader = new OsmReader();
569 try {
570 progressMonitor.beginTask(tr("Prepare OSM data...", 2));
571 progressMonitor.indeterminateSubTask(tr("Parsing OSM data..."));
572 InputSource inputSource = new InputSource(new InputStreamReader(source, "UTF-8"));
573 SAXParserFactory.newInstance().newSAXParser().parse(inputSource, reader.new Parser());
574 progressMonitor.worked(1);
575
576 progressMonitor.indeterminateSubTask(tr("Preparing data set..."));
577 reader.ds.beginUpdate();
578 try {
579 reader.processNodesAfterParsing();
580 reader.processWaysAfterParsing();
581 reader.processRelationsAfterParsing();
582 } finally {
583 reader.ds.endUpdate();
584 }
585 progressMonitor.worked(1);
586 return reader.getDataSet();
587 } catch(IllegalDataException e) {
588 throw e;
589 } catch(ParserConfigurationException e) {
590 throw new IllegalDataException(e.getMessage(), e);
591 } catch (SAXParseException e) {
592 throw new IllegalDataException(tr("Line {0} column {1}: ", e.getLineNumber(), e.getColumnNumber()) + e.getMessage(), e);
593 } catch(SAXException e) {
594 throw new IllegalDataException(e.getMessage(), e);
595 } catch(Exception e) {
596 throw new IllegalDataException(e);
597 } finally {
598 progressMonitor.finishTask();
599 }
600 }
601}
Note: See TracBrowser for help on using the repository browser.