source: josm/trunk/src/org/openstreetmap/josm/io/OsmReader.java@ 4268

Last change on this file since 4268 was 4268, checked in by stoecker, 13 years ago

fix #6647 - patch by Don-vip - fix changeset downloading
  • Property svn:eol-style set to native
File size: 27.0 KB
Line 
1// License: GPL. See LICENSE file for details.
2package org.openstreetmap.josm.io;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.InputStream;
7import java.text.MessageFormat;
8import java.util.ArrayList;
9import java.util.Collection;
10import java.util.HashMap;
11import java.util.LinkedList;
12import java.util.List;
13import java.util.Map;
14
15import javax.xml.parsers.ParserConfigurationException;
16import javax.xml.parsers.SAXParserFactory;
17
18import org.openstreetmap.josm.data.Bounds;
19import org.openstreetmap.josm.data.coor.LatLon;
20import org.openstreetmap.josm.data.osm.DataSet;
21import org.openstreetmap.josm.data.osm.DataSource;
22import org.openstreetmap.josm.data.osm.Node;
23import org.openstreetmap.josm.data.osm.NodeData;
24import org.openstreetmap.josm.data.osm.OsmPrimitive;
25import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
26import org.openstreetmap.josm.data.osm.PrimitiveData;
27import org.openstreetmap.josm.data.osm.PrimitiveId;
28import org.openstreetmap.josm.data.osm.Relation;
29import org.openstreetmap.josm.data.osm.RelationData;
30import org.openstreetmap.josm.data.osm.RelationMember;
31import org.openstreetmap.josm.data.osm.SimplePrimitiveId;
32import org.openstreetmap.josm.data.osm.User;
33import org.openstreetmap.josm.data.osm.Way;
34import org.openstreetmap.josm.data.osm.WayData;
35import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
36import org.openstreetmap.josm.gui.progress.ProgressMonitor;
37import org.openstreetmap.josm.tools.CheckParameterUtil;
38import org.openstreetmap.josm.tools.DateUtils;
39import org.xml.sax.Attributes;
40import org.xml.sax.InputSource;
41import org.xml.sax.Locator;
42import org.xml.sax.SAXException;
43import org.xml.sax.SAXParseException;
44import org.xml.sax.helpers.DefaultHandler;
45
46/**
47 * Parser for the Osm Api. Read from an input stream and construct a dataset out of it.
48 *
49 */
50public class OsmReader {
51 /**
52 * The dataset to add parsed objects to.
53 */
54 private DataSet ds = new DataSet();
55
56 /**
57 * Replies the parsed data set
58 *
59 * @return the parsed data set
60 */
61 public DataSet getDataSet() {
62 return ds;
63 }
64
65 /** the map from external ids to read OsmPrimitives. External ids are
66 * longs too, but in contrast to internal ids negative values are used
67 * to identify primitives unknown to the OSM server
68 */
69 private Map<PrimitiveId, OsmPrimitive> externalIdMap = new HashMap<PrimitiveId, OsmPrimitive>();
70
71 /**
72 * constructor (for private use only)
73 *
74 * @see #parseDataSet(InputStream, DataSet, ProgressMonitor)
75 * @see #parseDataSetOsm(InputStream, DataSet, ProgressMonitor)
76 */
77 private OsmReader() {
78 externalIdMap = new HashMap<PrimitiveId, OsmPrimitive>();
79 }
80
81 /**
82 * Used as a temporary storage for relation members, before they
83 * are resolved into pointers to real objects.
84 */
85 private static class RelationMemberData {
86 public OsmPrimitiveType type;
87 public long id;
88 public String role;
89 }
90
91 /**
92 * Data structure for the remaining way objects
93 */
94 private Map<Long, Collection<Long>> ways = new HashMap<Long, Collection<Long>>();
95
96 /**
97 * Data structure for relation objects
98 */
99 private Map<Long, Collection<RelationMemberData>> relations = new HashMap<Long, Collection<RelationMemberData>>();
100
101 private class Parser extends DefaultHandler {
102 private Locator locator;
103
104 @Override
105 public void setDocumentLocator(Locator locator) {
106 this.locator = locator;
107 }
108
109 protected void throwException(String msg) throws OsmDataParsingException{
110 throw new OsmDataParsingException(msg).rememberLocation(locator);
111 }
112 /**
113 * The current osm primitive to be read.
114 */
115 private OsmPrimitive currentPrimitive;
116 private long currentExternalId;
117 private String generator;
118
119 @Override public void startElement(String namespaceURI, String localName, String qName, Attributes atts) throws SAXException {
120
121 try {
122 if (qName.equals("osm") || qName.equals("osmChange")) {
123 if (atts == null) {
124 throwException(tr("Missing mandatory attribute ''{0}'' of XML element {1}.", "version", qName));
125 }
126 String v = atts.getValue("version");
127 if (v == null) {
128 throwException(tr("Missing mandatory attribute ''{0}''.", "version"));
129 }
130 if (!(v.equals("0.5") || v.equals("0.6"))) {
131 throwException(tr("Unsupported version: {0}", v));
132 }
133 // save generator attribute for later use when creating DataSource objects
134 generator = atts.getValue("generator");
135 ds.setVersion(v);
136
137 } else if (qName.equals("bounds")) {
138 // new style bounds.
139 String minlon = atts.getValue("minlon");
140 String minlat = atts.getValue("minlat");
141 String maxlon = atts.getValue("maxlon");
142 String maxlat = atts.getValue("maxlat");
143 String origin = atts.getValue("origin");
144 if (minlon != null && maxlon != null && minlat != null && maxlat != null) {
145 if (origin == null) {
146 origin = generator;
147 }
148 Bounds bounds = new Bounds(
149 Double.parseDouble(minlat), Double.parseDouble(minlon),
150 Double.parseDouble(maxlat), Double.parseDouble(maxlon));
151 if (bounds.isOutOfTheWorld()) {
152 Bounds copy = new Bounds(bounds);
153 bounds.normalize();
154 System.out.println("Bbox " + copy + " is out of the world, normalized to " + bounds);
155 }
156 DataSource src = new DataSource(bounds, origin);
157 ds.dataSources.add(src);
158 } else {
159 throwException(tr(
160 "Missing manadatory attributes on element ''bounds''. Got minlon=''{0}'',minlat=''{1}'',maxlon=''{3}'',maxlat=''{4}'', origin=''{5}''.",
161 minlon, minlat, maxlon, maxlat, origin
162 ));
163 }
164
165 // ---- PARSING NODES AND WAYS ----
166
167 } else if (qName.equals("node")) {
168 NodeData nd = new NodeData();
169 nd.setCoor(new LatLon(getDouble(atts, "lat"), getDouble(atts, "lon")));
170 readCommon(atts, nd);
171 Node n = new Node(nd.getId(), nd.getVersion());
172 n.setVisible(nd.isVisible());
173 n.load(nd);
174 externalIdMap.put(nd.getPrimitiveId(), n);
175 currentPrimitive = n;
176 currentExternalId = nd.getUniqueId();
177 } else if (qName.equals("way")) {
178 WayData wd = new WayData();
179 readCommon(atts, wd);
180 Way w = new Way(wd.getId(), wd.getVersion());
181 w.setVisible(wd.isVisible());
182 w.load(wd);
183 externalIdMap.put(wd.getPrimitiveId(), w);
184 ways.put(wd.getUniqueId(), new ArrayList<Long>());
185 currentPrimitive = w;
186 currentExternalId = wd.getUniqueId();
187 } else if (qName.equals("nd")) {
188 Collection<Long> list = ways.get(currentExternalId);
189 if (list == null) {
190 throwException(
191 tr("Found XML element <nd> not as direct child of element <way>.")
192 );
193 }
194 if (atts.getValue("ref") == null) {
195 throwException(
196 tr("Missing mandatory attribute ''{0}'' on <nd> of way {1}.", "ref", currentPrimitive.getUniqueId())
197 );
198 }
199 long id = getLong(atts, "ref");
200 if (id == 0) {
201 throwException(
202 tr("Illegal value of attribute ''ref'' of element <nd>. Got {0}.", id)
203 );
204 }
205 if (currentPrimitive.isDeleted()) {
206 System.out.println(tr("Deleted way {0} contains nodes", currentPrimitive.getUniqueId()));
207 } else {
208 list.add(id);
209 }
210
211 // ---- PARSING RELATIONS ----
212
213 } else if (qName.equals("relation")) {
214 RelationData rd = new RelationData();
215 readCommon(atts, rd);
216 Relation r = new Relation(rd.getId(), rd.getVersion());
217 r.setVisible(rd.isVisible());
218 r.load(rd);
219 externalIdMap.put(rd.getPrimitiveId(), r);
220 relations.put(rd.getUniqueId(), new LinkedList<RelationMemberData>());
221 currentPrimitive = r;
222 currentExternalId = rd.getUniqueId();
223 } else if (qName.equals("member")) {
224 Collection<RelationMemberData> list = relations.get(currentExternalId);
225 if (list == null) {
226 throwException(
227 tr("Found XML element <member> not as direct child of element <relation>.")
228 );
229 }
230 RelationMemberData emd = new RelationMemberData();
231 String value = atts.getValue("ref");
232 if (value == null) {
233 throwException(tr("Missing attribute ''ref'' on member in relation {0}.",currentPrimitive.getUniqueId()));
234 }
235 try {
236 emd.id = Long.parseLong(value);
237 } catch(NumberFormatException e) {
238 throwException(tr("Illegal value for attribute ''ref'' on member in relation {0}. Got {1}", Long.toString(currentPrimitive.getUniqueId()),value));
239 }
240 value = atts.getValue("type");
241 if (value == null) {
242 throwException(tr("Missing attribute ''type'' on member {0} in relation {1}.", Long.toString(emd.id), Long.toString(currentPrimitive.getUniqueId())));
243 }
244 try {
245 emd.type = OsmPrimitiveType.fromApiTypeName(value);
246 } catch(IllegalArgumentException e) {
247 throwException(tr("Illegal value for attribute ''type'' on member {0} in relation {1}. Got {2}.", Long.toString(emd.id), Long.toString(currentPrimitive.getUniqueId()), value));
248 }
249 value = atts.getValue("role");
250 emd.role = value;
251
252 if (emd.id == 0) {
253 throwException(tr("Incomplete <member> specification with ref=0"));
254 }
255
256 if (currentPrimitive.isDeleted()) {
257 System.out.println(tr("Deleted relation {0} contains members", currentPrimitive.getUniqueId()));
258 } else {
259 list.add(emd);
260 }
261
262 // ---- PARSING TAGS (applicable to all objects) ----
263
264 } else if (qName.equals("tag")) {
265 String key = atts.getValue("k");
266 String value = atts.getValue("v");
267 if (key == null || value == null) {
268 throwException(tr("Missing key or value attribute in tag."));
269 }
270 currentPrimitive.put(key.intern(), value.intern());
271
272 } else {
273 System.out.println(tr("Undefined element ''{0}'' found in input stream. Skipping.", qName));
274 }
275 } catch (Exception e) {
276 throw new SAXParseException(e.getMessage(), locator, e);
277 }
278 }
279
280 private double getDouble(Attributes atts, String value) {
281 return Double.parseDouble(atts.getValue(value));
282 }
283
284 private User createUser(String uid, String name) throws SAXException {
285 if (uid == null) {
286 if (name == null)
287 return null;
288 return User.createLocalUser(name);
289 }
290 try {
291 long id = Long.parseLong(uid);
292 return User.createOsmUser(id, name);
293 } catch(NumberFormatException e) {
294 throwException(MessageFormat.format("Illegal value for attribute ''uid''. Got ''{0}''.", uid));
295 }
296 return null;
297 }
298 /**
299 * Read out the common attributes from atts and put them into this.current.
300 */
301 void readCommon(Attributes atts, PrimitiveData current) throws SAXException {
302 current.setId(getLong(atts, "id"));
303 if (current.getUniqueId() == 0) {
304 throwException(tr("Illegal object with ID=0."));
305 }
306
307 String time = atts.getValue("timestamp");
308 if (time != null && time.length() != 0) {
309 current.setTimestamp(DateUtils.fromString(time));
310 }
311
312 // user attribute added in 0.4 API
313 String user = atts.getValue("user");
314 // uid attribute added in 0.6 API
315 String uid = atts.getValue("uid");
316 current.setUser(createUser(uid, user));
317
318 // visible attribute added in 0.4 API
319 String visible = atts.getValue("visible");
320 if (visible != null) {
321 current.setVisible(Boolean.parseBoolean(visible));
322 }
323
324 String versionString = atts.getValue("version");
325 int version = 0;
326 if (versionString != null) {
327 try {
328 version = Integer.parseInt(versionString);
329 } catch(NumberFormatException e) {
330 throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.", Long.toString(current.getUniqueId()), versionString));
331 }
332 if (ds.getVersion().equals("0.6")){
333 if (version <= 0 && current.getUniqueId() > 0) {
334 throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.", Long.toString(current.getUniqueId()), versionString));
335 } else if (version < 0 && current.getUniqueId() <= 0) {
336 System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 0, "0.6"));
337 version = 0;
338 }
339 } else if (ds.getVersion().equals("0.5")) {
340 if (version <= 0 && current.getUniqueId() > 0) {
341 System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 1, "0.5"));
342 version = 1;
343 } else if (version < 0 && current.getUniqueId() <= 0) {
344 System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 0, "0.5"));
345 version = 0;
346 }
347 } else {
348 // should not happen. API version has been checked before
349 throwException(tr("Unknown or unsupported API version. Got {0}.", ds.getVersion()));
350 }
351 } else {
352 // version expected for OSM primitives with an id assigned by the server (id > 0), since API 0.6
353 //
354 if (current.getUniqueId() > 0 && ds.getVersion() != null && ds.getVersion().equals("0.6")) {
355 throwException(tr("Missing attribute ''version'' on OSM primitive with ID {0}.", Long.toString(current.getUniqueId())));
356 } else if (current.getUniqueId() > 0 && ds.getVersion() != null && ds.getVersion().equals("0.5")) {
357 // default version in 0.5 files for existing primitives
358 System.out.println(tr("WARNING: Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 1, "0.5"));
359 version= 1;
360 } else if (current.getUniqueId() <= 0 && ds.getVersion() != null && ds.getVersion().equals("0.5")) {
361 // default version in 0.5 files for new primitives, no warning necessary. This is
362 // (was) legal in API 0.5
363 version= 0;
364 }
365 }
366 current.setVersion(version);
367
368 String action = atts.getValue("action");
369 if (action == null) {
370 // do nothing
371 } else if (action.equals("delete")) {
372 current.setDeleted(true);
373 current.setModified(current.isVisible());
374 } else if (action.equals("modify")) {
375 current.setModified(true);
376 }
377
378 String v = atts.getValue("changeset");
379 if (v == null) {
380 current.setChangesetId(0);
381 } else {
382 try {
383 current.setChangesetId(Integer.parseInt(v));
384 } catch(NumberFormatException e) {
385 if (current.getUniqueId() <= 0) {
386 // for a new primitive we just log a warning
387 System.out.println(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", v, current.getUniqueId()));
388 current.setChangesetId(0);
389 } else {
390 // for an existing primitive this is a problem
391 throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v));
392 }
393 }
394 if (current.getChangesetId() <=0) {
395 if (current.getUniqueId() <= 0) {
396 // for a new primitive we just log a warning
397 System.out.println(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", v, current.getUniqueId()));
398 current.setChangesetId(0);
399 } else {
400 // for an existing primitive this is a problem
401 throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v));
402 }
403 }
404 }
405 }
406
407 private long getLong(Attributes atts, String name) throws SAXException {
408 String value = atts.getValue(name);
409 if (value == null) {
410 throwException(tr("Missing required attribute ''{0}''.",name));
411 }
412 try {
413 return Long.parseLong(value);
414 } catch(NumberFormatException e) {
415 throwException(tr("Illegal long value for attribute ''{0}''. Got ''{1}''.",name, value));
416 }
417 return 0; // should not happen
418 }
419 }
420
421 /**
422 * Processes the ways after parsing. Rebuilds the list of nodes of each way and
423 * adds the way to the dataset
424 *
425 * @throws IllegalDataException thrown if a data integrity problem is detected
426 */
427 protected void processWaysAfterParsing() throws IllegalDataException{
428 for (Long externalWayId: ways.keySet()) {
429 Way w = (Way)externalIdMap.get(new SimplePrimitiveId(externalWayId, OsmPrimitiveType.WAY));
430 List<Node> wayNodes = new ArrayList<Node>();
431 for (long id : ways.get(externalWayId)) {
432 Node n = (Node)externalIdMap.get(new SimplePrimitiveId(id, OsmPrimitiveType.NODE));
433 if (n == null) {
434 if (id <= 0)
435 throw new IllegalDataException (
436 tr("Way with external ID ''{0}'' includes missing node with external ID ''{1}''.",
437 externalWayId,
438 id));
439 // create an incomplete node if necessary
440 //
441 n = (Node)ds.getPrimitiveById(id,OsmPrimitiveType.NODE);
442 if (n == null) {
443 n = new Node(id);
444 ds.addPrimitive(n);
445 }
446 }
447 if (n.isDeleted()) {
448 System.out.println(tr("Deleted node {0} is part of way {1}", id, w.getId()));
449 } else {
450 wayNodes.add(n);
451 }
452 }
453 w.setNodes(wayNodes);
454 if (w.hasIncompleteNodes()) {
455 System.out.println(tr("Way {0} with {1} nodes has incomplete nodes because at least one node was missing in the loaded data.",
456 externalWayId, w.getNodesCount()));
457 }
458 ds.addPrimitive(w);
459 }
460 }
461
462 /**
463 * Processes the parsed nodes after parsing. Just adds them to
464 * the dataset
465 *
466 */
467 protected void processNodesAfterParsing() {
468 for (OsmPrimitive primitive: externalIdMap.values()) {
469 if (primitive instanceof Node) {
470 this.ds.addPrimitive(primitive);
471 }
472 }
473 }
474
475 /**
476 * Completes the parsed relations with its members.
477 *
478 * @throws IllegalDataException thrown if a data integrity problem is detected, i.e. if a
479 * relation member refers to a local primitive which wasn't available in the data
480 *
481 */
482 private void processRelationsAfterParsing() throws IllegalDataException {
483
484 // First add all relations to make sure that when relation reference other relation, the referenced will be already in dataset
485 for (Long externalRelationId : relations.keySet()) {
486 Relation relation = (Relation) externalIdMap.get(
487 new SimplePrimitiveId(externalRelationId, OsmPrimitiveType.RELATION)
488 );
489 ds.addPrimitive(relation);
490 }
491
492 for (Long externalRelationId : relations.keySet()) {
493 Relation relation = (Relation) externalIdMap.get(
494 new SimplePrimitiveId(externalRelationId, OsmPrimitiveType.RELATION)
495 );
496 List<RelationMember> relationMembers = new ArrayList<RelationMember>();
497 for (RelationMemberData rm : relations.get(externalRelationId)) {
498 OsmPrimitive primitive = null;
499
500 // lookup the member from the map of already created primitives
501 primitive = externalIdMap.get(new SimplePrimitiveId(rm.id, rm.type));
502
503 if (primitive == null) {
504 if (rm.id <= 0)
505 // relation member refers to a primitive with a negative id which was not
506 // found in the data. This is always a data integrity problem and we abort
507 // with an exception
508 //
509 throw new IllegalDataException(
510 tr("Relation with external id ''{0}'' refers to a missing primitive with external id ''{1}''.",
511 externalRelationId,
512 rm.id));
513
514 // member refers to OSM primitive which was not present in the parsed data
515 // -> create a new incomplete primitive and add it to the dataset
516 //
517 primitive = ds.getPrimitiveById(rm.id, rm.type);
518 if (primitive == null) {
519 switch (rm.type) {
520 case NODE:
521 primitive = new Node(rm.id); break;
522 case WAY:
523 primitive = new Way(rm.id); break;
524 case RELATION:
525 primitive = new Relation(rm.id); break;
526 default: throw new AssertionError(); // can't happen
527 }
528
529 ds.addPrimitive(primitive);
530 externalIdMap.put(new SimplePrimitiveId(rm.id, rm.type), primitive);
531 }
532 }
533 if (primitive.isDeleted()) {
534 System.out.println(tr("Deleted member {0} is used by relation {1}", primitive.getId(), relation.getId()));
535 } else {
536 relationMembers.add(new RelationMember(rm.role, primitive));
537 }
538 }
539 relation.setMembers(relationMembers);
540 }
541 }
542
543 /**
544 * Parse the given input source and return the dataset.
545 *
546 * @param source the source input stream. Must not be null.
547 * @param progressMonitor the progress monitor. If null, {@see NullProgressMonitor#INSTANCE} is assumed
548 *
549 * @return the dataset with the parsed data
550 * @throws IllegalDataException thrown if the an error was found while parsing the data from the source
551 * @throws IllegalArgumentException thrown if source is null
552 */
553 public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
554 if (progressMonitor == null) {
555 progressMonitor = NullProgressMonitor.INSTANCE;
556 }
557 CheckParameterUtil.ensureParameterNotNull(source, "source");
558 OsmReader reader = new OsmReader();
559 try {
560 progressMonitor.beginTask(tr("Prepare OSM data...", 2));
561 progressMonitor.indeterminateSubTask(tr("Parsing OSM data..."));
562
563 InputSource inputSource = new InputSource(UTFInputStreamReader.create(source, "UTF-8"));
564 SAXParserFactory.newInstance().newSAXParser().parse(inputSource, reader.new Parser());
565 progressMonitor.worked(1);
566
567 progressMonitor.indeterminateSubTask(tr("Preparing data set..."));
568 reader.ds.beginUpdate();
569 try {
570 reader.processNodesAfterParsing();
571 reader.processWaysAfterParsing();
572 reader.processRelationsAfterParsing();
573 } finally {
574 reader.ds.endUpdate();
575 }
576 progressMonitor.worked(1);
577 return reader.getDataSet();
578 } catch(IllegalDataException e) {
579 throw e;
580 } catch(ParserConfigurationException e) {
581 throw new IllegalDataException(e.getMessage(), e);
582 } catch (SAXParseException e) {
583 throw new IllegalDataException(tr("Line {0} column {1}: ", e.getLineNumber(), e.getColumnNumber()) + e.getMessage(), e);
584 } catch(SAXException e) {
585 throw new IllegalDataException(e.getMessage(), e);
586 } catch(Exception e) {
587 throw new IllegalDataException(e);
588 } finally {
589 progressMonitor.finishTask();
590 }
591 }
592}
Note: See TracBrowser for help on using the repository browser.