source: josm/trunk/src/org/openstreetmap/josm/io/OsmReader.java@ 1724

Last change on this file since 1724 was 1690, checked in by Gubaer, 15 years ago

new: MultiFetchServerObjectReader using APIs Multi Fetch method
update: now uses Multi Fetch to check for deleted primitives on the server
update: now uses Multi Fetch to update the selected primitives with the state from the server
fixed: cleaned up merging in MergeVisitor
new: conflict resolution dialog; now resolves conflicts due to different visibilities
new: replacement for realEqual() on OsmPrimitive and derived classes; realEqual now @deprecated
fixed: cleaning up OsmReader
fixed: progress indication in OsmApi

  • Property svn:eol-style set to native
File size: 19.8 KB
Line 
1// License: GPL. Copyright 2007 by Immanuel Scholz and others
2package org.openstreetmap.josm.io;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.IOException;
7import java.io.InputStream;
8import java.io.InputStreamReader;
9import java.util.ArrayList;
10import java.util.Collection;
11import java.util.Date;
12import java.util.HashMap;
13import java.util.HashSet;
14import java.util.LinkedList;
15import java.util.Map;
16import java.util.Set;
17import java.util.Map.Entry;
18
19import javax.xml.parsers.ParserConfigurationException;
20import javax.xml.parsers.SAXParserFactory;
21
22import org.openstreetmap.josm.Main;
23import org.openstreetmap.josm.data.Bounds;
24import org.openstreetmap.josm.data.coor.LatLon;
25import org.openstreetmap.josm.data.osm.DataSet;
26import org.openstreetmap.josm.data.osm.DataSource;
27import org.openstreetmap.josm.data.osm.Node;
28import org.openstreetmap.josm.data.osm.OsmPrimitive;
29import org.openstreetmap.josm.data.osm.Relation;
30import org.openstreetmap.josm.data.osm.RelationMember;
31import org.openstreetmap.josm.data.osm.User;
32import org.openstreetmap.josm.data.osm.Way;
33import org.openstreetmap.josm.data.osm.visitor.AddVisitor;
34import org.openstreetmap.josm.gui.PleaseWaitDialog;
35import org.openstreetmap.josm.tools.DateUtils;
36import org.xml.sax.Attributes;
37import org.xml.sax.InputSource;
38import org.xml.sax.SAXException;
39import org.xml.sax.helpers.DefaultHandler;
40
41/**
42 * Parser for the Osm Api. Read from an input stream and construct a dataset out of it.
43 *
44 * Reading process takes place in three phases. During the first phase (including xml parse),
45 * all nodes are read and stored. Other information than nodes are stored in a raw list
46 *
47 * The second phase read all ways out of the remaining objects in the raw list.
48 *
49 * @author Imi
50 */
51public class OsmReader {
52
53 /**
54 * This is used as (readonly) source for finding missing references when not transferred in the
55 * file.
56 */
57 private DataSet references;
58
59 /**
60 * The dataset to add parsed objects to.
61 */
62 private DataSet ds = new DataSet();
63 public DataSet getDs() { return ds; }
64
65 /**
66 * Record warnings. If there were any data inconsistencies, append
67 * a newline-terminated string.
68 */
69 private String parseNotes = new String();
70 private int parseNotesCount = 0;
71 public String getParseNotes() {
72 return parseNotes;
73 }
74
75 /** the list of ids of skipped {@see Way}s, i.e. ways which referred to nodes
76 * not included in the parsed data
77 */
78 private Set<Long> skippedWayIds = new HashSet<Long>();
79
80 /**
81 * The visitor to use to add the data to the set.
82 */
83 private AddVisitor adder = new AddVisitor(ds);
84
85 /**
86 * All read nodes after phase 1.
87 */
88 private Map<Long, Node> nodes = new HashMap<Long, Node>();
89
90
91 private static class OsmPrimitiveData {
92 public long id = 0;
93 public Map<String,String> keys = new HashMap<String, String>();
94 public boolean modified = false;
95 public boolean selected = false;
96 public boolean deleted = false;
97 public Date timestamp = new Date();
98 public User user = null;
99 public boolean visible = true;
100 public int version = -1;
101 public LatLon latlon = new LatLon(0,0);
102
103 public void copyTo(OsmPrimitive osm) {
104 osm.id = id;
105 osm.keys = keys;
106 osm.modified = modified;
107 osm.selected = selected;
108 osm.deleted = deleted;
109 osm.setTimestamp(timestamp);
110 osm.user = user;
111 osm.visible = visible;
112 osm.version = version;
113 osm.mappaintStyle = null;
114 }
115
116 public Node createNode() {
117 Node node = new Node(latlon);
118 copyTo(node);
119 return node;
120 }
121
122 public Way createWay() {
123 Way way = new Way(id);
124 copyTo(way);
125 return way;
126 }
127
128 public Relation createRelation() {
129 Relation rel = new Relation(id);
130 copyTo(rel);
131 return rel;
132 }
133 }
134
135 /**
136 * Used as a temporary storage for relation members, before they
137 * are resolved into pointers to real objects.
138 */
139 private static class RelationMemberData {
140 public String type;
141 public long id;
142 public RelationMember relationMember;
143 }
144
145 /**
146 * Data structure for the remaining way objects
147 */
148 private Map<OsmPrimitiveData, Collection<Long>> ways = new HashMap<OsmPrimitiveData, Collection<Long>>();
149
150 /**
151 * Data structure for relation objects
152 */
153 private Map<OsmPrimitiveData, Collection<RelationMemberData>> relations = new HashMap<OsmPrimitiveData, Collection<RelationMemberData>>();
154
155 private class Parser extends DefaultHandler {
156 /**
157 * The current osm primitive to be read.
158 */
159 private OsmPrimitiveData current;
160 private String generator;
161
162 @Override public void startElement(String namespaceURI, String localName, String qName, Attributes atts) throws SAXException {
163 try {
164 if (qName.equals("osm")) {
165 if (atts == null)
166 throw new SAXException(tr("Unknown version"));
167 String v = atts.getValue("version");
168 if (v == null)
169 throw new SAXException(tr("Version number missing from OSM data"));
170 if (!(v.equals("0.5") || v.equals("0.6")))
171 throw new SAXException(tr("Unknown version: {0}", v));
172 // save generator attribute for later use when creating DataSource objects
173 generator = atts.getValue("generator");
174 ds.version = v;
175
176 } else if (qName.equals("bounds")) {
177 // new style bounds.
178 String minlon = atts.getValue("minlon");
179 String minlat = atts.getValue("minlat");
180 String maxlon = atts.getValue("maxlon");
181 String maxlat = atts.getValue("maxlat");
182 String origin = atts.getValue("origin");
183 if (minlon != null && maxlon != null && minlat != null && maxlat != null) {
184 if (origin == null) {
185 origin = generator;
186 }
187 Bounds bounds = new Bounds(
188 new LatLon(Double.parseDouble(minlat), Double.parseDouble(minlon)),
189 new LatLon(Double.parseDouble(maxlat), Double.parseDouble(maxlon)));
190 DataSource src = new DataSource(bounds, origin);
191 ds.dataSources.add(src);
192 }
193
194 // ---- PARSING NODES AND WAYS ----
195
196 } else if (qName.equals("node")) {
197 current = new OsmPrimitiveData();
198 current.latlon = new LatLon(getDouble(atts, "lat"), getDouble(atts, "lon"));
199 readCommon(atts, current);
200 } else if (qName.equals("way")) {
201 current = new OsmPrimitiveData();
202 readCommon(atts, current);
203 ways.put(current, new ArrayList<Long>());
204 } else if (qName.equals("nd")) {
205 Collection<Long> list = ways.get(current);
206 if (list == null)
207 throw new SAXException(tr("Found <nd> element in non-way."));
208 long id = getLong(atts, "ref");
209 if (id == 0)
210 throw new SAXException(tr("<nd> has zero ref"));
211 list.add(id);
212
213 // ---- PARSING RELATIONS ----
214
215 } else if (qName.equals("relation")) {
216 current = new OsmPrimitiveData();
217 readCommon(atts, current);
218 relations.put(current, new LinkedList<RelationMemberData>());
219 } else if (qName.equals("member")) {
220 Collection<RelationMemberData> list = relations.get(current);
221 if (list == null)
222 throw new SAXException(tr("Found <member> element in non-relation."));
223 RelationMemberData emd = new RelationMemberData();
224 emd.relationMember = new RelationMember();
225 String value = atts.getValue("ref");
226 if (value == null)
227 throw new SAXException(tr("Missing attribute \"ref\" on member in relation {0}",current.id));
228 try {
229 emd.id = Long.parseLong(value);
230 } catch(NumberFormatException e) {
231 throw new SAXException(tr("Illegal value for attribute \"ref\" on member in relation {0}, got {1}", Long.toString(current.id),value));
232 }
233 value = atts.getValue("type");
234 if (value == null)
235 throw new SAXException(tr("Missing attribute \"type\" on member {0} in relation {1}", Long.toString(emd.id), Long.toString(current.id)));
236 if (! (value.equals("way") || value.equals("node") || value.equals("relation")))
237 throw new SAXException(tr("Unexpected \"type\" on member {0} in relation {1}, got {2}.", Long.toString(emd.id), Long.toString(current.id), value));
238 emd.type= value;
239 value = atts.getValue("role");
240 emd.relationMember.role = value;
241
242 if (emd.id == 0)
243 throw new SAXException(tr("Incomplete <member> specification with ref=0"));
244
245 list.add(emd);
246
247 // ---- PARSING TAGS (applicable to all objects) ----
248
249 } else if (qName.equals("tag")) {
250 String key = atts.getValue("k");
251 String value = atts.getValue("v");
252 current.keys.put(key,value);
253 }
254 } catch (NumberFormatException x) {
255 x.printStackTrace(); // SAXException does not chain correctly
256 throw new SAXException(x.getMessage(), x);
257 } catch (NullPointerException x) {
258 x.printStackTrace(); // SAXException does not chain correctly
259 throw new SAXException(tr("NullPointerException, possibly some missing tags."), x);
260 }
261 }
262
263 @Override
264 public void endElement(String uri, String localName, String qName) throws SAXException {
265 if (qName.equals("node")) {
266 nodes.put(current.id, current.createNode());
267 }
268 }
269
270 private double getDouble(Attributes atts, String value) {
271 return Double.parseDouble(atts.getValue(value));
272 }
273 }
274
275 /**
276 * Read out the common attributes from atts and put them into this.current.
277 */
278 void readCommon(Attributes atts, OsmPrimitiveData current) throws SAXException {
279 current.id = getLong(atts, "id");
280 if (current.id == 0)
281 throw new SAXException(tr("Illegal object with id=0"));
282
283 String time = atts.getValue("timestamp");
284 if (time != null && time.length() != 0) {
285 current.timestamp = DateUtils.fromString(time);
286 }
287
288 // user attribute added in 0.4 API
289 String user = atts.getValue("user");
290 if (user != null) {
291 // do not store literally; get object reference for string
292 current.user = User.get(user);
293 }
294
295 // uid attribute added in 0.6 API
296 String uid = atts.getValue("uid");
297 if (uid != null) {
298 if (current.user != null) {
299 current.user.uid = uid;
300 }
301 }
302
303 // visible attribute added in 0.4 API
304 String visible = atts.getValue("visible");
305 if (visible != null) {
306 current.visible = Boolean.parseBoolean(visible);
307 }
308
309 String version = atts.getValue("version");
310 current.version = 0;
311 if (version != null) {
312 try {
313 current.version = Integer.parseInt(version);
314 } catch(NumberFormatException e) {
315 throw new SAXException(tr("Illegal value for attribute \"version\" on OSM primitive with id {0}, got {1}", Long.toString(current.id), version));
316 }
317 } else {
318 // version expected for OSM primitives with an id assigned by the server (id > 0), since API 0.6
319 //
320 if (current.id > 0 && ds.version != null && ds.version.equals("0.6"))
321 throw new SAXException(tr("Missing attribute \"version\" on OSM primitive with id {0}", Long.toString(current.id)));
322 }
323
324 String action = atts.getValue("action");
325 if (action == null)
326 return;
327 if (action.equals("delete")) {
328 current.deleted = true;
329 } else if (action.startsWith("modify")) {
330 current.modified = true;
331 }
332 }
333 private long getLong(Attributes atts, String value) throws SAXException {
334 String s = atts.getValue(value);
335 if (s == null)
336 throw new SAXException(tr("Missing required attribute \"{0}\".",value));
337 return Long.parseLong(s);
338 }
339
340 private Node findNode(long id) {
341 Node n = nodes.get(id);
342 if (n != null)
343 return n;
344 for (Node node : references.nodes)
345 if (node.id == id)
346 return node;
347 // TODO: This has to be changed to support multiple layers.
348 for (Node node : Main.ds.nodes)
349 if (node.id == id)
350 return new Node(node);
351 return null;
352 }
353
354 protected void createWays() {
355 for (Entry<OsmPrimitiveData, Collection<Long>> e : ways.entrySet()) {
356 Way w = new Way();
357 boolean failed = false;
358 for (long id : e.getValue()) {
359 Node n = findNode(id);
360 if (n == null) {
361 /* don't report ALL of them, just a few */
362 if (parseNotesCount++ < 6) {
363 parseNotes += tr("Skipping a way because it includes a node that doesn''t exist: {0}\n", id);
364 } else if (parseNotesCount == 6) {
365 parseNotes += "...\n";
366 }
367 failed = true;
368 break;
369 }
370 w.nodes.add(n);
371 }
372 if (failed) {
373 skippedWayIds.add(e.getKey().id);
374 continue;
375 }
376 e.getKey().copyTo(w);
377 adder.visit(w);
378 }
379
380 }
381
382 /**
383 * Return the Way object with the given id, or null if it doesn't
384 * exist yet. This method only looks at ways stored in the data set.
385 *
386 * @param id
387 * @return way object or null
388 */
389 private Way findWay(long id) {
390 for (Way wy : Main.ds.ways)
391 if (wy.id == id)
392 return wy;
393 return null;
394 }
395
396 /**
397 * Return the Relation object with the given id, or null if it doesn't
398 * exist yet. This method only looks at relations stored in the data set.
399 *
400 * @param id
401 * @return relation object or null
402 */
403 private Relation findRelation(long id) {
404 for (Relation e : ds.relations)
405 if (e.id == id)
406 return e;
407 for (Relation e : Main.ds.relations)
408 if (e.id == id)
409 return e;
410 return null;
411 }
412
413 /**
414 * Create relations. This is slightly different than n/s/w because
415 * unlike other objects, relations may reference other relations; it
416 * is not guaranteed that a referenced relation will have been created
417 * before it is referenced. So we have to create all relations first,
418 * and populate them later.
419 */
420 private void createRelations() {
421
422 // pass 1 - create all relations
423 for (Entry<OsmPrimitiveData, Collection<RelationMemberData>> e : relations.entrySet()) {
424 Relation en = new Relation();
425 e.getKey().copyTo(en);
426 adder.visit(en);
427 }
428
429 // Cache the ways here for much better search performance
430 HashMap<Long, Way> hm = new HashMap<Long, Way>(10000);
431 for (Way wy : ds.ways) {
432 hm.put(wy.id, wy);
433 }
434
435 // pass 2 - sort out members
436 for (Entry<OsmPrimitiveData, Collection<RelationMemberData>> e : relations.entrySet()) {
437 Relation en = findRelation(e.getKey().id);
438 if (en == null) throw new Error("Failed to create relation " + e.getKey().id);
439
440 for (RelationMemberData emd : e.getValue()) {
441 RelationMember em = emd.relationMember;
442 if (emd.type.equals("node")) {
443 em.member = findNode(emd.id);
444 if (em.member == null) {
445 em.member = new Node(emd.id);
446 adder.visit((Node)em.member);
447 }
448 } else if (emd.type.equals("way")) {
449 em.member = hm.get(emd.id);
450 if (em.member == null) {
451 em.member = findWay(emd.id);
452 }
453 if (em.member == null) {
454 em.member = new Way(emd.id);
455 adder.visit((Way)em.member);
456 }
457 } else if (emd.type.equals("relation")) {
458 em.member = findRelation(emd.id);
459 if (em.member == null) {
460 em.member = new Relation(emd.id);
461 adder.visit((Relation)em.member);
462 }
463 } else {
464 // this is an error.
465 }
466 en.members.add(em);
467 }
468 }
469 hm = null;
470 }
471
472 /**
473 * Parse the given input source and return the dataset.
474 * @param ref The dataset that is search in for references first. If
475 * the Reference is not found here, Main.ds is searched and a copy of the
476 * element found there is returned.
477 */
478 public static DataSet parseDataSet(InputStream source, DataSet ref, PleaseWaitDialog pleaseWaitDlg) throws SAXException, IOException {
479 return parseDataSetOsm(source, ref, pleaseWaitDlg).ds;
480 }
481
482 public static OsmReader parseDataSetOsm(InputStream source, DataSet ref, PleaseWaitDialog pleaseWaitDlg) throws SAXException, IOException {
483 OsmReader osm = new OsmReader();
484 osm.references = ref == null ? new DataSet() : ref;
485
486 // phase 1: Parse nodes and read in raw ways
487 InputSource inputSource = new InputSource(new InputStreamReader(source, "UTF-8"));
488 try {
489 SAXParserFactory.newInstance().newSAXParser().parse(inputSource, osm.new Parser());
490 } catch (ParserConfigurationException e1) {
491 e1.printStackTrace(); // broken SAXException chaining
492 throw new SAXException(e1);
493 }
494
495 Main.pleaseWaitDlg.currentAction.setText(tr("Prepare OSM data..."));
496 Main.pleaseWaitDlg.setIndeterminate(true);
497
498 for (Node n : osm.nodes.values()) {
499 osm.adder.visit(n);
500 }
501
502 try {
503 osm.createWays();
504 osm.createRelations();
505 } catch (NumberFormatException e) {
506 e.printStackTrace();
507 throw new SAXException(tr("Ill-formed node id"));
508 }
509
510 // clear all negative ids (new to this file)
511 for (OsmPrimitive o : osm.ds.allPrimitives())
512 if (o.id < 0) {
513 o.id = 0;
514 }
515
516 Main.pleaseWaitDlg.setIndeterminate(false);
517 Main.pleaseWaitDlg.progress.setValue(0);
518
519 return osm;
520 }
521
522 /**
523 * replies a set of ids of skipped {@see Way}s, i.e. ways which were included in the downloaded
524 * data but which referred to nodes <strong>not</strong> available in the downloaded data
525 *
526 * @return the set of ids
527 */
528 public Set<Long> getSkippedWayIds() {
529 return skippedWayIds;
530 }
531}
Note: See TracBrowser for help on using the repository browser.