source: josm/trunk/src/org/openstreetmap/josm/io/OsmReader.java@ 1499

Last change on this file since 1499 was 1499, checked in by stoecker, 15 years ago

close #2302 - patch by jttt - optimizations and encapsulation
  • Property svn:eol-style set to native
File size: 21.6 KB
Line 
1// License: GPL. Copyright 2007 by Immanuel Scholz and others
2package org.openstreetmap.josm.io;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.IOException;
7import java.io.InputStream;
8import java.io.InputStreamReader;
9import java.util.ArrayList;
10import java.util.Arrays;
11import java.util.Collection;
12import java.util.HashMap;
13import java.util.HashSet;
14import java.util.LinkedList;
15import java.util.Map;
16import java.util.Map.Entry;
17
18import javax.xml.parsers.ParserConfigurationException;
19import javax.xml.parsers.SAXParserFactory;
20
21import org.openstreetmap.josm.Main;
22import org.openstreetmap.josm.data.Bounds;
23import org.openstreetmap.josm.data.coor.LatLon;
24import org.openstreetmap.josm.data.osm.DataSet;
25import org.openstreetmap.josm.data.osm.DataSource;
26import org.openstreetmap.josm.data.osm.Node;
27import org.openstreetmap.josm.data.osm.OsmPrimitive;
28import org.openstreetmap.josm.data.osm.Relation;
29import org.openstreetmap.josm.data.osm.RelationMember;
30import org.openstreetmap.josm.data.osm.User;
31import org.openstreetmap.josm.data.osm.Way;
32import org.openstreetmap.josm.data.osm.visitor.AddVisitor;
33import org.openstreetmap.josm.data.osm.visitor.Visitor;
34import org.openstreetmap.josm.gui.PleaseWaitDialog;
35import org.openstreetmap.josm.tools.DateUtils;
36import org.xml.sax.Attributes;
37import org.xml.sax.InputSource;
38import org.xml.sax.SAXException;
39import org.xml.sax.helpers.DefaultHandler;
40
41/**
42 * Parser for the Osm Api. Read from an input stream and construct a dataset out of it.
43 *
44 * Reading process takes place in three phases. During the first phase (including xml parse),
45 * all nodes are read and stored. Other information than nodes are stored in a raw list
46 *
47 * The second phase read all ways out of the remaining objects in the raw list.
48 *
49 * @author Imi
50 */
51public class OsmReader {
52
53// static long tagsN = 0;
54// static long nodesN = 0;
55// static long waysN = 0;
56// static long relationsN = 0;
57// static long membersN = 0;
58
59 static InputStream currSource;
60
61 /**
62 * This is used as (readonly) source for finding missing references when not transferred in the
63 * file.
64 */
65 private DataSet references;
66
67 /**
68 * The dataset to add parsed objects to.
69 */
70 private DataSet ds = new DataSet();
71 public DataSet getDs() { return ds; }
72
73 /**
74 * Record warnings. If there were any data inconsistencies, append
75 * a newline-terminated string.
76 */
77 private String parseNotes = new String();
78 private int parseNotesCount = 0;
79 public String getParseNotes() {
80 return parseNotes;
81 }
82
83 /**
84 * The visitor to use to add the data to the set.
85 */
86 private AddVisitor adder = new AddVisitor(ds);
87
88 /**
89 * All read nodes after phase 1.
90 */
91 private Map<Long, Node> nodes = new HashMap<Long, Node>();
92
93 // TODO: What the hack? Is this really from me? Please, clean this up!
94 private static class OsmPrimitiveData extends OsmPrimitive {
95 @Override public void visit(Visitor visitor) {}
96 public int compareTo(OsmPrimitive o) {return 0;}
97
98 public void copyTo(OsmPrimitive osm) {
99 osm.id = id;
100 osm.keys = keys;
101 osm.modified = modified;
102 osm.selected = selected;
103 osm.deleted = deleted;
104 osm.setTimestamp(getTimestamp());
105 osm.user = user;
106 osm.visible = visible;
107 osm.version = version;
108 osm.mappaintStyle = null;
109 }
110 }
111
112 /**
113 * Used as a temporary storage for relation members, before they
114 * are resolved into pointers to real objects.
115 */
116 private static class RelationMemberData {
117 public String type;
118 public long id;
119 public RelationMember relationMember;
120 }
121
122 /**
123 * Data structure for the remaining way objects
124 */
125 private Map<OsmPrimitiveData, Collection<Long>> ways = new HashMap<OsmPrimitiveData, Collection<Long>>();
126
127 /**
128 * Data structure for relation objects
129 */
130 private Map<OsmPrimitiveData, Collection<RelationMemberData>> relations = new HashMap<OsmPrimitiveData, Collection<RelationMemberData>>();
131
132 /**
133 * List of protocol versions that will be accepted on reading
134 */
135 private HashSet<String> allowedVersions = new HashSet<String>();
136
137 private class Parser extends DefaultHandler {
138 /**
139 * The current osm primitive to be read.
140 */
141 private OsmPrimitive current;
142 private String generator;
143 private Map<String, String> keys = new HashMap<String, String>();
144// int n = 0;
145
146 @Override public void startElement(String namespaceURI, String localName, String qName, Attributes atts) throws SAXException {
147 try {
148// if(n%100000 == 0) {
149// try {
150// FileInputStream fis = (FileInputStream)currSource;
151// FileChannel channel = fis.getChannel();
152// double perc = (((double)channel.position()) / ((double)channel.size()) * 100.0);
153// System.out.format(" " + (int)perc + "%%");
154// }
155// catch(java.lang.ClassCastException cce) {
156// }
157// catch(IOException e) {
158// System.out.format("Error reading file position " + e);
159// }
160// }
161// n++;
162
163 if (qName.equals("osm")) {
164 if (atts == null)
165 throw new SAXException(tr("Unknown version"));
166 if (!allowedVersions.contains(atts.getValue("version")))
167 throw new SAXException(tr("Unknown version")+": "+atts.getValue("version"));
168 // save generator attribute for later use when creating DataSource objects
169 generator = atts.getValue("generator");
170
171
172 } else if (qName.equals("bound")) {
173 // old style bounds.
174 // TODO: remove this around 1st October 2008.
175 // - this is a bit of a hack; since we still write out old style bound objects,
176 // we don't want to load them both. so when writing, we add a "note" tag the our
177 // old-style bound, and when reading, ignore those with a "note".
178 String note = atts.getValue("note");
179 if (note == null) {
180 System.out.println("Notice: old style <bound> element detected; support for these will be dropped in a future release.");
181 String bbox = atts.getValue("box");
182 String origin = atts.getValue("origin");
183 if (origin == null) origin = "";
184 if (bbox != null) {
185 String[] b = bbox.split(",");
186 Bounds bounds = new Bounds();
187 if (b.length == 4)
188 bounds = new Bounds(
189 new LatLon(Double.parseDouble(b[0]),Double.parseDouble(b[1])),
190 new LatLon(Double.parseDouble(b[2]),Double.parseDouble(b[3])));
191 DataSource src = new DataSource(bounds, origin);
192 ds.dataSources.add(src);
193 }
194 }
195 } else if (qName.equals("bounds")) {
196 // new style bounds.
197 String minlon = atts.getValue("minlon");
198 String minlat = atts.getValue("minlat");
199 String maxlon = atts.getValue("maxlon");
200 String maxlat = atts.getValue("maxlat");
201 String origin = atts.getValue("origin");
202 if (minlon != null && maxlon != null && minlat != null && maxlat != null) {
203 if (origin == null) origin = generator;
204 Bounds bounds = new Bounds(
205 new LatLon(Double.parseDouble(minlat), Double.parseDouble(minlon)),
206 new LatLon(Double.parseDouble(maxlat), Double.parseDouble(maxlon)));
207 DataSource src = new DataSource(bounds, origin);
208 ds.dataSources.add(src);
209 }
210
211 // ---- PARSING NODES AND WAYS ----
212
213 } else if (qName.equals("node")) {
214// nodesN++;
215 current = new Node(new LatLon(getDouble(atts, "lat"), getDouble(atts, "lon")));
216 readCommon(atts, current);
217 nodes.put(current.id, (Node)current);
218 } else if (qName.equals("way")) {
219// waysN++;
220 current = new OsmPrimitiveData();
221 readCommon(atts, current);
222 ways.put((OsmPrimitiveData)current, new ArrayList<Long>());
223 } else if (qName.equals("nd")) {
224 Collection<Long> list = ways.get(current);
225 if (list == null)
226 throw new SAXException(tr("Found <nd> element in non-way."));
227 long id = getLong(atts, "ref");
228 if (id == 0)
229 throw new SAXException(tr("<nd> has zero ref"));
230 list.add(id);
231
232 // ---- PARSING RELATIONS ----
233
234 } else if (qName.equals("relation")) {
235// relationsN++;
236 current = new OsmPrimitiveData();
237 readCommon(atts, current);
238 relations.put((OsmPrimitiveData)current, new LinkedList<RelationMemberData>());
239 } else if (qName.equals("member")) {
240// membersN++;
241 Collection<RelationMemberData> list = relations.get(current);
242 if (list == null)
243 throw new SAXException(tr("Found <member> element in non-relation."));
244 RelationMemberData emd = new RelationMemberData();
245 emd.relationMember = new RelationMember();
246 emd.id = getLong(atts, "ref");
247 emd.type=atts.getValue("type");
248 emd.relationMember.role = atts.getValue("role");
249
250 if (emd.id == 0)
251 throw new SAXException(tr("Incomplete <member> specification with ref=0"));
252
253 list.add(emd);
254
255 // ---- PARSING TAGS (applicable to all objects) ----
256
257 } else if (qName.equals("tag")) {
258// tagsN++;
259 String key = atts.getValue("k");
260 String internedKey = keys.get(key);
261 if (internedKey == null) {
262 internedKey = key;
263 keys.put(key, key);
264 }
265 current.put(internedKey, atts.getValue("v"));
266 }
267 } catch (NumberFormatException x) {
268 x.printStackTrace(); // SAXException does not chain correctly
269 throw new SAXException(x.getMessage(), x);
270 } catch (NullPointerException x) {
271 x.printStackTrace(); // SAXException does not chain correctly
272 throw new SAXException(tr("NullPointerException, possibly some missing tags."), x);
273 }
274 }
275
276 private double getDouble(Attributes atts, String value) {
277 return Double.parseDouble(atts.getValue(value));
278 }
279 }
280
281 /**
282 * Constructor initializes list of allowed protocol versions.
283 */
284 public OsmReader() {
285 // first add the main server version
286 allowedVersions.add(Main.pref.get("osm-server.version", "0.5"));
287 // now also add all compatible versions
288 String[] additionalVersions =
289 Main.pref.get("osm-server.additional-versions", "").split("/,/");
290 if (additionalVersions.length == 1 && additionalVersions[0].length() == 0)
291 additionalVersions = new String[] {};
292 allowedVersions.addAll(Arrays.asList(additionalVersions));
293 }
294
295 /**
296 * Read out the common attributes from atts and put them into this.current.
297 */
298 void readCommon(Attributes atts, OsmPrimitive current) throws SAXException {
299 current.id = getLong(atts, "id");
300 if (current.id == 0)
301 throw new SAXException(tr("Illegal object with id=0"));
302
303 String time = atts.getValue("timestamp");
304 if (time != null && time.length() != 0) {
305 current.setTimestamp(DateUtils.fromString(time));
306 }
307
308 // user attribute added in 0.4 API
309 String user = atts.getValue("user");
310 if (user != null) {
311 // do not store literally; get object reference for string
312 current.user = User.get(user);
313 }
314
315 // visible attribute added in 0.4 API
316 String visible = atts.getValue("visible");
317 if (visible != null) {
318 current.visible = Boolean.parseBoolean(visible);
319 }
320
321 // oldversion attribute added in 0.6 API
322
323 // Note there is an asymmetry here: the server will send
324 // the version as "version" the client sends it as
325 // "oldversion". So we take both since which we receive will
326 // depend on reading from a file or reading from the server
327
328 String version = atts.getValue("version");
329 if (version != null) {
330 current.version = Integer.parseInt(version);
331 }
332 version = atts.getValue("old_version");
333 if (version != null) {
334 current.version = Integer.parseInt(version);
335 }
336
337 String action = atts.getValue("action");
338 if (action == null)
339 return;
340 if (action.equals("delete"))
341 current.delete(true);
342 else if (action.startsWith("modify"))
343 current.modified = true;
344 }
345 private long getLong(Attributes atts, String value) throws SAXException {
346 String s = atts.getValue(value);
347 if (s == null)
348 throw new SAXException(tr("Missing required attribute \"{0}\".",value));
349 return Long.parseLong(s);
350 }
351
352 private Node findNode(long id) {
353 Node n = nodes.get(id);
354 if (n != null)
355 return n;
356 for (Node node : references.nodes)
357 if (node.id == id)
358 return node;
359 // TODO: This has to be changed to support multiple layers.
360 for (Node node : Main.ds.nodes)
361 if (node.id == id)
362 return new Node(node);
363 return null;
364 }
365
366 private void createWays() {
367 for (Entry<OsmPrimitiveData, Collection<Long>> e : ways.entrySet()) {
368 Way w = new Way();
369 boolean failed = false;
370 for (long id : e.getValue()) {
371 Node n = findNode(id);
372 if (n == null) {
373 /* don't report ALL of them, just a few */
374 if (parseNotesCount++ < 6) {
375 parseNotes += tr("Skipping a way because it includes a node that doesn''t exist: {0}\n", id);
376 } else if (parseNotesCount == 6) {
377 parseNotes += "...\n";
378 }
379 failed = true;
380 break;
381 }
382 w.nodes.add(n);
383 }
384 if (failed) continue;
385 e.getKey().copyTo(w);
386 adder.visit(w);
387 }
388
389 }
390
391 /**
392 * Return the Way object with the given id, or null if it doesn't
393 * exist yet. This method only looks at ways stored in the data set.
394 *
395 * @param id
396 * @return way object or null
397 */
398 private Way findWay(long id) {
399 for (Way wy : Main.ds.ways)
400 if (wy.id == id)
401 return wy;
402 return null;
403 }
404
405 /**
406 * Return the Relation object with the given id, or null if it doesn't
407 * exist yet. This method only looks at relations stored in the data set.
408 *
409 * @param id
410 * @return relation object or null
411 */
412 private Relation findRelation(long id) {
413 for (Relation e : ds.relations)
414 if (e.id == id)
415 return e;
416 for (Relation e : Main.ds.relations)
417 if (e.id == id)
418 return e;
419 return null;
420 }
421
422 /**
423 * Create relations. This is slightly different than n/s/w because
424 * unlike other objects, relations may reference other relations; it
425 * is not guaranteed that a referenced relation will have been created
426 * before it is referenced. So we have to create all relations first,
427 * and populate them later.
428 */
429 private void createRelations() {
430
431 // pass 1 - create all relations
432 for (Entry<OsmPrimitiveData, Collection<RelationMemberData>> e : relations.entrySet()) {
433 Relation en = new Relation();
434 e.getKey().copyTo(en);
435 adder.visit(en);
436 }
437
438 // Cache the ways here for much better search performance
439 HashMap<Long, Way> hm = new HashMap<Long, Way>(10000);
440 for (Way wy : ds.ways)
441 hm.put(wy.id, wy);
442
443 // pass 2 - sort out members
444 for (Entry<OsmPrimitiveData, Collection<RelationMemberData>> e : relations.entrySet()) {
445 Relation en = findRelation(e.getKey().id);
446 if (en == null) throw new Error("Failed to create relation " + e.getKey().id);
447
448 for (RelationMemberData emd : e.getValue()) {
449 RelationMember em = emd.relationMember;
450 if (emd.type.equals("node")) {
451 em.member = findNode(emd.id);
452 if (em.member == null) {
453 em.member = new Node(emd.id);
454 adder.visit((Node)em.member);
455 }
456 } else if (emd.type.equals("way")) {
457 em.member = hm.get(emd.id);
458 if (em.member == null)
459 em.member = findWay(emd.id);
460 if (em.member == null) {
461 em.member = new Way(emd.id);
462 adder.visit((Way)em.member);
463 }
464 } else if (emd.type.equals("relation")) {
465 em.member = findRelation(emd.id);
466 if (em.member == null) {
467 em.member = new Relation(emd.id);
468 adder.visit((Relation)em.member);
469 }
470 } else {
471 // this is an error.
472 }
473 en.members.add(em);
474 }
475 }
476 hm = null;
477 }
478
479 /**
480 * Parse the given input source and return the dataset.
481 * @param ref The dataset that is search in for references first. If
482 * the Reference is not found here, Main.ds is searched and a copy of the
483 * element found there is returned.
484 */
485 public static DataSet parseDataSet(InputStream source, DataSet ref, PleaseWaitDialog pleaseWaitDlg) throws SAXException, IOException {
486 return parseDataSetOsm(source, ref, pleaseWaitDlg).ds;
487 }
488
489 public static OsmReader parseDataSetOsm(InputStream source, DataSet ref, PleaseWaitDialog pleaseWaitDlg) throws SAXException, IOException {
490 OsmReader osm = new OsmReader();
491 osm.references = ref == null ? new DataSet() : ref;
492
493
494 currSource = source;
495
496 // phase 1: Parse nodes and read in raw ways
497 InputSource inputSource = new InputSource(new InputStreamReader(source, "UTF-8"));
498 try {
499 SAXParserFactory.newInstance().newSAXParser().parse(inputSource, osm.new Parser());
500 } catch (ParserConfigurationException e1) {
501 e1.printStackTrace(); // broken SAXException chaining
502 throw new SAXException(e1);
503 }
504
505 Main.pleaseWaitDlg.currentAction.setText(tr("Prepare OSM data..."));
506 Main.pleaseWaitDlg.setIndeterminate(true);
507
508// System.out.println("Parser finished: Tags " + tagsN + " Nodes " + nodesN + " Ways " + waysN +
509// " Relations " + relationsN + " Members " + membersN);
510
511 for (Node n : osm.nodes.values())
512 osm.adder.visit(n);
513
514 try {
515 osm.createWays();
516 osm.createRelations();
517 } catch (NumberFormatException e) {
518 e.printStackTrace();
519 throw new SAXException(tr("Ill-formed node id"));
520 }
521
522 // clear all negative ids (new to this file)
523 for (OsmPrimitive o : osm.ds.allPrimitives())
524 if (o.id < 0)
525 o.id = 0;
526
527// System.out.println("Data loaded!");
528 Main.pleaseWaitDlg.setIndeterminate(false);
529 Main.pleaseWaitDlg.progress.setValue(0);
530
531 return osm;
532 }
533}
Note: See TracBrowser for help on using the repository browser.