source: josm/trunk/src/org/openstreetmap/josm/io/OsmReader.java@ 1494

Last change on this file since 1494 was 1494, checked in by stoecker, 15 years ago

fix #2312 - performance optimization
  • Property svn:eol-style set to native
File size: 22.1 KB
Line 
1// License: GPL. Copyright 2007 by Immanuel Scholz and others
2package org.openstreetmap.josm.io;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.IOException;
7import java.io.InputStream;
8import java.io.InputStreamReader;
9import java.util.ArrayList;
10import java.util.Arrays;
11import java.util.Collection;
12import java.util.HashMap;
13import java.util.HashSet;
14import java.util.LinkedList;
15import java.util.Map;
16import java.util.Map.Entry;
17
18import javax.xml.parsers.ParserConfigurationException;
19import javax.xml.parsers.SAXParserFactory;
20
21import org.openstreetmap.josm.Main;
22import org.openstreetmap.josm.data.Bounds;
23import org.openstreetmap.josm.data.coor.LatLon;
24import org.openstreetmap.josm.data.osm.DataSet;
25import org.openstreetmap.josm.data.osm.DataSource;
26import org.openstreetmap.josm.data.osm.Node;
27import org.openstreetmap.josm.data.osm.OsmPrimitive;
28import org.openstreetmap.josm.data.osm.Relation;
29import org.openstreetmap.josm.data.osm.RelationMember;
30import org.openstreetmap.josm.data.osm.User;
31import org.openstreetmap.josm.data.osm.Way;
32import org.openstreetmap.josm.data.osm.visitor.AddVisitor;
33import org.openstreetmap.josm.data.osm.visitor.Visitor;
34import org.openstreetmap.josm.gui.PleaseWaitDialog;
35import org.xml.sax.Attributes;
36import org.xml.sax.InputSource;
37import org.xml.sax.SAXException;
38import org.xml.sax.helpers.DefaultHandler;
39
40/**
41 * Parser for the Osm Api. Read from an input stream and construct a dataset out of it.
42 *
43 * Reading process takes place in three phases. During the first phase (including xml parse),
44 * all nodes are read and stored. Other information than nodes are stored in a raw list
45 *
46 * The second phase read all ways out of the remaining objects in the raw list.
47 *
48 * @author Imi
49 */
50public class OsmReader {
51
52// static long tagsN = 0;
53// static long nodesN = 0;
54// static long waysN = 0;
55// static long relationsN = 0;
56// static long membersN = 0;
57
58 static InputStream currSource;
59
60 /**
61 * This is used as (readonly) source for finding missing references when not transferred in the
62 * file.
63 */
64 private DataSet references;
65
66 /**
67 * The dataset to add parsed objects to.
68 */
69 private DataSet ds = new DataSet();
70 public DataSet getDs() { return ds; }
71
72 /**
73 * Record warnings. If there were any data inconsistencies, append
74 * a newline-terminated string.
75 */
76 private String parseNotes = new String();
77 private int parseNotesCount = 0;
78 public String getParseNotes() {
79 return parseNotes;
80 }
81
82 /**
83 * The visitor to use to add the data to the set.
84 */
85 private AddVisitor adder = new AddVisitor(ds);
86
87 /**
88 * All read nodes after phase 1.
89 */
90 private Map<Long, Node> nodes = new HashMap<Long, Node>();
91
92 // TODO: What the hack? Is this really from me? Please, clean this up!
93 private static class OsmPrimitiveData extends OsmPrimitive {
94 @Override public void visit(Visitor visitor) {}
95 public int compareTo(OsmPrimitive o) {return 0;}
96
97 public void copyTo(OsmPrimitive osm) {
98 osm.id = id;
99 osm.keys = keys;
100 osm.modified = modified;
101 osm.selected = selected;
102 osm.deleted = deleted;
103 osm.timestamp = timestamp;
104 osm.user = user;
105 osm.visible = visible;
106 osm.version = version;
107 osm.checkTagged();
108 osm.checkDirectionTagged();
109 osm.mappaintStyle = null;
110 }
111 }
112
113 /**
114 * Used as a temporary storage for relation members, before they
115 * are resolved into pointers to real objects.
116 */
117 private static class RelationMemberData {
118 public String type;
119 public long id;
120 public RelationMember relationMember;
121 }
122
123 /**
124 * Data structure for the remaining way objects
125 */
126 private Map<OsmPrimitiveData, Collection<Long>> ways = new HashMap<OsmPrimitiveData, Collection<Long>>();
127
128 /**
129 * Data structure for relation objects
130 */
131 private Map<OsmPrimitiveData, Collection<RelationMemberData>> relations = new HashMap<OsmPrimitiveData, Collection<RelationMemberData>>();
132
133 /**
134 * List of protocol versions that will be accepted on reading
135 */
136 private HashSet<String> allowedVersions = new HashSet<String>();
137
138 private class Parser extends DefaultHandler {
139 /**
140 * The current osm primitive to be read.
141 */
142 private OsmPrimitive current;
143 private String generator;
144 private Map<String, String> keys = new HashMap<String, String>();
145// int n = 0;
146
147 @Override public void startElement(String namespaceURI, String localName, String qName, Attributes atts) throws SAXException {
148 try {
149// if(n%100000 == 0) {
150// try {
151// FileInputStream fis = (FileInputStream)currSource;
152// FileChannel channel = fis.getChannel();
153// double perc = (((double)channel.position()) / ((double)channel.size()) * 100.0);
154// System.out.format(" " + (int)perc + "%%");
155// }
156// catch(java.lang.ClassCastException cce) {
157// }
158// catch(IOException e) {
159// System.out.format("Error reading file position " + e);
160// }
161// }
162// n++;
163
164 if (qName.equals("osm")) {
165 if (atts == null)
166 throw new SAXException(tr("Unknown version"));
167 if (!allowedVersions.contains(atts.getValue("version")))
168 throw new SAXException(tr("Unknown version")+": "+atts.getValue("version"));
169 // save generator attribute for later use when creating DataSource objects
170 generator = atts.getValue("generator");
171
172
173 } else if (qName.equals("bound")) {
174 // old style bounds.
175 // TODO: remove this around 1st October 2008.
176 // - this is a bit of a hack; since we still write out old style bound objects,
177 // we don't want to load them both. so when writing, we add a "note" tag the our
178 // old-style bound, and when reading, ignore those with a "note".
179 String note = atts.getValue("note");
180 if (note == null) {
181 System.out.println("Notice: old style <bound> element detected; support for these will be dropped in a future release.");
182 String bbox = atts.getValue("box");
183 String origin = atts.getValue("origin");
184 if (origin == null) origin = "";
185 if (bbox != null) {
186 String[] b = bbox.split(",");
187 Bounds bounds = new Bounds();
188 if (b.length == 4)
189 bounds = new Bounds(
190 new LatLon(Double.parseDouble(b[0]),Double.parseDouble(b[1])),
191 new LatLon(Double.parseDouble(b[2]),Double.parseDouble(b[3])));
192 DataSource src = new DataSource(bounds, origin);
193 ds.dataSources.add(src);
194 }
195 }
196 } else if (qName.equals("bounds")) {
197 // new style bounds.
198 String minlon = atts.getValue("minlon");
199 String minlat = atts.getValue("minlat");
200 String maxlon = atts.getValue("maxlon");
201 String maxlat = atts.getValue("maxlat");
202 String origin = atts.getValue("origin");
203 if (minlon != null && maxlon != null && minlat != null && maxlat != null) {
204 if (origin == null) origin = generator;
205 Bounds bounds = new Bounds(
206 new LatLon(Double.parseDouble(minlat), Double.parseDouble(minlon)),
207 new LatLon(Double.parseDouble(maxlat), Double.parseDouble(maxlon)));
208 DataSource src = new DataSource(bounds, origin);
209 ds.dataSources.add(src);
210 }
211
212 // ---- PARSING NODES AND WAYS ----
213
214 } else if (qName.equals("node")) {
215// nodesN++;
216 current = new Node(new LatLon(getDouble(atts, "lat"), getDouble(atts, "lon")));
217 readCommon(atts, current);
218 nodes.put(current.id, (Node)current);
219 } else if (qName.equals("way")) {
220// waysN++;
221 current = new OsmPrimitiveData();
222 readCommon(atts, current);
223 ways.put((OsmPrimitiveData)current, new ArrayList<Long>());
224 } else if (qName.equals("nd")) {
225 Collection<Long> list = ways.get(current);
226 if (list == null)
227 throw new SAXException(tr("Found <nd> element in non-way."));
228 long id = getLong(atts, "ref");
229 if (id == 0)
230 throw new SAXException(tr("<nd> has zero ref"));
231 list.add(id);
232
233 // ---- PARSING RELATIONS ----
234
235 } else if (qName.equals("relation")) {
236// relationsN++;
237 current = new OsmPrimitiveData();
238 readCommon(atts, current);
239 relations.put((OsmPrimitiveData)current, new LinkedList<RelationMemberData>());
240 } else if (qName.equals("member")) {
241// membersN++;
242 Collection<RelationMemberData> list = relations.get(current);
243 if (list == null)
244 throw new SAXException(tr("Found <member> element in non-relation."));
245 RelationMemberData emd = new RelationMemberData();
246 emd.relationMember = new RelationMember();
247 emd.id = getLong(atts, "ref");
248 emd.type=atts.getValue("type");
249 emd.relationMember.role = atts.getValue("role");
250
251 if (emd.id == 0)
252 throw new SAXException(tr("Incomplete <member> specification with ref=0"));
253
254 list.add(emd);
255
256 // ---- PARSING TAGS (applicable to all objects) ----
257
258 } else if (qName.equals("tag")) {
259// tagsN++;
260 String key = atts.getValue("k");
261 String internedKey = keys.get(key);
262 if (internedKey == null) {
263 internedKey = key;
264 keys.put(key, key);
265 }
266 current.put(internedKey, atts.getValue("v"));
267 }
268 } catch (NumberFormatException x) {
269 x.printStackTrace(); // SAXException does not chain correctly
270 throw new SAXException(x.getMessage(), x);
271 } catch (NullPointerException x) {
272 x.printStackTrace(); // SAXException does not chain correctly
273 throw new SAXException(tr("NullPointerException, possibly some missing tags."), x);
274 }
275 }
276
277 private double getDouble(Attributes atts, String value) {
278 return Double.parseDouble(atts.getValue(value));
279 }
280 }
281
282 /**
283 * Constructor initializes list of allowed protocol versions.
284 */
285 public OsmReader() {
286 // first add the main server version
287 allowedVersions.add(Main.pref.get("osm-server.version", "0.5"));
288 // now also add all compatible versions
289 String[] additionalVersions =
290 Main.pref.get("osm-server.additional-versions", "").split("/,/");
291 if (additionalVersions.length == 1 && additionalVersions[0].length() == 0)
292 additionalVersions = new String[] {};
293 allowedVersions.addAll(Arrays.asList(additionalVersions));
294 }
295
296 /**
297 * Read out the common attributes from atts and put them into this.current.
298 */
299 void readCommon(Attributes atts, OsmPrimitive current) throws SAXException {
300 current.id = getLong(atts, "id");
301 if (current.id == 0)
302 throw new SAXException(tr("Illegal object with id=0"));
303
304 String time = atts.getValue("timestamp");
305 if (time != null && time.length() != 0) {
306 /* Do not parse the date here since it wastes a HUGE amount of time.
307 * Moved into OsmPrimitive.
308 try {
309 current.timestamp = DateParser.parse(time);
310 } catch (ParseException e) {
311 e.printStackTrace();
312 throw new SAXException(tr("Couldn''t read time format \"{0}\".",time));
313 }
314 */
315 current.timestamp = time;
316 }
317
318 // user attribute added in 0.4 API
319 String user = atts.getValue("user");
320 if (user != null) {
321 // do not store literally; get object reference for string
322 current.user = User.get(user);
323 }
324
325 // visible attribute added in 0.4 API
326 String visible = atts.getValue("visible");
327 if (visible != null) {
328 current.visible = Boolean.parseBoolean(visible);
329 }
330
331 // oldversion attribute added in 0.6 API
332
333 // Note there is an asymmetry here: the server will send
334 // the version as "version" the client sends it as
335 // "oldversion". So we take both since which we receive will
336 // depend on reading from a file or reading from the server
337
338 String version = atts.getValue("version");
339 if (version != null) {
340 current.version = Integer.parseInt(version);
341 }
342 version = atts.getValue("old_version");
343 if (version != null) {
344 current.version = Integer.parseInt(version);
345 }
346
347 String action = atts.getValue("action");
348 if (action == null)
349 return;
350 if (action.equals("delete"))
351 current.delete(true);
352 else if (action.startsWith("modify"))
353 current.modified = true;
354 }
355 private long getLong(Attributes atts, String value) throws SAXException {
356 String s = atts.getValue(value);
357 if (s == null)
358 throw new SAXException(tr("Missing required attribute \"{0}\".",value));
359 return Long.parseLong(s);
360 }
361
362 private Node findNode(long id) {
363 Node n = nodes.get(id);
364 if (n != null)
365 return n;
366 for (Node node : references.nodes)
367 if (node.id == id)
368 return node;
369 // TODO: This has to be changed to support multiple layers.
370 for (Node node : Main.ds.nodes)
371 if (node.id == id)
372 return new Node(node);
373 return null;
374 }
375
376 private void createWays() {
377 for (Entry<OsmPrimitiveData, Collection<Long>> e : ways.entrySet()) {
378 Way w = new Way();
379 boolean failed = false;
380 for (long id : e.getValue()) {
381 Node n = findNode(id);
382 if (n == null) {
383 /* don't report ALL of them, just a few */
384 if (parseNotesCount++ < 6) {
385 parseNotes += tr("Skipping a way because it includes a node that doesn''t exist: {0}\n", id);
386 } else if (parseNotesCount == 6) {
387 parseNotes += "...\n";
388 }
389 failed = true;
390 break;
391 }
392 w.nodes.add(n);
393 }
394 if (failed) continue;
395 e.getKey().copyTo(w);
396 adder.visit(w);
397 }
398
399 }
400
401 /**
402 * Return the Way object with the given id, or null if it doesn't
403 * exist yet. This method only looks at ways stored in the data set.
404 *
405 * @param id
406 * @return way object or null
407 */
408 private Way findWay(long id) {
409 for (Way wy : Main.ds.ways)
410 if (wy.id == id)
411 return wy;
412 return null;
413 }
414
415 /**
416 * Return the Relation object with the given id, or null if it doesn't
417 * exist yet. This method only looks at relations stored in the data set.
418 *
419 * @param id
420 * @return relation object or null
421 */
422 private Relation findRelation(long id) {
423 for (Relation e : ds.relations)
424 if (e.id == id)
425 return e;
426 for (Relation e : Main.ds.relations)
427 if (e.id == id)
428 return e;
429 return null;
430 }
431
432 /**
433 * Create relations. This is slightly different than n/s/w because
434 * unlike other objects, relations may reference other relations; it
435 * is not guaranteed that a referenced relation will have been created
436 * before it is referenced. So we have to create all relations first,
437 * and populate them later.
438 */
439 private void createRelations() {
440
441 // pass 1 - create all relations
442 for (Entry<OsmPrimitiveData, Collection<RelationMemberData>> e : relations.entrySet()) {
443 Relation en = new Relation();
444 e.getKey().copyTo(en);
445 adder.visit(en);
446 }
447
448 // Cache the ways here for much better search performance
449 HashMap<Long, Way> hm = new HashMap<Long, Way>(10000);
450 for (Way wy : ds.ways)
451 hm.put(wy.id, wy);
452
453 // pass 2 - sort out members
454 for (Entry<OsmPrimitiveData, Collection<RelationMemberData>> e : relations.entrySet()) {
455 Relation en = findRelation(e.getKey().id);
456 if (en == null) throw new Error("Failed to create relation " + e.getKey().id);
457
458 for (RelationMemberData emd : e.getValue()) {
459 RelationMember em = emd.relationMember;
460 if (emd.type.equals("node")) {
461 em.member = findNode(emd.id);
462 if (em.member == null) {
463 em.member = new Node(emd.id);
464 adder.visit((Node)em.member);
465 }
466 } else if (emd.type.equals("way")) {
467 em.member = hm.get(emd.id);
468 if (em.member == null)
469 em.member = findWay(emd.id);
470 if (em.member == null) {
471 em.member = new Way(emd.id);
472 adder.visit((Way)em.member);
473 }
474 } else if (emd.type.equals("relation")) {
475 em.member = findRelation(emd.id);
476 if (em.member == null) {
477 em.member = new Relation(emd.id);
478 adder.visit((Relation)em.member);
479 }
480 } else {
481 // this is an error.
482 }
483 en.members.add(em);
484 }
485 }
486 hm = null;
487 }
488
489 /**
490 * Parse the given input source and return the dataset.
491 * @param ref The dataset that is search in for references first. If
492 * the Reference is not found here, Main.ds is searched and a copy of the
493 * element found there is returned.
494 */
495 public static DataSet parseDataSet(InputStream source, DataSet ref, PleaseWaitDialog pleaseWaitDlg) throws SAXException, IOException {
496 return parseDataSetOsm(source, ref, pleaseWaitDlg).ds;
497 }
498
499 public static OsmReader parseDataSetOsm(InputStream source, DataSet ref, PleaseWaitDialog pleaseWaitDlg) throws SAXException, IOException {
500 OsmReader osm = new OsmReader();
501 osm.references = ref == null ? new DataSet() : ref;
502
503
504 currSource = source;
505
506 // phase 1: Parse nodes and read in raw ways
507 InputSource inputSource = new InputSource(new InputStreamReader(source, "UTF-8"));
508 try {
509 SAXParserFactory.newInstance().newSAXParser().parse(inputSource, osm.new Parser());
510 } catch (ParserConfigurationException e1) {
511 e1.printStackTrace(); // broken SAXException chaining
512 throw new SAXException(e1);
513 }
514
515 Main.pleaseWaitDlg.currentAction.setText(tr("Prepare OSM data..."));
516 Main.pleaseWaitDlg.setIndeterminate(true);
517
518// System.out.println("Parser finished: Tags " + tagsN + " Nodes " + nodesN + " Ways " + waysN +
519// " Relations " + relationsN + " Members " + membersN);
520
521 for (Node n : osm.nodes.values())
522 osm.adder.visit(n);
523
524 try {
525 osm.createWays();
526 osm.createRelations();
527 } catch (NumberFormatException e) {
528 e.printStackTrace();
529 throw new SAXException(tr("Ill-formed node id"));
530 }
531
532 // clear all negative ids (new to this file)
533 for (OsmPrimitive o : osm.ds.allPrimitives())
534 if (o.id < 0)
535 o.id = 0;
536
537// System.out.println("Data loaded!");
538 Main.pleaseWaitDlg.setIndeterminate(false);
539 Main.pleaseWaitDlg.progress.setValue(0);
540
541 return osm;
542 }
543}
Note: See TracBrowser for help on using the repository browser.