source: josm/trunk/src/org/openstreetmap/josm/io/OsmReader.java@ 15735

Last change on this file since 15735 was 15470, checked in by Don-vip, 5 years ago

fix #18249 - Allow unknown xml attributes to be added as tags (patch by taylor.smock)
  • Property svn:eol-style set to native
File size: 19.2 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.io;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.InputStream;
7import java.util.Collection;
8import java.util.Objects;
9import java.util.Set;
10import java.util.TreeSet;
11import java.util.regex.Matcher;
12import java.util.regex.Pattern;
13
14import javax.xml.stream.Location;
15import javax.xml.stream.XMLStreamConstants;
16import javax.xml.stream.XMLStreamException;
17import javax.xml.stream.XMLStreamReader;
18
19import org.openstreetmap.josm.data.osm.Changeset;
20import org.openstreetmap.josm.data.osm.DataSet;
21import org.openstreetmap.josm.data.osm.Node;
22import org.openstreetmap.josm.data.osm.PrimitiveData;
23import org.openstreetmap.josm.data.osm.Relation;
24import org.openstreetmap.josm.data.osm.RelationMemberData;
25import org.openstreetmap.josm.data.osm.Tagged;
26import org.openstreetmap.josm.data.osm.Way;
27import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
28import org.openstreetmap.josm.gui.progress.ProgressMonitor;
29import org.openstreetmap.josm.tools.Logging;
30import org.openstreetmap.josm.tools.UncheckedParseException;
31import org.openstreetmap.josm.tools.XmlUtils;
32
33/**
34 * Parser for the Osm API (XML output). Read from an input stream and construct a dataset out of it.
35 *
36 * For each xml element, there is a dedicated method.
37 * The XMLStreamReader cursor points to the start of the element, when the method is
38 * entered, and it must point to the end of the same element, when it is exited.
39 */
40public class OsmReader extends AbstractReader {
41
42 protected XMLStreamReader parser;
43
44 protected boolean convertUnknownToTags;
45
46 private static final Set<String> COMMON_XML_ATTRIBUTES = new TreeSet<>();
47
48 static {
49 COMMON_XML_ATTRIBUTES.add("id");
50 COMMON_XML_ATTRIBUTES.add("timestamp");
51 COMMON_XML_ATTRIBUTES.add("user");
52 COMMON_XML_ATTRIBUTES.add("uid");
53 COMMON_XML_ATTRIBUTES.add("visible");
54 COMMON_XML_ATTRIBUTES.add("version");
55 COMMON_XML_ATTRIBUTES.add("action");
56 COMMON_XML_ATTRIBUTES.add("changeset");
57 COMMON_XML_ATTRIBUTES.add("lat");
58 COMMON_XML_ATTRIBUTES.add("lon");
59 }
60
61 /**
62 * constructor (for private and subclasses use only)
63 *
64 * @see #parseDataSet(InputStream, ProgressMonitor)
65 */
66 protected OsmReader() {
67 this(false);
68 }
69
70 /**
71 * constructor (for private and subclasses use only)
72 * @param convertUnknownToTags if true, keep unknown xml attributes as tags
73 *
74 * @see #parseDataSet(InputStream, ProgressMonitor)
75 * @since 15470
76 */
77 protected OsmReader(boolean convertUnknownToTags) {
78 // Restricts visibility
79 this.convertUnknownToTags = convertUnknownToTags;
80 }
81
82 protected void setParser(XMLStreamReader parser) {
83 this.parser = parser;
84 }
85
86 protected void throwException(Throwable th) throws XMLStreamException {
87 throw new XmlStreamParsingException(th.getMessage(), parser.getLocation(), th);
88 }
89
90 protected void throwException(String msg, Throwable th) throws XMLStreamException {
91 throw new XmlStreamParsingException(msg, parser.getLocation(), th);
92 }
93
94 protected void throwException(String msg) throws XMLStreamException {
95 throw new XmlStreamParsingException(msg, parser.getLocation());
96 }
97
98 protected void parse() throws XMLStreamException {
99 int event = parser.getEventType();
100 while (true) {
101 if (event == XMLStreamConstants.START_ELEMENT) {
102 parseRoot();
103 } else if (event == XMLStreamConstants.END_ELEMENT)
104 return;
105 if (parser.hasNext()) {
106 event = parser.next();
107 } else {
108 break;
109 }
110 }
111 parser.close();
112 }
113
114 protected void parseRoot() throws XMLStreamException {
115 if ("osm".equals(parser.getLocalName())) {
116 parseOsm();
117 } else {
118 parseUnknown();
119 }
120 }
121
122 private void parseOsm() throws XMLStreamException {
123 try {
124 parseVersion(parser.getAttributeValue(null, "version"));
125 parseDownloadPolicy("download", parser.getAttributeValue(null, "download"));
126 parseUploadPolicy("upload", parser.getAttributeValue(null, "upload"));
127 parseLocked(parser.getAttributeValue(null, "locked"));
128 } catch (IllegalDataException e) {
129 throwException(e);
130 }
131 String generator = parser.getAttributeValue(null, "generator");
132 Long uploadChangesetId = null;
133 if (parser.getAttributeValue(null, "upload-changeset") != null) {
134 uploadChangesetId = getLong("upload-changeset");
135 }
136 while (parser.hasNext()) {
137 int event = parser.next();
138
139 if (cancel) {
140 cancel = false;
141 throw new OsmParsingCanceledException(tr("Reading was canceled"), parser.getLocation());
142 }
143
144 if (event == XMLStreamConstants.START_ELEMENT) {
145 switch (parser.getLocalName()) {
146 case "bounds":
147 parseBounds(generator);
148 break;
149 case "node":
150 parseNode();
151 break;
152 case "way":
153 parseWay();
154 break;
155 case "relation":
156 parseRelation();
157 break;
158 case "changeset":
159 parseChangeset(uploadChangesetId);
160 break;
161 case "remark": // Used by Overpass API
162 parseRemark();
163 break;
164 default:
165 parseUnknown();
166 }
167 } else if (event == XMLStreamConstants.END_ELEMENT) {
168 return;
169 }
170 }
171 }
172
173 private void handleIllegalDataException(IllegalDataException e) throws XMLStreamException {
174 Throwable cause = e.getCause();
175 if (cause instanceof XMLStreamException) {
176 throw (XMLStreamException) cause;
177 } else {
178 throwException(e);
179 }
180 }
181
182 private void parseRemark() throws XMLStreamException {
183 while (parser.hasNext()) {
184 int event = parser.next();
185 if (event == XMLStreamConstants.CHARACTERS) {
186 ds.setRemark(parser.getText());
187 } else if (event == XMLStreamConstants.END_ELEMENT) {
188 return;
189 }
190 }
191 }
192
193 private void parseBounds(String generator) throws XMLStreamException {
194 String minlon = parser.getAttributeValue(null, "minlon");
195 String minlat = parser.getAttributeValue(null, "minlat");
196 String maxlon = parser.getAttributeValue(null, "maxlon");
197 String maxlat = parser.getAttributeValue(null, "maxlat");
198 String origin = parser.getAttributeValue(null, "origin");
199 try {
200 parseBounds(generator, minlon, minlat, maxlon, maxlat, origin);
201 } catch (IllegalDataException e) {
202 handleIllegalDataException(e);
203 }
204 jumpToEnd();
205 }
206
207 protected Node parseNode() throws XMLStreamException {
208 String lat = parser.getAttributeValue(null, "lat");
209 String lon = parser.getAttributeValue(null, "lon");
210 try {
211 return parseNode(lat, lon, this::readCommon, this::parseNodeTags);
212 } catch (IllegalDataException e) {
213 handleIllegalDataException(e);
214 }
215 return null;
216 }
217
218 private void parseNodeTags(Node n) throws IllegalDataException {
219 try {
220 while (parser.hasNext()) {
221 int event = parser.next();
222 if (event == XMLStreamConstants.START_ELEMENT) {
223 if ("tag".equals(parser.getLocalName())) {
224 parseTag(n);
225 } else {
226 parseUnknown();
227 }
228 } else if (event == XMLStreamConstants.END_ELEMENT) {
229 return;
230 }
231 }
232 } catch (XMLStreamException e) {
233 throw new IllegalDataException(e);
234 }
235 }
236
237 protected Way parseWay() throws XMLStreamException {
238 try {
239 return parseWay(this::readCommon, this::parseWayNodesAndTags);
240 } catch (IllegalDataException e) {
241 handleIllegalDataException(e);
242 }
243 return null;
244 }
245
246 private void parseWayNodesAndTags(Way w, Collection<Long> nodeIds) throws IllegalDataException {
247 try {
248 while (parser.hasNext()) {
249 int event = parser.next();
250 if (event == XMLStreamConstants.START_ELEMENT) {
251 switch (parser.getLocalName()) {
252 case "nd":
253 nodeIds.add(parseWayNode(w));
254 break;
255 case "tag":
256 parseTag(w);
257 break;
258 default:
259 parseUnknown();
260 }
261 } else if (event == XMLStreamConstants.END_ELEMENT) {
262 break;
263 }
264 }
265 } catch (XMLStreamException e) {
266 throw new IllegalDataException(e);
267 }
268 }
269
270 private long parseWayNode(Way w) throws XMLStreamException {
271 if (parser.getAttributeValue(null, "ref") == null) {
272 throwException(
273 tr("Missing mandatory attribute ''{0}'' on <nd> of way {1}.", "ref", Long.toString(w.getUniqueId()))
274 );
275 }
276 long id = getLong("ref");
277 if (id == 0) {
278 throwException(
279 tr("Illegal value of attribute ''ref'' of element <nd>. Got {0}.", Long.toString(id))
280 );
281 }
282 jumpToEnd();
283 return id;
284 }
285
286 protected Relation parseRelation() throws XMLStreamException {
287 try {
288 return parseRelation(this::readCommon, this::parseRelationMembersAndTags);
289 } catch (IllegalDataException e) {
290 handleIllegalDataException(e);
291 }
292 return null;
293 }
294
295 private void parseRelationMembersAndTags(Relation r, Collection<RelationMemberData> members) throws IllegalDataException {
296 try {
297 while (parser.hasNext()) {
298 int event = parser.next();
299 if (event == XMLStreamConstants.START_ELEMENT) {
300 switch (parser.getLocalName()) {
301 case "member":
302 members.add(parseRelationMember(r));
303 break;
304 case "tag":
305 parseTag(r);
306 break;
307 default:
308 parseUnknown();
309 }
310 } else if (event == XMLStreamConstants.END_ELEMENT) {
311 break;
312 }
313 }
314 } catch (XMLStreamException e) {
315 throw new IllegalDataException(e);
316 }
317 }
318
319 private RelationMemberData parseRelationMember(Relation r) throws XMLStreamException {
320 RelationMemberData result = null;
321 try {
322 String ref = parser.getAttributeValue(null, "ref");
323 String type = parser.getAttributeValue(null, "type");
324 String role = parser.getAttributeValue(null, "role");
325 result = parseRelationMember(r, ref, type, role);
326 jumpToEnd();
327 } catch (IllegalDataException e) {
328 handleIllegalDataException(e);
329 }
330 return result;
331 }
332
333 private void parseChangeset(Long uploadChangesetId) throws XMLStreamException {
334
335 Long id = null;
336 if (parser.getAttributeValue(null, "id") != null) {
337 id = getLong("id");
338 }
339 // Read changeset info if neither upload-changeset nor id are set, or if they are both set to the same value
340 if (Objects.equals(id, uploadChangesetId)) {
341 uploadChangeset = new Changeset(id != null ? id.intValue() : 0);
342 while (true) {
343 int event = parser.next();
344 if (event == XMLStreamConstants.START_ELEMENT) {
345 if ("tag".equals(parser.getLocalName())) {
346 parseTag(uploadChangeset);
347 } else {
348 parseUnknown();
349 }
350 } else if (event == XMLStreamConstants.END_ELEMENT)
351 return;
352 }
353 } else {
354 jumpToEnd(false);
355 }
356 }
357
358 private void parseTag(Tagged t) throws XMLStreamException {
359 String key = parser.getAttributeValue(null, "k");
360 String value = parser.getAttributeValue(null, "v");
361 try {
362 parseTag(t, key, value);
363 } catch (IllegalDataException e) {
364 throwException(e);
365 }
366 jumpToEnd();
367 }
368
369 protected void parseUnknown(boolean printWarning) throws XMLStreamException {
370 final String element = parser.getLocalName();
371 if (printWarning && ("note".equals(element) || "meta".equals(element))) {
372 // we know that Overpass API returns those elements
373 Logging.debug(tr("Undefined element ''{0}'' found in input stream. Skipping.", element));
374 } else if (printWarning) {
375 Logging.info(tr("Undefined element ''{0}'' found in input stream. Skipping.", element));
376 }
377 while (true) {
378 int event = parser.next();
379 if (event == XMLStreamConstants.START_ELEMENT) {
380 parseUnknown(false); /* no more warning for inner elements */
381 } else if (event == XMLStreamConstants.END_ELEMENT)
382 return;
383 }
384 }
385
386 protected void parseUnknown() throws XMLStreamException {
387 parseUnknown(true);
388 }
389
390 /**
391 * When cursor is at the start of an element, moves it to the end tag of that element.
392 * Nested content is skipped.
393 *
394 * This is basically the same code as parseUnknown(), except for the warnings, which
395 * are displayed for inner elements and not at top level.
396 * @param printWarning if {@code true}, a warning message will be printed if an unknown element is met
397 * @throws XMLStreamException if there is an error processing the underlying XML source
398 */
399 protected final void jumpToEnd(boolean printWarning) throws XMLStreamException {
400 while (true) {
401 int event = parser.next();
402 if (event == XMLStreamConstants.START_ELEMENT) {
403 parseUnknown(printWarning);
404 } else if (event == XMLStreamConstants.END_ELEMENT)
405 return;
406 }
407 }
408
409 protected final void jumpToEnd() throws XMLStreamException {
410 jumpToEnd(true);
411 }
412
413 /**
414 * Read out the common attributes and put them into current OsmPrimitive.
415 * @param current primitive to update
416 * @throws IllegalDataException if there is an error processing the underlying XML source
417 */
418 private void readCommon(PrimitiveData current) throws IllegalDataException {
419 try {
420 parseId(current, getLong("id"));
421 parseTimestamp(current, parser.getAttributeValue(null, "timestamp"));
422 parseUser(current, parser.getAttributeValue(null, "user"), parser.getAttributeValue(null, "uid"));
423 parseVisible(current, parser.getAttributeValue(null, "visible"));
424 parseVersion(current, parser.getAttributeValue(null, "version"));
425 parseAction(current, parser.getAttributeValue(null, "action"));
426 parseChangeset(current, parser.getAttributeValue(null, "changeset"));
427
428 if (convertUnknownToTags) {
429 for (int i = 0; i < parser.getAttributeCount(); i++) {
430 if (!COMMON_XML_ATTRIBUTES.contains(parser.getAttributeLocalName(i))) {
431 parseTag(current, parser.getAttributeLocalName(i), parser.getAttributeValue(i));
432 }
433 }
434 }
435 } catch (UncheckedParseException | XMLStreamException e) {
436 throw new IllegalDataException(e);
437 }
438 }
439
440 private long getLong(String name) throws XMLStreamException {
441 String value = parser.getAttributeValue(null, name);
442 try {
443 return getLong(name, value);
444 } catch (IllegalDataException e) {
445 throwException(e);
446 }
447 return 0; // should not happen
448 }
449
450 /**
451 * Exception thrown after user cancelation.
452 */
453 private static final class OsmParsingCanceledException extends XmlStreamParsingException implements ImportCancelException {
454 /**
455 * Constructs a new {@code OsmParsingCanceledException}.
456 * @param msg The error message
457 * @param location The parser location
458 */
459 OsmParsingCanceledException(String msg, Location location) {
460 super(msg, location);
461 }
462 }
463
464 @Override
465 protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
466 return doParseDataSet(source, progressMonitor, ir -> {
467 try {
468 setParser(XmlUtils.newSafeXMLInputFactory().createXMLStreamReader(ir));
469 parse();
470 } catch (XmlStreamParsingException | UncheckedParseException e) {
471 throw new IllegalDataException(e.getMessage(), e);
472 } catch (XMLStreamException e) {
473 String msg = e.getMessage();
474 Pattern p = Pattern.compile("Message: (.+)");
475 Matcher m = p.matcher(msg);
476 if (m.find()) {
477 msg = m.group(1);
478 }
479 if (e.getLocation() != null)
480 throw new IllegalDataException(tr("Line {0} column {1}: ",
481 e.getLocation().getLineNumber(), e.getLocation().getColumnNumber()) + msg, e);
482 else
483 throw new IllegalDataException(msg, e);
484 }
485 });
486 }
487
488 /**
489 * Parse the given input source and return the dataset.
490 *
491 * @param source the source input stream. Must not be null.
492 * @param progressMonitor the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed
493 *
494 * @return the dataset with the parsed data
495 * @throws IllegalDataException if an error was found while parsing the data from the source
496 * @throws IllegalArgumentException if source is null
497 */
498 public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
499 return parseDataSet(source, progressMonitor, false);
500 }
501
502 /**
503 * Parse the given input source and return the dataset.
504 *
505 * @param source the source input stream. Must not be null.
506 * @param progressMonitor the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed
507 * @param convertUnknownToTags true if unknown xml attributes should be kept as tags
508 *
509 * @return the dataset with the parsed data
510 * @throws IllegalDataException if an error was found while parsing the data from the source
511 * @throws IllegalArgumentException if source is null
512 * @since 15470
513 */
514 public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor, boolean convertUnknownToTags)
515 throws IllegalDataException {
516 return new OsmReader(convertUnknownToTags).doParseDataSet(source, progressMonitor);
517 }
518}
Note: See TracBrowser for help on using the repository browser.