source: josm/trunk/src/org/openstreetmap/josm/io/GeoJSONReader.java@ 17646

Last change on this file since 17646 was 17646, checked in by Don-vip, 3 years ago

fix #20653 - catch IllegalArgumentException when parsing geojson
  • Property svn:eol-style set to native
File size: 19.2 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.io;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.BufferedInputStream;
7import java.io.BufferedReader;
8import java.io.IOException;
9import java.io.InputStream;
10import java.io.InputStreamReader;
11import java.io.StringReader;
12import java.nio.charset.StandardCharsets;
13import java.util.ArrayList;
14import java.util.List;
15import java.util.Map;
16import java.util.Objects;
17import java.util.Optional;
18import java.util.TreeMap;
19import java.util.stream.Collectors;
20
21import javax.json.Json;
22import javax.json.JsonArray;
23import javax.json.JsonNumber;
24import javax.json.JsonObject;
25import javax.json.JsonString;
26import javax.json.JsonValue;
27import javax.json.stream.JsonParser;
28import javax.json.stream.JsonParser.Event;
29import javax.json.stream.JsonParsingException;
30
31import org.openstreetmap.josm.data.coor.EastNorth;
32import org.openstreetmap.josm.data.coor.LatLon;
33import org.openstreetmap.josm.data.osm.BBox;
34import org.openstreetmap.josm.data.osm.DataSet;
35import org.openstreetmap.josm.data.osm.Node;
36import org.openstreetmap.josm.data.osm.OsmPrimitive;
37import org.openstreetmap.josm.data.osm.Relation;
38import org.openstreetmap.josm.data.osm.RelationMember;
39import org.openstreetmap.josm.data.osm.UploadPolicy;
40import org.openstreetmap.josm.data.osm.Way;
41import org.openstreetmap.josm.data.projection.Projection;
42import org.openstreetmap.josm.data.projection.Projections;
43import org.openstreetmap.josm.data.validation.TestError;
44import org.openstreetmap.josm.data.validation.tests.DuplicateWay;
45import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
46import org.openstreetmap.josm.gui.progress.ProgressMonitor;
47import org.openstreetmap.josm.tools.CheckParameterUtil;
48import org.openstreetmap.josm.tools.Logging;
49import org.openstreetmap.josm.tools.Utils;
50
51/**
52 * Reader that reads GeoJSON files. See <a href="https://tools.ietf.org/html/rfc7946">RFC7946</a> for more information.
53 * @since 15424
54 */
55public class GeoJSONReader extends AbstractReader {
56
57 private static final String CRS = "crs";
58 private static final String NAME = "name";
59 private static final String LINK = "link";
60 private static final String COORDINATES = "coordinates";
61 private static final String FEATURES = "features";
62 private static final String PROPERTIES = "properties";
63 private static final String GEOMETRY = "geometry";
64 private static final String TYPE = "type";
65 /** The record separator is 0x1E per RFC 7464 */
66 private static final byte RECORD_SEPARATOR_BYTE = 0x1E;
67 private Projection projection = Projections.getProjectionByCode("EPSG:4326"); // WGS 84
68
69 GeoJSONReader() {
70 // Restricts visibility
71 }
72
73 private void parse(final JsonParser parser) throws IllegalDataException {
74 while (parser.hasNext()) {
75 Event event = parser.next();
76 if (event == Event.START_OBJECT) {
77 parseRoot(parser.getObject());
78 }
79 }
80 parser.close();
81 }
82
83 private void parseRoot(final JsonObject object) throws IllegalDataException {
84 parseCrs(object.getJsonObject(CRS));
85 switch (Optional.ofNullable(object.getJsonString(TYPE))
86 .orElseThrow(() -> new IllegalDataException("No type")).getString()) {
87 case "FeatureCollection":
88 JsonValue.ValueType valueType = object.get(FEATURES).getValueType();
89 CheckParameterUtil.ensureThat(valueType == JsonValue.ValueType.ARRAY, "features must be ARRAY, but is " + valueType);
90 parseFeatureCollection(object.getJsonArray(FEATURES));
91 break;
92 case "Feature":
93 parseFeature(object);
94 break;
95 case "GeometryCollection":
96 parseGeometryCollection(null, object);
97 break;
98 default:
99 parseGeometry(null, object);
100 }
101 }
102
103 /**
104 * Parse CRS as per https://geojson.org/geojson-spec.html#coordinate-reference-system-objects.
105 * CRS are obsolete in RFC7946 but still allowed for interoperability with older applications.
106 * Only named CRS are supported.
107 *
108 * @param crs CRS JSON object
109 * @throws IllegalDataException in case of error
110 */
111 private void parseCrs(final JsonObject crs) throws IllegalDataException {
112 if (crs != null) {
113 // Inspired by https://github.com/JOSM/geojson/commit/f13ceed4645244612a63581c96e20da802779c56
114 JsonObject properties = crs.getJsonObject("properties");
115 if (properties != null) {
116 switch (crs.getString(TYPE)) {
117 case NAME:
118 String crsName = properties.getString(NAME);
119 if ("urn:ogc:def:crs:OGC:1.3:CRS84".equals(crsName)) {
120 // https://osgeo-org.atlassian.net/browse/GEOT-1710
121 crsName = "EPSG:4326";
122 } else if (crsName.startsWith("urn:ogc:def:crs:EPSG:")) {
123 crsName = crsName.replace("urn:ogc:def:crs:", "");
124 }
125 projection = Optional.ofNullable(Projections.getProjectionByCode(crsName))
126 .orElse(Projections.getProjectionByCode("EPSG:4326")); // WGS84
127 break;
128 case LINK: // Not supported (security risk)
129 default:
130 throw new IllegalDataException(crs.toString());
131 }
132 }
133 }
134 }
135
136 private void parseFeatureCollection(final JsonArray features) {
137 for (JsonValue feature : features) {
138 if (feature instanceof JsonObject) {
139 parseFeature((JsonObject) feature);
140 }
141 }
142 }
143
144 private void parseFeature(final JsonObject feature) {
145 JsonValue geometry = feature.get(GEOMETRY);
146 if (geometry != null && geometry.getValueType() == JsonValue.ValueType.OBJECT) {
147 parseGeometry(feature, geometry.asJsonObject());
148 } else {
149 JsonValue properties = feature.get(PROPERTIES);
150 if (properties != null && properties.getValueType() == JsonValue.ValueType.OBJECT) {
151 parseNonGeometryFeature(feature, properties.asJsonObject());
152 } else {
153 Logging.warn(tr("Relation/non-geometry feature without properties found: {0}", feature));
154 }
155 }
156 }
157
158 private void parseNonGeometryFeature(final JsonObject feature, final JsonObject properties) {
159 // get relation type
160 JsonValue type = properties.get(TYPE);
161 if (type == null || properties.getValueType() == JsonValue.ValueType.STRING) {
162 Logging.warn(tr("Relation/non-geometry feature without type found: {0}", feature));
163 return;
164 }
165
166 // create misc. non-geometry feature
167 final Relation relation = new Relation();
168 fillTagsFromFeature(feature, relation);
169 relation.put(TYPE, type.toString());
170 getDataSet().addPrimitive(relation);
171 }
172
173 private void parseGeometryCollection(final JsonObject feature, final JsonObject geometry) {
174 for (JsonValue jsonValue : geometry.getJsonArray("geometries")) {
175 parseGeometry(feature, jsonValue.asJsonObject());
176 }
177 }
178
179 private void parseGeometry(final JsonObject feature, final JsonObject geometry) {
180 if (geometry == null) {
181 parseNullGeometry(feature);
182 return;
183 }
184
185 switch (geometry.getString(TYPE)) {
186 case "Point":
187 parsePoint(feature, geometry.getJsonArray(COORDINATES));
188 break;
189 case "MultiPoint":
190 parseMultiPoint(feature, geometry);
191 break;
192 case "LineString":
193 parseLineString(feature, geometry.getJsonArray(COORDINATES));
194 break;
195 case "MultiLineString":
196 parseMultiLineString(feature, geometry);
197 break;
198 case "Polygon":
199 parsePolygon(feature, geometry.getJsonArray(COORDINATES));
200 break;
201 case "MultiPolygon":
202 parseMultiPolygon(feature, geometry);
203 break;
204 case "GeometryCollection":
205 parseGeometryCollection(feature, geometry);
206 break;
207 default:
208 parseUnknown(geometry);
209 }
210 }
211
212 private LatLon getLatLon(final JsonArray coordinates) {
213 return projection.eastNorth2latlon(new EastNorth(
214 parseCoordinate(coordinates.get(0)),
215 parseCoordinate(coordinates.get(1))));
216 }
217
218 private static double parseCoordinate(JsonValue coordinate) {
219 if (coordinate instanceof JsonString) {
220 return Double.parseDouble(((JsonString) coordinate).getString());
221 } else if (coordinate instanceof JsonNumber) {
222 return ((JsonNumber) coordinate).doubleValue();
223 } else {
224 throw new IllegalArgumentException(Objects.toString(coordinate));
225 }
226 }
227
228 private void parsePoint(final JsonObject feature, final JsonArray coordinates) {
229 fillTagsFromFeature(feature, createNode(getLatLon(coordinates)));
230 }
231
232 private void parseMultiPoint(final JsonObject feature, final JsonObject geometry) {
233 for (JsonValue coordinate : geometry.getJsonArray(COORDINATES)) {
234 parsePoint(feature, coordinate.asJsonArray());
235 }
236 }
237
238 private void parseLineString(final JsonObject feature, final JsonArray coordinates) {
239 if (!coordinates.isEmpty()) {
240 createWay(coordinates, false)
241 .ifPresent(way -> fillTagsFromFeature(feature, way));
242 }
243 }
244
245 private void parseMultiLineString(final JsonObject feature, final JsonObject geometry) {
246 for (JsonValue coordinate : geometry.getJsonArray(COORDINATES)) {
247 parseLineString(feature, coordinate.asJsonArray());
248 }
249 }
250
251 private void parsePolygon(final JsonObject feature, final JsonArray coordinates) {
252 final int size = coordinates.size();
253 if (size == 1) {
254 createWay(coordinates.getJsonArray(0), true)
255 .ifPresent(way -> fillTagsFromFeature(feature, way));
256 } else if (size > 1) {
257 // create multipolygon
258 final Relation multipolygon = new Relation();
259 createWay(coordinates.getJsonArray(0), true)
260 .ifPresent(way -> multipolygon.addMember(new RelationMember("outer", way)));
261
262 for (JsonValue interiorRing : coordinates.subList(1, size)) {
263 createWay(interiorRing.asJsonArray(), true)
264 .ifPresent(way -> multipolygon.addMember(new RelationMember("inner", way)));
265 }
266
267 fillTagsFromFeature(feature, multipolygon);
268 multipolygon.put(TYPE, "multipolygon");
269 getDataSet().addPrimitive(multipolygon);
270 }
271 }
272
273 private void parseMultiPolygon(final JsonObject feature, final JsonObject geometry) {
274 for (JsonValue coordinate : geometry.getJsonArray(COORDINATES)) {
275 parsePolygon(feature, coordinate.asJsonArray());
276 }
277 }
278
279 private Node createNode(final LatLon latlon) {
280 final List<Node> existingNodes = getDataSet().searchNodes(new BBox(latlon, latlon));
281 if (!existingNodes.isEmpty()) {
282 // reuse existing node, avoid multiple nodes on top of each other
283 return existingNodes.get(0);
284 }
285 final Node node = new Node(latlon);
286 getDataSet().addPrimitive(node);
287 return node;
288 }
289
290 private Optional<Way> createWay(final JsonArray coordinates, final boolean autoClose) {
291 if (coordinates.isEmpty()) {
292 return Optional.empty();
293 }
294
295 final List<LatLon> latlons = coordinates.stream()
296 .map(coordinate -> getLatLon(coordinate.asJsonArray()))
297 .collect(Collectors.toList());
298
299 final int size = latlons.size();
300 final boolean doAutoclose;
301 if (size > 1) {
302 if (latlons.get(0).equals(latlons.get(size - 1))) {
303 doAutoclose = false; // already closed
304 } else {
305 doAutoclose = autoClose;
306 }
307 } else {
308 doAutoclose = false;
309 }
310
311 final Way way = new Way();
312 getDataSet().addPrimitive(way);
313 final List<Node> rawNodes = latlons.stream().map(this::createNode).collect(Collectors.toList());
314 if (doAutoclose) {
315 rawNodes.add(rawNodes.get(0));
316 }
317 // see #19833: remove duplicated references to the same node
318 final List<Node> wayNodes = new ArrayList<>(rawNodes.size());
319 Node last = null;
320 for (Node curr : rawNodes) {
321 if (last != curr)
322 wayNodes.add(curr);
323 last = curr;
324 }
325 way.setNodes(wayNodes);
326
327 return Optional.of(way);
328 }
329
330 /**
331 * Replace all existing tags in primitive by the values given in the GeoJSON feature.
332 * @param feature the GeoJSON feature
333 * @param primitive the OSM primitive
334 */
335 private static void fillTagsFromFeature(final JsonObject feature, final OsmPrimitive primitive) {
336 if (feature != null) {
337 primitive.setKeys(getTags(feature));
338 }
339 }
340
341 private static void parseUnknown(final JsonObject object) {
342 Logging.warn(tr("Unknown json object found {0}", object));
343 }
344
345 private static void parseNullGeometry(JsonObject feature) {
346 Logging.warn(tr("Geometry of feature {0} is null", feature));
347 }
348
349 private static Map<String, String> getTags(final JsonObject feature) {
350 final Map<String, String> tags = new TreeMap<>();
351
352 if (feature.containsKey(PROPERTIES) && !feature.isNull(PROPERTIES)) {
353 JsonValue properties = feature.get(PROPERTIES);
354 if (properties != null && properties.getValueType() == JsonValue.ValueType.OBJECT) {
355 for (Map.Entry<String, JsonValue> stringJsonValueEntry : properties.asJsonObject().entrySet()) {
356 final JsonValue value = stringJsonValueEntry.getValue();
357
358 if (value instanceof JsonString) {
359 tags.put(stringJsonValueEntry.getKey(), ((JsonString) value).getString());
360 } else if (value instanceof JsonObject) {
361 Logging.warn(
362 "The GeoJSON contains an object with property '" + stringJsonValueEntry.getKey()
363 + "' whose value has the unsupported type '" + value.getClass().getSimpleName()
364 + "'. That key-value pair is ignored!"
365 );
366 } else if (value.getValueType() != JsonValue.ValueType.NULL) {
367 tags.put(stringJsonValueEntry.getKey(), value.toString());
368 }
369 }
370 }
371 }
372 return tags;
373 }
374
375 /**
376 * Check if the inputstream follows RFC 7464
377 * @param source The source to check (should be at the beginning)
378 * @return {@code true} if the initial character is {@link GeoJSONReader#RECORD_SEPARATOR_BYTE}.
379 */
380 private static boolean isLineDelimited(InputStream source) {
381 source.mark(2);
382 try {
383 int start = source.read();
384 if (RECORD_SEPARATOR_BYTE == start) {
385 return true;
386 }
387 source.reset();
388 } catch (IOException e) {
389 Logging.error(e);
390 }
391 return false;
392 }
393
394 @Override
395 protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
396 try (InputStream markSupported = source.markSupported() ? source : new BufferedInputStream(source)) {
397 ds.setUploadPolicy(UploadPolicy.DISCOURAGED);
398 if (isLineDelimited(markSupported)) {
399 try (BufferedReader reader = new BufferedReader(new InputStreamReader(markSupported, StandardCharsets.UTF_8))) {
400 String line;
401 String rs = new String(new byte[]{RECORD_SEPARATOR_BYTE}, StandardCharsets.US_ASCII);
402 while ((line = reader.readLine()) != null) {
403 line = Utils.strip(line, rs);
404 try (JsonParser parser = Json.createParser(new StringReader(line))) {
405 parse(parser);
406 }
407 }
408 }
409 } else {
410 try (JsonParser parser = Json.createParser(markSupported)) {
411 parse(parser);
412 }
413 }
414 mergeEqualMultipolygonWays();
415 } catch (IOException | IllegalArgumentException | JsonParsingException e) {
416 throw new IllegalDataException(e);
417 }
418 return getDataSet();
419 }
420
421 /**
422 * Import may create duplicate ways were one is member of a multipolygon and untagged and the other is tagged.
423 * Try to merge them here.
424 */
425 private void mergeEqualMultipolygonWays() {
426 DuplicateWay test = new DuplicateWay();
427 test.startTest(null);
428 for (Way w: getDataSet().getWays()) {
429 test.visit(w);
430 }
431 test.endTest();
432
433 if (test.getErrors().isEmpty())
434 return;
435
436 for (TestError e : test.getErrors()) {
437 if (e.getPrimitives().size() == 2 && !e.isFixable()) {
438 List<Way> mpWays = new ArrayList<>();
439 Way replacement = null;
440 for (OsmPrimitive p : e.getPrimitives()) {
441 if (p.isTagged() && p.referrers(Relation.class).count() == 0)
442 replacement = (Way) p;
443 else if (p.referrers(Relation.class).anyMatch(Relation::isMultipolygon))
444 mpWays.add((Way) p);
445 }
446 if (replacement == null && mpWays.size() == 2) {
447 replacement = mpWays.remove(1);
448 }
449 if (replacement != null && mpWays.size() == 1) {
450 Way mpWay = mpWays.get(0);
451 for (Relation r : mpWay.referrers(Relation.class).filter(Relation::isMultipolygon)
452 .collect(Collectors.toList())) {
453 for (int i = 0; i < r.getMembersCount(); i++) {
454 if (r.getMember(i).getMember().equals(mpWay)) {
455 r.setMember(i, new RelationMember(r.getRole(i), replacement));
456 }
457 }
458 }
459 mpWay.setDeleted(true);
460 }
461 }
462 }
463 ds.cleanupDeletedPrimitives();
464 }
465
466 /**
467 * Parse the given input source and return the dataset.
468 *
469 * @param source the source input stream. Must not be null.
470 * @param progressMonitor the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed
471 * @return the dataset with the parsed data
472 * @throws IllegalDataException if an error was found while parsing the data from the source
473 * @throws IllegalArgumentException if source is null
474 */
475 public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
476 return new GeoJSONReader().doParseDataSet(source, progressMonitor);
477 }
478}
Note: See TracBrowser for help on using the repository browser.