source: josm/trunk/src/org/openstreetmap/josm/io/GeoJSONReader.java@ 17054

Last change on this file since 17054 was 17054, checked in by GerdP, 4 years ago

fix #19833: Duplicated way nodes after GeoJSON import

  • filter duplicated nodes before creating an OSM way
  • Property svn:eol-style set to native
File size: 16.7 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.io;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.BufferedInputStream;
7import java.io.BufferedReader;
8import java.io.IOException;
9import java.io.InputStream;
10import java.io.InputStreamReader;
11import java.io.StringReader;
12import java.nio.charset.StandardCharsets;
13import java.util.ArrayList;
14import java.util.List;
15import java.util.Map;
16import java.util.Objects;
17import java.util.Optional;
18import java.util.TreeMap;
19import java.util.stream.Collectors;
20
21import javax.json.Json;
22import javax.json.JsonArray;
23import javax.json.JsonNumber;
24import javax.json.JsonObject;
25import javax.json.JsonString;
26import javax.json.JsonValue;
27import javax.json.stream.JsonParser;
28import javax.json.stream.JsonParser.Event;
29import javax.json.stream.JsonParsingException;
30
31import org.openstreetmap.josm.data.coor.EastNorth;
32import org.openstreetmap.josm.data.coor.LatLon;
33import org.openstreetmap.josm.data.osm.BBox;
34import org.openstreetmap.josm.data.osm.DataSet;
35import org.openstreetmap.josm.data.osm.Node;
36import org.openstreetmap.josm.data.osm.OsmPrimitive;
37import org.openstreetmap.josm.data.osm.Relation;
38import org.openstreetmap.josm.data.osm.RelationMember;
39import org.openstreetmap.josm.data.osm.UploadPolicy;
40import org.openstreetmap.josm.data.osm.Way;
41import org.openstreetmap.josm.data.projection.Projection;
42import org.openstreetmap.josm.data.projection.Projections;
43import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
44import org.openstreetmap.josm.gui.progress.ProgressMonitor;
45import org.openstreetmap.josm.tools.Logging;
46import org.openstreetmap.josm.tools.Utils;
47
48/**
49 * Reader that reads GeoJSON files. See <a href="https://tools.ietf.org/html/rfc7946">RFC7946</a> for more information.
50 * @since 15424
51 */
52public class GeoJSONReader extends AbstractReader {
53
54 private static final String CRS = "crs";
55 private static final String NAME = "name";
56 private static final String LINK = "link";
57 private static final String COORDINATES = "coordinates";
58 private static final String FEATURES = "features";
59 private static final String PROPERTIES = "properties";
60 private static final String GEOMETRY = "geometry";
61 private static final String TYPE = "type";
62 /** The record separator is 0x1E per RFC 7464 */
63 private static final byte RECORD_SEPARATOR_BYTE = 0x1E;
64 private Projection projection = Projections.getProjectionByCode("EPSG:4326"); // WGS 84
65
66 GeoJSONReader() {
67 // Restricts visibility
68 }
69
70 private void parse(final JsonParser parser) throws IllegalDataException {
71 while (parser.hasNext()) {
72 Event event = parser.next();
73 if (event == Event.START_OBJECT) {
74 parseRoot(parser.getObject());
75 }
76 }
77 parser.close();
78 }
79
80 private void parseRoot(final JsonObject object) throws IllegalDataException {
81 parseCrs(object.getJsonObject(CRS));
82 switch (Optional.ofNullable(object.getJsonString(TYPE))
83 .orElseThrow(() -> new IllegalDataException("No type")).getString()) {
84 case "FeatureCollection":
85 parseFeatureCollection(object.getJsonArray(FEATURES));
86 break;
87 case "Feature":
88 parseFeature(object);
89 break;
90 case "GeometryCollection":
91 parseGeometryCollection(null, object);
92 break;
93 default:
94 parseGeometry(null, object);
95 }
96 }
97
98 /**
99 * Parse CRS as per https://geojson.org/geojson-spec.html#coordinate-reference-system-objects.
100 * CRS are obsolete in RFC7946 but still allowed for interoperability with older applications.
101 * Only named CRS are supported.
102 *
103 * @param crs CRS JSON object
104 * @throws IllegalDataException in case of error
105 */
106 private void parseCrs(final JsonObject crs) throws IllegalDataException {
107 if (crs != null) {
108 // Inspired by https://github.com/JOSM/geojson/commit/f13ceed4645244612a63581c96e20da802779c56
109 JsonObject properties = crs.getJsonObject("properties");
110 if (properties != null) {
111 switch (crs.getString(TYPE)) {
112 case NAME:
113 String crsName = properties.getString(NAME);
114 if ("urn:ogc:def:crs:OGC:1.3:CRS84".equals(crsName)) {
115 // https://osgeo-org.atlassian.net/browse/GEOT-1710
116 crsName = "EPSG:4326";
117 } else if (crsName.startsWith("urn:ogc:def:crs:EPSG:")) {
118 crsName = crsName.replace("urn:ogc:def:crs:", "");
119 }
120 projection = Optional.ofNullable(Projections.getProjectionByCode(crsName))
121 .orElse(Projections.getProjectionByCode("EPSG:4326")); // WGS84
122 break;
123 case LINK: // Not supported (security risk)
124 default:
125 throw new IllegalDataException(crs.toString());
126 }
127 }
128 }
129 }
130
131 private void parseFeatureCollection(final JsonArray features) {
132 for (JsonValue feature : features) {
133 if (feature instanceof JsonObject) {
134 parseFeature((JsonObject) feature);
135 }
136 }
137 }
138
139 private void parseFeature(final JsonObject feature) {
140 JsonValue geometry = feature.get(GEOMETRY);
141 if (geometry != null && geometry.getValueType() == JsonValue.ValueType.OBJECT) {
142 parseGeometry(feature, geometry.asJsonObject());
143 } else {
144 JsonValue properties = feature.get(PROPERTIES);
145 if (properties != null && properties.getValueType() == JsonValue.ValueType.OBJECT) {
146 parseNonGeometryFeature(feature, properties.asJsonObject());
147 } else {
148 Logging.warn(tr("Relation/non-geometry feature without properties found: {0}", feature));
149 }
150 }
151 }
152
153 private void parseNonGeometryFeature(final JsonObject feature, final JsonObject properties) {
154 // get relation type
155 JsonValue type = properties.get(TYPE);
156 if (type == null || properties.getValueType() == JsonValue.ValueType.STRING) {
157 Logging.warn(tr("Relation/non-geometry feature without type found: {0}", feature));
158 return;
159 }
160
161 // create misc. non-geometry feature
162 final Relation relation = new Relation();
163 relation.put(TYPE, type.toString());
164 fillTagsFromFeature(feature, relation);
165 getDataSet().addPrimitive(relation);
166 }
167
168 private void parseGeometryCollection(final JsonObject feature, final JsonObject geometry) {
169 for (JsonValue jsonValue : geometry.getJsonArray("geometries")) {
170 parseGeometry(feature, jsonValue.asJsonObject());
171 }
172 }
173
174 private void parseGeometry(final JsonObject feature, final JsonObject geometry) {
175 if (geometry == null) {
176 parseNullGeometry(feature);
177 return;
178 }
179
180 switch (geometry.getString(TYPE)) {
181 case "Point":
182 parsePoint(feature, geometry.getJsonArray(COORDINATES));
183 break;
184 case "MultiPoint":
185 parseMultiPoint(feature, geometry);
186 break;
187 case "LineString":
188 parseLineString(feature, geometry.getJsonArray(COORDINATES));
189 break;
190 case "MultiLineString":
191 parseMultiLineString(feature, geometry);
192 break;
193 case "Polygon":
194 parsePolygon(feature, geometry.getJsonArray(COORDINATES));
195 break;
196 case "MultiPolygon":
197 parseMultiPolygon(feature, geometry);
198 break;
199 case "GeometryCollection":
200 parseGeometryCollection(feature, geometry);
201 break;
202 default:
203 parseUnknown(geometry);
204 }
205 }
206
207 private LatLon getLatLon(final JsonArray coordinates) {
208 return projection.eastNorth2latlon(new EastNorth(
209 parseCoordinate(coordinates.get(0)),
210 parseCoordinate(coordinates.get(1))));
211 }
212
213 private static double parseCoordinate(JsonValue coordinate) {
214 if (coordinate instanceof JsonString) {
215 return Double.parseDouble(((JsonString) coordinate).getString());
216 } else if (coordinate instanceof JsonNumber) {
217 return ((JsonNumber) coordinate).doubleValue();
218 } else {
219 throw new IllegalArgumentException(Objects.toString(coordinate));
220 }
221 }
222
223 private void parsePoint(final JsonObject feature, final JsonArray coordinates) {
224 fillTagsFromFeature(feature, createNode(getLatLon(coordinates)));
225 }
226
227 private void parseMultiPoint(final JsonObject feature, final JsonObject geometry) {
228 for (JsonValue coordinate : geometry.getJsonArray(COORDINATES)) {
229 parsePoint(feature, coordinate.asJsonArray());
230 }
231 }
232
233 private void parseLineString(final JsonObject feature, final JsonArray coordinates) {
234 if (!coordinates.isEmpty()) {
235 createWay(coordinates, false)
236 .ifPresent(way -> fillTagsFromFeature(feature, way));
237 }
238 }
239
240 private void parseMultiLineString(final JsonObject feature, final JsonObject geometry) {
241 for (JsonValue coordinate : geometry.getJsonArray(COORDINATES)) {
242 parseLineString(feature, coordinate.asJsonArray());
243 }
244 }
245
246 private void parsePolygon(final JsonObject feature, final JsonArray coordinates) {
247 final int size = coordinates.size();
248 if (size == 1) {
249 createWay(coordinates.getJsonArray(0), true)
250 .ifPresent(way -> fillTagsFromFeature(feature, way));
251 } else if (size > 1) {
252 // create multipolygon
253 final Relation multipolygon = new Relation();
254 multipolygon.put(TYPE, "multipolygon");
255 createWay(coordinates.getJsonArray(0), true)
256 .ifPresent(way -> multipolygon.addMember(new RelationMember("outer", way)));
257
258 for (JsonValue interiorRing : coordinates.subList(1, size)) {
259 createWay(interiorRing.asJsonArray(), true)
260 .ifPresent(way -> multipolygon.addMember(new RelationMember("inner", way)));
261 }
262
263 fillTagsFromFeature(feature, multipolygon);
264 getDataSet().addPrimitive(multipolygon);
265 }
266 }
267
268 private void parseMultiPolygon(final JsonObject feature, final JsonObject geometry) {
269 for (JsonValue coordinate : geometry.getJsonArray(COORDINATES)) {
270 parsePolygon(feature, coordinate.asJsonArray());
271 }
272 }
273
274 private Node createNode(final LatLon latlon) {
275 final List<Node> existingNodes = getDataSet().searchNodes(new BBox(latlon, latlon));
276 if (!existingNodes.isEmpty()) {
277 // reuse existing node, avoid multiple nodes on top of each other
278 return existingNodes.get(0);
279 }
280 final Node node = new Node(latlon);
281 getDataSet().addPrimitive(node);
282 return node;
283 }
284
285 private Optional<Way> createWay(final JsonArray coordinates, final boolean autoClose) {
286 if (coordinates.isEmpty()) {
287 return Optional.empty();
288 }
289
290 final List<LatLon> latlons = coordinates.stream()
291 .map(coordinate -> getLatLon(coordinate.asJsonArray()))
292 .collect(Collectors.toList());
293
294 final int size = latlons.size();
295 final boolean doAutoclose;
296 if (size > 1) {
297 if (latlons.get(0).equals(latlons.get(size - 1))) {
298 doAutoclose = false; // already closed
299 } else {
300 doAutoclose = autoClose;
301 }
302 } else {
303 doAutoclose = false;
304 }
305
306 final Way way = new Way();
307 getDataSet().addPrimitive(way);
308 final List<Node> rawNodes = latlons.stream().map(this::createNode).collect(Collectors.toList());
309 if (doAutoclose) {
310 rawNodes.add(rawNodes.get(0));
311 }
312 // see #19833: remove duplicated references to the same node
313 final List<Node> wayNodes = new ArrayList<>(rawNodes.size());
314 Node last = null;
315 for (Node curr : rawNodes) {
316 if (last != curr)
317 wayNodes.add(curr);
318 last = curr;
319 }
320 way.setNodes(wayNodes);
321
322 return Optional.of(way);
323 }
324
325 private static void fillTagsFromFeature(final JsonObject feature, final OsmPrimitive primitive) {
326 if (feature != null) {
327 primitive.setKeys(getTags(feature));
328 }
329 }
330
331 private static void parseUnknown(final JsonObject object) {
332 Logging.warn(tr("Unknown json object found {0}", object));
333 }
334
335 private static void parseNullGeometry(JsonObject feature) {
336 Logging.warn(tr("Geometry of feature {0} is null", feature));
337 }
338
339 private static Map<String, String> getTags(final JsonObject feature) {
340 final Map<String, String> tags = new TreeMap<>();
341
342 if (feature.containsKey(PROPERTIES) && !feature.isNull(PROPERTIES)) {
343 JsonValue properties = feature.get(PROPERTIES);
344 if (properties != null && properties.getValueType() == JsonValue.ValueType.OBJECT) {
345 for (Map.Entry<String, JsonValue> stringJsonValueEntry : properties.asJsonObject().entrySet()) {
346 final JsonValue value = stringJsonValueEntry.getValue();
347
348 if (value instanceof JsonString) {
349 tags.put(stringJsonValueEntry.getKey(), ((JsonString) value).getString());
350 } else if (value instanceof JsonObject) {
351 Logging.warn(
352 "The GeoJSON contains an object with property '" + stringJsonValueEntry.getKey()
353 + "' whose value has the unsupported type '" + value.getClass().getSimpleName()
354 + "'. That key-value pair is ignored!"
355 );
356 } else if (value.getValueType() != JsonValue.ValueType.NULL) {
357 tags.put(stringJsonValueEntry.getKey(), value.toString());
358 }
359 }
360 }
361 }
362 return tags;
363 }
364
365 /**
366 * Check if the inputstream follows RFC 7464
367 * @param source The source to check (should be at the beginning)
368 * @return {@code true} if the initial character is {@link GeoJSONReader#RECORD_SEPARATOR_BYTE}.
369 */
370 private static boolean isLineDelimited(InputStream source) {
371 source.mark(2);
372 try {
373 int start = source.read();
374 if (RECORD_SEPARATOR_BYTE == start) {
375 return true;
376 }
377 source.reset();
378 } catch (IOException e) {
379 Logging.error(e);
380 }
381 return false;
382 }
383
384 @Override
385 protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
386 try (InputStream markSupported = source.markSupported() ? source : new BufferedInputStream(source)) {
387 ds.setUploadPolicy(UploadPolicy.DISCOURAGED);
388 if (isLineDelimited(markSupported)) {
389 try (BufferedReader reader = new BufferedReader(new InputStreamReader(markSupported, StandardCharsets.UTF_8))) {
390 String line;
391 String rs = new String(new byte[]{RECORD_SEPARATOR_BYTE}, StandardCharsets.US_ASCII);
392 while ((line = reader.readLine()) != null) {
393 line = Utils.strip(line, rs);
394 try (JsonParser parser = Json.createParser(new StringReader(line))) {
395 parse(parser);
396 }
397 }
398 }
399 } else {
400 try (JsonParser parser = Json.createParser(markSupported)) {
401 parse(parser);
402 }
403 }
404 } catch (IOException | JsonParsingException e) {
405 throw new IllegalDataException(e);
406 }
407 return getDataSet();
408 }
409
410 /**
411 * Parse the given input source and return the dataset.
412 *
413 * @param source the source input stream. Must not be null.
414 * @param progressMonitor the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed
415 * @return the dataset with the parsed data
416 * @throws IllegalDataException if an error was found while parsing the data from the source
417 * @throws IllegalArgumentException if source is null
418 */
419 public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
420 return new GeoJSONReader().doParseDataSet(source, progressMonitor);
421 }
422}
Note: See TracBrowser for help on using the repository browser.