source: josm/trunk/src/org/openstreetmap/josm/io/GeoJSONReader.java@ 18007

Last change on this file since 18007 was 18007, checked in by Don-vip, 3 years ago

fix #21044 - keep all tags when merging geojson nodes at the same location
  • Property svn:eol-style set to native
File size: 20.2 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.io;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.BufferedInputStream;
7import java.io.BufferedReader;
8import java.io.IOException;
9import java.io.InputStream;
10import java.io.InputStreamReader;
11import java.io.StringReader;
12import java.nio.charset.StandardCharsets;
13import java.util.ArrayList;
14import java.util.Arrays;
15import java.util.List;
16import java.util.Map;
17import java.util.Objects;
18import java.util.Optional;
19import java.util.stream.Collectors;
20
21import javax.json.Json;
22import javax.json.JsonArray;
23import javax.json.JsonNumber;
24import javax.json.JsonObject;
25import javax.json.JsonString;
26import javax.json.JsonValue;
27import javax.json.stream.JsonParser;
28import javax.json.stream.JsonParser.Event;
29import javax.json.stream.JsonParsingException;
30
31import org.openstreetmap.josm.data.coor.EastNorth;
32import org.openstreetmap.josm.data.coor.LatLon;
33import org.openstreetmap.josm.data.osm.BBox;
34import org.openstreetmap.josm.data.osm.DataSet;
35import org.openstreetmap.josm.data.osm.Node;
36import org.openstreetmap.josm.data.osm.OsmPrimitive;
37import org.openstreetmap.josm.data.osm.Relation;
38import org.openstreetmap.josm.data.osm.RelationMember;
39import org.openstreetmap.josm.data.osm.Tag;
40import org.openstreetmap.josm.data.osm.TagCollection;
41import org.openstreetmap.josm.data.osm.TagMap;
42import org.openstreetmap.josm.data.osm.UploadPolicy;
43import org.openstreetmap.josm.data.osm.Way;
44import org.openstreetmap.josm.data.projection.Projection;
45import org.openstreetmap.josm.data.projection.Projections;
46import org.openstreetmap.josm.data.validation.TestError;
47import org.openstreetmap.josm.data.validation.tests.DuplicateWay;
48import org.openstreetmap.josm.gui.conflict.tags.TagConflictResolutionUtil;
49import org.openstreetmap.josm.gui.conflict.tags.TagConflictResolverModel;
50import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
51import org.openstreetmap.josm.gui.progress.ProgressMonitor;
52import org.openstreetmap.josm.tools.CheckParameterUtil;
53import org.openstreetmap.josm.tools.Logging;
54import org.openstreetmap.josm.tools.Utils;
55
56/**
57 * Reader that reads GeoJSON files. See <a href="https://tools.ietf.org/html/rfc7946">RFC7946</a> for more information.
58 * @since 15424
59 */
60public class GeoJSONReader extends AbstractReader {
61
62 private static final String CRS = "crs";
63 private static final String NAME = "name";
64 private static final String LINK = "link";
65 private static final String COORDINATES = "coordinates";
66 private static final String FEATURES = "features";
67 private static final String PROPERTIES = "properties";
68 private static final String GEOMETRY = "geometry";
69 private static final String TYPE = "type";
70 /** The record separator is 0x1E per RFC 7464 */
71 private static final byte RECORD_SEPARATOR_BYTE = 0x1E;
72 private Projection projection = Projections.getProjectionByCode("EPSG:4326"); // WGS 84
73
74 GeoJSONReader() {
75 // Restricts visibility
76 }
77
78 private void parse(final JsonParser parser) throws IllegalDataException {
79 while (parser.hasNext()) {
80 Event event = parser.next();
81 if (event == Event.START_OBJECT) {
82 parseRoot(parser.getObject());
83 }
84 }
85 parser.close();
86 }
87
88 private void parseRoot(final JsonObject object) throws IllegalDataException {
89 parseCrs(object.getJsonObject(CRS));
90 switch (Optional.ofNullable(object.getJsonString(TYPE))
91 .orElseThrow(() -> new IllegalDataException("No type")).getString()) {
92 case "FeatureCollection":
93 JsonValue.ValueType valueType = object.get(FEATURES).getValueType();
94 CheckParameterUtil.ensureThat(valueType == JsonValue.ValueType.ARRAY, "features must be ARRAY, but is " + valueType);
95 parseFeatureCollection(object.getJsonArray(FEATURES));
96 break;
97 case "Feature":
98 parseFeature(object);
99 break;
100 case "GeometryCollection":
101 parseGeometryCollection(null, object);
102 break;
103 default:
104 parseGeometry(null, object);
105 }
106 }
107
108 /**
109 * Parse CRS as per https://geojson.org/geojson-spec.html#coordinate-reference-system-objects.
110 * CRS are obsolete in RFC7946 but still allowed for interoperability with older applications.
111 * Only named CRS are supported.
112 *
113 * @param crs CRS JSON object
114 * @throws IllegalDataException in case of error
115 */
116 private void parseCrs(final JsonObject crs) throws IllegalDataException {
117 if (crs != null) {
118 // Inspired by https://github.com/JOSM/geojson/commit/f13ceed4645244612a63581c96e20da802779c56
119 JsonObject properties = crs.getJsonObject("properties");
120 if (properties != null) {
121 switch (crs.getString(TYPE)) {
122 case NAME:
123 String crsName = properties.getString(NAME);
124 if ("urn:ogc:def:crs:OGC:1.3:CRS84".equals(crsName)) {
125 // https://osgeo-org.atlassian.net/browse/GEOT-1710
126 crsName = "EPSG:4326";
127 } else if (crsName.startsWith("urn:ogc:def:crs:EPSG:")) {
128 crsName = crsName.replace("urn:ogc:def:crs:", "");
129 }
130 projection = Optional.ofNullable(Projections.getProjectionByCode(crsName))
131 .orElse(Projections.getProjectionByCode("EPSG:4326")); // WGS84
132 break;
133 case LINK: // Not supported (security risk)
134 default:
135 throw new IllegalDataException(crs.toString());
136 }
137 }
138 }
139 }
140
141 private void parseFeatureCollection(final JsonArray features) {
142 for (JsonValue feature : features) {
143 if (feature instanceof JsonObject) {
144 parseFeature((JsonObject) feature);
145 }
146 }
147 }
148
149 private void parseFeature(final JsonObject feature) {
150 JsonValue geometry = feature.get(GEOMETRY);
151 if (geometry != null && geometry.getValueType() == JsonValue.ValueType.OBJECT) {
152 parseGeometry(feature, geometry.asJsonObject());
153 } else {
154 JsonValue properties = feature.get(PROPERTIES);
155 if (properties != null && properties.getValueType() == JsonValue.ValueType.OBJECT) {
156 parseNonGeometryFeature(feature, properties.asJsonObject());
157 } else {
158 Logging.warn(tr("Relation/non-geometry feature without properties found: {0}", feature));
159 }
160 }
161 }
162
163 private void parseNonGeometryFeature(final JsonObject feature, final JsonObject properties) {
164 // get relation type
165 JsonValue type = properties.get(TYPE);
166 if (type == null || properties.getValueType() == JsonValue.ValueType.STRING) {
167 Logging.warn(tr("Relation/non-geometry feature without type found: {0}", feature));
168 return;
169 }
170
171 // create misc. non-geometry feature
172 final Relation relation = new Relation();
173 fillTagsFromFeature(feature, relation);
174 relation.put(TYPE, type.toString());
175 getDataSet().addPrimitive(relation);
176 }
177
178 private void parseGeometryCollection(final JsonObject feature, final JsonObject geometry) {
179 for (JsonValue jsonValue : geometry.getJsonArray("geometries")) {
180 parseGeometry(feature, jsonValue.asJsonObject());
181 }
182 }
183
184 private void parseGeometry(final JsonObject feature, final JsonObject geometry) {
185 if (geometry == null) {
186 parseNullGeometry(feature);
187 return;
188 }
189
190 switch (geometry.getString(TYPE)) {
191 case "Point":
192 parsePoint(feature, geometry.getJsonArray(COORDINATES));
193 break;
194 case "MultiPoint":
195 parseMultiPoint(feature, geometry);
196 break;
197 case "LineString":
198 parseLineString(feature, geometry.getJsonArray(COORDINATES));
199 break;
200 case "MultiLineString":
201 parseMultiLineString(feature, geometry);
202 break;
203 case "Polygon":
204 parsePolygon(feature, geometry.getJsonArray(COORDINATES));
205 break;
206 case "MultiPolygon":
207 parseMultiPolygon(feature, geometry);
208 break;
209 case "GeometryCollection":
210 parseGeometryCollection(feature, geometry);
211 break;
212 default:
213 parseUnknown(geometry);
214 }
215 }
216
217 private LatLon getLatLon(final JsonArray coordinates) {
218 return projection.eastNorth2latlon(new EastNorth(
219 parseCoordinate(coordinates.get(0)),
220 parseCoordinate(coordinates.get(1))));
221 }
222
223 private static double parseCoordinate(JsonValue coordinate) {
224 if (coordinate instanceof JsonString) {
225 return Double.parseDouble(((JsonString) coordinate).getString());
226 } else if (coordinate instanceof JsonNumber) {
227 return ((JsonNumber) coordinate).doubleValue();
228 } else {
229 throw new IllegalArgumentException(Objects.toString(coordinate));
230 }
231 }
232
233 private void parsePoint(final JsonObject feature, final JsonArray coordinates) {
234 fillTagsFromFeature(feature, createNode(getLatLon(coordinates)));
235 }
236
237 private void parseMultiPoint(final JsonObject feature, final JsonObject geometry) {
238 for (JsonValue coordinate : geometry.getJsonArray(COORDINATES)) {
239 parsePoint(feature, coordinate.asJsonArray());
240 }
241 }
242
243 private void parseLineString(final JsonObject feature, final JsonArray coordinates) {
244 if (!coordinates.isEmpty()) {
245 createWay(coordinates, false)
246 .ifPresent(way -> fillTagsFromFeature(feature, way));
247 }
248 }
249
250 private void parseMultiLineString(final JsonObject feature, final JsonObject geometry) {
251 for (JsonValue coordinate : geometry.getJsonArray(COORDINATES)) {
252 parseLineString(feature, coordinate.asJsonArray());
253 }
254 }
255
256 private void parsePolygon(final JsonObject feature, final JsonArray coordinates) {
257 final int size = coordinates.size();
258 if (size == 1) {
259 createWay(coordinates.getJsonArray(0), true)
260 .ifPresent(way -> fillTagsFromFeature(feature, way));
261 } else if (size > 1) {
262 // create multipolygon
263 final Relation multipolygon = new Relation();
264 createWay(coordinates.getJsonArray(0), true)
265 .ifPresent(way -> multipolygon.addMember(new RelationMember("outer", way)));
266
267 for (JsonValue interiorRing : coordinates.subList(1, size)) {
268 createWay(interiorRing.asJsonArray(), true)
269 .ifPresent(way -> multipolygon.addMember(new RelationMember("inner", way)));
270 }
271
272 fillTagsFromFeature(feature, multipolygon);
273 multipolygon.put(TYPE, "multipolygon");
274 getDataSet().addPrimitive(multipolygon);
275 }
276 }
277
278 private void parseMultiPolygon(final JsonObject feature, final JsonObject geometry) {
279 for (JsonValue coordinate : geometry.getJsonArray(COORDINATES)) {
280 parsePolygon(feature, coordinate.asJsonArray());
281 }
282 }
283
284 private Node createNode(final LatLon latlon) {
285 final List<Node> existingNodes = getDataSet().searchNodes(new BBox(latlon, latlon));
286 if (!existingNodes.isEmpty()) {
287 // reuse existing node, avoid multiple nodes on top of each other
288 return existingNodes.get(0);
289 }
290 final Node node = new Node(latlon);
291 getDataSet().addPrimitive(node);
292 return node;
293 }
294
295 private Optional<Way> createWay(final JsonArray coordinates, final boolean autoClose) {
296 if (coordinates.isEmpty()) {
297 return Optional.empty();
298 }
299
300 final List<LatLon> latlons = coordinates.stream()
301 .map(coordinate -> getLatLon(coordinate.asJsonArray()))
302 .collect(Collectors.toList());
303
304 final int size = latlons.size();
305 final boolean doAutoclose;
306 if (size > 1) {
307 if (latlons.get(0).equals(latlons.get(size - 1))) {
308 doAutoclose = false; // already closed
309 } else {
310 doAutoclose = autoClose;
311 }
312 } else {
313 doAutoclose = false;
314 }
315
316 final Way way = new Way();
317 getDataSet().addPrimitive(way);
318 final List<Node> rawNodes = latlons.stream().map(this::createNode).collect(Collectors.toList());
319 if (doAutoclose) {
320 rawNodes.add(rawNodes.get(0));
321 }
322 // see #19833: remove duplicated references to the same node
323 final List<Node> wayNodes = new ArrayList<>(rawNodes.size());
324 Node last = null;
325 for (Node curr : rawNodes) {
326 if (last != curr)
327 wayNodes.add(curr);
328 last = curr;
329 }
330 way.setNodes(wayNodes);
331
332 return Optional.of(way);
333 }
334
335 /**
336 * Merge existing tags in primitive (if any) with the values given in the GeoJSON feature.
337 * @param feature the GeoJSON feature
338 * @param primitive the OSM primitive
339 */
340 private static void fillTagsFromFeature(final JsonObject feature, final OsmPrimitive primitive) {
341 if (feature != null) {
342 TagCollection featureTags = getTags(feature);
343 primitive.setKeys(new TagMap(primitive.isTagged() ? mergeAllTagValues(primitive, featureTags) : featureTags));
344 }
345 }
346
347 private static TagCollection mergeAllTagValues(final OsmPrimitive primitive, TagCollection featureTags) {
348 TagCollection tags = TagCollection.from(primitive).union(featureTags);
349 TagConflictResolutionUtil.applyAutomaticTagConflictResolution(tags);
350 TagConflictResolutionUtil.normalizeTagCollectionBeforeEditing(tags, Arrays.asList(primitive));
351 TagConflictResolverModel tagModel = new TagConflictResolverModel();
352 tagModel.populate(new TagCollection(tags), tags.getKeysWithMultipleValues());
353 tagModel.actOnDecisions((k, d) -> d.keepAll());
354 return tagModel.getAllResolutions();
355 }
356
357 private static void parseUnknown(final JsonObject object) {
358 Logging.warn(tr("Unknown json object found {0}", object));
359 }
360
361 private static void parseNullGeometry(JsonObject feature) {
362 Logging.warn(tr("Geometry of feature {0} is null", feature));
363 }
364
365 private static TagCollection getTags(final JsonObject feature) {
366 final TagCollection tags = new TagCollection();
367
368 if (feature.containsKey(PROPERTIES) && !feature.isNull(PROPERTIES)) {
369 JsonValue properties = feature.get(PROPERTIES);
370 if (properties != null && properties.getValueType() == JsonValue.ValueType.OBJECT) {
371 for (Map.Entry<String, JsonValue> stringJsonValueEntry : properties.asJsonObject().entrySet()) {
372 final JsonValue value = stringJsonValueEntry.getValue();
373
374 if (value instanceof JsonString) {
375 tags.add(new Tag(stringJsonValueEntry.getKey(), ((JsonString) value).getString()));
376 } else if (value instanceof JsonObject) {
377 Logging.warn(
378 "The GeoJSON contains an object with property '" + stringJsonValueEntry.getKey()
379 + "' whose value has the unsupported type '" + value.getClass().getSimpleName()
380 + "'. That key-value pair is ignored!"
381 );
382 } else if (value.getValueType() != JsonValue.ValueType.NULL) {
383 tags.add(new Tag(stringJsonValueEntry.getKey(), value.toString()));
384 }
385 }
386 }
387 }
388 return tags;
389 }
390
391 /**
392 * Check if the inputstream follows RFC 7464
393 * @param source The source to check (should be at the beginning)
394 * @return {@code true} if the initial character is {@link GeoJSONReader#RECORD_SEPARATOR_BYTE}.
395 */
396 private static boolean isLineDelimited(InputStream source) {
397 source.mark(2);
398 try {
399 int start = source.read();
400 if (RECORD_SEPARATOR_BYTE == start) {
401 return true;
402 }
403 source.reset();
404 } catch (IOException e) {
405 Logging.error(e);
406 }
407 return false;
408 }
409
410 @Override
411 protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
412 try (InputStream markSupported = source.markSupported() ? source : new BufferedInputStream(source)) {
413 ds.setUploadPolicy(UploadPolicy.DISCOURAGED);
414 if (isLineDelimited(markSupported)) {
415 try (BufferedReader reader = new BufferedReader(new InputStreamReader(markSupported, StandardCharsets.UTF_8))) {
416 String line;
417 String rs = new String(new byte[]{RECORD_SEPARATOR_BYTE}, StandardCharsets.US_ASCII);
418 while ((line = reader.readLine()) != null) {
419 line = Utils.strip(line, rs);
420 try (JsonParser parser = Json.createParser(new StringReader(line))) {
421 parse(parser);
422 }
423 }
424 }
425 } else {
426 try (JsonParser parser = Json.createParser(markSupported)) {
427 parse(parser);
428 }
429 }
430 mergeEqualMultipolygonWays();
431 } catch (IOException | IllegalArgumentException | JsonParsingException e) {
432 throw new IllegalDataException(e);
433 }
434 return getDataSet();
435 }
436
437 /**
438 * Import may create duplicate ways were one is member of a multipolygon and untagged and the other is tagged.
439 * Try to merge them here.
440 */
441 private void mergeEqualMultipolygonWays() {
442 DuplicateWay test = new DuplicateWay();
443 test.startTest(null);
444 for (Way w: getDataSet().getWays()) {
445 test.visit(w);
446 }
447 test.endTest();
448
449 if (test.getErrors().isEmpty())
450 return;
451
452 for (TestError e : test.getErrors()) {
453 if (e.getPrimitives().size() == 2 && !e.isFixable()) {
454 List<Way> mpWays = new ArrayList<>();
455 Way replacement = null;
456 for (OsmPrimitive p : e.getPrimitives()) {
457 if (p.isTagged() && p.referrers(Relation.class).count() == 0)
458 replacement = (Way) p;
459 else if (p.referrers(Relation.class).anyMatch(Relation::isMultipolygon))
460 mpWays.add((Way) p);
461 }
462 if (replacement == null && mpWays.size() == 2) {
463 replacement = mpWays.remove(1);
464 }
465 if (replacement != null && mpWays.size() == 1) {
466 Way mpWay = mpWays.get(0);
467 for (Relation r : mpWay.referrers(Relation.class).filter(Relation::isMultipolygon)
468 .collect(Collectors.toList())) {
469 for (int i = 0; i < r.getMembersCount(); i++) {
470 if (r.getMember(i).getMember().equals(mpWay)) {
471 r.setMember(i, new RelationMember(r.getRole(i), replacement));
472 }
473 }
474 }
475 mpWay.setDeleted(true);
476 }
477 }
478 }
479 ds.cleanupDeletedPrimitives();
480 }
481
482 /**
483 * Parse the given input source and return the dataset.
484 *
485 * @param source the source input stream. Must not be null.
486 * @param progressMonitor the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed
487 * @return the dataset with the parsed data
488 * @throws IllegalDataException if an error was found while parsing the data from the source
489 * @throws IllegalArgumentException if source is null
490 */
491 public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
492 return new GeoJSONReader().doParseDataSet(source, progressMonitor);
493 }
494}
Note: See TracBrowser for help on using the repository browser.