source: josm/trunk/src/org/openstreetmap/josm/io/GeoJSONWriter.java

Last change on this file was 18754, checked in by taylor.smock, 11 months ago

Significantly reduce allocations in GeoJSONWriter

When using josm validate -i MesaCountyCO.osm.gz, where Mesa County, CO is the
data inside Mesa County, Colorado, USA, this patch reduces overall runtime by
~75%. This is largely done by reducing the amount of resources it takes to write
errors to file. So GeoJSONWriter#write now takes ~99% fewer CPU cycles and ~97%
fewer memory allocations. Time spent in JVM specific methods (like GC) was
reduced by ~50%.
  • Property svn:eol-style set to native
File size: 20.3 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.io;
3
4import java.io.StringReader;
5import java.io.StringWriter;
6import java.io.Writer;
7import java.math.BigDecimal;
8import java.math.RoundingMode;
9import java.time.Instant;
10import java.util.ArrayList;
11import java.util.Arrays;
12import java.util.Collection;
13import java.util.Collections;
14import java.util.EnumSet;
15import java.util.HashSet;
16import java.util.Iterator;
17import java.util.List;
18import java.util.Map;
19import java.util.Set;
20import java.util.stream.Collectors;
21import java.util.stream.Stream;
22
23import jakarta.json.Json;
24import jakarta.json.JsonArrayBuilder;
25import jakarta.json.JsonObject;
26import jakarta.json.JsonObjectBuilder;
27import jakarta.json.JsonValue;
28import jakarta.json.JsonWriter;
29import jakarta.json.spi.JsonProvider;
30import jakarta.json.stream.JsonGenerator;
31import jakarta.json.stream.JsonParser;
32import jakarta.json.stream.JsonParsingException;
33
34import org.openstreetmap.josm.data.Bounds;
35import org.openstreetmap.josm.data.coor.EastNorth;
36import org.openstreetmap.josm.data.coor.ILatLon;
37import org.openstreetmap.josm.data.coor.LatLon;
38import org.openstreetmap.josm.data.osm.DataSet;
39import org.openstreetmap.josm.data.osm.INode;
40import org.openstreetmap.josm.data.osm.IWay;
41import org.openstreetmap.josm.data.osm.MultipolygonBuilder;
42import org.openstreetmap.josm.data.osm.Node;
43import org.openstreetmap.josm.data.osm.OsmPrimitive;
44import org.openstreetmap.josm.data.osm.Relation;
45import org.openstreetmap.josm.data.osm.RelationMember;
46import org.openstreetmap.josm.data.osm.Way;
47import org.openstreetmap.josm.data.osm.visitor.OsmPrimitiveVisitor;
48import org.openstreetmap.josm.data.preferences.BooleanProperty;
49import org.openstreetmap.josm.data.projection.Projection;
50import org.openstreetmap.josm.data.projection.Projections;
51import org.openstreetmap.josm.gui.mappaint.ElemStyles;
52import org.openstreetmap.josm.tools.Geometry;
53import org.openstreetmap.josm.tools.Logging;
54import org.openstreetmap.josm.tools.Pair;
55import org.openstreetmap.josm.tools.Utils;
56
57/**
58 * Writes OSM data as a GeoJSON string, using JSR 353: Java API for JSON Processing (JSON-P).
59 * <p>
60 * See <a href="https://tools.ietf.org/html/rfc7946">RFC7946: The GeoJSON Format</a>
61 */
62public class GeoJSONWriter {
63
64 enum Options {
65 /** If using the right hand rule, we have to ensure that the "right" side is the interior of the object. */
66 RIGHT_HAND_RULE,
67 /** Write OSM information to the feature properties field. This tries to follow the Overpass turbo format. */
68 WRITE_OSM_INFORMATION,
69 /** Skip empty nodes */
70 SKIP_EMPTY_NODES
71 }
72
73 private final DataSet data;
74 private static final Projection projection = Projections.getProjectionByCode("EPSG:4326"); // WGS 84
75 private static final BooleanProperty SKIP_EMPTY_NODES = new BooleanProperty("geojson.export.skip-empty-nodes", true);
76 private static final BooleanProperty UNTAGGED_CLOSED_IS_POLYGON = new BooleanProperty("geojson.export.untagged-closed-is-polygon", false);
77
78 /**
79 * Used to avoid many calls to {@link JsonProvider#provider} in {@link #getCoorArray(JsonArrayBuilder, EastNorth)}.
80 * For validating Mesa County, CO, this reduces CPU and memory usage of {@link #write()} by ~80%. By using this for
81 * other {@link Json} calls, {@link #write()} takes ~95% less resources than the original. And the entire process
82 * takes 1/4 of the time (38 minutes -&gt; <10 minutes).
83 * <p>
84 * For more details, see <a href="https://github.com/jakartaee/jsonp-api/issues/346">JSONP #346</a>.
85 */
86 protected static final JsonProvider JSON_PROVIDER = JsonProvider.provider();
87 private static final Set<Way> processedMultipolygonWays = new HashSet<>();
88 private final EnumSet<Options> options = EnumSet.noneOf(Options.class);
89
90 /**
91 * This is used to determine that a tag should be interpreted as a json
92 * object or array. The tag should have both {@link #JSON_VALUE_START_MARKER}
93 * and {@link #JSON_VALUE_END_MARKER}.
94 */
95 static final String JSON_VALUE_START_MARKER = "{";
96 /**
97 * This is used to determine that a tag should be interpreted as a json
98 * object or array. The tag should have both {@link #JSON_VALUE_START_MARKER}
99 * and {@link #JSON_VALUE_END_MARKER}.
100 */
101 static final String JSON_VALUE_END_MARKER = "}";
102
103 /**
104 * Constructs a new {@code GeoJSONWriter}.
105 * @param ds The OSM data set to save
106 * @since 12806
107 */
108 public GeoJSONWriter(DataSet ds) {
109 this.data = ds;
110 if (Boolean.TRUE.equals(SKIP_EMPTY_NODES.get())) {
111 this.options.add(Options.SKIP_EMPTY_NODES);
112 }
113 }
114
115 /**
116 * Set the options for this writer. See {@link Options}.
117 * @param options The options to set.
118 */
119 void setOptions(final Options... options) {
120 this.options.clear();
121 this.options.addAll(Arrays.asList(options));
122 }
123
124 /**
125 * Writes OSM data as a GeoJSON string (prettified).
126 * @return The GeoJSON data
127 */
128 public String write() {
129 return write(true);
130 }
131
132 /**
133 * Writes OSM data as a GeoJSON string (prettified or not).
134 * @param pretty {@code true} to have pretty output, {@code false} otherwise
135 * @return The GeoJSON data
136 * @since 6756
137 */
138 public String write(boolean pretty) {
139 StringWriter stringWriter = new StringWriter();
140 write(pretty, stringWriter);
141 return stringWriter.toString();
142 }
143
144 /**
145 * Writes OSM data as a GeoJSON string (prettified or not).
146 * @param pretty {@code true} to have pretty output, {@code false} otherwise
147 * @param writer The writer used to write results
148 */
149 public void write(boolean pretty, Writer writer) {
150 Map<String, Object> config = Collections.singletonMap(JsonGenerator.PRETTY_PRINTING, pretty);
151 try (JsonWriter jsonWriter = JSON_PROVIDER.createWriterFactory(config).createWriter(writer)) {
152 JsonObjectBuilder object = JSON_PROVIDER.createObjectBuilder()
153 .add("type", "FeatureCollection")
154 .add("generator", "JOSM");
155 appendLayerBounds(data, object);
156 appendLayerFeatures(data, object);
157 jsonWriter.writeObject(object.build());
158 }
159 }
160
161 /**
162 * Convert a primitive to a json object
163 */
164 private class GeometryPrimitiveVisitor implements OsmPrimitiveVisitor {
165
166 private final JsonObjectBuilder geomObj;
167
168 GeometryPrimitiveVisitor(JsonObjectBuilder geomObj) {
169 this.geomObj = geomObj;
170 }
171
172 @Override
173 public void visit(Node n) {
174 geomObj.add("type", "Point");
175 LatLon ll = n.getCoor();
176 if (ll != null) {
177 geomObj.add("coordinates", getCoorArray(null, ll));
178 }
179 }
180
181 @Override
182 public void visit(Way w) {
183 if (w != null) {
184 if (!w.isTagged() && processedMultipolygonWays.contains(w)) {
185 // no need to write this object again
186 return;
187 }
188 boolean writeAsPolygon = w.isClosed() && ((!w.isTagged() && UNTAGGED_CLOSED_IS_POLYGON.get())
189 || ElemStyles.hasAreaElemStyle(w, false));
190 final List<Node> nodes = w.getNodes();
191 if (writeAsPolygon && options.contains(Options.RIGHT_HAND_RULE) && Geometry.isClockwise(nodes)) {
192 Collections.reverse(nodes);
193 }
194 final JsonArrayBuilder array = getCoorsArray(nodes);
195 if (writeAsPolygon) {
196 geomObj.add("type", "Polygon");
197 geomObj.add("coordinates", JSON_PROVIDER.createArrayBuilder().add(array));
198 } else {
199 geomObj.add("type", "LineString");
200 geomObj.add("coordinates", array);
201 }
202 }
203 }
204
205 @Override
206 public void visit(Relation r) {
207 if (r == null || !r.isMultipolygon() || r.hasIncompleteMembers()) {
208 return;
209 }
210 if (r.isMultipolygon()) {
211 try {
212 this.visitMultipolygon(r);
213 return;
214 } catch (MultipolygonBuilder.JoinedPolygonCreationException ex) {
215 Logging.warn("GeoJSON: Failed to export multipolygon {0}, falling back to other multi geometry types", r.getUniqueId());
216 Logging.warn(ex);
217 }
218 }
219 // These are run if (a) r is not a multipolygon or (b) r is not a well-formed multipolygon.
220 if (r.getMemberPrimitives().stream().allMatch(IWay.class::isInstance)) {
221 this.visitMultiLineString(r);
222 } else if (r.getMemberPrimitives().stream().allMatch(INode.class::isInstance)) {
223 this.visitMultiPoints(r);
224 } else {
225 this.visitMultiGeometry(r);
226 }
227 }
228
229 /**
230 * Visit a multi-part geometry.
231 * Note: Does not currently recurse down relations. RFC 7946 indicates that we
232 * should avoid nested geometry collections. This behavior may change any time in the future!
233 * @param r The relation to visit.
234 */
235 private void visitMultiGeometry(final Relation r) {
236 final JsonArrayBuilder jsonArrayBuilder = JSON_PROVIDER.createArrayBuilder();
237 r.getMemberPrimitives().stream().filter(p -> !(p instanceof Relation))
238 .map(p -> {
239 final JsonObjectBuilder tempGeomObj = JSON_PROVIDER.createObjectBuilder();
240 p.accept(new GeometryPrimitiveVisitor(tempGeomObj));
241 return tempGeomObj.build();
242 }).forEach(jsonArrayBuilder::add);
243 geomObj.add("type", "GeometryCollection");
244 geomObj.add("geometries", jsonArrayBuilder);
245 }
246
247 /**
248 * Visit a relation that only contains points
249 * @param r The relation to visit
250 */
251 private void visitMultiPoints(final Relation r) {
252 final JsonArrayBuilder multiPoint = JSON_PROVIDER.createArrayBuilder();
253 r.getMembers().stream().map(RelationMember::getMember).filter(Node.class::isInstance).map(Node.class::cast)
254 .map(latLon -> getCoorArray(null, latLon))
255 .forEach(multiPoint::add);
256 geomObj.add("type", "MultiPoint");
257 geomObj.add("coordinates", multiPoint);
258 }
259
260 /**
261 * Visit a relation that is a multi line string
262 * @param r The relation to convert
263 */
264 private void visitMultiLineString(final Relation r) {
265 final JsonArrayBuilder multiLine = JSON_PROVIDER.createArrayBuilder();
266 r.getMembers().stream().map(RelationMember::getMember).filter(Way.class::isInstance).map(Way.class::cast)
267 .map(Way::getNodes).map(p -> {
268 JsonArrayBuilder array = getCoorsArray(p);
269 ILatLon ll = p.get(0);
270 // since first node is not duplicated as last node
271 return ll.isLatLonKnown() ? array.add(getCoorArray(null, ll)) : array;
272 }).forEach(multiLine::add);
273 geomObj.add("type", "MultiLineString");
274 geomObj.add("coordinates", multiLine);
275 processedMultipolygonWays.addAll(r.getMemberPrimitives(Way.class));
276 }
277
278 /**
279 * Convert a multipolygon to geojson
280 * @param r The relation to convert
281 * @throws MultipolygonBuilder.JoinedPolygonCreationException See {@link MultipolygonBuilder#joinWays(Relation)}.
282 * Note that if the exception is thrown, {@link #geomObj} will not have been modified.
283 */
284 private void visitMultipolygon(final Relation r) throws MultipolygonBuilder.JoinedPolygonCreationException {
285 final Pair<List<MultipolygonBuilder.JoinedPolygon>, List<MultipolygonBuilder.JoinedPolygon>> mp =
286 MultipolygonBuilder.joinWays(r);
287 final JsonArrayBuilder polygon = JSON_PROVIDER.createArrayBuilder();
288 // Peek would theoretically be better for these two streams, but SonarLint doesn't like it.
289 // java:S3864: "Stream.peek" should be used with caution
290 final Stream<List<Node>> outer = mp.a.stream().map(MultipolygonBuilder.JoinedPolygon::getNodes).map(nodes -> {
291 final ArrayList<Node> tempNodes = new ArrayList<>(nodes);
292 tempNodes.add(tempNodes.get(0));
293 if (options.contains(Options.RIGHT_HAND_RULE) && Geometry.isClockwise(tempNodes)) {
294 Collections.reverse(nodes);
295 }
296 return nodes;
297 });
298 final Stream<List<Node>> inner = mp.b.stream().map(MultipolygonBuilder.JoinedPolygon::getNodes).map(nodes -> {
299 final ArrayList<Node> tempNodes = new ArrayList<>(nodes);
300 tempNodes.add(tempNodes.get(0));
301 // Note that we are checking !Geometry.isClockwise, which is different from the outer
302 // ring check.
303 if (options.contains(Options.RIGHT_HAND_RULE) && !Geometry.isClockwise(tempNodes)) {
304 Collections.reverse(nodes);
305 }
306 return nodes;
307 });
308 Stream.concat(outer, inner)
309 .map(p -> {
310 JsonArrayBuilder array = getCoorsArray(p);
311 ILatLon ll = p.get(0);
312 // since first node is not duplicated as last node
313 return ll.isLatLonKnown() ? array.add(getCoorArray(null, ll)) : array;
314 })
315 .forEach(polygon::add);
316 final JsonArrayBuilder multiPolygon = JSON_PROVIDER.createArrayBuilder().add(polygon);
317 geomObj.add("type", "MultiPolygon");
318 geomObj.add("coordinates", multiPolygon);
319 processedMultipolygonWays.addAll(r.getMemberPrimitives(Way.class));
320 }
321
322 private JsonArrayBuilder getCoorsArray(Iterable<Node> nodes) {
323 final JsonArrayBuilder builder = JSON_PROVIDER.createArrayBuilder();
324 for (Node n : nodes) {
325 if (n.isLatLonKnown()) {
326 builder.add(getCoorArray(null, n));
327 }
328 }
329 return builder;
330 }
331 }
332
333 private JsonArrayBuilder getCoorArray(JsonArrayBuilder builder, ILatLon c) {
334 return getCoorArray(builder, projection.latlon2eastNorth(c));
335 }
336
337 private static JsonArrayBuilder getCoorArray(JsonArrayBuilder builder, EastNorth c) {
338 return (builder != null ? builder : JSON_PROVIDER.createArrayBuilder())
339 .add(BigDecimal.valueOf(c.getX()).setScale(11, RoundingMode.HALF_UP))
340 .add(BigDecimal.valueOf(c.getY()).setScale(11, RoundingMode.HALF_UP));
341 }
342
343 protected void appendPrimitive(OsmPrimitive p, JsonArrayBuilder array) {
344 if (p.isIncomplete() ||
345 (this.options.contains(Options.SKIP_EMPTY_NODES) && p instanceof Node && p.getKeys().isEmpty())) {
346 return;
347 }
348
349 // Properties
350 final JsonObjectBuilder propObj = JSON_PROVIDER.createObjectBuilder();
351 for (Map.Entry<String, String> t : p.getKeys().entrySet()) {
352 // If writing OSM information, follow Overpass syntax (escape `@` with another `@`)
353 final String key = options.contains(Options.WRITE_OSM_INFORMATION) && t.getKey().startsWith("@")
354 ? '@' + t.getKey() : t.getKey();
355 propObj.add(key, convertValueToJson(t.getValue()));
356 }
357 if (options.contains(Options.WRITE_OSM_INFORMATION)) {
358 // Use the same format as Overpass
359 propObj.add("@id", p.getPrimitiveId().getType().getAPIName() + '/' + p.getUniqueId()); // type/id
360 if (!p.isNew()) {
361 propObj.add("@timestamp", Instant.ofEpochSecond(p.getRawTimestamp()).toString());
362 propObj.add("@version", Integer.toString(p.getVersion()));
363 propObj.add("@changeset", Long.toString(p.getChangesetId()));
364 }
365 if (p.getUser() != null) {
366 propObj.add("@user", p.getUser().getName());
367 propObj.add("@uid", p.getUser().getId());
368 }
369 if (options.contains(Options.WRITE_OSM_INFORMATION) && p.getReferrers(true).stream().anyMatch(Relation.class::isInstance)) {
370 final JsonArrayBuilder jsonArrayBuilder = JSON_PROVIDER.createArrayBuilder();
371 for (Relation relation : Utils.filteredCollection(p.getReferrers(), Relation.class)) {
372 final JsonObjectBuilder relationObject = JSON_PROVIDER.createObjectBuilder();
373 relationObject.add("rel", relation.getId());
374 Collection<RelationMember> members = relation.getMembersFor(Collections.singleton(p));
375 // Each role is a separate object in overpass-turbo geojson export. For now, just concat them.
376 relationObject.add("role",
377 members.stream().map(RelationMember::getRole).collect(Collectors.joining(";")));
378 final JsonObjectBuilder relationKeys = JSON_PROVIDER.createObjectBuilder();
379 // Uncertain if the @relation reltags need to be @ escaped. I don't think so, as example output
380 // didn't have any metadata in it.
381 for (Map.Entry<String, String> tag : relation.getKeys().entrySet()) {
382 relationKeys.add(tag.getKey(), convertValueToJson(tag.getValue()));
383 }
384 relationObject.add("reltags", relationKeys);
385 }
386 propObj.add("@relations", jsonArrayBuilder);
387 }
388 }
389 final JsonObject prop = propObj.build();
390
391 // Geometry
392 final JsonObjectBuilder geomObj = JSON_PROVIDER.createObjectBuilder();
393 p.accept(new GeometryPrimitiveVisitor(geomObj));
394 final JsonObject geom = geomObj.build();
395
396 if (!geom.isEmpty()) {
397 // Build primitive JSON object
398 array.add(JSON_PROVIDER.createObjectBuilder()
399 .add("type", "Feature")
400 .add("properties", prop.isEmpty() ? JsonValue.NULL : prop)
401 .add("geometry", geom.isEmpty() ? JsonValue.NULL : geom));
402 }
403 }
404
405 private static JsonValue convertValueToJson(String value) {
406 if (value.startsWith(JSON_VALUE_START_MARKER) && value.endsWith(JSON_VALUE_END_MARKER)) {
407 try (JsonParser parser = JSON_PROVIDER.createParser(new StringReader(value))) {
408 if (parser.hasNext() && parser.next() != null) {
409 return parser.getValue();
410 }
411 } catch (JsonParsingException e) {
412 Logging.warn(e);
413 }
414 }
415 return JSON_PROVIDER.createValue(value);
416 }
417
418 protected void appendLayerBounds(DataSet ds, JsonObjectBuilder object) {
419 if (ds != null) {
420 Iterator<Bounds> it = ds.getDataSourceBounds().iterator();
421 if (it.hasNext()) {
422 Bounds b = new Bounds(it.next());
423 while (it.hasNext()) {
424 b.extend(it.next());
425 }
426 appendBounds(b, object);
427 }
428 }
429 }
430
431 protected void appendBounds(Bounds b, JsonObjectBuilder object) {
432 if (b != null) {
433 JsonArrayBuilder builder = JSON_PROVIDER.createArrayBuilder();
434 getCoorArray(builder, b.getMin());
435 getCoorArray(builder, b.getMax());
436 object.add("bbox", builder);
437 }
438 }
439
440 protected void appendLayerFeatures(DataSet ds, JsonObjectBuilder object) {
441 JsonArrayBuilder array = JSON_PROVIDER.createArrayBuilder();
442 if (ds != null) {
443 processedMultipolygonWays.clear();
444 Collection<OsmPrimitive> primitives = ds.allNonDeletedPrimitives();
445 // Relations first
446 for (OsmPrimitive p : primitives) {
447 if (p instanceof Relation)
448 appendPrimitive(p, array);
449 }
450 for (OsmPrimitive p : primitives) {
451 if (!(p instanceof Relation))
452 appendPrimitive(p, array);
453 }
454 processedMultipolygonWays.clear();
455 }
456 object.add("features", array);
457 }
458}
Note: See TracBrowser for help on using the repository browser.