diff --git a/src/org/openstreetmap/josm/data/validation/ValidatorCLI.java b/src/org/openstreetmap/josm/data/validation/ValidatorCLI.java new file mode 100644 index 0000000000..6428bb1ba7 --- /dev/null +++ b/src/org/openstreetmap/josm/data/validation/ValidatorCLI.java @@ -0,0 +1,247 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.validation; + +import static org.openstreetmap.josm.tools.I18n.tr; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Collection; +import java.util.Locale; +import java.util.Optional; +import java.util.logging.Level; + +import org.openstreetmap.josm.cli.CLIModule; +import org.openstreetmap.josm.data.osm.DataSet; +import org.openstreetmap.josm.data.preferences.JosmBaseDirectories; +import org.openstreetmap.josm.data.preferences.JosmUrls; +import org.openstreetmap.josm.data.projection.ProjectionRegistry; +import org.openstreetmap.josm.data.projection.Projections; +import org.openstreetmap.josm.gui.MainApplication; +import org.openstreetmap.josm.gui.layer.OsmDataLayer; +import org.openstreetmap.josm.gui.progress.NullProgressMonitor; +import org.openstreetmap.josm.io.Compression; +import org.openstreetmap.josm.io.GeoJSONMapRouletteWriter; +import org.openstreetmap.josm.io.IllegalDataException; +import org.openstreetmap.josm.io.OsmReader; +import org.openstreetmap.josm.spi.preferences.Config; +import org.openstreetmap.josm.spi.preferences.MemoryPreferences; +import org.openstreetmap.josm.tools.Http1Client; +import org.openstreetmap.josm.tools.HttpClient; +import org.openstreetmap.josm.tools.JosmRuntimeException; +import org.openstreetmap.josm.tools.Logging; +import org.openstreetmap.josm.tools.OptionParser; +import org.openstreetmap.josm.tools.Stopwatch; +import org.openstreetmap.josm.tools.Territories; + +/** + * Add a validate command to the JOSM command line interface. + * @author Taylor Smock + * @since xxx + */ +public class ValidatorCLI implements CLIModule { + public static ValidatorCLI INSTANCE = new ValidatorCLI(); + + /** The input file */ + private String input; + /** The output file */ + private String output; + + /** The log level */ + private Level logLevel; + + private enum Option { + HELP(false, 'h'), + INPUT(true, 'i'), + OUTPUT(true, 'o'), + DEBUG(false, '*'), + TRACE(false, '*'); + + private final String name; + private final boolean requiresArgument; + private final char shortOption; + Option(final boolean requiresArgument, final char shortOption) { + this.name = name().toLowerCase(Locale.ROOT).replace('_', '-'); + this.requiresArgument = requiresArgument; + this.shortOption = shortOption; + } + + /** + * Replies the option name + * @return The option name, in lowercase + */ + public String getName() { + return this.name; + } + + /** + * Determines if this option requires an argument. + * @return {@code true} if this option requires an argument, {@code false} otherwise + */ + public boolean requiresArgument() { + return this.requiresArgument; + } + + /** + * Replies the short option (single letter) associated with this option. + * @return the short option or '*' if there is no short option + */ + public char getShortOption() { + return this.shortOption; + } + } + + @Override + public String getActionKeyword() { + return "validate"; + } + + @Override + public void processArguments(final String[] argArray) { + try { + Logging.setLogLevel(Level.INFO); + this.parseArguments(argArray); + this.initialize(); + final Stopwatch stopwatch = Stopwatch.createStarted(); + final String task = tr("Validating {0}, saving output to {1}", this.input, this.output); + Logging.info(task); + OsmValidator.initializeTests(); + final DataSet dataSet = this.loadDataSet(); + // Adding a layer specifically for UntaggedWay.class (it wants the active dataset, which is from the + // current layer). + MainApplication.getLayerManager().addLayer(new OsmDataLayer(dataSet, this.input, new File(this.input))); + Collection tests = OsmValidator.getEnabledTests(false); + tests.parallelStream().forEach(test -> test.startTest(NullProgressMonitor.INSTANCE)); + tests.parallelStream().forEach(test -> test.visit(dataSet.allPrimitives())); + tests.parallelStream().forEach(Test::endTest); + if (Files.isRegularFile(Paths.get(this.output)) && !Files.deleteIfExists(Paths.get(this.output))) { + Logging.error("Could not delete {0}, attempting to append", this.output); + } + GeoJSONMapRouletteWriter geoJSONMapRouletteWriter = new GeoJSONMapRouletteWriter(dataSet); + try (FileOutputStream fileOutputStream = new FileOutputStream(this.output)) { + for (Test test : tests) { + test.getErrors().stream().map(geoJSONMapRouletteWriter::write) + .filter(Optional::isPresent).map(Optional::get) + .map(jsonObject -> jsonObject.toString().getBytes(StandardCharsets.UTF_8)).forEach(bytes -> { + try { + // Write the ASCII Record Separator character + fileOutputStream.write(0x1e); + fileOutputStream.write(bytes); + // Write the ASCII Line Feed character + fileOutputStream.write(0x0a); + } catch (IOException e) { + throw new JosmRuntimeException(e); + } + }); + } + } + Logging.info(stopwatch.toString(task)); + } catch (JosmRuntimeException | IllegalArgumentException | IllegalDataException | IOException e) { + Logging.info(e); + System.exit(1); + } + System.exit(0); + } + + /** + * Initialize everything that might be needed + * + * Arguments may need to be parsed first. + */ + void initialize() { + Logging.setLogLevel(this.logLevel); + HttpClient.setFactory(Http1Client::new); + Config.setBaseDirectoriesProvider(JosmBaseDirectories.getInstance()); // for right-left-hand traffic cache file + Config.setPreferencesInstance(new MemoryPreferences()); + Config.setUrlsProvider(JosmUrls.getInstance()); + ProjectionRegistry.setProjection(Projections.getProjectionByCode("epsg:3857".toUpperCase(Locale.ROOT))); + + Territories.initializeInternalData(); + OsmValidator.initialize(); + } + + /** + * Parse command line arguments and do some low-level error checking. + * @param argArray the arguments array + */ + void parseArguments(String[] argArray) { + Logging.setLogLevel(Level.INFO); + + OptionParser parser = new OptionParser("JOSM rendering"); + for (Option o : Option.values()) { + if (o.requiresArgument()) { + parser.addArgumentParameter(o.getName(), + OptionParser.OptionCount.REQUIRED, + arg -> handleOption(o, arg)); + } else { + parser.addFlagParameter(o.getName(), () -> handleOption(o)); + } + if (o.getShortOption() != '*') { + parser.addShortAlias(o.getName(), Character.toString(o.getShortOption())); + } + } + parser.parseOptionsOrExit(Arrays.asList(argArray)); + } + + private void handleOption(final Option option) { + switch (option) { + case HELP: + showHelp(); + System.exit(0); + break; + case DEBUG: + this.logLevel = Logging.LEVEL_DEBUG; + break; + case TRACE: + this.logLevel = Logging.LEVEL_TRACE; + break; + default: + throw new AssertionError("Unexpected option: " + option); + } + } + + private void handleOption(final Option option, final String argument) { + switch (option) { + case INPUT: + this.input = argument; + break; + case OUTPUT: + this.output = argument; + break; + default: + throw new AssertionError("Unexpected option: " + option); + } + } + + private DataSet loadDataSet() throws IOException, IllegalDataException { + if (this.input == null) { + throw new IllegalArgumentException(tr("Missing argument - input data file ({0})", "--input|-i")); + } + try (InputStream inputStream = Compression.getUncompressedFileInputStream(Paths.get(this.input))) { + return OsmReader.parseDataSet(inputStream, null); + } catch (IllegalDataException e) { + throw new IllegalDataException(tr("In .osm data file ''{0}'' - ", this.input) + e.getMessage(), e); + } + } + + private static void showHelp() { + System.out.println(getHelp()); + } + + private static String getHelp() { + return tr("JOSM Validation command line interface") + "\n\n" + + tr("Usage") + ":\n" + + "\tjava -jar josm.jar validate \n\n" + + tr("Description") + ":\n" + + tr("Validates data and saves the result to a file.") + "\n\n"+ + tr("Options") + ":\n" + + "\t--help|-h " + tr("Show this help") + "\n" + + "\t--input|-i " + tr("Input data file name (.osm)") + "\n" + + "\t--output|-o " + tr("Output data file name (.geojson, line-by-line delimited for MapRoulette)"); + } +} diff --git a/src/org/openstreetmap/josm/gui/MainApplication.java b/src/org/openstreetmap/josm/gui/MainApplication.java index a4572b8b9b..438efd8a07 100644 --- a/src/org/openstreetmap/josm/gui/MainApplication.java +++ b/src/org/openstreetmap/josm/gui/MainApplication.java @@ -98,6 +98,7 @@ import org.openstreetmap.josm.data.projection.ProjectionRegistry; import org.openstreetmap.josm.data.projection.datum.NTV2GridShiftFileSource; import org.openstreetmap.josm.data.projection.datum.NTV2GridShiftFileWrapper; import org.openstreetmap.josm.data.projection.datum.NTV2Proj4DirGridShiftFileSource; +import org.openstreetmap.josm.data.validation.ValidatorCLI; import org.openstreetmap.josm.data.validation.tests.MapCSSTagChecker; import org.openstreetmap.josm.gui.ProgramArguments.Option; import org.openstreetmap.josm.gui.SplashScreen.SplashProgressMonitor; @@ -311,6 +312,7 @@ public class MainApplication { registerCLIModule(JOSM_CLI_MODULE); registerCLIModule(ProjectionCLI.INSTANCE); registerCLIModule(RenderingCLI.INSTANCE); + registerCLIModule(ValidatorCLI.INSTANCE); } /** @@ -660,7 +662,8 @@ public class MainApplication { tr("commands")+":\n"+ "\trunjosm "+tr("launch JOSM (default, performed when no command is specified)")+'\n'+ "\trender "+tr("render data and save the result to an image file")+'\n'+ - "\tproject "+tr("convert coordinates from one coordinate reference system to another")+"\n\n"+ + "\tproject " + tr("convert coordinates from one coordinate reference system to another")+ '\n' + + "\tvalidate " + tr("validate data") + "\n\n" + tr("For details on the {0} and {1} commands, run them with the {2} option.", "render", "project", "--help")+'\n'+ tr("The remainder of this help page documents the {0} command.", "runjosm")+"\n\n"+ tr("options")+":\n"+ diff --git a/src/org/openstreetmap/josm/gui/util/GuiHelper.java b/src/org/openstreetmap/josm/gui/util/GuiHelper.java index 89ab9f84b0..4c17334af2 100644 --- a/src/org/openstreetmap/josm/gui/util/GuiHelper.java +++ b/src/org/openstreetmap/josm/gui/util/GuiHelper.java @@ -288,7 +288,7 @@ public final class GuiHelper { * @since 10271 */ public static void assertCallFromEdt() { - if (!SwingUtilities.isEventDispatchThread()) { + if (!SwingUtilities.isEventDispatchThread() && !GraphicsEnvironment.isHeadless()) { throw new IllegalStateException( "Needs to be called from the EDT thread, not from " + Thread.currentThread().getName()); } diff --git a/src/org/openstreetmap/josm/io/GeoJSONMapRouletteWriter.java b/src/org/openstreetmap/josm/io/GeoJSONMapRouletteWriter.java new file mode 100644 index 0000000000..d61be93cd8 --- /dev/null +++ b/src/org/openstreetmap/josm/io/GeoJSONMapRouletteWriter.java @@ -0,0 +1,90 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.io; + +import java.util.Objects; +import java.util.Optional; + +import javax.json.Json; +import javax.json.JsonArray; +import javax.json.JsonArrayBuilder; +import javax.json.JsonObject; +import javax.json.JsonObjectBuilder; +import javax.json.JsonValue; + +import org.openstreetmap.josm.data.osm.DataSet; +import org.openstreetmap.josm.data.osm.OsmPrimitive; +import org.openstreetmap.josm.data.osm.WaySegment; +import org.openstreetmap.josm.data.validation.TestError; +import org.openstreetmap.josm.tools.Logging; + +/** + * Convert {@link TestError} to MapRoulette Tasks + * @author Taylor Smock + * @since xxx + */ +public class GeoJSONMapRouletteWriter extends GeoJSONWriter { + + /** + * Constructs a new {@code GeoJSONWriter}. + * @param ds The originating OSM dataset + */ + public GeoJSONMapRouletteWriter(DataSet ds) { + super(ds); + super.setOptions(Options.RIGHT_HAND_RULE, Options.WRITE_OSM_INFORMATION); + } + + /** + * Convert a test error to a string + * @param testError The test error to convert + * @return The MapRoulette challenge object + */ + public Optional write(final TestError testError) { + final JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder(); + final JsonArrayBuilder featuresBuilder = Json.createArrayBuilder(); + final JsonObjectBuilder propertiesBuilder = Json.createObjectBuilder(); + propertiesBuilder.add("message", testError.getMessage()); + Optional.ofNullable(testError.getDescription()).ifPresent(description -> propertiesBuilder.add("description", description)); + propertiesBuilder.add("code", testError.getCode()); + propertiesBuilder.add("fixable", testError.isFixable()); + propertiesBuilder.add("severity", testError.getSeverity().toString()); + propertiesBuilder.add("severityInteger", testError.getSeverity().getLevel()); + propertiesBuilder.add("test", testError.getTester().getName()); + testError.getPrimitives().forEach(primitive -> super.appendPrimitive(primitive, featuresBuilder)); + testError.getHighlighted().stream().map(p -> { + if (p instanceof OsmPrimitive) { + return p; + } else if (p instanceof WaySegment) { + return ((WaySegment) p).toWay(); + } + Logging.error("Could not convert {0} to an OsmPrimitive", p); + return null; + }).filter(Objects::nonNull).map(OsmPrimitive.class::cast) + .forEach(primitive -> super.appendPrimitive(primitive, featuresBuilder)); + final JsonArray featureArray = featuresBuilder.build(); + final JsonArrayBuilder featuresMessageBuilder = Json.createArrayBuilder(); + if (featureArray.isEmpty()) { + Logging.error("Could not generate task for {0}", testError.getMessage()); + return Optional.empty(); + } + JsonObject primitive = featureArray.getJsonObject(0); + JsonObjectBuilder replacementPrimitive = Json.createObjectBuilder(primitive); + final JsonObjectBuilder properties; + if (primitive.containsKey("properties") && primitive.get("properties").getValueType() == JsonValue.ValueType.OBJECT) { + properties = Json.createObjectBuilder(primitive.getJsonObject("properties")); + } else { + properties = Json.createObjectBuilder(); + } + properties.addAll(propertiesBuilder); + replacementPrimitive.add("properties", properties); + featuresMessageBuilder.add(replacementPrimitive); + for (int i = 1; i < featureArray.size(); i++) { + featuresMessageBuilder.add(featureArray.get(i)); + } + // For now, don't add any cooperativeWork objects, as JOSM should be able to find the fixes. + // This should change if the ValidatorCLI can use plugins (especially those introducing external data, like + // the ElevationProfile plugin (which provides elevation data)). + jsonObjectBuilder.add("type", "FeatureCollection"); + jsonObjectBuilder.add("features", featuresMessageBuilder); + return Optional.of(jsonObjectBuilder.build()); + } +} diff --git a/src/org/openstreetmap/josm/io/GeoJSONWriter.java b/src/org/openstreetmap/josm/io/GeoJSONWriter.java index d2eb644846..cb87f7b767 100644 --- a/src/org/openstreetmap/josm/io/GeoJSONWriter.java +++ b/src/org/openstreetmap/josm/io/GeoJSONWriter.java @@ -6,14 +6,18 @@ import java.io.StringWriter; import java.io.Writer; import java.math.BigDecimal; import java.math.RoundingMode; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.EnumSet; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; +import java.util.stream.Collectors; import java.util.stream.Stream; import javax.json.Json; @@ -35,14 +39,17 @@ import org.openstreetmap.josm.data.osm.MultipolygonBuilder.JoinedPolygon; import org.openstreetmap.josm.data.osm.Node; import org.openstreetmap.josm.data.osm.OsmPrimitive; import org.openstreetmap.josm.data.osm.Relation; +import org.openstreetmap.josm.data.osm.RelationMember; import org.openstreetmap.josm.data.osm.Way; import org.openstreetmap.josm.data.osm.visitor.OsmPrimitiveVisitor; import org.openstreetmap.josm.data.preferences.BooleanProperty; import org.openstreetmap.josm.data.projection.Projection; import org.openstreetmap.josm.data.projection.Projections; import org.openstreetmap.josm.gui.mappaint.ElemStyles; +import org.openstreetmap.josm.tools.Geometry; import org.openstreetmap.josm.tools.Logging; import org.openstreetmap.josm.tools.Pair; +import org.openstreetmap.josm.tools.Utils; /** * Writes OSM data as a GeoJSON string, using JSR 353: Java API for JSON Processing (JSON-P). @@ -51,11 +58,19 @@ import org.openstreetmap.josm.tools.Pair; */ public class GeoJSONWriter { + enum Options { + /** If using the right hand rule, we have to ensure that the "right" side is the interior of the object. */ + RIGHT_HAND_RULE, + /** Write OSM information to the feature properties field. This tries to follow the Overpass turbo format. */ + WRITE_OSM_INFORMATION; + } + private final DataSet data; - private final Projection projection; + private static final Projection projection = Projections.getProjectionByCode("EPSG:4326"); // WGS 84 private static final BooleanProperty SKIP_EMPTY_NODES = new BooleanProperty("geojson.export.skip-empty-nodes", true); private static final BooleanProperty UNTAGGED_CLOSED_IS_POLYGON = new BooleanProperty("geojson.export.untagged-closed-is-polygon", false); private static final Set processedMultipolygonWays = new HashSet<>(); + private EnumSet options = EnumSet.noneOf(Options.class); /** * This is used to determine that a tag should be interpreted as a json @@ -77,7 +92,15 @@ public class GeoJSONWriter { */ public GeoJSONWriter(DataSet ds) { this.data = ds; - this.projection = Projections.getProjectionByCode("EPSG:4326"); // WGS 84 + } + + /** + * Set the options for this writer. See {@link Options}. + * @param options The options to set. + */ + void setOptions(final Options... options) { + this.options.clear(); + this.options.addAll(Arrays.asList(options)); } /** @@ -117,6 +140,9 @@ public class GeoJSONWriter { } } + /** + * Convert a primitive to a json object + */ private class GeometryPrimitiveVisitor implements OsmPrimitiveVisitor { private final JsonObjectBuilder geomObj; @@ -141,9 +167,13 @@ public class GeoJSONWriter { // no need to write this object again return; } - final JsonArrayBuilder array = getCoorsArray(w.getNodes()); boolean writeAsPolygon = w.isClosed() && ((!w.isTagged() && UNTAGGED_CLOSED_IS_POLYGON.get()) || ElemStyles.hasAreaElemStyle(w, false)); + final List nodes = w.getNodes(); + if (writeAsPolygon && options.contains(Options.RIGHT_HAND_RULE) && Geometry.isClockwise(nodes)) { + Collections.reverse(nodes); + } + final JsonArrayBuilder array = getCoorsArray(nodes); if (writeAsPolygon) { geomObj.add("type", "Polygon"); geomObj.add("coordinates", Json.createArrayBuilder().add(array)); @@ -162,10 +192,26 @@ public class GeoJSONWriter { try { final Pair, List> mp = MultipolygonBuilder.joinWays(r); final JsonArrayBuilder polygon = Json.createArrayBuilder(); - Stream.concat(mp.a.stream(), mp.b.stream()) + final Stream> outer = mp.a.stream().map(JoinedPolygon::getNodes).map(nodes -> { + final ArrayList tempNodes = new ArrayList<>(nodes); + tempNodes.add(tempNodes.get(0)); + if (options.contains(Options.RIGHT_HAND_RULE) && Geometry.isClockwise(tempNodes)) { + Collections.reverse(nodes); + } + return nodes; + }); + final Stream> inner = mp.b.stream().map(JoinedPolygon::getNodes).map(nodes -> { + final ArrayList tempNodes = new ArrayList<>(nodes); + tempNodes.add(tempNodes.get(0)); + if (options.contains(Options.RIGHT_HAND_RULE) && !Geometry.isClockwise(tempNodes)) { + Collections.reverse(nodes); + } + return nodes; + }); + Stream.concat(outer, inner) .map(p -> { - JsonArrayBuilder array = getCoorsArray(p.getNodes()); - LatLon ll = p.getNodes().get(0).getCoor(); + JsonArrayBuilder array = getCoorsArray(p); + LatLon ll = p.get(0).getCoor(); // since first node is not duplicated as last node return ll != null ? array.add(getCoorArray(null, ll)) : array; }) @@ -210,8 +256,43 @@ public class GeoJSONWriter { // Properties final JsonObjectBuilder propObj = Json.createObjectBuilder(); - for (Entry t : p.getKeys().entrySet()) { - propObj.add(t.getKey(), convertValueToJson(t.getValue())); + for (Map.Entry t : p.getKeys().entrySet()) { + // If writing OSM information, follow Overpass syntax (escape `@` with another `@`) + final String key = options.contains(Options.WRITE_OSM_INFORMATION) && t.getKey().startsWith("@") + ? '@' + t.getKey() : t.getKey(); + propObj.add(key, convertValueToJson(t.getValue())); + } + if (options.contains(Options.WRITE_OSM_INFORMATION)) { + // Use the same format as Overpass + propObj.add("@id", p.getPrimitiveId().getType().getAPIName() + '/' + p.getId()); // type/id + if (!p.isNew()) { + propObj.add("@timestamp", Instant.ofEpochSecond(p.getRawTimestamp()).toString()); + propObj.add("@version", Integer.toString(p.getVersion())); + propObj.add("@changeset", Long.toString(p.getChangesetId())); + } + if (p.getUser() != null) { + propObj.add("@user", p.getUser().getName()); + propObj.add("@uid", p.getUser().getId()); + } + if (options.contains(Options.WRITE_OSM_INFORMATION) && p.getReferrers(true).stream().anyMatch(Relation.class::isInstance)) { + final JsonArrayBuilder jsonArrayBuilder = Json.createArrayBuilder(); + for (Relation relation : Utils.filteredCollection(p.getReferrers(), Relation.class)) { + final JsonObjectBuilder relationObject = Json.createObjectBuilder(); + relationObject.add("rel", relation.getId()); + Collection members = relation.getMembersFor(Collections.singleton(p)); + // Each role is a separate object in overpass-turbo geojson export. For now, just concat them. + relationObject.add("role", + members.stream().map(RelationMember::getRole).collect(Collectors.joining(";"))); + final JsonObjectBuilder relationKeys = Json.createObjectBuilder(); + // Uncertain if the @relation reltags need to be @ escaped. I don't think so, as example output + // didn't have any metadata in it. + for (Map.Entry tag : relation.getKeys().entrySet()) { + relationKeys.add(tag.getKey(), convertValueToJson(tag.getValue())); + } + relationObject.add("reltags", relationKeys); + } + propObj.add("@relations", jsonArrayBuilder); + } } final JsonObject prop = propObj.build();