source: josm/trunk/src/org/openstreetmap/josm/data/validation/ValidatorCLI.java@ 18366

Last change on this file since 18366 was 18366, checked in by taylor.smock, 2 years ago

Replace usages of System.exit with Lifecycle.exitJosm

This fixes some coverity scan issues (spotbugs) with respect to exiting the VM.

See #15182: Standalone JOSM validator
File size: 20.8 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.validation;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5import static org.openstreetmap.josm.tools.I18n.trn;
6
7import java.io.File;
8import java.io.IOException;
9import java.io.InputStream;
10import java.io.OutputStream;
11import java.nio.charset.StandardCharsets;
12import java.nio.file.Files;
13import java.nio.file.Paths;
14import java.util.ArrayList;
15import java.util.Arrays;
16import java.util.Collection;
17import java.util.Collections;
18import java.util.HashMap;
19import java.util.List;
20import java.util.Locale;
21import java.util.Map;
22import java.util.Optional;
23import java.util.concurrent.atomic.AtomicReference;
24import java.util.function.Supplier;
25import java.util.logging.Level;
26import java.util.stream.Collectors;
27
28import org.apache.commons.compress.utils.FileNameUtils;
29import org.openstreetmap.josm.actions.ExtensionFileFilter;
30import org.openstreetmap.josm.cli.CLIModule;
31import org.openstreetmap.josm.data.Preferences;
32import org.openstreetmap.josm.data.osm.DataSet;
33import org.openstreetmap.josm.data.preferences.JosmBaseDirectories;
34import org.openstreetmap.josm.data.preferences.JosmUrls;
35import org.openstreetmap.josm.data.projection.ProjectionRegistry;
36import org.openstreetmap.josm.data.projection.Projections;
37import org.openstreetmap.josm.data.validation.tests.MapCSSTagChecker;
38import org.openstreetmap.josm.gui.MainApplication;
39import org.openstreetmap.josm.gui.io.CustomConfigurator;
40import org.openstreetmap.josm.gui.io.importexport.FileImporter;
41import org.openstreetmap.josm.gui.layer.OsmDataLayer;
42import org.openstreetmap.josm.gui.mappaint.MapPaintStyles;
43import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleSource;
44import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.ParseException;
45import org.openstreetmap.josm.gui.progress.CLIProgressMonitor;
46import org.openstreetmap.josm.gui.progress.ProgressMonitor;
47import org.openstreetmap.josm.io.Compression;
48import org.openstreetmap.josm.io.GeoJSONMapRouletteWriter;
49import org.openstreetmap.josm.io.IllegalDataException;
50import org.openstreetmap.josm.io.OsmChangeReader;
51import org.openstreetmap.josm.spi.lifecycle.Lifecycle;
52import org.openstreetmap.josm.spi.preferences.Config;
53import org.openstreetmap.josm.spi.preferences.IPreferences;
54import org.openstreetmap.josm.spi.preferences.MemoryPreferences;
55import org.openstreetmap.josm.tools.Http1Client;
56import org.openstreetmap.josm.tools.HttpClient;
57import org.openstreetmap.josm.tools.I18n;
58import org.openstreetmap.josm.tools.JosmRuntimeException;
59import org.openstreetmap.josm.tools.Logging;
60import org.openstreetmap.josm.tools.OptionParser;
61import org.openstreetmap.josm.tools.Stopwatch;
62import org.openstreetmap.josm.tools.Territories;
63import org.openstreetmap.josm.tools.Utils;
64
65/**
66 * Add a validate command to the JOSM command line interface.
67 * @author Taylor Smock
68 * @since 18365
69 */
70public class ValidatorCLI implements CLIModule {
71 /**
72 * The unique instance.
73 */
74 public static final ValidatorCLI INSTANCE = new ValidatorCLI();
75
76 /** The input file(s) */
77 private final List<String> input = new ArrayList<>();
78 /** The change files. input file -> list of change files */
79 private final Map<String, List<String>> changeFiles = new HashMap<>();
80 /** The output file(s). If {@code null}, use input filename as base (replace extension with geojson). input -> output */
81 private final Map<String, String> output = new HashMap<>();
82
83 private static final Supplier<ProgressMonitor> progressMonitorFactory = CLIProgressMonitor::new;
84
85 /** The log level */
86 private Level logLevel;
87
88 private enum Option {
89 /** --help Show the help for validate */
90 HELP(false, 'h'),
91 /** --input=&lt;input-file&gt; Set the current input file */
92 INPUT(true, 'i', OptionParser.OptionCount.MULTIPLE),
93 /** --output=&lt;output-file&gt; Set the output file for the current input file */
94 OUTPUT(true, 'o', OptionParser.OptionCount.MULTIPLE),
95 /** --change-file=&lt;change-file&gt; Add a change file */
96 CHANGE_FILE(true, 'c', OptionParser.OptionCount.MULTIPLE),
97 /** --debug Set logging level to debug */
98 DEBUG(false, '*'),
99 /** --trace Set logging level to trace */
100 TRACE(false, '*'),
101 /** --language=&lt;language&gt; Set the language */
102 LANGUAGE(true, 'l'),
103 /** --load-preferences=&lt;url-to-xml&gt; Changes preferences according to the XML file */
104 LOAD_PREFERENCES(true, 'p'),
105 /** --set=&lt;key&gt;=&lt;value&gt; Set preference key to value */
106 SET(true, 's');
107
108 private final String name;
109 private final boolean requiresArgument;
110 private final char shortOption;
111 private final OptionParser.OptionCount optionCount;
112 Option(final boolean requiresArgument, final char shortOption) {
113 this(requiresArgument, shortOption, OptionParser.OptionCount.OPTIONAL);
114 }
115
116 Option(final boolean requiresArgument, final char shortOption, final OptionParser.OptionCount optionCount) {
117 this.name = name().toLowerCase(Locale.ROOT).replace('_', '-');
118 this.requiresArgument = requiresArgument;
119 this.shortOption = shortOption;
120 this.optionCount = optionCount;
121 }
122
123 /**
124 * Replies the option name
125 * @return The option name, in lowercase
126 */
127 public String getName() {
128 return this.name;
129 }
130
131 /**
132 * Get the number of times this option should be seen
133 * @return The option count
134 */
135 public OptionParser.OptionCount getOptionCount() {
136 return this.optionCount;
137 }
138
139 /**
140 * Replies the short option (single letter) associated with this option.
141 * @return the short option or '*' if there is no short option
142 */
143 public char getShortOption() {
144 return this.shortOption;
145 }
146
147 /**
148 * Determines if this option requires an argument.
149 * @return {@code true} if this option requires an argument, {@code false} otherwise
150 */
151 public boolean requiresArgument() {
152 return this.requiresArgument;
153 }
154
155 }
156
157 @Override
158 public String getActionKeyword() {
159 return "validate";
160 }
161
162 @Override
163 public void processArguments(final String[] argArray) {
164 try {
165 // Ensure that preferences are only in memory
166 Config.setPreferencesInstance(new MemoryPreferences());
167 Logging.setLogLevel(Level.INFO);
168 this.parseArguments(argArray);
169 if (this.input.isEmpty()) {
170 throw new IllegalArgumentException(tr("Missing argument - input data file ({0})", "--input|-i"));
171 }
172 this.initialize();
173 final ProgressMonitor fileMonitor = progressMonitorFactory.get();
174 fileMonitor.beginTask(tr("Processing files..."), this.input.size());
175 for (String inputFile : this.input) {
176 if (inputFile.endsWith(".validator.mapcss")) {
177 this.processValidatorFile(inputFile);
178 } else if (inputFile.endsWith(".mapcss")) {
179 this.processMapcssFile(inputFile);
180 } else {
181 this.processFile(inputFile);
182 }
183 fileMonitor.worked(1);
184 }
185 fileMonitor.finishTask();
186 } catch (Exception e) {
187 Logging.info(e);
188 Lifecycle.exitJosm(true, 1);
189 }
190 Lifecycle.exitJosm(true, 0);
191 }
192
193 /**
194 * Process a standard mapcss file
195 * @param inputFile The mapcss file to validate
196 * @throws ParseException if the file does not match the mapcss syntax
197 */
198 private void processMapcssFile(final String inputFile) throws ParseException {
199 final MapCSSStyleSource styleSource = new MapCSSStyleSource(new File(inputFile).toURI().getPath(), inputFile, inputFile);
200 styleSource.loadStyleSource();
201 if (!styleSource.getErrors().isEmpty()) {
202 throw new ParseException(trn("{0} had {1} error", "{0} had {1} errors", styleSource.getErrors().size(),
203 inputFile, styleSource.getErrors().size()));
204 } else {
205 Logging.info(tr("{0} had no errors", inputFile));
206 }
207 }
208
209 /**
210 * Process a validator file
211 * @param inputFile The file to check
212 * @throws IOException if there is a problem reading the file
213 * @throws ParseException if the file does not match the validator mapcss syntax
214 */
215 private void processValidatorFile(final String inputFile) throws ParseException, IOException {
216 // Check asserts
217 Config.getPref().putBoolean("validator.check_assert_local_rules", true);
218 final MapCSSTagChecker mapCSSTagChecker = new MapCSSTagChecker();
219 final Collection<String> assertionErrors = new ArrayList<>();
220 final MapCSSTagChecker.ParseResult result = mapCSSTagChecker.addMapCSS(new File(inputFile).toURI().getPath(),
221 assertionErrors::add);
222 if (!result.parseErrors.isEmpty() || !assertionErrors.isEmpty()) {
223 for (Throwable throwable : result.parseErrors) {
224 Logging.error(throwable);
225 }
226 for (String error : assertionErrors) {
227 Logging.error(error);
228 }
229 throw new ParseException(trn("{0} had {1} error", "{0} had {1} errors", result.parseErrors.size() + assertionErrors.size(),
230 inputFile, result.parseErrors.size() + assertionErrors.size()));
231 } else {
232 Logging.info(tr("{0} had no errors"), inputFile);
233 }
234 }
235
236 /**
237 * Process an OSM file
238 * @param inputFile The input filename
239 * @throws IllegalArgumentException If an argument is not valid
240 * @throws IllegalDataException If there is bad data
241 * @throws IOException If a file could not be read or written
242 */
243 private void processFile(final String inputFile) throws IllegalDataException, IOException {
244 final File inputFileFile = new File(inputFile);
245 final List<FileImporter> inputFileImporters = ExtensionFileFilter.getImporters().stream()
246 .filter(importer -> importer.acceptFile(inputFileFile)).collect(Collectors.toList());
247 final Stopwatch stopwatch = Stopwatch.createStarted();
248 if (inputFileImporters.stream().noneMatch(fileImporter ->
249 fileImporter.importDataHandleExceptions(inputFileFile, progressMonitorFactory.get()))) {
250 throw new IOException(tr("Could not load input file: {0}", inputFile));
251 }
252 final String outputFile = Optional.ofNullable(this.output.get(inputFile)).orElseGet(() -> getDefaultOutputName(inputFile));
253 final String task = tr("Validating {0}, saving output to {1}", inputFile, outputFile);
254 OsmDataLayer dataLayer = null;
255 try {
256 Logging.info(task);
257 OsmValidator.initializeTests();
258 dataLayer = MainApplication.getLayerManager().getLayersOfType(OsmDataLayer.class)
259 .stream().filter(layer -> inputFileFile.equals(layer.getAssociatedFile()))
260 .findFirst().orElseThrow(() -> new JosmRuntimeException(tr("Could not find a layer for {0}", inputFile)));
261 final DataSet dataSet = dataLayer.getDataSet();
262 if (this.changeFiles.containsKey(inputFile)) {
263 ProgressMonitor changeFilesMonitor = progressMonitorFactory.get();
264 for (String changeFile : this.changeFiles.getOrDefault(inputFile, Collections.emptyList())) {
265 try (InputStream changeStream = Compression.getUncompressedFileInputStream(Paths.get(changeFile))) {
266 dataSet.mergeFrom(OsmChangeReader.parseDataSet(changeStream, changeFilesMonitor));
267 }
268 }
269 }
270 Collection<Test> tests = OsmValidator.getEnabledTests(false);
271 if (Files.isRegularFile(Paths.get(outputFile)) && !Files.deleteIfExists(Paths.get(outputFile))) {
272 Logging.error("Could not delete {0}, attempting to append", outputFile);
273 }
274 GeoJSONMapRouletteWriter geoJSONMapRouletteWriter = new GeoJSONMapRouletteWriter(dataSet);
275 try (OutputStream fileOutputStream = Files.newOutputStream(Paths.get(outputFile))) {
276 tests.parallelStream().forEach(test -> runTest(test, geoJSONMapRouletteWriter, fileOutputStream, dataSet));
277 }
278 } finally {
279 if (dataLayer != null) {
280 MainApplication.getLayerManager().removeLayer(dataLayer);
281 }
282 Logging.info(stopwatch.toString(task));
283 }
284 }
285
286 /**
287 * Get the default output name
288 * @param inputString The input file
289 * @return The default output name for the input file (extension stripped, ".geojson" added)
290 */
291 private static String getDefaultOutputName(final String inputString) {
292 final String extension = FileNameUtils.getExtension(inputString);
293 if (!Arrays.asList("zip", "bz", "xz", "geojson").contains(extension)) {
294 return FileNameUtils.getBaseName(inputString) + ".geojson";
295 } else if ("geojson".equals(extension)) {
296 // Account for geojson input files
297 return FileNameUtils.getBaseName(inputString) + ".validated.geojson";
298 }
299 return FileNameUtils.getBaseName(FileNameUtils.getBaseName(inputString)) + ".geojson";
300 }
301
302 /**
303 * Run a test
304 * @param test The test to run
305 * @param geoJSONMapRouletteWriter The object to use to create challenges
306 * @param fileOutputStream The location to write data to
307 * @param dataSet The dataset to check
308 */
309 private void runTest(final Test test, final GeoJSONMapRouletteWriter geoJSONMapRouletteWriter,
310 final OutputStream fileOutputStream, DataSet dataSet) {
311 test.startTest(progressMonitorFactory.get());
312 test.visit(dataSet.allPrimitives());
313 test.endTest();
314 test.getErrors().stream().map(geoJSONMapRouletteWriter::write)
315 .filter(Optional::isPresent).map(Optional::get)
316 .map(jsonObject -> jsonObject.toString().getBytes(StandardCharsets.UTF_8)).forEach(bytes -> {
317 try {
318 writeToFile(fileOutputStream, bytes);
319 } catch (IOException e) {
320 throw new JosmRuntimeException(e);
321 }
322 });
323 test.clear();
324 }
325
326 /**
327 * Write to a file. Synchronized to avoid writing to the same file in different threads.
328 *
329 * @param fileOutputStream The file output stream to read
330 * @param bytes The bytes to write (surrounded by RS and LF)
331 * @throws IOException If we couldn't write to file
332 */
333 private synchronized void writeToFile(final OutputStream fileOutputStream, final byte[] bytes)
334 throws IOException {
335 // Write the ASCII Record Separator character
336 fileOutputStream.write(0x1e);
337 fileOutputStream.write(bytes);
338 // Write the ASCII Line Feed character
339 fileOutputStream.write(0x0a);
340 }
341
342 /**
343 * Initialize everything that might be needed
344 *
345 * Arguments may need to be parsed first.
346 */
347 void initialize() {
348 Logging.setLogLevel(this.logLevel);
349 HttpClient.setFactory(Http1Client::new);
350 Config.setBaseDirectoriesProvider(JosmBaseDirectories.getInstance()); // for right-left-hand traffic cache file
351 Config.setUrlsProvider(JosmUrls.getInstance());
352 ProjectionRegistry.setProjection(Projections.getProjectionByCode("epsg:3857".toUpperCase(Locale.ROOT)));
353
354 Territories.initializeInternalData();
355 OsmValidator.initialize();
356 MapPaintStyles.readFromPreferences();
357 }
358
359 /**
360 * Parse command line arguments and do some low-level error checking.
361 * @param argArray the arguments array
362 */
363 void parseArguments(String[] argArray) {
364 Logging.setLogLevel(Level.INFO);
365
366 OptionParser parser = new OptionParser("JOSM validate");
367 final AtomicReference<String> currentInput = new AtomicReference<>(null);
368 for (Option o : Option.values()) {
369 if (o.requiresArgument()) {
370 parser.addArgumentParameter(o.getName(),
371 o.getOptionCount(),
372 arg -> handleOption(currentInput.get(), o, arg).ifPresent(currentInput::set));
373 } else {
374 parser.addFlagParameter(o.getName(), () -> handleOption(o));
375 }
376 if (o.getShortOption() != '*') {
377 parser.addShortAlias(o.getName(), Character.toString(o.getShortOption()));
378 }
379 }
380 parser.parseOptionsOrExit(Arrays.asList(argArray));
381 }
382
383 private void handleOption(final Option option) {
384 switch (option) {
385 case HELP:
386 showHelp();
387 Lifecycle.exitJosm(true, 0);
388 break;
389 case DEBUG:
390 this.logLevel = Logging.LEVEL_DEBUG;
391 break;
392 case TRACE:
393 this.logLevel = Logging.LEVEL_TRACE;
394 break;
395 default:
396 throw new AssertionError("Unexpected option: " + option);
397 }
398 }
399
400 /**
401 * Handle an option
402 * @param currentInput The current input file, if any. May be {@code null}.
403 * @param option The option to parse
404 * @param argument The argument for the option
405 * @return The new input file, if any.
406 */
407 private Optional<String> handleOption(final String currentInput, final Option option, final String argument) {
408 switch (option) {
409 case INPUT:
410 this.input.add(argument);
411 return Optional.of(argument);
412 case OUTPUT:
413 this.output.put(currentInput, argument);
414 break;
415 case CHANGE_FILE:
416 this.changeFiles.computeIfAbsent(currentInput, key -> new ArrayList<>()).add(argument);
417 break;
418 case LANGUAGE:
419 I18n.set(argument);
420 break;
421 case LOAD_PREFERENCES:
422 final Preferences tempPreferences = new Preferences();
423 tempPreferences.enableSaveOnPut(false);
424 CustomConfigurator.XMLCommandProcessor config = new CustomConfigurator.XMLCommandProcessor(tempPreferences);
425 try (InputStream is = Utils.openStream(new File(argument).toURI().toURL())) {
426 config.openAndReadXML(is);
427 } catch (IOException e) {
428 throw new JosmRuntimeException(e);
429 }
430 final IPreferences pref = Config.getPref();
431 if (pref instanceof MemoryPreferences) {
432 final MemoryPreferences memoryPreferences = (MemoryPreferences) pref;
433 tempPreferences.getAllSettings().entrySet().stream().filter(entry -> entry.getValue().isNew())
434 .forEach(entry -> memoryPreferences.putSetting(entry.getKey(), entry.getValue()));
435 } else {
436 throw new JosmRuntimeException(tr("Preferences are not the expected type"));
437 }
438 break;
439 case SET:
440
441 default:
442 throw new AssertionError("Unexpected option: " + option);
443 }
444 return Optional.empty();
445 }
446
447 private static void showHelp() {
448 System.out.println(getHelp());
449 }
450
451 private static String getHelp() {
452 final String helpPadding = "\t ";
453 // CHECKSTYLE.OFF: SingleSpaceSeparator
454 return tr("JOSM Validation command line interface") + "\n\n" +
455 tr("Usage") + ":\n" +
456 "\tjava -jar josm.jar validate <options>\n\n" +
457 tr("Description") + ":\n" +
458 tr("Validates data and saves the result to a file.") + "\n\n"+
459 tr("Options") + ":\n" +
460 "\t--help|-h " + tr("Show this help") + "\n" +
461 "\t--input|-i <file> " + tr("Input data file name (.osm, .validator.mapcss, .mapcss).") + '\n' +
462 helpPadding + tr("OSM files can be specified multiple times. Required.") + '\n' +
463 helpPadding + tr(".validator.mapcss and .mapcss files will stop processing on first error.") + '\n' +
464 helpPadding + tr("Non-osm files do not use --output or --change-file") + '\n' +
465 "\t--output|-o <file> " + tr("Output data file name (.geojson, line-by-line delimited for MapRoulette). Optional.")
466 + '\n' +
467 "\t--change-file|-c <file> " + tr("Change file name (.osc). Can be specified multiple times per input.") + '\n' +
468 helpPadding + tr("Changes will be applied in the specified order. Optional.");
469 // CHECKSTYLE.ON: SingleSpaceSeparator
470 }
471}
Note: See TracBrowser for help on using the repository browser.