source: josm/trunk/src/org/openstreetmap/josm/data/validation/OsmValidator.java@ 14828

Last change on this file since 14828 was 14828, checked in by GerdP, 5 years ago

fix #17268: There should be a method to clear ignored errors

patch clear_ignored_errors_v32.patch by taylor.smock adapted to r14827
  • Property svn:eol-style set to native
File size: 22.9 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.validation;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.awt.GraphicsEnvironment;
7import java.io.File;
8import java.io.FileNotFoundException;
9import java.io.IOException;
10import java.nio.charset.StandardCharsets;
11import java.nio.file.Files;
12import java.nio.file.Path;
13import java.nio.file.Paths;
14import java.util.ArrayList;
15import java.util.Arrays;
16import java.util.Collection;
17import java.util.Collections;
18import java.util.EnumMap;
19import java.util.Enumeration;
20import java.util.HashMap;
21import java.util.Iterator;
22import java.util.List;
23import java.util.Map;
24import java.util.Map.Entry;
25import java.util.SortedMap;
26import java.util.TreeMap;
27import java.util.TreeSet;
28import java.util.function.Predicate;
29import java.util.stream.Collectors;
30
31import javax.swing.JOptionPane;
32import javax.swing.JTree;
33import javax.swing.tree.DefaultMutableTreeNode;
34import javax.swing.tree.TreeModel;
35import javax.swing.tree.TreeNode;
36
37import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper;
38import org.openstreetmap.josm.data.projection.ProjectionRegistry;
39import org.openstreetmap.josm.data.validation.tests.Addresses;
40import org.openstreetmap.josm.data.validation.tests.ApiCapabilitiesTest;
41import org.openstreetmap.josm.data.validation.tests.BarriersEntrances;
42import org.openstreetmap.josm.data.validation.tests.Coastlines;
43import org.openstreetmap.josm.data.validation.tests.ConditionalKeys;
44import org.openstreetmap.josm.data.validation.tests.CrossingWays;
45import org.openstreetmap.josm.data.validation.tests.DuplicateNode;
46import org.openstreetmap.josm.data.validation.tests.DuplicateRelation;
47import org.openstreetmap.josm.data.validation.tests.DuplicateWay;
48import org.openstreetmap.josm.data.validation.tests.DuplicatedWayNodes;
49import org.openstreetmap.josm.data.validation.tests.Highways;
50import org.openstreetmap.josm.data.validation.tests.InternetTags;
51import org.openstreetmap.josm.data.validation.tests.Lanes;
52import org.openstreetmap.josm.data.validation.tests.LongSegment;
53import org.openstreetmap.josm.data.validation.tests.MapCSSTagChecker;
54import org.openstreetmap.josm.data.validation.tests.MultipolygonTest;
55import org.openstreetmap.josm.data.validation.tests.NameMismatch;
56import org.openstreetmap.josm.data.validation.tests.OpeningHourTest;
57import org.openstreetmap.josm.data.validation.tests.OverlappingWays;
58import org.openstreetmap.josm.data.validation.tests.PowerLines;
59import org.openstreetmap.josm.data.validation.tests.PublicTransportRouteTest;
60import org.openstreetmap.josm.data.validation.tests.RelationChecker;
61import org.openstreetmap.josm.data.validation.tests.RightAngleBuildingTest;
62import org.openstreetmap.josm.data.validation.tests.SelfIntersectingWay;
63import org.openstreetmap.josm.data.validation.tests.SimilarNamedWays;
64import org.openstreetmap.josm.data.validation.tests.TagChecker;
65import org.openstreetmap.josm.data.validation.tests.TurnrestrictionTest;
66import org.openstreetmap.josm.data.validation.tests.UnclosedWays;
67import org.openstreetmap.josm.data.validation.tests.UnconnectedWays;
68import org.openstreetmap.josm.data.validation.tests.UntaggedNode;
69import org.openstreetmap.josm.data.validation.tests.UntaggedWay;
70import org.openstreetmap.josm.data.validation.tests.WayConnectedToArea;
71import org.openstreetmap.josm.data.validation.tests.WronglyOrderedWays;
72import org.openstreetmap.josm.gui.MainApplication;
73import org.openstreetmap.josm.gui.layer.ValidatorLayer;
74import org.openstreetmap.josm.gui.preferences.projection.ProjectionPreference;
75import org.openstreetmap.josm.gui.util.GuiHelper;
76import org.openstreetmap.josm.spi.preferences.Config;
77import org.openstreetmap.josm.tools.AlphanumComparator;
78import org.openstreetmap.josm.tools.Logging;
79import org.openstreetmap.josm.tools.Utils;
80
81/**
82 * A OSM data validator.
83 *
84 * @author Francisco R. Santos <frsantos@gmail.com>
85 */
86public final class OsmValidator {
87
88 private OsmValidator() {
89 // Hide default constructor for utilities classes
90 }
91
92 private static volatile ValidatorLayer errorLayer;
93
94 /** Grid detail, multiplier of east,north values for valuable cell sizing */
95 private static double griddetail;
96
97 private static final SortedMap<String, String> ignoredErrors = new TreeMap<>();
98 /**
99 * All registered tests
100 */
101 private static final Collection<Class<? extends Test>> allTests = new ArrayList<>();
102 private static final Map<String, Test> allTestsMap = new HashMap<>();
103
104 /**
105 * All available tests in core
106 */
107 @SuppressWarnings("unchecked")
108 private static final Class<Test>[] CORE_TEST_CLASSES = new Class[] {
109 /* FIXME - unique error numbers for tests aren't properly unique - ignoring will not work as expected */
110 DuplicateNode.class, // ID 1 .. 99
111 OverlappingWays.class, // ID 101 .. 199
112 UntaggedNode.class, // ID 201 .. 299
113 UntaggedWay.class, // ID 301 .. 399
114 SelfIntersectingWay.class, // ID 401 .. 499
115 DuplicatedWayNodes.class, // ID 501 .. 599
116 CrossingWays.Ways.class, // ID 601 .. 699
117 CrossingWays.Boundaries.class, // ID 601 .. 699
118 CrossingWays.Barrier.class, // ID 601 .. 699
119 CrossingWays.SelfCrossing.class, // ID 601 .. 699
120 SimilarNamedWays.class, // ID 701 .. 799
121 Coastlines.class, // ID 901 .. 999
122 WronglyOrderedWays.class, // ID 1001 .. 1099
123 UnclosedWays.class, // ID 1101 .. 1199
124 TagChecker.class, // ID 1201 .. 1299
125 UnconnectedWays.UnconnectedHighways.class, // ID 1301 .. 1399
126 UnconnectedWays.UnconnectedRailways.class, // ID 1301 .. 1399
127 UnconnectedWays.UnconnectedWaterways.class, // ID 1301 .. 1399
128 UnconnectedWays.UnconnectedNaturalOrLanduse.class, // ID 1301 .. 1399
129 UnconnectedWays.UnconnectedPower.class, // ID 1301 .. 1399
130 DuplicateWay.class, // ID 1401 .. 1499
131 NameMismatch.class, // ID 1501 .. 1599
132 MultipolygonTest.class, // ID 1601 .. 1699
133 RelationChecker.class, // ID 1701 .. 1799
134 TurnrestrictionTest.class, // ID 1801 .. 1899
135 DuplicateRelation.class, // ID 1901 .. 1999
136 WayConnectedToArea.class, // ID 2301 .. 2399
137 PowerLines.class, // ID 2501 .. 2599
138 Addresses.class, // ID 2601 .. 2699
139 Highways.class, // ID 2701 .. 2799
140 BarriersEntrances.class, // ID 2801 .. 2899
141 OpeningHourTest.class, // 2901 .. 2999
142 MapCSSTagChecker.class, // 3000 .. 3099
143 Lanes.class, // 3100 .. 3199
144 ConditionalKeys.class, // 3200 .. 3299
145 InternetTags.class, // 3300 .. 3399
146 ApiCapabilitiesTest.class, // 3400 .. 3499
147 LongSegment.class, // 3500 .. 3599
148 PublicTransportRouteTest.class, // 3600 .. 3699
149 RightAngleBuildingTest.class, // 3700 .. 3799
150 };
151
152 /**
153 * Adds a test to the list of available tests
154 * @param testClass The test class
155 */
156 public static void addTest(Class<? extends Test> testClass) {
157 allTests.add(testClass);
158 try {
159 allTestsMap.put(testClass.getName(), testClass.getConstructor().newInstance());
160 } catch (ReflectiveOperationException e) {
161 Logging.error(e);
162 }
163 }
164
165 static {
166 for (Class<? extends Test> testClass : CORE_TEST_CLASSES) {
167 addTest(testClass);
168 }
169 }
170
171 /**
172 * Initializes {@code OsmValidator}.
173 */
174 public static void initialize() {
175 checkValidatorDir();
176 initializeGridDetail();
177 loadIgnoredErrors();
178 }
179
180 /**
181 * Returns the validator directory.
182 *
183 * @return The validator directory
184 */
185 public static String getValidatorDir() {
186 File dir = new File(Config.getDirs().getUserDataDirectory(true), "validator");
187 try {
188 return dir.getAbsolutePath();
189 } catch (SecurityException e) {
190 Logging.log(Logging.LEVEL_ERROR, null, e);
191 return dir.getPath();
192 }
193 }
194
195 /**
196 * Check if validator directory exists (store ignored errors file)
197 */
198 private static void checkValidatorDir() {
199 File pathDir = new File(getValidatorDir());
200 try {
201 if (!pathDir.exists()) {
202 Utils.mkDirs(pathDir);
203 }
204 } catch (SecurityException e) {
205 Logging.log(Logging.LEVEL_ERROR, "Unable to check validator directory", e);
206 }
207 }
208
209 private static void loadIgnoredErrors() {
210 ignoredErrors.clear();
211 if (ValidatorPrefHelper.PREF_USE_IGNORE.get()) {
212 Config.getPref().getListOfMaps(ValidatorPrefHelper.PREF_IGNORELIST).forEach(ignoredErrors::putAll);
213 Path path = Paths.get(getValidatorDir()).resolve("ignorederrors");
214 try {
215 if (path.toFile().exists()) {
216 try {
217 TreeSet<String> treeSet = new TreeSet<>();
218 treeSet.addAll(Files.readAllLines(path, StandardCharsets.UTF_8));
219 treeSet.forEach(ignore -> ignoredErrors.putIfAbsent(ignore, ""));
220
221 saveIgnoredErrors();
222 Files.deleteIfExists(path);
223
224 } catch (FileNotFoundException e) {
225 Logging.debug(Logging.getErrorMessage(e));
226 } catch (IOException e) {
227 Logging.error(e);
228 }
229 }
230 } catch (SecurityException e) {
231 Logging.log(Logging.LEVEL_ERROR, "Unable to load ignored errors", e);
232 }
233 }
234 }
235
236 /**
237 * Adds an ignored error
238 * @param s The ignore group / sub group name
239 * @see TestError#getIgnoreGroup()
240 * @see TestError#getIgnoreSubGroup()
241 */
242 public static void addIgnoredError(String s) {
243 addIgnoredError(s, "");
244 }
245
246 /**
247 * Adds an ignored error
248 * @param s The ignore group / sub group name
249 * @param description What the error actually is
250 * @see TestError#getIgnoreGroup()
251 * @see TestError#getIgnoreSubGroup()
252 */
253 public static void addIgnoredError(String s, String description) {
254 if (description == null) description = "";
255 ignoredErrors.put(s, description);
256 }
257
258 /**
259 * Make sure that we don't keep single entries for a "group ignore" or
260 * multiple different entries for the single entries that are in the same group.
261 */
262 private static void cleanupIgnoredErrors() {
263 if (ignoredErrors.size() > 1) {
264 List<String> toRemove = new ArrayList<>();
265
266 Iterator<Entry<String, String>> iter = ignoredErrors.entrySet().iterator();
267 Entry<String, String> last = iter.next();
268 while (iter.hasNext()) {
269 Entry<String, String> entry = iter.next();
270 if (entry.getKey().startsWith(last.getKey())) {
271 toRemove.add(entry.getKey());
272 } else {
273 last = entry;
274 }
275 }
276 toRemove.forEach(ignoredErrors::remove);
277 }
278
279 Map<String, String> tmap = buildIgnore(buildJTreeList());
280 if (tmap != null && !tmap.isEmpty()) {
281 ignoredErrors.clear();
282 ignoredErrors.putAll(tmap);
283 }
284 }
285
286 /**
287 * Check if a error should be ignored
288 * @param s The ignore group / sub group name
289 * @return <code>true</code> to ignore that error
290 */
291 public static boolean hasIgnoredError(String s) {
292 return ignoredErrors.containsKey(s);
293 }
294
295 /**
296 * Get the list of all ignored errors
297 * @return The <code>Collection&ltString&gt</code> of errors that are ignored
298 */
299 public static SortedMap<String, String> getIgnoredErrors() {
300 return ignoredErrors;
301 }
302
303 /**
304 * Build a JTree with a list
305 * @return &lttype&gtlist as a {@code JTree}
306 */
307 public static JTree buildJTreeList() {
308 DefaultMutableTreeNode root = new DefaultMutableTreeNode(tr("Ignore list"));
309 for (Entry<String, String> e: ignoredErrors.entrySet()) {
310 String key = e.getKey();
311 String value = e.getValue();
312 ArrayList<String> ignoredWayList = new ArrayList<>();
313 String[] osmobjects = key.split(":(r|w|n)_");
314 for (int i = 1; i < osmobjects.length; i++) {
315 String osmid = osmobjects[i];
316 if (osmid.matches("^[0-9]+$")) {
317 osmid = '_' + osmid;
318 int index = key.indexOf(osmid);
319 if (index < key.lastIndexOf(']')) continue;
320 char type = key.charAt(index - 1);
321 ignoredWayList.add(type + osmid);
322 }
323 }
324 for (String osmignore : ignoredWayList) {
325 key = key.replace(':' + osmignore, "");
326 }
327
328 DefaultMutableTreeNode trunk;
329 DefaultMutableTreeNode branch;
330
331 if (value != null && !value.isEmpty()) {
332 trunk = inTree(root, value);
333 branch = inTree(trunk, key);
334 trunk.add(branch);
335 } else {
336 trunk = inTree(root, key);
337 branch = trunk;
338 }
339 ignoredWayList.forEach(osmignore -> branch.add(new DefaultMutableTreeNode(osmignore)));
340
341 root.add(trunk);
342 }
343 return new JTree(root);
344 }
345
346 private static DefaultMutableTreeNode inTree(DefaultMutableTreeNode root, String name) {
347 @SuppressWarnings("unchecked")
348 Enumeration<TreeNode> trunks = root.children();
349 while (trunks.hasMoreElements()) {
350 TreeNode ttrunk = trunks.nextElement();
351 if (ttrunk instanceof DefaultMutableTreeNode) {
352 DefaultMutableTreeNode trunk = (DefaultMutableTreeNode) ttrunk;
353 if (name.equals(trunk.getUserObject())) {
354 return trunk;
355 }
356 }
357 }
358 return new DefaultMutableTreeNode(name);
359 }
360
361 /**
362 * Build a {@code HashMap} from a tree of ignored errors
363 * @param tree The JTree of ignored errors
364 * @return A {@code HashMap} of the ignored errors for comparison
365 */
366 public static Map<String, String> buildIgnore(JTree tree) {
367 TreeModel model = tree.getModel();
368 DefaultMutableTreeNode root = (DefaultMutableTreeNode) model.getRoot();
369 return buildIgnore(model, root);
370 }
371
372 private static Map<String, String> buildIgnore(TreeModel model, DefaultMutableTreeNode node) {
373 HashMap<String, String> rHashMap = new HashMap<>();
374
375 String osmids = node.getUserObject().toString();
376 String description = "";
377
378 if (!model.getRoot().equals(node)) {
379 description = ((DefaultMutableTreeNode) node.getParent()).getUserObject().toString();
380 } else {
381 description = node.getUserObject().toString();
382 }
383 if (tr("Ignore list").equals(description)) description = "";
384 if (!osmids.matches("^[0-9]+(_.*|$)")) {
385 description = osmids;
386 osmids = "";
387 }
388
389
390 for (int i = 0; i < model.getChildCount(node); i++) {
391 DefaultMutableTreeNode child = (DefaultMutableTreeNode) model.getChild(node, i);
392 if (model.getChildCount(child) == 0) {
393 String ignoreName = child.getUserObject().toString();
394 if (ignoreName.matches("^(r|w|n)_.*")) {
395 osmids += ":" + child.getUserObject().toString();
396 } else if (ignoreName.matches("^[0-9]+(_.*|)$")) {
397 rHashMap.put(ignoreName, description);
398 }
399 } else {
400 rHashMap.putAll(buildIgnore(model, child));
401 }
402 }
403 if (!osmids.isEmpty() && osmids.indexOf(':') != 0) rHashMap.put(osmids, description);
404 return rHashMap;
405 }
406
407 /**
408 * Reset the error list by deleting {@code validator.ignorelist}
409 */
410 public static void resetErrorList() {
411 saveIgnoredErrors();
412 Config.getPref().putListOfMaps(ValidatorPrefHelper.PREF_IGNORELIST, null);
413 OsmValidator.initialize();
414 }
415
416 /**
417 * Saves the names of the ignored errors to a preference
418 */
419 public static void saveIgnoredErrors() {
420 List<Map<String, String>> list = new ArrayList<>();
421 cleanupIgnoredErrors();
422 list.add(ignoredErrors);
423 int i = 0;
424 while (i < list.size()) {
425 if (list.get(i) == null || list.get(i).isEmpty()) {
426 list.remove(i);
427 continue;
428 }
429 i++;
430 }
431 if (list.isEmpty()) list = null;
432 Config.getPref().putListOfMaps(ValidatorPrefHelper.PREF_IGNORELIST, list);
433 }
434
435 /**
436 * Initializes error layer.
437 */
438 public static synchronized void initializeErrorLayer() {
439 if (!ValidatorPrefHelper.PREF_LAYER.get())
440 return;
441 if (errorLayer == null) {
442 errorLayer = new ValidatorLayer();
443 MainApplication.getLayerManager().addLayer(errorLayer);
444 }
445 }
446
447 /**
448 * Resets error layer.
449 * @since 11852
450 */
451 public static synchronized void resetErrorLayer() {
452 errorLayer = null;
453 }
454
455 /**
456 * Gets a map from simple names to all tests.
457 * @return A map of all tests, indexed and sorted by the name of their Java class
458 */
459 public static SortedMap<String, Test> getAllTestsMap() {
460 applyPrefs(allTestsMap, false);
461 applyPrefs(allTestsMap, true);
462 return new TreeMap<>(allTestsMap);
463 }
464
465 /**
466 * Returns the instance of the given test class.
467 * @param <T> testClass type
468 * @param testClass The class of test to retrieve
469 * @return the instance of the given test class, if any, or {@code null}
470 * @since 6670
471 */
472 @SuppressWarnings("unchecked")
473 public static <T extends Test> T getTest(Class<T> testClass) {
474 if (testClass == null) {
475 return null;
476 }
477 return (T) allTestsMap.get(testClass.getName());
478 }
479
480 private static void applyPrefs(Map<String, Test> tests, boolean beforeUpload) {
481 for (String testName : Config.getPref().getList(beforeUpload
482 ? ValidatorPrefHelper.PREF_SKIP_TESTS_BEFORE_UPLOAD : ValidatorPrefHelper.PREF_SKIP_TESTS)) {
483 Test test = tests.get(testName);
484 if (test != null) {
485 if (beforeUpload) {
486 test.testBeforeUpload = false;
487 } else {
488 test.enabled = false;
489 }
490 }
491 }
492 }
493
494 /**
495 * Gets all tests that are possible
496 * @return The tests
497 */
498 public static Collection<Test> getTests() {
499 return getAllTestsMap().values();
500 }
501
502 /**
503 * Gets all tests that are run
504 * @param beforeUpload To get the ones that are run before upload
505 * @return The tests
506 */
507 public static Collection<Test> getEnabledTests(boolean beforeUpload) {
508 Collection<Test> enabledTests = getTests();
509 for (Test t : new ArrayList<>(enabledTests)) {
510 if (beforeUpload ? t.testBeforeUpload : t.enabled) {
511 continue;
512 }
513 enabledTests.remove(t);
514 }
515 return enabledTests;
516 }
517
518 /**
519 * Gets the list of all available test classes
520 *
521 * @return A collection of the test classes
522 */
523 public static Collection<Class<? extends Test>> getAllAvailableTestClasses() {
524 return Collections.unmodifiableCollection(allTests);
525 }
526
527 /**
528 * Initialize grid details based on current projection system. Values based on
529 * the original value fixed for EPSG:4326 (10000) using heuristics (that is, test&amp;error
530 * until most bugs were discovered while keeping the processing time reasonable)
531 */
532 public static void initializeGridDetail() {
533 String code = ProjectionRegistry.getProjection().toCode();
534 if (Arrays.asList(ProjectionPreference.wgs84.allCodes()).contains(code)) {
535 OsmValidator.griddetail = 10_000;
536 } else if (Arrays.asList(ProjectionPreference.mercator.allCodes()).contains(code)) {
537 OsmValidator.griddetail = 0.01;
538 } else if (Arrays.asList(ProjectionPreference.lambert.allCodes()).contains(code)) {
539 OsmValidator.griddetail = 0.1;
540 } else {
541 OsmValidator.griddetail = 1.0;
542 }
543 }
544
545 /**
546 * Returns grid detail, multiplier of east,north values for valuable cell sizing
547 * @return grid detail
548 * @since 11852
549 */
550 public static double getGridDetail() {
551 return griddetail;
552 }
553
554 private static boolean testsInitialized;
555
556 /**
557 * Initializes all tests if this operations hasn't been performed already.
558 */
559 public static synchronized void initializeTests() {
560 if (!testsInitialized) {
561 Logging.debug("Initializing validator tests");
562 final long startTime = System.currentTimeMillis();
563 initializeTests(getTests());
564 testsInitialized = true;
565 if (Logging.isDebugEnabled()) {
566 final long elapsedTime = System.currentTimeMillis() - startTime;
567 Logging.debug("Initializing validator tests completed in {0}", Utils.getDurationString(elapsedTime));
568 }
569 }
570 }
571
572 /**
573 * Initializes all tests
574 * @param allTests The tests to initialize
575 */
576 public static void initializeTests(Collection<? extends Test> allTests) {
577 for (Test test : allTests) {
578 try {
579 if (test.enabled) {
580 test.initialize();
581 }
582 } catch (Exception e) { // NOPMD
583 String message = tr("Error initializing test {0}:\n {1}", test.getClass().getSimpleName(), e);
584 Logging.error(message);
585 if (!GraphicsEnvironment.isHeadless()) {
586 GuiHelper.runInEDT(() ->
587 JOptionPane.showMessageDialog(MainApplication.getMainFrame(), message, tr("Error"), JOptionPane.ERROR_MESSAGE)
588 );
589 }
590 }
591 }
592 }
593
594 /**
595 * Groups the given collection of errors by severity, then message, then description.
596 * @param errors list of errors to group
597 * @param filterToUse optional filter
598 * @return collection of errors grouped by severity, then message, then description
599 * @since 12667
600 */
601 public static Map<Severity, Map<String, Map<String, List<TestError>>>> getErrorsBySeverityMessageDescription(
602 Collection<TestError> errors, Predicate<? super TestError> filterToUse) {
603 return errors.stream().filter(filterToUse).collect(
604 Collectors.groupingBy(TestError::getSeverity, () -> new EnumMap<>(Severity.class),
605 Collectors.groupingBy(TestError::getMessage, () -> new TreeMap<>(AlphanumComparator.getInstance()),
606 Collectors.groupingBy(e -> e.getDescription() == null ? "" : e.getDescription(),
607 () -> new TreeMap<>(AlphanumComparator.getInstance()),
608 Collectors.toList()
609 ))));
610 }
611}
Note: See TracBrowser for help on using the repository browser.