source: josm/trunk/src/org/openstreetmap/josm/data/validation/OsmValidator.java@ 14852

Last change on this file since 14852 was 14852, checked in by GerdP, 5 years ago

see #17268: fix more javadoc issues
  • Property svn:eol-style set to native
File size: 23.1 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.validation;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.awt.GraphicsEnvironment;
7import java.io.File;
8import java.io.FileNotFoundException;
9import java.io.IOException;
10import java.nio.charset.StandardCharsets;
11import java.nio.file.Files;
12import java.nio.file.Path;
13import java.nio.file.Paths;
14import java.util.ArrayList;
15import java.util.Arrays;
16import java.util.Collection;
17import java.util.Collections;
18import java.util.EnumMap;
19import java.util.Enumeration;
20import java.util.HashMap;
21import java.util.Iterator;
22import java.util.List;
23import java.util.Map;
24import java.util.Map.Entry;
25import java.util.SortedMap;
26import java.util.TreeMap;
27import java.util.TreeSet;
28import java.util.function.Predicate;
29import java.util.regex.Pattern;
30import java.util.stream.Collectors;
31
32import javax.swing.JOptionPane;
33import javax.swing.JTree;
34import javax.swing.tree.DefaultMutableTreeNode;
35import javax.swing.tree.TreeModel;
36import javax.swing.tree.TreeNode;
37
38import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper;
39import org.openstreetmap.josm.data.projection.ProjectionRegistry;
40import org.openstreetmap.josm.data.validation.tests.Addresses;
41import org.openstreetmap.josm.data.validation.tests.ApiCapabilitiesTest;
42import org.openstreetmap.josm.data.validation.tests.BarriersEntrances;
43import org.openstreetmap.josm.data.validation.tests.Coastlines;
44import org.openstreetmap.josm.data.validation.tests.ConditionalKeys;
45import org.openstreetmap.josm.data.validation.tests.CrossingWays;
46import org.openstreetmap.josm.data.validation.tests.DuplicateNode;
47import org.openstreetmap.josm.data.validation.tests.DuplicateRelation;
48import org.openstreetmap.josm.data.validation.tests.DuplicateWay;
49import org.openstreetmap.josm.data.validation.tests.DuplicatedWayNodes;
50import org.openstreetmap.josm.data.validation.tests.Highways;
51import org.openstreetmap.josm.data.validation.tests.InternetTags;
52import org.openstreetmap.josm.data.validation.tests.Lanes;
53import org.openstreetmap.josm.data.validation.tests.LongSegment;
54import org.openstreetmap.josm.data.validation.tests.MapCSSTagChecker;
55import org.openstreetmap.josm.data.validation.tests.MultipolygonTest;
56import org.openstreetmap.josm.data.validation.tests.NameMismatch;
57import org.openstreetmap.josm.data.validation.tests.OpeningHourTest;
58import org.openstreetmap.josm.data.validation.tests.OverlappingWays;
59import org.openstreetmap.josm.data.validation.tests.PowerLines;
60import org.openstreetmap.josm.data.validation.tests.PublicTransportRouteTest;
61import org.openstreetmap.josm.data.validation.tests.RelationChecker;
62import org.openstreetmap.josm.data.validation.tests.RightAngleBuildingTest;
63import org.openstreetmap.josm.data.validation.tests.SelfIntersectingWay;
64import org.openstreetmap.josm.data.validation.tests.SimilarNamedWays;
65import org.openstreetmap.josm.data.validation.tests.TagChecker;
66import org.openstreetmap.josm.data.validation.tests.TurnrestrictionTest;
67import org.openstreetmap.josm.data.validation.tests.UnclosedWays;
68import org.openstreetmap.josm.data.validation.tests.UnconnectedWays;
69import org.openstreetmap.josm.data.validation.tests.UntaggedNode;
70import org.openstreetmap.josm.data.validation.tests.UntaggedWay;
71import org.openstreetmap.josm.data.validation.tests.WayConnectedToArea;
72import org.openstreetmap.josm.data.validation.tests.WronglyOrderedWays;
73import org.openstreetmap.josm.gui.MainApplication;
74import org.openstreetmap.josm.gui.layer.ValidatorLayer;
75import org.openstreetmap.josm.gui.preferences.projection.ProjectionPreference;
76import org.openstreetmap.josm.gui.util.GuiHelper;
77import org.openstreetmap.josm.spi.preferences.Config;
78import org.openstreetmap.josm.tools.AlphanumComparator;
79import org.openstreetmap.josm.tools.Logging;
80import org.openstreetmap.josm.tools.Utils;
81
82/**
83 * A OSM data validator.
84 *
85 * @author Francisco R. Santos <frsantos@gmail.com>
86 */
87public final class OsmValidator {
88
89 private OsmValidator() {
90 // Hide default constructor for utilities classes
91 }
92
93 private static volatile ValidatorLayer errorLayer;
94
95 /** Grid detail, multiplier of east,north values for valuable cell sizing */
96 private static double griddetail;
97
98 private static final SortedMap<String, String> ignoredErrors = new TreeMap<>();
99 /**
100 * All registered tests
101 */
102 private static final Collection<Class<? extends Test>> allTests = new ArrayList<>();
103 private static final Map<String, Test> allTestsMap = new HashMap<>();
104
105 /**
106 * All available tests in core
107 */
108 @SuppressWarnings("unchecked")
109 private static final Class<Test>[] CORE_TEST_CLASSES = new Class[] {
110 /* FIXME - unique error numbers for tests aren't properly unique - ignoring will not work as expected */
111 DuplicateNode.class, // ID 1 .. 99
112 OverlappingWays.class, // ID 101 .. 199
113 UntaggedNode.class, // ID 201 .. 299
114 UntaggedWay.class, // ID 301 .. 399
115 SelfIntersectingWay.class, // ID 401 .. 499
116 DuplicatedWayNodes.class, // ID 501 .. 599
117 CrossingWays.Ways.class, // ID 601 .. 699
118 CrossingWays.Boundaries.class, // ID 601 .. 699
119 CrossingWays.Barrier.class, // ID 601 .. 699
120 CrossingWays.SelfCrossing.class, // ID 601 .. 699
121 SimilarNamedWays.class, // ID 701 .. 799
122 Coastlines.class, // ID 901 .. 999
123 WronglyOrderedWays.class, // ID 1001 .. 1099
124 UnclosedWays.class, // ID 1101 .. 1199
125 TagChecker.class, // ID 1201 .. 1299
126 UnconnectedWays.UnconnectedHighways.class, // ID 1301 .. 1399
127 UnconnectedWays.UnconnectedRailways.class, // ID 1301 .. 1399
128 UnconnectedWays.UnconnectedWaterways.class, // ID 1301 .. 1399
129 UnconnectedWays.UnconnectedNaturalOrLanduse.class, // ID 1301 .. 1399
130 UnconnectedWays.UnconnectedPower.class, // ID 1301 .. 1399
131 DuplicateWay.class, // ID 1401 .. 1499
132 NameMismatch.class, // ID 1501 .. 1599
133 MultipolygonTest.class, // ID 1601 .. 1699
134 RelationChecker.class, // ID 1701 .. 1799
135 TurnrestrictionTest.class, // ID 1801 .. 1899
136 DuplicateRelation.class, // ID 1901 .. 1999
137 WayConnectedToArea.class, // ID 2301 .. 2399
138 PowerLines.class, // ID 2501 .. 2599
139 Addresses.class, // ID 2601 .. 2699
140 Highways.class, // ID 2701 .. 2799
141 BarriersEntrances.class, // ID 2801 .. 2899
142 OpeningHourTest.class, // 2901 .. 2999
143 MapCSSTagChecker.class, // 3000 .. 3099
144 Lanes.class, // 3100 .. 3199
145 ConditionalKeys.class, // 3200 .. 3299
146 InternetTags.class, // 3300 .. 3399
147 ApiCapabilitiesTest.class, // 3400 .. 3499
148 LongSegment.class, // 3500 .. 3599
149 PublicTransportRouteTest.class, // 3600 .. 3699
150 RightAngleBuildingTest.class, // 3700 .. 3799
151 };
152
153 /**
154 * Adds a test to the list of available tests
155 * @param testClass The test class
156 */
157 public static void addTest(Class<? extends Test> testClass) {
158 allTests.add(testClass);
159 try {
160 allTestsMap.put(testClass.getName(), testClass.getConstructor().newInstance());
161 } catch (ReflectiveOperationException e) {
162 Logging.error(e);
163 }
164 }
165
166 static {
167 for (Class<? extends Test> testClass : CORE_TEST_CLASSES) {
168 addTest(testClass);
169 }
170 }
171
172 /**
173 * Initializes {@code OsmValidator}.
174 */
175 public static void initialize() {
176 checkValidatorDir();
177 initializeGridDetail();
178 loadIgnoredErrors();
179 }
180
181 /**
182 * Returns the validator directory.
183 *
184 * @return The validator directory
185 */
186 public static String getValidatorDir() {
187 File dir = new File(Config.getDirs().getUserDataDirectory(true), "validator");
188 try {
189 return dir.getAbsolutePath();
190 } catch (SecurityException e) {
191 Logging.log(Logging.LEVEL_ERROR, null, e);
192 return dir.getPath();
193 }
194 }
195
196 /**
197 * Check if validator directory exists (store ignored errors file)
198 */
199 private static void checkValidatorDir() {
200 File pathDir = new File(getValidatorDir());
201 try {
202 if (!pathDir.exists()) {
203 Utils.mkDirs(pathDir);
204 }
205 } catch (SecurityException e) {
206 Logging.log(Logging.LEVEL_ERROR, "Unable to check validator directory", e);
207 }
208 }
209
210 private static void loadIgnoredErrors() {
211 ignoredErrors.clear();
212 if (ValidatorPrefHelper.PREF_USE_IGNORE.get()) {
213 Config.getPref().getListOfMaps(ValidatorPrefHelper.PREF_IGNORELIST).forEach(ignoredErrors::putAll);
214 Path path = Paths.get(getValidatorDir()).resolve("ignorederrors");
215 try {
216 if (path.toFile().exists()) {
217 try {
218 TreeSet<String> treeSet = new TreeSet<>();
219 treeSet.addAll(Files.readAllLines(path, StandardCharsets.UTF_8));
220 treeSet.forEach(ignore -> ignoredErrors.putIfAbsent(ignore, ""));
221
222 saveIgnoredErrors();
223 Files.deleteIfExists(path);
224
225 } catch (FileNotFoundException e) {
226 Logging.debug(Logging.getErrorMessage(e));
227 } catch (IOException e) {
228 Logging.error(e);
229 }
230 }
231 } catch (SecurityException e) {
232 Logging.log(Logging.LEVEL_ERROR, "Unable to load ignored errors", e);
233 }
234 }
235 }
236
237 /**
238 * Adds an ignored error
239 * @param s The ignore group / sub group name
240 * @see TestError#getIgnoreGroup()
241 * @see TestError#getIgnoreSubGroup()
242 */
243 public static void addIgnoredError(String s) {
244 addIgnoredError(s, "");
245 }
246
247 /**
248 * Adds an ignored error
249 * @param s The ignore group / sub group name
250 * @param description What the error actually is
251 * @see TestError#getIgnoreGroup()
252 * @see TestError#getIgnoreSubGroup()
253 */
254 public static void addIgnoredError(String s, String description) {
255 if (description == null) description = "";
256 ignoredErrors.put(s, description);
257 }
258
259 /**
260 * Make sure that we don't keep single entries for a "group ignore" or
261 * multiple different entries for the single entries that are in the same group.
262 */
263 private static void cleanupIgnoredErrors() {
264 if (ignoredErrors.size() > 1) {
265 List<String> toRemove = new ArrayList<>();
266
267 Iterator<Entry<String, String>> iter = ignoredErrors.entrySet().iterator();
268 Entry<String, String> last = iter.next();
269 while (iter.hasNext()) {
270 Entry<String, String> entry = iter.next();
271 if (entry.getKey().startsWith(last.getKey())) {
272 toRemove.add(entry.getKey());
273 } else {
274 last = entry;
275 }
276 }
277 toRemove.forEach(ignoredErrors::remove);
278 }
279
280 Map<String, String> tmap = buildIgnore(buildJTreeList());
281 if (!tmap.isEmpty()) {
282 ignoredErrors.clear();
283 ignoredErrors.putAll(tmap);
284 }
285 }
286
287 /**
288 * Check if a error should be ignored
289 * @param s The ignore group / sub group name
290 * @return <code>true</code> to ignore that error
291 */
292 public static boolean hasIgnoredError(String s) {
293 return ignoredErrors.containsKey(s);
294 }
295
296 /**
297 * Get the list of all ignored errors
298 * @return The <code>Collection&lt;String&gt;</code> of errors that are ignored
299 */
300 public static SortedMap<String, String> getIgnoredErrors() {
301 return ignoredErrors;
302 }
303
304 /**
305 * Build a JTree with a list
306 * @return &lt;type&gt;list as a {@code JTree}
307 */
308 public static JTree buildJTreeList() {
309 DefaultMutableTreeNode root = new DefaultMutableTreeNode(tr("Ignore list"));
310 final Pattern elemId1Pattern = Pattern.compile(":(r|w|n)_");
311 final Pattern elemId2Pattern = Pattern.compile("^[0-9]+$");
312 for (Entry<String, String> e: ignoredErrors.entrySet()) {
313 String key = e.getKey();
314 String value = e.getValue();
315 ArrayList<String> ignoredWayList = new ArrayList<>();
316 String[] osmobjects = elemId1Pattern.split(key);
317 for (int i = 1; i < osmobjects.length; i++) {
318 String osmid = osmobjects[i];
319 if (elemId2Pattern.matcher(osmid).matches()) {
320 osmid = '_' + osmid;
321 int index = key.indexOf(osmid);
322 if (index < key.lastIndexOf(']')) continue;
323 char type = key.charAt(index - 1);
324 ignoredWayList.add(type + osmid);
325 }
326 }
327 for (String osmignore : ignoredWayList) {
328 key = key.replace(':' + osmignore, "");
329 }
330
331 DefaultMutableTreeNode trunk;
332 DefaultMutableTreeNode branch;
333
334 if (value != null && !value.isEmpty()) {
335 trunk = inTree(root, value);
336 branch = inTree(trunk, key);
337 trunk.add(branch);
338 } else {
339 trunk = inTree(root, key);
340 branch = trunk;
341 }
342 ignoredWayList.forEach(osmignore -> branch.add(new DefaultMutableTreeNode(osmignore)));
343
344 root.add(trunk);
345 }
346 return new JTree(root);
347 }
348
349 private static DefaultMutableTreeNode inTree(DefaultMutableTreeNode root, String name) {
350 @SuppressWarnings("unchecked")
351 Enumeration<TreeNode> trunks = root.children();
352 while (trunks.hasMoreElements()) {
353 TreeNode ttrunk = trunks.nextElement();
354 if (ttrunk instanceof DefaultMutableTreeNode) {
355 DefaultMutableTreeNode trunk = (DefaultMutableTreeNode) ttrunk;
356 if (name.equals(trunk.getUserObject())) {
357 return trunk;
358 }
359 }
360 }
361 return new DefaultMutableTreeNode(name);
362 }
363
364 /**
365 * Build a {@code HashMap} from a tree of ignored errors
366 * @param tree The JTree of ignored errors
367 * @return A {@code HashMap} of the ignored errors for comparison
368 */
369 public static Map<String, String> buildIgnore(JTree tree) {
370 TreeModel model = tree.getModel();
371 DefaultMutableTreeNode root = (DefaultMutableTreeNode) model.getRoot();
372 return buildIgnore(model, root);
373 }
374
375 private static Map<String, String> buildIgnore(TreeModel model, DefaultMutableTreeNode node) {
376 HashMap<String, String> rHashMap = new HashMap<>();
377
378 String osmids = node.getUserObject().toString();
379 String description = "";
380
381 if (!model.getRoot().equals(node)) {
382 description = ((DefaultMutableTreeNode) node.getParent()).getUserObject().toString();
383 } else {
384 description = node.getUserObject().toString();
385 }
386 if (tr("Ignore list").equals(description)) description = "";
387 if (!osmids.matches("^[0-9]+(_.*|$)")) {
388 description = osmids;
389 osmids = "";
390 }
391
392
393 StringBuilder sb = new StringBuilder();
394 for (int i = 0; i < model.getChildCount(node); i++) {
395 DefaultMutableTreeNode child = (DefaultMutableTreeNode) model.getChild(node, i);
396 if (model.getChildCount(child) == 0) {
397 String ignoreName = child.getUserObject().toString();
398 if (ignoreName.matches("^(r|w|n)_.*")) {
399 sb.append(":").append(child.getUserObject().toString());
400 } else if (ignoreName.matches("^[0-9]+(_.*|)$")) {
401 rHashMap.put(ignoreName, description);
402 }
403 } else {
404 rHashMap.putAll(buildIgnore(model, child));
405 }
406 }
407 osmids += sb.toString();
408 if (!osmids.isEmpty() && osmids.indexOf(':') != 0) rHashMap.put(osmids, description);
409 return rHashMap;
410 }
411
412 /**
413 * Reset the error list by deleting {@code validator.ignorelist}
414 */
415 public static void resetErrorList() {
416 saveIgnoredErrors();
417 Config.getPref().putListOfMaps(ValidatorPrefHelper.PREF_IGNORELIST, null);
418 OsmValidator.initialize();
419 }
420
421 /**
422 * Saves the names of the ignored errors to a preference
423 */
424 public static void saveIgnoredErrors() {
425 List<Map<String, String>> list = new ArrayList<>();
426 cleanupIgnoredErrors();
427 list.add(ignoredErrors);
428 int i = 0;
429 while (i < list.size()) {
430 if (list.get(i) == null || list.get(i).isEmpty()) {
431 list.remove(i);
432 continue;
433 }
434 i++;
435 }
436 if (list.isEmpty()) list = null;
437 Config.getPref().putListOfMaps(ValidatorPrefHelper.PREF_IGNORELIST, list);
438 }
439
440 /**
441 * Initializes error layer.
442 */
443 public static synchronized void initializeErrorLayer() {
444 if (!ValidatorPrefHelper.PREF_LAYER.get())
445 return;
446 if (errorLayer == null) {
447 errorLayer = new ValidatorLayer();
448 MainApplication.getLayerManager().addLayer(errorLayer);
449 }
450 }
451
452 /**
453 * Resets error layer.
454 * @since 11852
455 */
456 public static synchronized void resetErrorLayer() {
457 errorLayer = null;
458 }
459
460 /**
461 * Gets a map from simple names to all tests.
462 * @return A map of all tests, indexed and sorted by the name of their Java class
463 */
464 public static SortedMap<String, Test> getAllTestsMap() {
465 applyPrefs(allTestsMap, false);
466 applyPrefs(allTestsMap, true);
467 return new TreeMap<>(allTestsMap);
468 }
469
470 /**
471 * Returns the instance of the given test class.
472 * @param <T> testClass type
473 * @param testClass The class of test to retrieve
474 * @return the instance of the given test class, if any, or {@code null}
475 * @since 6670
476 */
477 @SuppressWarnings("unchecked")
478 public static <T extends Test> T getTest(Class<T> testClass) {
479 if (testClass == null) {
480 return null;
481 }
482 return (T) allTestsMap.get(testClass.getName());
483 }
484
485 private static void applyPrefs(Map<String, Test> tests, boolean beforeUpload) {
486 for (String testName : Config.getPref().getList(beforeUpload
487 ? ValidatorPrefHelper.PREF_SKIP_TESTS_BEFORE_UPLOAD : ValidatorPrefHelper.PREF_SKIP_TESTS)) {
488 Test test = tests.get(testName);
489 if (test != null) {
490 if (beforeUpload) {
491 test.testBeforeUpload = false;
492 } else {
493 test.enabled = false;
494 }
495 }
496 }
497 }
498
499 /**
500 * Gets all tests that are possible
501 * @return The tests
502 */
503 public static Collection<Test> getTests() {
504 return getAllTestsMap().values();
505 }
506
507 /**
508 * Gets all tests that are run
509 * @param beforeUpload To get the ones that are run before upload
510 * @return The tests
511 */
512 public static Collection<Test> getEnabledTests(boolean beforeUpload) {
513 Collection<Test> enabledTests = getTests();
514 for (Test t : new ArrayList<>(enabledTests)) {
515 if (beforeUpload ? t.testBeforeUpload : t.enabled) {
516 continue;
517 }
518 enabledTests.remove(t);
519 }
520 return enabledTests;
521 }
522
523 /**
524 * Gets the list of all available test classes
525 *
526 * @return A collection of the test classes
527 */
528 public static Collection<Class<? extends Test>> getAllAvailableTestClasses() {
529 return Collections.unmodifiableCollection(allTests);
530 }
531
532 /**
533 * Initialize grid details based on current projection system. Values based on
534 * the original value fixed for EPSG:4326 (10000) using heuristics (that is, test&amp;error
535 * until most bugs were discovered while keeping the processing time reasonable)
536 */
537 public static void initializeGridDetail() {
538 String code = ProjectionRegistry.getProjection().toCode();
539 if (Arrays.asList(ProjectionPreference.wgs84.allCodes()).contains(code)) {
540 OsmValidator.griddetail = 10_000;
541 } else if (Arrays.asList(ProjectionPreference.mercator.allCodes()).contains(code)) {
542 OsmValidator.griddetail = 0.01;
543 } else if (Arrays.asList(ProjectionPreference.lambert.allCodes()).contains(code)) {
544 OsmValidator.griddetail = 0.1;
545 } else {
546 OsmValidator.griddetail = 1.0;
547 }
548 }
549
550 /**
551 * Returns grid detail, multiplier of east,north values for valuable cell sizing
552 * @return grid detail
553 * @since 11852
554 */
555 public static double getGridDetail() {
556 return griddetail;
557 }
558
559 private static boolean testsInitialized;
560
561 /**
562 * Initializes all tests if this operations hasn't been performed already.
563 */
564 public static synchronized void initializeTests() {
565 if (!testsInitialized) {
566 Logging.debug("Initializing validator tests");
567 final long startTime = System.currentTimeMillis();
568 initializeTests(getTests());
569 testsInitialized = true;
570 if (Logging.isDebugEnabled()) {
571 final long elapsedTime = System.currentTimeMillis() - startTime;
572 Logging.debug("Initializing validator tests completed in {0}", Utils.getDurationString(elapsedTime));
573 }
574 }
575 }
576
577 /**
578 * Initializes all tests
579 * @param allTests The tests to initialize
580 */
581 public static void initializeTests(Collection<? extends Test> allTests) {
582 for (Test test : allTests) {
583 try {
584 if (test.enabled) {
585 test.initialize();
586 }
587 } catch (Exception e) { // NOPMD
588 String message = tr("Error initializing test {0}:\n {1}", test.getClass().getSimpleName(), e);
589 Logging.error(message);
590 if (!GraphicsEnvironment.isHeadless()) {
591 GuiHelper.runInEDT(() ->
592 JOptionPane.showMessageDialog(MainApplication.getMainFrame(), message, tr("Error"), JOptionPane.ERROR_MESSAGE)
593 );
594 }
595 }
596 }
597 }
598
599 /**
600 * Groups the given collection of errors by severity, then message, then description.
601 * @param errors list of errors to group
602 * @param filterToUse optional filter
603 * @return collection of errors grouped by severity, then message, then description
604 * @since 12667
605 */
606 public static Map<Severity, Map<String, Map<String, List<TestError>>>> getErrorsBySeverityMessageDescription(
607 Collection<TestError> errors, Predicate<? super TestError> filterToUse) {
608 return errors.stream().filter(filterToUse).collect(
609 Collectors.groupingBy(TestError::getSeverity, () -> new EnumMap<>(Severity.class),
610 Collectors.groupingBy(TestError::getMessage, () -> new TreeMap<>(AlphanumComparator.getInstance()),
611 Collectors.groupingBy(e -> e.getDescription() == null ? "" : e.getDescription(),
612 () -> new TreeMap<>(AlphanumComparator.getInstance()),
613 Collectors.toList()
614 ))));
615 }
616}
Note: See TracBrowser for help on using the repository browser.