source: josm/trunk/src/org/openstreetmap/josm/data/validation/OsmValidator.java@ 13669

Last change on this file since 13669 was 13647, checked in by Don-vip, 6 years ago

see #16204 - Allow to start and close JOSM in WebStart sandbox mode (where every external access is denied). This was very useful to reproduce some very tricky bugs that occured in real life but were almost impossible to diagnose.
  • Property svn:eol-style set to native
File size: 16.1 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.validation;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.awt.GraphicsEnvironment;
7import java.io.File;
8import java.io.FileNotFoundException;
9import java.io.IOException;
10import java.io.PrintWriter;
11import java.nio.charset.StandardCharsets;
12import java.nio.file.Files;
13import java.nio.file.Path;
14import java.nio.file.Paths;
15import java.util.ArrayList;
16import java.util.Arrays;
17import java.util.Collection;
18import java.util.Collections;
19import java.util.EnumMap;
20import java.util.HashMap;
21import java.util.List;
22import java.util.Map;
23import java.util.SortedMap;
24import java.util.TreeMap;
25import java.util.TreeSet;
26import java.util.function.Predicate;
27import java.util.stream.Collectors;
28
29import javax.swing.JOptionPane;
30
31import org.openstreetmap.josm.Main;
32import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper;
33import org.openstreetmap.josm.data.validation.tests.Addresses;
34import org.openstreetmap.josm.data.validation.tests.ApiCapabilitiesTest;
35import org.openstreetmap.josm.data.validation.tests.BarriersEntrances;
36import org.openstreetmap.josm.data.validation.tests.Coastlines;
37import org.openstreetmap.josm.data.validation.tests.ConditionalKeys;
38import org.openstreetmap.josm.data.validation.tests.CrossingWays;
39import org.openstreetmap.josm.data.validation.tests.DuplicateNode;
40import org.openstreetmap.josm.data.validation.tests.DuplicateRelation;
41import org.openstreetmap.josm.data.validation.tests.DuplicateWay;
42import org.openstreetmap.josm.data.validation.tests.DuplicatedWayNodes;
43import org.openstreetmap.josm.data.validation.tests.Highways;
44import org.openstreetmap.josm.data.validation.tests.InternetTags;
45import org.openstreetmap.josm.data.validation.tests.Lanes;
46import org.openstreetmap.josm.data.validation.tests.LongSegment;
47import org.openstreetmap.josm.data.validation.tests.MapCSSTagChecker;
48import org.openstreetmap.josm.data.validation.tests.MultipolygonTest;
49import org.openstreetmap.josm.data.validation.tests.NameMismatch;
50import org.openstreetmap.josm.data.validation.tests.OpeningHourTest;
51import org.openstreetmap.josm.data.validation.tests.OverlappingWays;
52import org.openstreetmap.josm.data.validation.tests.PowerLines;
53import org.openstreetmap.josm.data.validation.tests.PublicTransportRouteTest;
54import org.openstreetmap.josm.data.validation.tests.RelationChecker;
55import org.openstreetmap.josm.data.validation.tests.SelfIntersectingWay;
56import org.openstreetmap.josm.data.validation.tests.SimilarNamedWays;
57import org.openstreetmap.josm.data.validation.tests.TagChecker;
58import org.openstreetmap.josm.data.validation.tests.TurnrestrictionTest;
59import org.openstreetmap.josm.data.validation.tests.UnclosedWays;
60import org.openstreetmap.josm.data.validation.tests.UnconnectedWays;
61import org.openstreetmap.josm.data.validation.tests.UntaggedNode;
62import org.openstreetmap.josm.data.validation.tests.UntaggedWay;
63import org.openstreetmap.josm.data.validation.tests.WayConnectedToArea;
64import org.openstreetmap.josm.data.validation.tests.WronglyOrderedWays;
65import org.openstreetmap.josm.gui.MainApplication;
66import org.openstreetmap.josm.gui.layer.ValidatorLayer;
67import org.openstreetmap.josm.gui.preferences.projection.ProjectionPreference;
68import org.openstreetmap.josm.gui.util.GuiHelper;
69import org.openstreetmap.josm.spi.preferences.Config;
70import org.openstreetmap.josm.tools.AlphanumComparator;
71import org.openstreetmap.josm.tools.Logging;
72import org.openstreetmap.josm.tools.Utils;
73
74/**
75 * A OSM data validator.
76 *
77 * @author Francisco R. Santos <frsantos@gmail.com>
78 */
79public final class OsmValidator {
80
81 private OsmValidator() {
82 // Hide default constructor for utilities classes
83 }
84
85 private static volatile ValidatorLayer errorLayer;
86
87 /** Grid detail, multiplier of east,north values for valuable cell sizing */
88 private static double griddetail;
89
90 private static final Collection<String> ignoredErrors = new TreeSet<>();
91
92 /**
93 * All registered tests
94 */
95 private static final Collection<Class<? extends Test>> allTests = new ArrayList<>();
96 private static final Map<String, Test> allTestsMap = new HashMap<>();
97
98 /**
99 * All available tests in core
100 */
101 @SuppressWarnings("unchecked")
102 private static final Class<Test>[] CORE_TEST_CLASSES = new Class[] {
103 /* FIXME - unique error numbers for tests aren't properly unique - ignoring will not work as expected */
104 DuplicateNode.class, // ID 1 .. 99
105 OverlappingWays.class, // ID 101 .. 199
106 UntaggedNode.class, // ID 201 .. 299
107 UntaggedWay.class, // ID 301 .. 399
108 SelfIntersectingWay.class, // ID 401 .. 499
109 DuplicatedWayNodes.class, // ID 501 .. 599
110 CrossingWays.Ways.class, // ID 601 .. 699
111 CrossingWays.Boundaries.class, // ID 601 .. 699
112 CrossingWays.Barrier.class, // ID 601 .. 699
113 CrossingWays.SelfCrossing.class, // ID 601 .. 699
114 SimilarNamedWays.class, // ID 701 .. 799
115 Coastlines.class, // ID 901 .. 999
116 WronglyOrderedWays.class, // ID 1001 .. 1099
117 UnclosedWays.class, // ID 1101 .. 1199
118 TagChecker.class, // ID 1201 .. 1299
119 UnconnectedWays.UnconnectedHighways.class, // ID 1301 .. 1399
120 UnconnectedWays.UnconnectedRailways.class, // ID 1301 .. 1399
121 UnconnectedWays.UnconnectedWaterways.class, // ID 1301 .. 1399
122 UnconnectedWays.UnconnectedNaturalOrLanduse.class, // ID 1301 .. 1399
123 UnconnectedWays.UnconnectedPower.class, // ID 1301 .. 1399
124 DuplicateWay.class, // ID 1401 .. 1499
125 NameMismatch.class, // ID 1501 .. 1599
126 MultipolygonTest.class, // ID 1601 .. 1699
127 RelationChecker.class, // ID 1701 .. 1799
128 TurnrestrictionTest.class, // ID 1801 .. 1899
129 DuplicateRelation.class, // ID 1901 .. 1999
130 WayConnectedToArea.class, // ID 2301 .. 2399
131 PowerLines.class, // ID 2501 .. 2599
132 Addresses.class, // ID 2601 .. 2699
133 Highways.class, // ID 2701 .. 2799
134 BarriersEntrances.class, // ID 2801 .. 2899
135 OpeningHourTest.class, // 2901 .. 2999
136 MapCSSTagChecker.class, // 3000 .. 3099
137 Lanes.class, // 3100 .. 3199
138 ConditionalKeys.class, // 3200 .. 3299
139 InternetTags.class, // 3300 .. 3399
140 ApiCapabilitiesTest.class, // 3400 .. 3499
141 LongSegment.class, // 3500 .. 3599
142 PublicTransportRouteTest.class, // 3600 .. 3699
143 };
144
145 /**
146 * Adds a test to the list of available tests
147 * @param testClass The test class
148 */
149 public static void addTest(Class<? extends Test> testClass) {
150 allTests.add(testClass);
151 try {
152 allTestsMap.put(testClass.getName(), testClass.getConstructor().newInstance());
153 } catch (ReflectiveOperationException e) {
154 Logging.error(e);
155 }
156 }
157
158 static {
159 for (Class<? extends Test> testClass : CORE_TEST_CLASSES) {
160 addTest(testClass);
161 }
162 }
163
164 /**
165 * Initializes {@code OsmValidator}.
166 */
167 public static void initialize() {
168 checkValidatorDir();
169 initializeGridDetail();
170 loadIgnoredErrors(); //FIXME: load only when needed
171 }
172
173 /**
174 * Returns the validator directory.
175 *
176 * @return The validator directory
177 */
178 public static String getValidatorDir() {
179 File dir = new File(Config.getDirs().getUserDataDirectory(true), "validator");
180 try {
181 return dir.getAbsolutePath();
182 } catch (SecurityException e) {
183 Logging.log(Logging.LEVEL_ERROR, null, e);
184 return dir.getPath();
185 }
186 }
187
188 /**
189 * Check if validator directory exists (store ignored errors file)
190 */
191 private static void checkValidatorDir() {
192 File pathDir = new File(getValidatorDir());
193 try {
194 if (!pathDir.exists()) {
195 Utils.mkDirs(pathDir);
196 }
197 } catch (SecurityException e) {
198 Logging.log(Logging.LEVEL_ERROR, "Unable to check validator directory", e);
199 }
200 }
201
202 private static void loadIgnoredErrors() {
203 ignoredErrors.clear();
204 if (ValidatorPrefHelper.PREF_USE_IGNORE.get()) {
205 Path path = Paths.get(getValidatorDir()).resolve("ignorederrors");
206 try {
207 if (path.toFile().exists()) {
208 try {
209 ignoredErrors.addAll(Files.readAllLines(path, StandardCharsets.UTF_8));
210 } catch (FileNotFoundException e) {
211 Logging.debug(Logging.getErrorMessage(e));
212 } catch (IOException e) {
213 Logging.error(e);
214 }
215 }
216 } catch (SecurityException e) {
217 Logging.log(Logging.LEVEL_ERROR, "Unable to load ignored errors", e);
218 }
219 }
220 }
221
222 /**
223 * Adds an ignored error
224 * @param s The ignore group / sub group name
225 * @see TestError#getIgnoreGroup()
226 * @see TestError#getIgnoreSubGroup()
227 */
228 public static void addIgnoredError(String s) {
229 ignoredErrors.add(s);
230 }
231
232 /**
233 * Check if a error should be ignored
234 * @param s The ignore group / sub group name
235 * @return <code>true</code> to ignore that error
236 */
237 public static boolean hasIgnoredError(String s) {
238 return ignoredErrors.contains(s);
239 }
240
241 /**
242 * Saves the names of the ignored errors to a file
243 */
244 public static void saveIgnoredErrors() {
245 try (PrintWriter out = new PrintWriter(new File(getValidatorDir(), "ignorederrors"), StandardCharsets.UTF_8.name())) {
246 for (String e : ignoredErrors) {
247 out.println(e);
248 }
249 } catch (IOException e) {
250 Logging.error(e);
251 }
252 }
253
254 /**
255 * Initializes error layer.
256 */
257 public static synchronized void initializeErrorLayer() {
258 if (!ValidatorPrefHelper.PREF_LAYER.get())
259 return;
260 if (errorLayer == null) {
261 errorLayer = new ValidatorLayer();
262 MainApplication.getLayerManager().addLayer(errorLayer);
263 }
264 }
265
266 /**
267 * Resets error layer.
268 * @since 11852
269 */
270 public static synchronized void resetErrorLayer() {
271 errorLayer = null;
272 }
273
274 /**
275 * Gets a map from simple names to all tests.
276 * @return A map of all tests, indexed and sorted by the name of their Java class
277 */
278 public static SortedMap<String, Test> getAllTestsMap() {
279 applyPrefs(allTestsMap, false);
280 applyPrefs(allTestsMap, true);
281 return new TreeMap<>(allTestsMap);
282 }
283
284 /**
285 * Returns the instance of the given test class.
286 * @param <T> testClass type
287 * @param testClass The class of test to retrieve
288 * @return the instance of the given test class, if any, or {@code null}
289 * @since 6670
290 */
291 @SuppressWarnings("unchecked")
292 public static <T extends Test> T getTest(Class<T> testClass) {
293 if (testClass == null) {
294 return null;
295 }
296 return (T) allTestsMap.get(testClass.getName());
297 }
298
299 private static void applyPrefs(Map<String, Test> tests, boolean beforeUpload) {
300 for (String testName : Config.getPref().getList(beforeUpload
301 ? ValidatorPrefHelper.PREF_SKIP_TESTS_BEFORE_UPLOAD : ValidatorPrefHelper.PREF_SKIP_TESTS)) {
302 Test test = tests.get(testName);
303 if (test != null) {
304 if (beforeUpload) {
305 test.testBeforeUpload = false;
306 } else {
307 test.enabled = false;
308 }
309 }
310 }
311 }
312
313 /**
314 * Gets all tests that are possible
315 * @return The tests
316 */
317 public static Collection<Test> getTests() {
318 return getAllTestsMap().values();
319 }
320
321 /**
322 * Gets all tests that are run
323 * @param beforeUpload To get the ones that are run before upload
324 * @return The tests
325 */
326 public static Collection<Test> getEnabledTests(boolean beforeUpload) {
327 Collection<Test> enabledTests = getTests();
328 for (Test t : new ArrayList<>(enabledTests)) {
329 if (beforeUpload ? t.testBeforeUpload : t.enabled) {
330 continue;
331 }
332 enabledTests.remove(t);
333 }
334 return enabledTests;
335 }
336
337 /**
338 * Gets the list of all available test classes
339 *
340 * @return A collection of the test classes
341 */
342 public static Collection<Class<? extends Test>> getAllAvailableTestClasses() {
343 return Collections.unmodifiableCollection(allTests);
344 }
345
346 /**
347 * Initialize grid details based on current projection system. Values based on
348 * the original value fixed for EPSG:4326 (10000) using heuristics (that is, test&amp;error
349 * until most bugs were discovered while keeping the processing time reasonable)
350 */
351 public static void initializeGridDetail() {
352 String code = Main.getProjection().toCode();
353 if (Arrays.asList(ProjectionPreference.wgs84.allCodes()).contains(code)) {
354 OsmValidator.griddetail = 10_000;
355 } else if (Arrays.asList(ProjectionPreference.mercator.allCodes()).contains(code)) {
356 OsmValidator.griddetail = 0.01;
357 } else if (Arrays.asList(ProjectionPreference.lambert.allCodes()).contains(code)) {
358 OsmValidator.griddetail = 0.1;
359 } else {
360 OsmValidator.griddetail = 1.0;
361 }
362 }
363
364 /**
365 * Returns grid detail, multiplier of east,north values for valuable cell sizing
366 * @return grid detail
367 * @since 11852
368 */
369 public static double getGridDetail() {
370 return griddetail;
371 }
372
373 private static boolean testsInitialized;
374
375 /**
376 * Initializes all tests if this operations hasn't been performed already.
377 */
378 public static synchronized void initializeTests() {
379 if (!testsInitialized) {
380 Logging.debug("Initializing validator tests");
381 final long startTime = System.currentTimeMillis();
382 initializeTests(getTests());
383 testsInitialized = true;
384 if (Logging.isDebugEnabled()) {
385 final long elapsedTime = System.currentTimeMillis() - startTime;
386 Logging.debug("Initializing validator tests completed in {0}", Utils.getDurationString(elapsedTime));
387 }
388 }
389 }
390
391 /**
392 * Initializes all tests
393 * @param allTests The tests to initialize
394 */
395 public static void initializeTests(Collection<? extends Test> allTests) {
396 for (Test test : allTests) {
397 try {
398 if (test.enabled) {
399 test.initialize();
400 }
401 } catch (Exception e) { // NOPMD
402 String message = tr("Error initializing test {0}:\n {1}", test.getClass().getSimpleName(), e);
403 Logging.error(message);
404 if (!GraphicsEnvironment.isHeadless()) {
405 GuiHelper.runInEDT(() -> {
406 JOptionPane.showMessageDialog(Main.parent, message, tr("Error"), JOptionPane.ERROR_MESSAGE);
407 });
408 }
409 }
410 }
411 }
412
413 /**
414 * Groups the given collection of errors by severity, then message, then description.
415 * @param errors list of errors to group
416 * @param filterToUse optional filter
417 * @return collection of errors grouped by severity, then message, then description
418 * @since 12667
419 */
420 public static Map<Severity, Map<String, Map<String, List<TestError>>>> getErrorsBySeverityMessageDescription(
421 Collection<TestError> errors, Predicate<? super TestError> filterToUse) {
422 return errors.stream().filter(filterToUse).collect(
423 Collectors.groupingBy(TestError::getSeverity, () -> new EnumMap<>(Severity.class),
424 Collectors.groupingBy(TestError::getMessage, () -> new TreeMap<>(AlphanumComparator.getInstance()),
425 Collectors.groupingBy(e -> e.getDescription() == null ? "" : e.getDescription(),
426 () -> new TreeMap<>(AlphanumComparator.getInstance()),
427 Collectors.toList()
428 ))));
429 }
430}
Note: See TracBrowser for help on using the repository browser.