source: josm/trunk/src/org/openstreetmap/josm/data/validation/tests/MapCSSTagChecker.java

Last change on this file was 18801, checked in by taylor.smock, 8 months ago

Fix #22832: Code cleanup and some simplification, documentation fixes (patch by gaben)

There should not be any functional changes in this patch; it is intended to do
the following:

  • Simplify and cleanup code (example: Arrays.asList(item) -> Collections.singletonList(item))
  • Fix typos in documentation (which also corrects the documentation to match what actually happens, in some cases)
  • Property svn:eol-style set to native
File size: 20.2 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.validation.tests;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.awt.geom.Area;
7import java.io.BufferedReader;
8import java.io.File;
9import java.io.IOException;
10import java.io.InputStream;
11import java.io.Reader;
12import java.util.ArrayList;
13import java.util.Collection;
14import java.util.HashMap;
15import java.util.HashSet;
16import java.util.Iterator;
17import java.util.List;
18import java.util.Map;
19import java.util.Map.Entry;
20import java.util.Objects;
21import java.util.Set;
22import java.util.function.Consumer;
23import java.util.function.Predicate;
24import java.util.stream.Stream;
25
26import org.openstreetmap.josm.data.osm.IPrimitive;
27import org.openstreetmap.josm.data.osm.OsmPrimitive;
28import org.openstreetmap.josm.data.preferences.BooleanProperty;
29import org.openstreetmap.josm.data.preferences.CachingProperty;
30import org.openstreetmap.josm.data.preferences.sources.SourceEntry;
31import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper;
32import org.openstreetmap.josm.data.validation.OsmValidator;
33import org.openstreetmap.josm.data.validation.Severity;
34import org.openstreetmap.josm.data.validation.Test;
35import org.openstreetmap.josm.data.validation.TestError;
36import org.openstreetmap.josm.gui.mappaint.Environment;
37import org.openstreetmap.josm.gui.mappaint.MultiCascade;
38import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule;
39import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleIndex;
40import org.openstreetmap.josm.gui.mappaint.mapcss.Selector;
41import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.ParseException;
42import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.TokenMgrError;
43import org.openstreetmap.josm.gui.progress.ProgressMonitor;
44import org.openstreetmap.josm.io.CachedFile;
45import org.openstreetmap.josm.io.FileWatcher;
46import org.openstreetmap.josm.io.UTFInputStreamReader;
47import org.openstreetmap.josm.spi.preferences.Config;
48import org.openstreetmap.josm.tools.CheckParameterUtil;
49import org.openstreetmap.josm.tools.I18n;
50import org.openstreetmap.josm.tools.Logging;
51import org.openstreetmap.josm.tools.MultiMap;
52import org.openstreetmap.josm.tools.Stopwatch;
53import org.openstreetmap.josm.tools.Utils;
54
55/**
56 * MapCSS-based tag checker/fixer.
57 * @since 6506
58 */
59public class MapCSSTagChecker extends Test.TagTest {
60 private MapCSSStyleIndex indexData;
61 private final Map<MapCSSRule, MapCSSTagCheckerAndRule> ruleToCheckMap = new HashMap<>();
62 private static final Map<IPrimitive, Area> mpAreaCache = new HashMap<>();
63 private static final Set<IPrimitive> toMatchForSurrounding = new HashSet<>();
64 static final boolean ALL_TESTS = true;
65 static final boolean ONLY_SELECTED_TESTS = false;
66
67 /**
68 * Cached version of {@link ValidatorPrefHelper#PREF_OTHER}, see #20745.
69 */
70 private static final CachingProperty<Boolean> PREF_OTHER = new BooleanProperty("validator.other", false).cached();
71
72 /**
73 * The preference key for tag checker source entries.
74 * @since 6670
75 */
76 public static final String ENTRIES_PREF_KEY = "validator." + MapCSSTagChecker.class.getName() + ".entries";
77
78 /**
79 * Constructs a new {@code MapCSSTagChecker}.
80 */
81 public MapCSSTagChecker() {
82 super(tr("Tag checker (MapCSS based)"), tr("This test checks for errors in tag keys and values."));
83 }
84
85 final MultiMap<String, MapCSSTagCheckerRule> checks = new MultiMap<>();
86
87 /** maps the source URL for a test to the title shown in the dialog where known */
88 private final Map<String, String> urlTitles = new HashMap<>();
89
90 /**
91 * Result of {@link MapCSSTagCheckerRule#readMapCSS}
92 * @since 8936
93 */
94 public static class ParseResult {
95 /** Checks successfully parsed */
96 public final List<MapCSSTagCheckerRule> parseChecks;
97 /** Errors that occurred during parsing */
98 public final Collection<Throwable> parseErrors;
99
100 /**
101 * Constructs a new {@code ParseResult}.
102 * @param parseChecks Checks successfully parsed
103 * @param parseErrors Errors that occurred during parsing
104 */
105 public ParseResult(List<MapCSSTagCheckerRule> parseChecks, Collection<Throwable> parseErrors) {
106 this.parseChecks = parseChecks;
107 this.parseErrors = parseErrors;
108 }
109 }
110
111 static class MapCSSTagCheckerAndRule extends MapCSSTagChecker {
112 public final MapCSSRule rule;
113 private final MapCSSTagCheckerRule tagCheck;
114 private final String source;
115
116 MapCSSTagCheckerAndRule(MapCSSRule rule) {
117 this.rule = rule;
118 this.tagCheck = null;
119 this.source = "";
120 }
121
122 MapCSSTagCheckerAndRule(MapCSSTagCheckerRule tagCheck, String source) {
123 this.rule = tagCheck.rule;
124 this.tagCheck = tagCheck;
125 this.source = source;
126 }
127
128 @Override
129 public String toString() {
130 return "MapCSSTagCheckerAndRule [rule=" + rule + ']';
131 }
132
133 @Override
134 public String getSource() {
135 return source;
136 }
137 }
138
139 static MapCSSStyleIndex createMapCSSTagCheckerIndex(
140 MultiMap<String, MapCSSTagCheckerRule> checks, boolean includeOtherSeverity, boolean allTests) {
141 final MapCSSStyleIndex index = new MapCSSStyleIndex();
142 final Stream<MapCSSRule> ruleStream = checks.values().stream()
143 .flatMap(Collection::stream)
144 // Ignore "information" level checks if not wanted, unless they also set a MapCSS class
145 .filter(c -> includeOtherSeverity || Severity.OTHER != c.getSeverity() || !c.setClassExpressions.isEmpty())
146 .filter(c -> allTests || c.rule.selectors.stream().anyMatch(Selector.ChildOrParentSelector.class::isInstance))
147 .map(c -> c.rule);
148 index.buildIndex(ruleStream);
149 return index;
150 }
151
152 /**
153 * Obtains all {@link TestError}s for the {@link OsmPrimitive} {@code p}.
154 * @param p The OSM primitive
155 * @param includeOtherSeverity if {@code true}, errors of severity {@link Severity#OTHER} (info) will also be returned
156 * @return all errors for the given primitive, with or without those of "info" severity
157 */
158 public synchronized Collection<TestError> getErrorsForPrimitive(OsmPrimitive p, boolean includeOtherSeverity) {
159 final List<TestError> res = new ArrayList<>();
160 if (indexData == null) {
161 indexData = createMapCSSTagCheckerIndex(checks, includeOtherSeverity, ALL_TESTS);
162 }
163
164 final Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null);
165 env.mpAreaCache = mpAreaCache;
166 env.toMatchForSurrounding = toMatchForSurrounding;
167
168 Iterator<MapCSSRule> candidates = indexData.getRuleCandidates(p);
169 while (candidates.hasNext()) {
170 MapCSSRule r = candidates.next();
171 for (Selector selector : r.selectors) {
172 env.clearSelectorMatchingInformation();
173 if (!selector.matches(env)) { // as side effect env.parent will be set (if s is a child selector)
174 continue;
175 }
176 MapCSSTagCheckerAndRule test = ruleToCheckMap.computeIfAbsent(r, rule -> checks.entrySet().stream()
177 .map(e -> e.getValue().stream()
178 // rule.selectors might be different due to MapCSSStyleIndex, however, the declarations are the same object
179 .filter(c -> c.rule.declaration == rule.declaration)
180 .findFirst()
181 .map(c -> new MapCSSTagCheckerAndRule(c, getTitle(e.getKey())))
182 .orElse(null))
183 .filter(Objects::nonNull)
184 .findFirst()
185 .orElse(null));
186 MapCSSTagCheckerRule check = test == null ? null : test.tagCheck;
187 if (check != null) {
188 r.declaration.execute(env);
189 if (!check.errors.isEmpty()) {
190 for (TestError e: check.getErrorsForPrimitive(p, selector, env, test)) {
191 addIfNotSimilar(e, res);
192 }
193 }
194 }
195 }
196 }
197 return res;
198 }
199
200 private String getTitle(String url) {
201 return urlTitles.getOrDefault(url, tr("unknown"));
202 }
203
204 /**
205 * See #12627
206 * Add error to given list if list doesn't already contain a similar error.
207 * Similar means same code and description and same combination of primitives and same combination of highlighted objects,
208 * but maybe with different orders.
209 * @param toAdd the error to add
210 * @param errors the list of errors
211 */
212 private static void addIfNotSimilar(TestError toAdd, List<TestError> errors) {
213 final boolean isDup = toAdd.getPrimitives().size() >= 2 && errors.stream().anyMatch(toAdd::isSimilar);
214 if (!isDup)
215 errors.add(toAdd);
216 }
217
218 static Collection<TestError> getErrorsForPrimitive(
219 OsmPrimitive p, boolean includeOtherSeverity, Collection<Set<MapCSSTagCheckerRule>> checksCol) {
220 // this variant is only used by the assertion tests
221 final List<TestError> r = new ArrayList<>();
222 final Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null);
223 env.mpAreaCache = mpAreaCache;
224 env.toMatchForSurrounding = toMatchForSurrounding;
225 for (Set<MapCSSTagCheckerRule> schecks : checksCol) {
226 for (MapCSSTagCheckerRule check : schecks) {
227 boolean ignoreError = Severity.OTHER == check.getSeverity() && !includeOtherSeverity;
228 // Do not run "information" level checks if not wanted, unless they also set a MapCSS class
229 if (ignoreError && check.setClassExpressions.isEmpty()) {
230 continue;
231 }
232 final Selector selector = check.whichSelectorMatchesEnvironment(env);
233 if (selector != null) {
234 final Environment envWithSelector = env.withSelector(selector);
235 check.rule.declaration.execute(envWithSelector);
236 if (!ignoreError && !check.errors.isEmpty()) {
237 r.addAll(check.getErrorsForPrimitive(p, selector, envWithSelector, new MapCSSTagCheckerAndRule(check.rule)));
238 }
239 }
240 }
241 }
242 return r;
243 }
244
245 /**
246 * Visiting call for primitives.
247 *
248 * @param p The primitive to inspect.
249 */
250 @Override
251 public void check(OsmPrimitive p) {
252 for (TestError e : getErrorsForPrimitive(p, PREF_OTHER.get())) {
253 addIfNotSimilar(e, errors);
254 }
255 }
256
257 /**
258 * Adds a new MapCSS config file from the given URL.
259 * @param url The unique URL of the MapCSS config file
260 * @return List of tag checks and parsing errors, or null
261 * @throws ParseException if the config file does not match MapCSS syntax
262 * @throws IOException if any I/O error occurs
263 * @since 7275
264 */
265 public synchronized ParseResult addMapCSS(String url) throws ParseException, IOException {
266 // Check assertions, useful for development of local files
267 final boolean checkAssertions = Config.getPref().getBoolean("validator.check_assert_local_rules", false) && Utils.isLocalUrl(url);
268 return addMapCSS(url, checkAssertions ? Logging::warn : null);
269 }
270
271 /**
272 * Adds a new MapCSS config file from the given URL. <br />
273 * NOTE: You should prefer {@link #addMapCSS(String)} unless you <i>need</i> to know what the assertions return.
274 *
275 * @param url The unique URL of the MapCSS config file
276 * @param assertionConsumer A string consumer for error messages.
277 * @return List of tag checks and parsing errors, or null
278 * @throws ParseException if the config file does not match MapCSS syntax
279 * @throws IOException if any I/O error occurs
280 * @since 18365 (public, primarily for ValidatorCLI)
281 */
282 public synchronized ParseResult addMapCSS(String url, Consumer<String> assertionConsumer) throws ParseException, IOException {
283 CheckParameterUtil.ensureParameterNotNull(url, "url");
284 ParseResult result;
285 try (CachedFile cache = new CachedFile(url);
286 InputStream zip = cache.findZipEntryInputStream("validator.mapcss", "");
287 InputStream s = zip != null ? zip : cache.getInputStream();
288 Reader reader = new BufferedReader(UTFInputStreamReader.create(s))) {
289 if (zip != null)
290 I18n.addTexts(cache.getFile());
291 result = MapCSSTagCheckerRule.readMapCSS(reader, assertionConsumer);
292 checks.remove(url);
293 checks.putAll(url, result.parseChecks);
294 urlTitles.put(url, findURLTitle(url));
295 indexData = null;
296 }
297 return result;
298 }
299
300 /** Find a user friendly string for the url.
301 *
302 * @param url the source for the set of rules
303 * @return a value that can be used in tool tip or progress bar.
304 */
305 private static String findURLTitle(String url) {
306 for (SourceEntry source : new ValidatorPrefHelper().get()) {
307 if (url.equals(source.url) && !Utils.isEmpty(source.title)) {
308 return source.title;
309 }
310 }
311 if (url.endsWith(".mapcss")) // do we have others?
312 url = new File(url).getName();
313 if (url.length() > 33) {
314 url = "..." + url.substring(url.length() - 30);
315 }
316 return url;
317 }
318
319 @Override
320 public synchronized void initialize() throws Exception {
321 checks.clear();
322 urlTitles.clear();
323 indexData = null;
324 for (SourceEntry source : new ValidatorPrefHelper().get()) {
325 if (!source.active) {
326 continue;
327 }
328 String i = source.url;
329 try {
330 if (!i.startsWith("resource:")) {
331 Logging.info(tr("Adding {0} to tag checker", i));
332 } else if (Logging.isDebugEnabled()) {
333 Logging.debug(tr("Adding {0} to tag checker", i));
334 }
335 addMapCSS(i);
336 if (Config.getPref().getBoolean("validator.auto_reload_local_rules", true) && source.isLocal()) {
337 FileWatcher.getDefaultInstance().registerSource(source);
338 }
339 } catch (IOException | IllegalStateException | IllegalArgumentException ex) {
340 Logging.warn(tr("Failed to add {0} to tag checker", i));
341 Logging.log(Logging.LEVEL_WARN, ex);
342 } catch (ParseException | TokenMgrError ex) {
343 Logging.warn(tr("Failed to add {0} to tag checker", i));
344 Logging.warn(ex);
345 }
346 }
347 MapCSSTagCheckerAsserts.clear();
348 }
349
350 /**
351 * Reload tagchecker rule.
352 * @param rule tagchecker rule to reload
353 * @since 12825
354 */
355 public static void reloadRule(SourceEntry rule) {
356 MapCSSTagChecker tagChecker = OsmValidator.getTest(MapCSSTagChecker.class);
357 if (tagChecker != null) {
358 try {
359 tagChecker.addMapCSS(rule.url);
360 } catch (IOException | ParseException | TokenMgrError e) {
361 Logging.warn(e);
362 }
363 }
364 }
365
366 @Override
367 public synchronized void startTest(ProgressMonitor progressMonitor) {
368 super.startTest(progressMonitor);
369 super.setShowElements(true);
370 }
371
372 @Override
373 public synchronized void endTest() {
374 // no need to keep the index, it is quickly build and doubles the memory needs
375 indexData = null;
376 // always clear the cache to make sure that we catch changes in geometry
377 mpAreaCache.clear();
378 ruleToCheckMap.clear();
379 toMatchForSurrounding.clear();
380 super.endTest();
381 }
382
383 @Override
384 public void visit(Collection<OsmPrimitive> selection) {
385 visit(selection, null);
386 }
387
388 /**
389 * Execute the rules from the URLs matching the given predicate.
390 * @param selection collection of primitives
391 * @param urlPredicate a predicate deciding whether the rules from the given URL shall be executed
392 */
393 void visit(Collection<OsmPrimitive> selection, Predicate<String> urlPredicate) {
394 if (urlPredicate == null && progressMonitor != null) {
395 progressMonitor.setTicksCount(selection.size() * checks.size());
396 }
397
398 mpAreaCache.clear();
399 toMatchForSurrounding.clear();
400
401 Set<OsmPrimitive> surrounding = new HashSet<>();
402 for (Entry<String, Set<MapCSSTagCheckerRule>> entry : checks.entrySet()) {
403 if (isCanceled()) {
404 break;
405 }
406 if (urlPredicate != null && !urlPredicate.test(entry.getKey())) {
407 continue;
408 }
409 visit(entry.getKey(), entry.getValue(), selection, surrounding);
410 }
411 }
412
413 /**
414 * Perform the checks for one check url
415 * @param url the url for the checks
416 * @param checksForUrl the checks to perform
417 * @param selection collection primitives
418 * @param surrounding surrounding primitives, evtl. filled by this routine
419 */
420 private void visit(String url, Set<MapCSSTagCheckerRule> checksForUrl, Collection<OsmPrimitive> selection, Set<OsmPrimitive> surrounding) {
421 MultiMap<String, MapCSSTagCheckerRule> currentCheck = new MultiMap<>();
422 currentCheck.putAll(url, checksForUrl);
423 indexData = createMapCSSTagCheckerIndex(currentCheck, includeOtherSeverityChecks(), ALL_TESTS);
424 Set<OsmPrimitive> tested = new HashSet<>();
425
426
427 String title = getTitle(url);
428 if (progressMonitor != null) {
429 progressMonitor.setExtraText(tr(" {0}", title));
430 }
431 long cnt = 0;
432 Stopwatch stopwatch = Stopwatch.createStarted();
433 for (OsmPrimitive p : selection) {
434 if (isCanceled()) {
435 break;
436 }
437 if (isPrimitiveUsable(p)) {
438 check(p);
439 if (partialSelection) {
440 tested.add(p);
441 }
442 }
443 if (progressMonitor != null) {
444 progressMonitor.worked(1);
445 cnt++;
446 // add frequently changing info to progress monitor so that it
447 // doesn't seem to hang when test takes longer than 0.5 seconds
448 if (cnt % 10_000 == 0 && stopwatch.elapsed() >= 500) {
449 progressMonitor.setExtraText(tr(" {0}: {1} of {2} elements done", title, cnt, selection.size()));
450 }
451 }
452 }
453
454 if (partialSelection && !tested.isEmpty()) {
455 testPartial(currentCheck, tested, surrounding);
456 }
457 }
458
459 private void testPartial(MultiMap<String, MapCSSTagCheckerRule> currentCheck, Set<OsmPrimitive> tested, Set<OsmPrimitive> surrounding) {
460
461 // #14287: see https://josm.openstreetmap.de/ticket/14287#comment:15
462 // execute tests for objects which might contain or cross previously tested elements
463
464 final boolean includeOtherSeverity = includeOtherSeverityChecks();
465 // rebuild index with a reduced set of rules (those that use ChildOrParentSelector) and thus may have left selectors
466 // matching the previously tested elements
467 indexData = createMapCSSTagCheckerIndex(currentCheck, includeOtherSeverity, ONLY_SELECTED_TESTS);
468 if (indexData.isEmpty())
469 return; // performance: some *.mapcss rule files don't use ChildOrParentSelector
470
471 if (surrounding.isEmpty()) {
472 for (OsmPrimitive p : tested) {
473 if (p.getDataSet() != null) {
474 surrounding.addAll(p.getDataSet().searchWays(p.getBBox()));
475 surrounding.addAll(p.getDataSet().searchRelations(p.getBBox()));
476 }
477 }
478 }
479
480 toMatchForSurrounding.clear();
481 toMatchForSurrounding.addAll(tested);
482 for (OsmPrimitive p : surrounding) {
483 if (tested.contains(p))
484 continue;
485 Collection<TestError> additionalErrors = getErrorsForPrimitive(p, includeOtherSeverity);
486 for (TestError e : additionalErrors) {
487 if (e.getPrimitives().stream().anyMatch(tested::contains))
488 addIfNotSimilar(e, errors);
489 }
490 }
491
492 }
493
494}
Note: See TracBrowser for help on using the repository browser.