source: josm/trunk/src/org/openstreetmap/josm/data/validation/tests/MapCSSTagChecker.java@ 17756

Last change on this file since 17756 was 17756, checked in by simon04, 3 years ago

see #20745 - Avoid heap allocations due to ValidatorPrefHelper.PREF_OTHER (fix NPE)
  • Property svn:eol-style set to native
File size: 19.4 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.validation.tests;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.awt.geom.Area;
7import java.io.BufferedReader;
8import java.io.File;
9import java.io.IOException;
10import java.io.InputStream;
11import java.io.Reader;
12import java.util.ArrayList;
13import java.util.Collection;
14import java.util.HashMap;
15import java.util.HashSet;
16import java.util.Iterator;
17import java.util.List;
18import java.util.Map;
19import java.util.Map.Entry;
20import java.util.Objects;
21import java.util.Set;
22import java.util.function.Consumer;
23import java.util.function.Predicate;
24import java.util.stream.Stream;
25
26import org.openstreetmap.josm.data.osm.IPrimitive;
27import org.openstreetmap.josm.data.osm.OsmPrimitive;
28import org.openstreetmap.josm.data.preferences.BooleanProperty;
29import org.openstreetmap.josm.data.preferences.CachingProperty;
30import org.openstreetmap.josm.data.preferences.sources.SourceEntry;
31import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper;
32import org.openstreetmap.josm.data.validation.OsmValidator;
33import org.openstreetmap.josm.data.validation.Severity;
34import org.openstreetmap.josm.data.validation.Test;
35import org.openstreetmap.josm.data.validation.TestError;
36import org.openstreetmap.josm.gui.mappaint.Environment;
37import org.openstreetmap.josm.gui.mappaint.MultiCascade;
38import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule;
39import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleIndex;
40import org.openstreetmap.josm.gui.mappaint.mapcss.Selector;
41import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.ParseException;
42import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.TokenMgrError;
43import org.openstreetmap.josm.gui.progress.ProgressMonitor;
44import org.openstreetmap.josm.io.CachedFile;
45import org.openstreetmap.josm.io.FileWatcher;
46import org.openstreetmap.josm.io.UTFInputStreamReader;
47import org.openstreetmap.josm.spi.preferences.Config;
48import org.openstreetmap.josm.tools.CheckParameterUtil;
49import org.openstreetmap.josm.tools.I18n;
50import org.openstreetmap.josm.tools.Logging;
51import org.openstreetmap.josm.tools.MultiMap;
52import org.openstreetmap.josm.tools.Stopwatch;
53import org.openstreetmap.josm.tools.Utils;
54
55/**
56 * MapCSS-based tag checker/fixer.
57 * @since 6506
58 */
59public class MapCSSTagChecker extends Test.TagTest {
60 private MapCSSStyleIndex indexData;
61 private final Map<MapCSSRule, MapCSSTagCheckerAndRule> ruleToCheckMap = new HashMap<>();
62 private static final Map<IPrimitive, Area> mpAreaCache = new HashMap<>();
63 static final boolean ALL_TESTS = true;
64 static final boolean ONLY_SELECTED_TESTS = false;
65
66 /**
67 * Cached version of {@link ValidatorPrefHelper#PREF_OTHER}, see #20745.
68 */
69 private static final CachingProperty<Boolean> PREF_OTHER = new BooleanProperty("validator.other", false).cached();
70
71 /**
72 * The preference key for tag checker source entries.
73 * @since 6670
74 */
75 public static final String ENTRIES_PREF_KEY = "validator." + MapCSSTagChecker.class.getName() + ".entries";
76
77 /**
78 * Constructs a new {@code MapCSSTagChecker}.
79 */
80 public MapCSSTagChecker() {
81 super(tr("Tag checker (MapCSS based)"), tr("This test checks for errors in tag keys and values."));
82 }
83
84 final MultiMap<String, MapCSSTagCheckerRule> checks = new MultiMap<>();
85
86 /** maps the source URL for a test to the title shown in the dialog where known */
87 private final Map<String, String> urlTitles = new HashMap<>();
88
89 /**
90 * Result of {@link MapCSSTagCheckerRule#readMapCSS}
91 * @since 8936
92 */
93 public static class ParseResult {
94 /** Checks successfully parsed */
95 public final List<MapCSSTagCheckerRule> parseChecks;
96 /** Errors that occurred during parsing */
97 public final Collection<Throwable> parseErrors;
98
99 /**
100 * Constructs a new {@code ParseResult}.
101 * @param parseChecks Checks successfully parsed
102 * @param parseErrors Errors that occurred during parsing
103 */
104 public ParseResult(List<MapCSSTagCheckerRule> parseChecks, Collection<Throwable> parseErrors) {
105 this.parseChecks = parseChecks;
106 this.parseErrors = parseErrors;
107 }
108 }
109
110 static class MapCSSTagCheckerAndRule extends MapCSSTagChecker {
111 public final MapCSSRule rule;
112 private final MapCSSTagCheckerRule tagCheck;
113 private final String source;
114
115 MapCSSTagCheckerAndRule(MapCSSRule rule) {
116 this.rule = rule;
117 this.tagCheck = null;
118 this.source = "";
119 }
120
121 MapCSSTagCheckerAndRule(MapCSSTagCheckerRule tagCheck, String source) {
122 this.rule = tagCheck.rule;
123 this.tagCheck = tagCheck;
124 this.source = source;
125 }
126
127 @Override
128 public String toString() {
129 return "MapCSSTagCheckerAndRule [rule=" + rule + ']';
130 }
131
132 @Override
133 public String getSource() {
134 return source;
135 }
136 }
137
138 static MapCSSStyleIndex createMapCSSTagCheckerIndex(
139 MultiMap<String, MapCSSTagCheckerRule> checks, boolean includeOtherSeverity, boolean allTests) {
140 final MapCSSStyleIndex index = new MapCSSStyleIndex();
141 final Stream<MapCSSRule> ruleStream = checks.values().stream()
142 .flatMap(Collection::stream)
143 // Ignore "information" level checks if not wanted, unless they also set a MapCSS class
144 .filter(c -> includeOtherSeverity || Severity.OTHER != c.getSeverity() || !c.setClassExpressions.isEmpty())
145 .filter(c -> allTests || c.rule.selectors.stream().anyMatch(Selector.ChildOrParentSelector.class::isInstance))
146 .map(c -> c.rule);
147 index.buildIndex(ruleStream);
148 return index;
149 }
150
151 /**
152 * Obtains all {@link TestError}s for the {@link OsmPrimitive} {@code p}.
153 * @param p The OSM primitive
154 * @param includeOtherSeverity if {@code true}, errors of severity {@link Severity#OTHER} (info) will also be returned
155 * @return all errors for the given primitive, with or without those of "info" severity
156 */
157 public synchronized Collection<TestError> getErrorsForPrimitive(OsmPrimitive p, boolean includeOtherSeverity) {
158 final List<TestError> res = new ArrayList<>();
159 if (indexData == null) {
160 indexData = createMapCSSTagCheckerIndex(checks, includeOtherSeverity, ALL_TESTS);
161 }
162
163 Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null);
164 env.mpAreaCache = mpAreaCache;
165
166 Iterator<MapCSSRule> candidates = indexData.getRuleCandidates(p);
167 while (candidates.hasNext()) {
168 MapCSSRule r = candidates.next();
169 for (Selector selector : r.selectors) {
170 env.clearSelectorMatchingInformation();
171 if (!selector.matches(env)) { // as side effect env.parent will be set (if s is a child selector)
172 continue;
173 }
174 MapCSSTagCheckerAndRule test = ruleToCheckMap.computeIfAbsent(r, rule -> checks.entrySet().stream()
175 .map(e -> e.getValue().stream()
176 // rule.selectors might be different due to MapCSSStyleIndex, however, the declarations are the same object
177 .filter(c -> c.rule.declaration == rule.declaration)
178 .findFirst()
179 .map(c -> new MapCSSTagCheckerAndRule(c, getTitle(e.getKey())))
180 .orElse(null))
181 .filter(Objects::nonNull)
182 .findFirst()
183 .orElse(null));
184 MapCSSTagCheckerRule check = test == null ? null : test.tagCheck;
185 if (check != null) {
186 r.declaration.execute(env);
187 if (!check.errors.isEmpty()) {
188 for (TestError e: check.getErrorsForPrimitive(p, selector, env, test)) {
189 addIfNotSimilar(e, res);
190 }
191 }
192 }
193 }
194 }
195 return res;
196 }
197
198 private String getTitle(String url) {
199 return urlTitles.getOrDefault(url, tr("unknown"));
200 }
201
202 /**
203 * See #12627
204 * Add error to given list if list doesn't already contain a similar error.
205 * Similar means same code and description and same combination of primitives and same combination of highlighted objects,
206 * but maybe with different orders.
207 * @param toAdd the error to add
208 * @param errors the list of errors
209 */
210 private static void addIfNotSimilar(TestError toAdd, List<TestError> errors) {
211 final boolean isDup = toAdd.getPrimitives().size() >= 2 && errors.stream().anyMatch(toAdd::isSimilar);
212 if (!isDup)
213 errors.add(toAdd);
214 }
215
216 static Collection<TestError> getErrorsForPrimitive(
217 OsmPrimitive p, boolean includeOtherSeverity, Collection<Set<MapCSSTagCheckerRule>> checksCol) {
218 // this variant is only used by the assertion tests
219 final List<TestError> r = new ArrayList<>();
220 final Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null);
221 env.mpAreaCache = mpAreaCache;
222 for (Set<MapCSSTagCheckerRule> schecks : checksCol) {
223 for (MapCSSTagCheckerRule check : schecks) {
224 boolean ignoreError = Severity.OTHER == check.getSeverity() && !includeOtherSeverity;
225 // Do not run "information" level checks if not wanted, unless they also set a MapCSS class
226 if (ignoreError && check.setClassExpressions.isEmpty()) {
227 continue;
228 }
229 final Selector selector = check.whichSelectorMatchesEnvironment(env);
230 if (selector != null) {
231 check.rule.declaration.execute(env);
232 if (!ignoreError && !check.errors.isEmpty()) {
233 r.addAll(check.getErrorsForPrimitive(p, selector, env, new MapCSSTagCheckerAndRule(check.rule)));
234 }
235 }
236 }
237 }
238 return r;
239 }
240
241 /**
242 * Visiting call for primitives.
243 *
244 * @param p The primitive to inspect.
245 */
246 @Override
247 public void check(OsmPrimitive p) {
248 for (TestError e : getErrorsForPrimitive(p, PREF_OTHER.get())) {
249 addIfNotSimilar(e, errors);
250 }
251 }
252
253 /**
254 * A handler for assertion error messages (for not fulfilled "assertMatch", "assertNoMatch").
255 */
256 @FunctionalInterface
257 interface AssertionConsumer extends Consumer<String> {
258 }
259
260 /**
261 * Adds a new MapCSS config file from the given URL.
262 * @param url The unique URL of the MapCSS config file
263 * @return List of tag checks and parsing errors, or null
264 * @throws ParseException if the config file does not match MapCSS syntax
265 * @throws IOException if any I/O error occurs
266 * @since 7275
267 */
268 public synchronized ParseResult addMapCSS(String url) throws ParseException, IOException {
269 // Check assertions, useful for development of local files
270 final boolean checkAssertions = Config.getPref().getBoolean("validator.check_assert_local_rules", false) && Utils.isLocalUrl(url);
271 return addMapCSS(url, checkAssertions ? Logging::warn : null);
272 }
273
274 synchronized ParseResult addMapCSS(String url, AssertionConsumer assertionConsumer) throws ParseException, IOException {
275 CheckParameterUtil.ensureParameterNotNull(url, "url");
276 ParseResult result;
277 try (CachedFile cache = new CachedFile(url);
278 InputStream zip = cache.findZipEntryInputStream("validator.mapcss", "");
279 InputStream s = zip != null ? zip : cache.getInputStream();
280 Reader reader = new BufferedReader(UTFInputStreamReader.create(s))) {
281 if (zip != null)
282 I18n.addTexts(cache.getFile());
283 result = MapCSSTagCheckerRule.readMapCSS(reader, assertionConsumer);
284 checks.remove(url);
285 checks.putAll(url, result.parseChecks);
286 urlTitles.put(url, findURLTitle(url));
287 indexData = null;
288 }
289 return result;
290 }
291
292 /** Find a user friendly string for the url.
293 *
294 * @param url the source for the set of rules
295 * @return a value that can be used in tool tip or progress bar.
296 */
297 private static String findURLTitle(String url) {
298 for (SourceEntry source : new ValidatorPrefHelper().get()) {
299 if (url.equals(source.url) && source.title != null && !source.title.isEmpty()) {
300 return source.title;
301 }
302 }
303 if (url.endsWith(".mapcss")) // do we have others?
304 url = new File(url).getName();
305 if (url.length() > 33) {
306 url = "..." + url.substring(url.length() - 30);
307 }
308 return url;
309 }
310
311 @Override
312 public synchronized void initialize() throws Exception {
313 checks.clear();
314 urlTitles.clear();
315 indexData = null;
316 for (SourceEntry source : new ValidatorPrefHelper().get()) {
317 if (!source.active) {
318 continue;
319 }
320 String i = source.url;
321 try {
322 if (!i.startsWith("resource:")) {
323 Logging.info(tr("Adding {0} to tag checker", i));
324 } else if (Logging.isDebugEnabled()) {
325 Logging.debug(tr("Adding {0} to tag checker", i));
326 }
327 addMapCSS(i);
328 if (Config.getPref().getBoolean("validator.auto_reload_local_rules", true) && source.isLocal()) {
329 FileWatcher.getDefaultInstance().registerSource(source);
330 }
331 } catch (IOException | IllegalStateException | IllegalArgumentException ex) {
332 Logging.warn(tr("Failed to add {0} to tag checker", i));
333 Logging.log(Logging.LEVEL_WARN, ex);
334 } catch (ParseException | TokenMgrError ex) {
335 Logging.warn(tr("Failed to add {0} to tag checker", i));
336 Logging.warn(ex);
337 }
338 }
339 MapCSSTagCheckerAsserts.clear();
340 }
341
342 /**
343 * Reload tagchecker rule.
344 * @param rule tagchecker rule to reload
345 * @since 12825
346 */
347 public static void reloadRule(SourceEntry rule) {
348 MapCSSTagChecker tagChecker = OsmValidator.getTest(MapCSSTagChecker.class);
349 if (tagChecker != null) {
350 try {
351 tagChecker.addMapCSS(rule.url);
352 } catch (IOException | ParseException | TokenMgrError e) {
353 Logging.warn(e);
354 }
355 }
356 }
357
358 @Override
359 public synchronized void startTest(ProgressMonitor progressMonitor) {
360 super.startTest(progressMonitor);
361 super.setShowElements(true);
362 }
363
364 @Override
365 public synchronized void endTest() {
366 // no need to keep the index, it is quickly build and doubles the memory needs
367 indexData = null;
368 // always clear the cache to make sure that we catch changes in geometry
369 mpAreaCache.clear();
370 ruleToCheckMap.clear();
371 super.endTest();
372 }
373
374 @Override
375 public void visit(Collection<OsmPrimitive> selection) {
376 visit(selection, null);
377 }
378
379 /**
380 * Execute the rules from the URLs matching the given predicate.
381 * @param selection collection of primitives
382 * @param urlPredicate a predicate deciding whether the rules from the given URL shall be executed
383 */
384 void visit(Collection<OsmPrimitive> selection, Predicate<String> urlPredicate) {
385 if (urlPredicate == null && progressMonitor != null) {
386 progressMonitor.setTicksCount(selection.size() * checks.size());
387 }
388
389 mpAreaCache.clear();
390
391 Set<OsmPrimitive> surrounding = new HashSet<>();
392 for (Entry<String, Set<MapCSSTagCheckerRule>> entry : checks.entrySet()) {
393 if (isCanceled()) {
394 break;
395 }
396 if (urlPredicate != null && !urlPredicate.test(entry.getKey())) {
397 continue;
398 }
399 visit(entry.getKey(), entry.getValue(), selection, surrounding);
400 }
401 }
402
403 /**
404 * Perform the checks for one check url
405 * @param url the url for the checks
406 * @param checksForUrl the checks to perform
407 * @param selection collection primitives
408 * @param surrounding surrounding primitives, evtl. filled by this routine
409 */
410 private void visit(String url, Set<MapCSSTagCheckerRule> checksForUrl, Collection<OsmPrimitive> selection, Set<OsmPrimitive> surrounding) {
411 MultiMap<String, MapCSSTagCheckerRule> currentCheck = new MultiMap<>();
412 currentCheck.putAll(url, checksForUrl);
413 indexData = createMapCSSTagCheckerIndex(currentCheck, includeOtherSeverityChecks(), ALL_TESTS);
414 Set<OsmPrimitive> tested = new HashSet<>();
415
416
417 String title = getTitle(url);
418 if (progressMonitor != null) {
419 progressMonitor.setExtraText(tr(" {0}", title));
420 }
421 long cnt = 0;
422 Stopwatch stopwatch = Stopwatch.createStarted();
423 for (OsmPrimitive p : selection) {
424 if (isCanceled()) {
425 break;
426 }
427 if (isPrimitiveUsable(p)) {
428 check(p);
429 if (partialSelection) {
430 tested.add(p);
431 }
432 }
433 if (progressMonitor != null) {
434 progressMonitor.worked(1);
435 cnt++;
436 // add frequently changing info to progress monitor so that it
437 // doesn't seem to hang when test takes longer than 0.5 seconds
438 if (cnt % 10000 == 0 && stopwatch.elapsed() >= 500) {
439 progressMonitor.setExtraText(tr(" {0}: {1} of {2} elements done", title, cnt, selection.size()));
440 }
441 }
442 }
443
444 if (partialSelection && !tested.isEmpty()) {
445 testPartial(currentCheck, tested, surrounding);
446 }
447 }
448
449 private void testPartial(MultiMap<String, MapCSSTagCheckerRule> currentCheck, Set<OsmPrimitive> tested, Set<OsmPrimitive> surrounding) {
450
451 // #14287: see https://josm.openstreetmap.de/ticket/14287#comment:15
452 // execute tests for objects which might contain or cross previously tested elements
453
454 final boolean includeOtherSeverity = includeOtherSeverityChecks();
455 // rebuild index with a reduced set of rules (those that use ChildOrParentSelector) and thus may have left selectors
456 // matching the previously tested elements
457 indexData = createMapCSSTagCheckerIndex(currentCheck, includeOtherSeverity, ONLY_SELECTED_TESTS);
458 if (indexData.isEmpty())
459 return; // performance: some *.mapcss rule files don't use ChildOrParentSelector
460
461 if (surrounding.isEmpty()) {
462 for (OsmPrimitive p : tested) {
463 if (p.getDataSet() != null) {
464 surrounding.addAll(p.getDataSet().searchWays(p.getBBox()));
465 surrounding.addAll(p.getDataSet().searchRelations(p.getBBox()));
466 }
467 }
468 }
469
470 for (OsmPrimitive p : surrounding) {
471 if (tested.contains(p))
472 continue;
473 Collection<TestError> additionalErrors = getErrorsForPrimitive(p, includeOtherSeverity);
474 for (TestError e : additionalErrors) {
475 if (e.getPrimitives().stream().anyMatch(tested::contains))
476 addIfNotSimilar(e, errors);
477 }
478 }
479
480 }
481
482}
Note: See TracBrowser for help on using the repository browser.