source: josm/trunk/src/org/openstreetmap/josm/data/validation/tests/MapCSSTagChecker.java@ 17497

Last change on this file since 17497 was 17497, checked in by Don-vip, 3 years ago

fix #20498 - make sure fixChangeKey does not overwrite existing tag
  • Property svn:eol-style set to native
File size: 44.2 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.validation.tests;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.awt.geom.Area;
7import java.io.BufferedReader;
8import java.io.File;
9import java.io.IOException;
10import java.io.InputStream;
11import java.io.Reader;
12import java.io.StringReader;
13import java.util.ArrayList;
14import java.util.Collection;
15import java.util.HashMap;
16import java.util.HashSet;
17import java.util.Iterator;
18import java.util.List;
19import java.util.Map;
20import java.util.Map.Entry;
21import java.util.Objects;
22import java.util.Optional;
23import java.util.Set;
24import java.util.function.Consumer;
25import java.util.function.Predicate;
26import java.util.regex.Matcher;
27import java.util.regex.Pattern;
28import java.util.stream.Collectors;
29import java.util.stream.Stream;
30
31import org.openstreetmap.josm.command.ChangePropertyCommand;
32import org.openstreetmap.josm.command.ChangePropertyKeyCommand;
33import org.openstreetmap.josm.command.Command;
34import org.openstreetmap.josm.command.DeleteCommand;
35import org.openstreetmap.josm.command.SequenceCommand;
36import org.openstreetmap.josm.data.osm.IPrimitive;
37import org.openstreetmap.josm.data.osm.OsmPrimitive;
38import org.openstreetmap.josm.data.osm.Tag;
39import org.openstreetmap.josm.data.osm.Way;
40import org.openstreetmap.josm.data.osm.WaySegment;
41import org.openstreetmap.josm.data.preferences.sources.SourceEntry;
42import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper;
43import org.openstreetmap.josm.data.validation.OsmValidator;
44import org.openstreetmap.josm.data.validation.Severity;
45import org.openstreetmap.josm.data.validation.Test;
46import org.openstreetmap.josm.data.validation.TestError;
47import org.openstreetmap.josm.gui.mappaint.Environment;
48import org.openstreetmap.josm.gui.mappaint.Keyword;
49import org.openstreetmap.josm.gui.mappaint.MultiCascade;
50import org.openstreetmap.josm.gui.mappaint.mapcss.Condition;
51import org.openstreetmap.josm.gui.mappaint.mapcss.Expression;
52import org.openstreetmap.josm.gui.mappaint.mapcss.Instruction;
53import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule;
54import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleIndex;
55import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleSource;
56import org.openstreetmap.josm.gui.mappaint.mapcss.Selector;
57import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.GeneralSelector;
58import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.MapCSSParser;
59import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.ParseException;
60import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.TokenMgrError;
61import org.openstreetmap.josm.gui.progress.ProgressMonitor;
62import org.openstreetmap.josm.io.CachedFile;
63import org.openstreetmap.josm.io.FileWatcher;
64import org.openstreetmap.josm.io.IllegalDataException;
65import org.openstreetmap.josm.io.UTFInputStreamReader;
66import org.openstreetmap.josm.spi.preferences.Config;
67import org.openstreetmap.josm.tools.CheckParameterUtil;
68import org.openstreetmap.josm.tools.I18n;
69import org.openstreetmap.josm.tools.Logging;
70import org.openstreetmap.josm.tools.MultiMap;
71import org.openstreetmap.josm.tools.Stopwatch;
72import org.openstreetmap.josm.tools.Utils;
73
74/**
75 * MapCSS-based tag checker/fixer.
76 * @since 6506
77 */
78public class MapCSSTagChecker extends Test.TagTest {
79 private MapCSSStyleIndex indexData;
80 private final Map<MapCSSRule, MapCSSTagCheckerAndRule> ruleToCheckMap = new HashMap<>();
81 private static final Map<IPrimitive, Area> mpAreaCache = new HashMap<>();
82 static final boolean ALL_TESTS = true;
83 static final boolean ONLY_SELECTED_TESTS = false;
84
85 /**
86 * The preference key for tag checker source entries.
87 * @since 6670
88 */
89 public static final String ENTRIES_PREF_KEY = "validator." + MapCSSTagChecker.class.getName() + ".entries";
90
91 /**
92 * Constructs a new {@code MapCSSTagChecker}.
93 */
94 public MapCSSTagChecker() {
95 super(tr("Tag checker (MapCSS based)"), tr("This test checks for errors in tag keys and values."));
96 }
97
98 /**
99 * Represents a fix to a validation test. The fixing {@link Command} can be obtained by {@link #createCommand(OsmPrimitive, Selector)}.
100 */
101 @FunctionalInterface
102 interface FixCommand {
103 /**
104 * Creates the fixing {@link Command} for the given primitive. The {@code matchingSelector} is used to evaluate placeholders
105 * (cf. {@link MapCSSTagChecker.TagCheck#insertArguments(Selector, String, OsmPrimitive)}).
106 * @param p OSM primitive
107 * @param matchingSelector matching selector
108 * @return fix command, or {@code null} if if cannot be created
109 */
110 Command createCommand(OsmPrimitive p, Selector matchingSelector);
111
112 /**
113 * Checks that object is either an {@link Expression} or a {@link String}.
114 * @param obj object to check
115 * @throws IllegalArgumentException if object is not an {@code Expression} or a {@code String}
116 */
117 static void checkObject(final Object obj) {
118 CheckParameterUtil.ensureThat(obj instanceof Expression || obj instanceof String,
119 () -> "instance of Exception or String expected, but got " + obj);
120 }
121
122 /**
123 * Evaluates given object as {@link Expression} or {@link String} on the matched {@link OsmPrimitive} and {@code matchingSelector}.
124 * @param obj object to evaluate ({@link Expression} or {@link String})
125 * @param p OSM primitive
126 * @param matchingSelector matching selector
127 * @return result string
128 */
129 static String evaluateObject(final Object obj, final OsmPrimitive p, final Selector matchingSelector) {
130 final String s;
131 if (obj instanceof Expression) {
132 s = (String) ((Expression) obj).evaluate(new Environment(p));
133 } else if (obj instanceof String) {
134 s = (String) obj;
135 } else {
136 return null;
137 }
138 return TagCheck.insertArguments(matchingSelector, s, p);
139 }
140
141 /**
142 * Creates a fixing command which executes a {@link ChangePropertyCommand} on the specified tag.
143 * @param obj object to evaluate ({@link Expression} or {@link String})
144 * @return created fix command
145 */
146 static FixCommand fixAdd(final Object obj) {
147 checkObject(obj);
148 return new FixCommand() {
149 @Override
150 public Command createCommand(OsmPrimitive p, Selector matchingSelector) {
151 final Tag tag = Tag.ofString(FixCommand.evaluateObject(obj, p, matchingSelector));
152 return new ChangePropertyCommand(p, tag.getKey(), tag.getValue());
153 }
154
155 @Override
156 public String toString() {
157 return "fixAdd: " + obj;
158 }
159 };
160 }
161
162 /**
163 * Creates a fixing command which executes a {@link ChangePropertyCommand} to delete the specified key.
164 * @param obj object to evaluate ({@link Expression} or {@link String})
165 * @return created fix command
166 */
167 static FixCommand fixRemove(final Object obj) {
168 checkObject(obj);
169 return new FixCommand() {
170 @Override
171 public Command createCommand(OsmPrimitive p, Selector matchingSelector) {
172 final String key = FixCommand.evaluateObject(obj, p, matchingSelector);
173 return new ChangePropertyCommand(p, key, "");
174 }
175
176 @Override
177 public String toString() {
178 return "fixRemove: " + obj;
179 }
180 };
181 }
182
183 /**
184 * Creates a fixing command which executes a {@link ChangePropertyKeyCommand} on the specified keys
185 * if {@code p} does not already have {@code newKey}
186 * @param oldKey old key
187 * @param newKey new key
188 * @return created fix command, or {@code null}
189 */
190 static FixCommand fixChangeKey(final String oldKey, final String newKey) {
191 return new FixCommand() {
192 @Override
193 public Command createCommand(OsmPrimitive p, Selector matchingSelector) {
194 return p.hasKey(newKey) ? null : new ChangePropertyKeyCommand(p,
195 TagCheck.insertArguments(matchingSelector, oldKey, p),
196 TagCheck.insertArguments(matchingSelector, newKey, p));
197 }
198
199 @Override
200 public String toString() {
201 return "fixChangeKey: " + oldKey + " => " + newKey;
202 }
203 };
204 }
205 }
206
207 final MultiMap<String, TagCheck> checks = new MultiMap<>();
208
209 /** maps the source URL for a test to the title shown in the dialog where known */
210 private final Map<String, String> urlTitles = new HashMap<>();
211
212 /**
213 * Result of {@link TagCheck#readMapCSS}
214 * @since 8936
215 */
216 public static class ParseResult {
217 /** Checks successfully parsed */
218 public final List<TagCheck> parseChecks;
219 /** Errors that occurred during parsing */
220 public final Collection<Throwable> parseErrors;
221
222 /**
223 * Constructs a new {@code ParseResult}.
224 * @param parseChecks Checks successfully parsed
225 * @param parseErrors Errors that occurred during parsing
226 */
227 public ParseResult(List<TagCheck> parseChecks, Collection<Throwable> parseErrors) {
228 this.parseChecks = parseChecks;
229 this.parseErrors = parseErrors;
230 }
231 }
232
233 /**
234 * Tag check.
235 */
236 public static class TagCheck implements Predicate<OsmPrimitive> {
237 /** The selector of this {@code TagCheck} */
238 protected final MapCSSRule rule;
239 /** Commands to apply in order to fix a matching primitive */
240 protected final List<FixCommand> fixCommands;
241 /** Tags (or arbitrary strings) of alternatives to be presented to the user */
242 protected final List<String> alternatives;
243 /** An {@link org.openstreetmap.josm.gui.mappaint.mapcss.Instruction.AssignmentInstruction}-{@link Severity} pair.
244 * Is evaluated on the matching primitive to give the error message. Map is checked to contain exactly one element. */
245 protected final Map<Instruction.AssignmentInstruction, Severity> errors;
246 /** MapCSS Classes to set on matching primitives */
247 protected final Collection<String> setClassExpressions;
248 /** Denotes whether the object should be deleted for fixing it */
249 protected boolean deletion;
250 /** A string used to group similar tests */
251 protected String group;
252
253 TagCheck(MapCSSRule rule) {
254 this.rule = rule;
255 this.fixCommands = new ArrayList<>();
256 this.alternatives = new ArrayList<>();
257 this.errors = new HashMap<>();
258 this.setClassExpressions = new HashSet<>();
259 }
260
261 TagCheck(TagCheck check) {
262 this.rule = check.rule;
263 this.fixCommands = Utils.toUnmodifiableList(check.fixCommands);
264 this.alternatives = Utils.toUnmodifiableList(check.alternatives);
265 this.errors = Utils.toUnmodifiableMap(check.errors);
266 this.setClassExpressions = Utils.toUnmodifiableList(check.setClassExpressions);
267 this.deletion = check.deletion;
268 this.group = check.group;
269 }
270
271 TagCheck toImmutable() {
272 return new TagCheck(this);
273 }
274
275 private static final String POSSIBLE_THROWS = "throwError/throwWarning/throwOther";
276
277 static TagCheck ofMapCSSRule(final MapCSSRule rule, AssertionConsumer assertionConsumer) throws IllegalDataException {
278 final TagCheck check = new TagCheck(rule);
279 final Map<String, Boolean> assertions = new HashMap<>();
280 for (Instruction i : rule.declaration.instructions) {
281 if (i instanceof Instruction.AssignmentInstruction) {
282 final Instruction.AssignmentInstruction ai = (Instruction.AssignmentInstruction) i;
283 if (ai.isSetInstruction) {
284 check.setClassExpressions.add(ai.key);
285 continue;
286 }
287 try {
288 final String val = ai.val instanceof Expression
289 ? Optional.ofNullable(((Expression) ai.val).evaluate(new Environment()))
290 .map(Object::toString).map(String::intern).orElse(null)
291 : ai.val instanceof String
292 ? (String) ai.val
293 : ai.val instanceof Keyword
294 ? ((Keyword) ai.val).val
295 : null;
296 if ("throwError".equals(ai.key)) {
297 check.errors.put(ai, Severity.ERROR);
298 } else if ("throwWarning".equals(ai.key)) {
299 check.errors.put(ai, Severity.WARNING);
300 } else if ("throwOther".equals(ai.key)) {
301 check.errors.put(ai, Severity.OTHER);
302 } else if (ai.key.startsWith("throw")) {
303 Logging.log(Logging.LEVEL_WARN,
304 "Unsupported " + ai.key + " instruction. Allowed instructions are " + POSSIBLE_THROWS + '.', null);
305 } else if ("fixAdd".equals(ai.key)) {
306 check.fixCommands.add(FixCommand.fixAdd(ai.val));
307 } else if ("fixRemove".equals(ai.key)) {
308 CheckParameterUtil.ensureThat(!(ai.val instanceof String) || !(val != null && val.contains("=")),
309 "Unexpected '='. Please only specify the key to remove in: " + ai);
310 check.fixCommands.add(FixCommand.fixRemove(ai.val));
311 } else if (val != null && "fixChangeKey".equals(ai.key)) {
312 CheckParameterUtil.ensureThat(val.contains("=>"), "Separate old from new key by '=>'!");
313 final String[] x = val.split("=>", 2);
314 check.fixCommands.add(FixCommand.fixChangeKey(Utils.removeWhiteSpaces(x[0]), Utils.removeWhiteSpaces(x[1])));
315 } else if (val != null && "fixDeleteObject".equals(ai.key)) {
316 CheckParameterUtil.ensureThat("this".equals(val), "fixDeleteObject must be followed by 'this'");
317 check.deletion = true;
318 } else if (val != null && "suggestAlternative".equals(ai.key)) {
319 check.alternatives.add(val);
320 } else if (val != null && "assertMatch".equals(ai.key)) {
321 assertions.put(val, Boolean.TRUE);
322 } else if (val != null && "assertNoMatch".equals(ai.key)) {
323 assertions.put(val, Boolean.FALSE);
324 } else if (val != null && "group".equals(ai.key)) {
325 check.group = val;
326 } else if (ai.key.startsWith("-")) {
327 Logging.debug("Ignoring extension instruction: " + ai.key + ": " + ai.val);
328 } else {
329 throw new IllegalDataException("Cannot add instruction " + ai.key + ": " + ai.val + '!');
330 }
331 } catch (IllegalArgumentException e) {
332 throw new IllegalDataException(e);
333 }
334 }
335 }
336 if (check.errors.isEmpty() && check.setClassExpressions.isEmpty()) {
337 throw new IllegalDataException(
338 "No "+POSSIBLE_THROWS+" given! You should specify a validation error message for " + rule.selectors);
339 } else if (check.errors.size() > 1) {
340 throw new IllegalDataException(
341 "More than one "+POSSIBLE_THROWS+" given! You should specify a single validation error message for "
342 + rule.selectors);
343 }
344 if (assertionConsumer != null) {
345 MapCSSTagCheckerAsserts.checkAsserts(check, assertions, assertionConsumer);
346 }
347 return check.toImmutable();
348 }
349
350 static ParseResult readMapCSS(Reader css) throws ParseException {
351 return readMapCSS(css, null);
352 }
353
354 static ParseResult readMapCSS(Reader css, AssertionConsumer assertionConsumer) throws ParseException {
355 CheckParameterUtil.ensureParameterNotNull(css, "css");
356
357 final MapCSSStyleSource source = new MapCSSStyleSource("");
358 final MapCSSParser preprocessor = new MapCSSParser(css, MapCSSParser.LexicalState.PREPROCESSOR);
359 try (StringReader mapcss = new StringReader(preprocessor.pp_root(source))) {
360 new MapCSSParser(mapcss, MapCSSParser.LexicalState.DEFAULT).sheet(source);
361 }
362 // Ignore "meta" rule(s) from external rules of JOSM wiki
363 source.removeMetaRules();
364 List<TagCheck> parseChecks = new ArrayList<>();
365 for (MapCSSRule rule : source.rules) {
366 try {
367 parseChecks.add(TagCheck.ofMapCSSRule(rule, assertionConsumer));
368 } catch (IllegalDataException e) {
369 Logging.error("Cannot add MapCSS rule: "+e.getMessage());
370 source.logError(e);
371 }
372 }
373 return new ParseResult(parseChecks, source.getErrors());
374 }
375
376 @Override
377 public boolean test(OsmPrimitive primitive) {
378 // Tests whether the primitive contains a deprecated tag which is represented by this MapCSSTagChecker.
379 return whichSelectorMatchesPrimitive(primitive) != null;
380 }
381
382 Selector whichSelectorMatchesPrimitive(OsmPrimitive primitive) {
383 return whichSelectorMatchesEnvironment(new Environment(primitive));
384 }
385
386 Selector whichSelectorMatchesEnvironment(Environment env) {
387 return rule.selectors.stream()
388 .filter(i -> i.matches(env.clearSelectorMatchingInformation()))
389 .findFirst()
390 .orElse(null);
391 }
392
393 /**
394 * Determines the {@code index}-th key/value/tag (depending on {@code type}) of the
395 * {@link GeneralSelector}.
396 * @param matchingSelector matching selector
397 * @param index index
398 * @param type selector type ("key", "value" or "tag")
399 * @param p OSM primitive
400 * @return argument value, can be {@code null}
401 */
402 static String determineArgument(GeneralSelector matchingSelector, int index, String type, OsmPrimitive p) {
403 try {
404 final Condition c = matchingSelector.getConditions().get(index);
405 final Tag tag = c instanceof Condition.ToTagConvertable
406 ? ((Condition.ToTagConvertable) c).asTag(p)
407 : null;
408 if (tag == null) {
409 return null;
410 } else if ("key".equals(type)) {
411 return tag.getKey();
412 } else if ("value".equals(type)) {
413 return tag.getValue();
414 } else if ("tag".equals(type)) {
415 return tag.toString();
416 }
417 } catch (IndexOutOfBoundsException ignore) {
418 Logging.debug(ignore);
419 }
420 return null;
421 }
422
423 /**
424 * Replaces occurrences of <code>{i.key}</code>, <code>{i.value}</code>, <code>{i.tag}</code> in {@code s} by the corresponding
425 * key/value/tag of the {@code index}-th {@link Condition} of {@code matchingSelector}.
426 * @param matchingSelector matching selector
427 * @param s any string
428 * @param p OSM primitive
429 * @return string with arguments inserted
430 */
431 static String insertArguments(Selector matchingSelector, String s, OsmPrimitive p) {
432 if (s != null && matchingSelector instanceof Selector.ChildOrParentSelector) {
433 return insertArguments(((Selector.ChildOrParentSelector) matchingSelector).right, s, p);
434 } else if (s == null || !(matchingSelector instanceof GeneralSelector)) {
435 return s;
436 }
437 final Matcher m = Pattern.compile("\\{(\\d+)\\.(key|value|tag)\\}").matcher(s);
438 final StringBuffer sb = new StringBuffer();
439 while (m.find()) {
440 final String argument = determineArgument((GeneralSelector) matchingSelector,
441 Integer.parseInt(m.group(1)), m.group(2), p);
442 try {
443 // Perform replacement with null-safe + regex-safe handling
444 m.appendReplacement(sb, String.valueOf(argument).replace("^(", "").replace(")$", ""));
445 } catch (IndexOutOfBoundsException | IllegalArgumentException e) {
446 Logging.log(Logging.LEVEL_ERROR, tr("Unable to replace argument {0} in {1}: {2}", argument, sb, e.getMessage()), e);
447 }
448 }
449 m.appendTail(sb);
450 return sb.toString();
451 }
452
453 /**
454 * Constructs a fix in terms of a {@link org.openstreetmap.josm.command.Command} for the {@link OsmPrimitive}
455 * if the error is fixable, or {@code null} otherwise.
456 *
457 * @param p the primitive to construct the fix for
458 * @return the fix or {@code null}
459 */
460 Command fixPrimitive(OsmPrimitive p) {
461 if (p.getDataSet() == null || (fixCommands.isEmpty() && !deletion)) {
462 return null;
463 }
464 try {
465 final Selector matchingSelector = whichSelectorMatchesPrimitive(p);
466 Collection<Command> cmds = fixCommands.stream()
467 .map(fixCommand -> fixCommand.createCommand(p, matchingSelector))
468 .filter(Objects::nonNull)
469 .collect(Collectors.toList());
470 if (deletion && !p.isDeleted()) {
471 cmds.add(new DeleteCommand(p));
472 }
473 return cmds.isEmpty() ? null
474 : new SequenceCommand(tr("Fix of {0}", getDescriptionForMatchingSelector(p, matchingSelector)), cmds);
475 } catch (IllegalArgumentException e) {
476 Logging.error(e);
477 return null;
478 }
479 }
480
481 /**
482 * Constructs a (localized) message for this deprecation check.
483 * @param p OSM primitive
484 *
485 * @return a message
486 */
487 String getMessage(OsmPrimitive p) {
488 if (errors.isEmpty()) {
489 // Return something to avoid NPEs
490 return rule.declaration.toString();
491 } else {
492 final Object val = errors.keySet().iterator().next().val;
493 return String.valueOf(
494 val instanceof Expression
495 ? ((Expression) val).evaluate(new Environment(p))
496 : val
497 );
498 }
499 }
500
501 /**
502 * Constructs a (localized) description for this deprecation check.
503 * @param p OSM primitive
504 *
505 * @return a description (possibly with alternative suggestions)
506 * @see #getDescriptionForMatchingSelector
507 */
508 String getDescription(OsmPrimitive p) {
509 if (alternatives.isEmpty()) {
510 return getMessage(p);
511 } else {
512 /* I18N: {0} is the test error message and {1} is an alternative */
513 return tr("{0}, use {1} instead", getMessage(p), String.join(tr(" or "), alternatives));
514 }
515 }
516
517 /**
518 * Constructs a (localized) description for this deprecation check
519 * where any placeholders are replaced by values of the matched selector.
520 *
521 * @param matchingSelector matching selector
522 * @param p OSM primitive
523 * @return a description (possibly with alternative suggestions)
524 */
525 String getDescriptionForMatchingSelector(OsmPrimitive p, Selector matchingSelector) {
526 return insertArguments(matchingSelector, getDescription(p), p);
527 }
528
529 Severity getSeverity() {
530 return errors.isEmpty() ? null : errors.values().iterator().next();
531 }
532
533 @Override
534 public String toString() {
535 return getDescription(null);
536 }
537
538 /**
539 * Constructs a {@link TestError} for the given primitive, or returns null if the primitive does not give rise to an error.
540 *
541 * @param p the primitive to construct the error for
542 * @param matchingSelector the matching selector (e.g., obtained via {@link #whichSelectorMatchesPrimitive})
543 * @param env the environment
544 * @param tester the tester
545 * @return an instance of {@link TestError}, or returns null if the primitive does not give rise to an error.
546 */
547 protected List<TestError> getErrorsForPrimitive(OsmPrimitive p, Selector matchingSelector, Environment env, Test tester) {
548 List<TestError> res = new ArrayList<>();
549 if (matchingSelector != null && !errors.isEmpty()) {
550 final Command fix = fixPrimitive(p);
551 final String description = getDescriptionForMatchingSelector(p, matchingSelector);
552 final String description1 = group == null ? description : group;
553 final String description2 = group == null ? null : description;
554 final String selector = matchingSelector.toString();
555 TestError.Builder errorBuilder = TestError.builder(tester, getSeverity(), 3000)
556 .messageWithManuallyTranslatedDescription(description1, description2, selector);
557 if (fix != null) {
558 errorBuilder.fix(() -> fix);
559 }
560 if (env.child instanceof OsmPrimitive) {
561 res.add(errorBuilder.primitives(p, (OsmPrimitive) env.child).build());
562 } else if (env.children != null) {
563 for (IPrimitive c : env.children) {
564 if (c instanceof OsmPrimitive) {
565 errorBuilder = TestError.builder(tester, getSeverity(), 3000)
566 .messageWithManuallyTranslatedDescription(description1, description2, selector);
567 if (fix != null) {
568 errorBuilder.fix(() -> fix);
569 }
570 // check if we have special information about highlighted objects */
571 boolean hiliteFound = false;
572 if (env.intersections != null) {
573 Area is = env.intersections.get(c);
574 if (is != null) {
575 errorBuilder.highlight(is);
576 hiliteFound = true;
577 }
578 }
579 if (env.crossingWaysMap != null && !hiliteFound) {
580 Map<List<Way>, List<WaySegment>> is = env.crossingWaysMap.get(c);
581 if (is != null) {
582 Set<WaySegment> toHilite = new HashSet<>();
583 for (List<WaySegment> wsList : is.values()) {
584 toHilite.addAll(wsList);
585 }
586 errorBuilder.highlightWaySegments(toHilite);
587 }
588 }
589 res.add(errorBuilder.primitives(p, (OsmPrimitive) c).build());
590 }
591 }
592 } else {
593 res.add(errorBuilder.primitives(p).build());
594 }
595 }
596 return res;
597 }
598
599 }
600
601 static class MapCSSTagCheckerAndRule extends MapCSSTagChecker {
602 public final MapCSSRule rule;
603 private final TagCheck tagCheck;
604 private final String source;
605
606 MapCSSTagCheckerAndRule(MapCSSRule rule) {
607 this.rule = rule;
608 this.tagCheck = null;
609 this.source = "";
610 }
611
612 MapCSSTagCheckerAndRule(TagCheck tagCheck, String source) {
613 this.rule = tagCheck.rule;
614 this.tagCheck = tagCheck;
615 this.source = source;
616 }
617
618 @Override
619 public String toString() {
620 return "MapCSSTagCheckerAndRule [rule=" + rule + ']';
621 }
622
623 @Override
624 public String getSource() {
625 return source;
626 }
627 }
628
629 static MapCSSStyleIndex createMapCSSTagCheckerIndex(MultiMap<String, TagCheck> checks, boolean includeOtherSeverity, boolean allTests) {
630 final MapCSSStyleIndex index = new MapCSSStyleIndex();
631 final Stream<MapCSSRule> ruleStream = checks.values().stream()
632 .flatMap(Collection::stream)
633 // Ignore "information" level checks if not wanted, unless they also set a MapCSS class
634 .filter(c -> includeOtherSeverity || Severity.OTHER != c.getSeverity() || !c.setClassExpressions.isEmpty())
635 .filter(c -> allTests || c.rule.selectors.stream().anyMatch(Selector.ChildOrParentSelector.class::isInstance))
636 .map(c -> c.rule);
637 index.buildIndex(ruleStream);
638 return index;
639 }
640
641 /**
642 * Obtains all {@link TestError}s for the {@link OsmPrimitive} {@code p}.
643 * @param p The OSM primitive
644 * @param includeOtherSeverity if {@code true}, errors of severity {@link Severity#OTHER} (info) will also be returned
645 * @return all errors for the given primitive, with or without those of "info" severity
646 */
647 public synchronized Collection<TestError> getErrorsForPrimitive(OsmPrimitive p, boolean includeOtherSeverity) {
648 final List<TestError> res = new ArrayList<>();
649 if (indexData == null) {
650 indexData = createMapCSSTagCheckerIndex(checks, includeOtherSeverity, ALL_TESTS);
651 }
652
653 Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null);
654 env.mpAreaCache = mpAreaCache;
655
656 Iterator<MapCSSRule> candidates = indexData.getRuleCandidates(p);
657 while (candidates.hasNext()) {
658 MapCSSRule r = candidates.next();
659 for (Selector selector : r.selectors) {
660 env.clearSelectorMatchingInformation();
661 if (!selector.matches(env)) { // as side effect env.parent will be set (if s is a child selector)
662 continue;
663 }
664 MapCSSTagCheckerAndRule test = ruleToCheckMap.computeIfAbsent(r, rule -> checks.entrySet().stream()
665 .map(e -> e.getValue().stream()
666 // rule.selectors might be different due to MapCSSStyleIndex, however, the declarations are the same object
667 .filter(c -> c.rule.declaration == rule.declaration)
668 .findFirst()
669 .map(c -> new MapCSSTagCheckerAndRule(c, getTitle(e.getKey())))
670 .orElse(null))
671 .filter(Objects::nonNull)
672 .findFirst()
673 .orElse(null));
674 TagCheck check = test == null ? null : test.tagCheck;
675 if (check != null) {
676 r.declaration.execute(env);
677 if (!check.errors.isEmpty()) {
678 for (TestError e: check.getErrorsForPrimitive(p, selector, env, test)) {
679 addIfNotSimilar(e, res);
680 }
681 }
682 }
683 }
684 }
685 return res;
686 }
687
688 private String getTitle(String url) {
689 return urlTitles.getOrDefault(url, tr("unknown"));
690 }
691
692 /**
693 * See #12627
694 * Add error to given list if list doesn't already contain a similar error.
695 * Similar means same code and description and same combination of primitives and same combination of highlighted objects,
696 * but maybe with different orders.
697 * @param toAdd the error to add
698 * @param errors the list of errors
699 */
700 private static void addIfNotSimilar(TestError toAdd, List<TestError> errors) {
701 final boolean isDup = toAdd.getPrimitives().size() >= 2 && errors.stream()
702 .anyMatch(e -> e.getCode() == toAdd.getCode()
703 && e.getMessage().equals(toAdd.getMessage())
704 && e.getPrimitives().size() == toAdd.getPrimitives().size()
705 && e.getPrimitives().containsAll(toAdd.getPrimitives())
706 && highlightedIsEqual(e.getHighlighted(), toAdd.getHighlighted()));
707 if (!isDup)
708 errors.add(toAdd);
709 }
710
711 private static boolean highlightedIsEqual(Collection<?> highlighted, Collection<?> highlighted2) {
712 if (highlighted.size() == highlighted2.size()) {
713 if (!highlighted.isEmpty()) {
714 Object h1 = highlighted.iterator().next();
715 Object h2 = highlighted2.iterator().next();
716 if (h1 instanceof Area && h2 instanceof Area) {
717 return ((Area) h1).equals((Area) h2);
718 }
719 return highlighted.containsAll(highlighted2);
720 }
721 return true;
722 }
723 return false;
724 }
725
726 static Collection<TestError> getErrorsForPrimitive(OsmPrimitive p, boolean includeOtherSeverity,
727 Collection<Set<TagCheck>> checksCol) {
728 // this variant is only used by the assertion tests
729 final List<TestError> r = new ArrayList<>();
730 final Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null);
731 env.mpAreaCache = mpAreaCache;
732 for (Set<TagCheck> schecks : checksCol) {
733 for (TagCheck check : schecks) {
734 boolean ignoreError = Severity.OTHER == check.getSeverity() && !includeOtherSeverity;
735 // Do not run "information" level checks if not wanted, unless they also set a MapCSS class
736 if (ignoreError && check.setClassExpressions.isEmpty()) {
737 continue;
738 }
739 final Selector selector = check.whichSelectorMatchesEnvironment(env);
740 if (selector != null) {
741 check.rule.declaration.execute(env);
742 if (!ignoreError && !check.errors.isEmpty()) {
743 r.addAll(check.getErrorsForPrimitive(p, selector, env, new MapCSSTagCheckerAndRule(check.rule)));
744 }
745 }
746 }
747 }
748 return r;
749 }
750
751 /**
752 * Visiting call for primitives.
753 *
754 * @param p The primitive to inspect.
755 */
756 @Override
757 public void check(OsmPrimitive p) {
758 for (TestError e : getErrorsForPrimitive(p, ValidatorPrefHelper.PREF_OTHER.get())) {
759 addIfNotSimilar(e, errors);
760 }
761 }
762
763 /**
764 * A handler for assertion error messages (for not fulfilled "assertMatch", "assertNoMatch").
765 */
766 @FunctionalInterface
767 interface AssertionConsumer extends Consumer<String> {
768 }
769
770 /**
771 * Adds a new MapCSS config file from the given URL.
772 * @param url The unique URL of the MapCSS config file
773 * @return List of tag checks and parsing errors, or null
774 * @throws ParseException if the config file does not match MapCSS syntax
775 * @throws IOException if any I/O error occurs
776 * @since 7275
777 */
778 public synchronized ParseResult addMapCSS(String url) throws ParseException, IOException {
779 // Check assertions, useful for development of local files
780 final boolean checkAssertions = Config.getPref().getBoolean("validator.check_assert_local_rules", false) && Utils.isLocalUrl(url);
781 return addMapCSS(url, checkAssertions ? Logging::warn : null);
782 }
783
784 synchronized ParseResult addMapCSS(String url, AssertionConsumer assertionConsumer) throws ParseException, IOException {
785 CheckParameterUtil.ensureParameterNotNull(url, "url");
786 ParseResult result;
787 try (CachedFile cache = new CachedFile(url);
788 InputStream zip = cache.findZipEntryInputStream("validator.mapcss", "");
789 InputStream s = zip != null ? zip : cache.getInputStream();
790 Reader reader = new BufferedReader(UTFInputStreamReader.create(s))) {
791 if (zip != null)
792 I18n.addTexts(cache.getFile());
793 result = TagCheck.readMapCSS(reader, assertionConsumer);
794 checks.remove(url);
795 checks.putAll(url, result.parseChecks);
796 urlTitles.put(url, findURLTitle(url));
797 indexData = null;
798 }
799 return result;
800 }
801
802 /** Find a user friendly string for the url.
803 *
804 * @param url the source for the set of rules
805 * @return a value that can be used in tool tip or progress bar.
806 */
807 private static String findURLTitle(String url) {
808 for (SourceEntry source : new ValidatorPrefHelper().get()) {
809 if (url.equals(source.url) && source.title != null && !source.title.isEmpty()) {
810 return source.title;
811 }
812 }
813 if (url.endsWith(".mapcss")) // do we have others?
814 url = new File(url).getName();
815 if (url.length() > 33) {
816 url = "..." + url.substring(url.length() - 30);
817 }
818 return url;
819 }
820
821 @Override
822 public synchronized void initialize() throws Exception {
823 checks.clear();
824 urlTitles.clear();
825 indexData = null;
826 for (SourceEntry source : new ValidatorPrefHelper().get()) {
827 if (!source.active) {
828 continue;
829 }
830 String i = source.url;
831 try {
832 if (!i.startsWith("resource:")) {
833 Logging.info(tr("Adding {0} to tag checker", i));
834 } else if (Logging.isDebugEnabled()) {
835 Logging.debug(tr("Adding {0} to tag checker", i));
836 }
837 addMapCSS(i);
838 if (Config.getPref().getBoolean("validator.auto_reload_local_rules", true) && source.isLocal()) {
839 FileWatcher.getDefaultInstance().registerSource(source);
840 }
841 } catch (IOException | IllegalStateException | IllegalArgumentException ex) {
842 Logging.warn(tr("Failed to add {0} to tag checker", i));
843 Logging.log(Logging.LEVEL_WARN, ex);
844 } catch (ParseException | TokenMgrError ex) {
845 Logging.warn(tr("Failed to add {0} to tag checker", i));
846 Logging.warn(ex);
847 }
848 }
849 MapCSSTagCheckerAsserts.clear();
850 }
851
852 /**
853 * Reload tagchecker rule.
854 * @param rule tagchecker rule to reload
855 * @since 12825
856 */
857 public static void reloadRule(SourceEntry rule) {
858 MapCSSTagChecker tagChecker = OsmValidator.getTest(MapCSSTagChecker.class);
859 if (tagChecker != null) {
860 try {
861 tagChecker.addMapCSS(rule.url);
862 } catch (IOException | ParseException | TokenMgrError e) {
863 Logging.warn(e);
864 }
865 }
866 }
867
868 @Override
869 public synchronized void startTest(ProgressMonitor progressMonitor) {
870 super.startTest(progressMonitor);
871 super.setShowElements(true);
872 }
873
874 @Override
875 public synchronized void endTest() {
876 // no need to keep the index, it is quickly build and doubles the memory needs
877 indexData = null;
878 // always clear the cache to make sure that we catch changes in geometry
879 mpAreaCache.clear();
880 ruleToCheckMap.clear();
881 super.endTest();
882 }
883
884 @Override
885 public void visit(Collection<OsmPrimitive> selection) {
886 if (progressMonitor != null) {
887 progressMonitor.setTicksCount(selection.size() * checks.size());
888 }
889
890 mpAreaCache.clear();
891
892 Set<OsmPrimitive> surrounding = new HashSet<>();
893 for (Entry<String, Set<TagCheck>> entry : checks.entrySet()) {
894 if (isCanceled()) {
895 break;
896 }
897 visit(entry.getKey(), entry.getValue(), selection, surrounding);
898 }
899 }
900
901 /**
902 * Perform the checks for one check url
903 * @param url the url for the checks
904 * @param checksForUrl the checks to perform
905 * @param selection collection primitives
906 * @param surrounding surrounding primitives, evtl. filled by this routine
907 */
908 private void visit(String url, Set<TagCheck> checksForUrl, Collection<OsmPrimitive> selection,
909 Set<OsmPrimitive> surrounding) {
910 MultiMap<String, TagCheck> currentCheck = new MultiMap<>();
911 currentCheck.putAll(url, checksForUrl);
912 indexData = createMapCSSTagCheckerIndex(currentCheck, includeOtherSeverityChecks(), ALL_TESTS);
913 Set<OsmPrimitive> tested = new HashSet<>();
914
915
916 String title = getTitle(url);
917 if (progressMonitor != null) {
918 progressMonitor.setExtraText(tr(" {0}", title));
919 }
920 long cnt = 0;
921 Stopwatch stopwatch = Stopwatch.createStarted();
922 for (OsmPrimitive p : selection) {
923 if (isCanceled()) {
924 break;
925 }
926 if (isPrimitiveUsable(p)) {
927 check(p);
928 if (partialSelection) {
929 tested.add(p);
930 }
931 }
932 if (progressMonitor != null) {
933 progressMonitor.worked(1);
934 cnt++;
935 // add frequently changing info to progress monitor so that it
936 // doesn't seem to hang when test takes longer than 0.5 seconds
937 if (cnt % 10000 == 0 && stopwatch.elapsed() >= 500) {
938 progressMonitor.setExtraText(tr(" {0}: {1} of {2} elements done", title, cnt, selection.size()));
939 }
940 }
941 }
942
943 if (partialSelection && !tested.isEmpty()) {
944 testPartial(currentCheck, tested, surrounding);
945 }
946 }
947
948 private void testPartial(MultiMap<String, TagCheck> currentCheck, Set<OsmPrimitive> tested,
949 Set<OsmPrimitive> surrounding) {
950
951 // #14287: see https://josm.openstreetmap.de/ticket/14287#comment:15
952 // execute tests for objects which might contain or cross previously tested elements
953
954 final boolean includeOtherSeverity = includeOtherSeverityChecks();
955 // rebuild index with a reduced set of rules (those that use ChildOrParentSelector) and thus may have left selectors
956 // matching the previously tested elements
957 indexData = createMapCSSTagCheckerIndex(currentCheck, includeOtherSeverity, ONLY_SELECTED_TESTS);
958 if (indexData.isEmpty())
959 return; // performance: some *.mapcss rule files don't use ChildOrParentSelector
960
961 if (surrounding.isEmpty()) {
962 for (OsmPrimitive p : tested) {
963 if (p.getDataSet() != null) {
964 surrounding.addAll(p.getDataSet().searchWays(p.getBBox()));
965 surrounding.addAll(p.getDataSet().searchRelations(p.getBBox()));
966 }
967 }
968 }
969
970 for (OsmPrimitive p : surrounding) {
971 if (tested.contains(p))
972 continue;
973 Collection<TestError> additionalErrors = getErrorsForPrimitive(p, includeOtherSeverity);
974 for (TestError e : additionalErrors) {
975 if (e.getPrimitives().stream().anyMatch(tested::contains))
976 addIfNotSimilar(e, errors);
977 }
978 }
979
980 }
981
982 /**
983 * Execute only the rules for the rules matching the given file name. See #19180
984 * @param ruleFile the name of the mapcss file, e.g. deprecated.mapcss
985 * @param selection collection of primitives
986 * @since 16784
987 */
988 public void runOnly(String ruleFile, Collection<OsmPrimitive> selection) {
989 mpAreaCache.clear();
990
991 Set<OsmPrimitive> surrounding = new HashSet<>();
992 for (Entry<String, Set<TagCheck>> entry : checks.entrySet()) {
993 if (isCanceled()) {
994 break;
995 }
996 if (entry.getKey().endsWith(ruleFile)) {
997 visit(entry.getKey(), entry.getValue(), selection, surrounding);
998 }
999 }
1000
1001 }
1002}
Note: See TracBrowser for help on using the repository browser.