source: josm/trunk/src/org/openstreetmap/josm/data/validation/tests/MapCSSTagChecker.java@ 14486

Last change on this file since 14486 was 14486, checked in by Don-vip, 5 years ago

see #17058 - fix unit tests
  • Property svn:eol-style set to native
File size: 47.6 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.validation.tests;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.awt.Rectangle;
7import java.io.BufferedReader;
8import java.io.IOException;
9import java.io.InputStream;
10import java.io.Reader;
11import java.io.StringReader;
12import java.lang.reflect.Method;
13import java.text.MessageFormat;
14import java.util.ArrayList;
15import java.util.Arrays;
16import java.util.Collection;
17import java.util.Collections;
18import java.util.HashMap;
19import java.util.HashSet;
20import java.util.Iterator;
21import java.util.LinkedHashMap;
22import java.util.LinkedHashSet;
23import java.util.LinkedList;
24import java.util.List;
25import java.util.Locale;
26import java.util.Map;
27import java.util.Objects;
28import java.util.Optional;
29import java.util.Set;
30import java.util.function.Predicate;
31import java.util.regex.Matcher;
32import java.util.regex.Pattern;
33
34import org.openstreetmap.josm.command.ChangePropertyCommand;
35import org.openstreetmap.josm.command.ChangePropertyKeyCommand;
36import org.openstreetmap.josm.command.Command;
37import org.openstreetmap.josm.command.DeleteCommand;
38import org.openstreetmap.josm.command.SequenceCommand;
39import org.openstreetmap.josm.data.coor.LatLon;
40import org.openstreetmap.josm.data.osm.DataSet;
41import org.openstreetmap.josm.data.osm.INode;
42import org.openstreetmap.josm.data.osm.IRelation;
43import org.openstreetmap.josm.data.osm.IWay;
44import org.openstreetmap.josm.data.osm.OsmPrimitive;
45import org.openstreetmap.josm.data.osm.OsmUtils;
46import org.openstreetmap.josm.data.osm.Relation;
47import org.openstreetmap.josm.data.osm.Tag;
48import org.openstreetmap.josm.data.osm.Way;
49import org.openstreetmap.josm.data.preferences.sources.SourceEntry;
50import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper;
51import org.openstreetmap.josm.data.validation.OsmValidator;
52import org.openstreetmap.josm.data.validation.Severity;
53import org.openstreetmap.josm.data.validation.Test;
54import org.openstreetmap.josm.data.validation.TestError;
55import org.openstreetmap.josm.gui.mappaint.Environment;
56import org.openstreetmap.josm.gui.mappaint.Keyword;
57import org.openstreetmap.josm.gui.mappaint.MultiCascade;
58import org.openstreetmap.josm.gui.mappaint.mapcss.Condition;
59import org.openstreetmap.josm.gui.mappaint.mapcss.ConditionFactory.ClassCondition;
60import org.openstreetmap.josm.gui.mappaint.mapcss.ConditionFactory.ExpressionCondition;
61import org.openstreetmap.josm.gui.mappaint.mapcss.Expression;
62import org.openstreetmap.josm.gui.mappaint.mapcss.ExpressionFactory.Functions;
63import org.openstreetmap.josm.gui.mappaint.mapcss.ExpressionFactory.ParameterFunction;
64import org.openstreetmap.josm.gui.mappaint.mapcss.Instruction;
65import org.openstreetmap.josm.gui.mappaint.mapcss.LiteralExpression;
66import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule;
67import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule.Declaration;
68import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleSource;
69import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleSource.MapCSSRuleIndex;
70import org.openstreetmap.josm.gui.mappaint.mapcss.Selector;
71import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.AbstractSelector;
72import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.GeneralSelector;
73import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.OptimizedGeneralSelector;
74import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.MapCSSParser;
75import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.ParseException;
76import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.TokenMgrError;
77import org.openstreetmap.josm.gui.progress.ProgressMonitor;
78import org.openstreetmap.josm.io.CachedFile;
79import org.openstreetmap.josm.io.FileWatcher;
80import org.openstreetmap.josm.io.IllegalDataException;
81import org.openstreetmap.josm.io.UTFInputStreamReader;
82import org.openstreetmap.josm.spi.preferences.Config;
83import org.openstreetmap.josm.tools.CheckParameterUtil;
84import org.openstreetmap.josm.tools.DefaultGeoProperty;
85import org.openstreetmap.josm.tools.GeoProperty;
86import org.openstreetmap.josm.tools.GeoPropertyIndex;
87import org.openstreetmap.josm.tools.I18n;
88import org.openstreetmap.josm.tools.JosmRuntimeException;
89import org.openstreetmap.josm.tools.Logging;
90import org.openstreetmap.josm.tools.MultiMap;
91import org.openstreetmap.josm.tools.Territories;
92import org.openstreetmap.josm.tools.Utils;
93
94/**
95 * MapCSS-based tag checker/fixer.
96 * @since 6506
97 */
98public class MapCSSTagChecker extends Test.TagTest {
99 IndexData indexData;
100
101 /**
102 * Helper class to store indexes of rules.
103 * @author Gerd
104 *
105 */
106 private static class IndexData {
107 final Map<MapCSSRule, TagCheck> ruleToCheckMap = new HashMap<>();
108
109 /**
110 * Rules for nodes
111 */
112 final MapCSSRuleIndex nodeRules = new MapCSSRuleIndex();
113 /**
114 * Rules for ways without tag area=no
115 */
116 final MapCSSRuleIndex wayRules = new MapCSSRuleIndex();
117 /**
118 * Rules for ways with tag area=no
119 */
120 final MapCSSRuleIndex wayNoAreaRules = new MapCSSRuleIndex();
121 /**
122 * Rules for relations that are not multipolygon relations
123 */
124 final MapCSSRuleIndex relationRules = new MapCSSRuleIndex();
125 /**
126 * Rules for multipolygon relations
127 */
128 final MapCSSRuleIndex multipolygonRules = new MapCSSRuleIndex();
129
130 IndexData(MultiMap<String, TagCheck> checks) {
131 buildIndex(checks);
132 }
133
134 private void buildIndex(MultiMap<String, TagCheck> checks) {
135 List<TagCheck> allChecks = new ArrayList<>();
136 for (Set<TagCheck> cs : checks.values()) {
137 allChecks.addAll(cs);
138 }
139
140 ruleToCheckMap.clear();
141 nodeRules.clear();
142 wayRules.clear();
143 wayNoAreaRules.clear();
144 relationRules.clear();
145 multipolygonRules.clear();
146
147 // optimization: filter rules for different primitive types
148 for (TagCheck c : allChecks) {
149 for (Selector s : c.rule.selectors) {
150 // find the rightmost selector, this must be a GeneralSelector
151 Selector selRightmost = s;
152 while (selRightmost instanceof Selector.ChildOrParentSelector) {
153 selRightmost = ((Selector.ChildOrParentSelector) selRightmost).right;
154 }
155 MapCSSRule optRule = new MapCSSRule(s.optimizedBaseCheck(), c.rule.declaration);
156
157 ruleToCheckMap.put(optRule, c);
158 final String base = ((GeneralSelector) selRightmost).getBase();
159 switch (base) {
160 case Selector.BASE_NODE:
161 nodeRules.add(optRule);
162 break;
163 case Selector.BASE_WAY:
164 wayNoAreaRules.add(optRule);
165 wayRules.add(optRule);
166 break;
167 case Selector.BASE_AREA:
168 wayRules.add(optRule);
169 multipolygonRules.add(optRule);
170 break;
171 case Selector.BASE_RELATION:
172 relationRules.add(optRule);
173 multipolygonRules.add(optRule);
174 break;
175 case Selector.BASE_ANY:
176 nodeRules.add(optRule);
177 wayRules.add(optRule);
178 wayNoAreaRules.add(optRule);
179 relationRules.add(optRule);
180 multipolygonRules.add(optRule);
181 break;
182 case Selector.BASE_CANVAS:
183 case Selector.BASE_META:
184 case Selector.BASE_SETTING:
185 break;
186 default:
187 final RuntimeException e = new JosmRuntimeException(MessageFormat.format("Unknown MapCSS base selector {0}", base));
188 Logging.warn(tr("Failed to index validator rules. Error was: {0}", e.getMessage()));
189 Logging.error(e);
190 }
191 }
192 }
193 nodeRules.initIndex();
194 wayRules.initIndex();
195 wayNoAreaRules.initIndex();
196 relationRules.initIndex();
197 multipolygonRules.initIndex();
198 }
199
200 /**
201 * Get the index of rules for the given primitive.
202 * @param p the primitve
203 * @return index of rules for the given primitive
204 */
205 public MapCSSRuleIndex get(OsmPrimitive p) {
206 if (p instanceof INode) {
207 return nodeRules;
208 } else if (p instanceof IWay) {
209 if (OsmUtils.isFalse(p.get("area"))) {
210 return wayNoAreaRules;
211 } else {
212 return wayRules;
213 }
214 } else if (p instanceof IRelation) {
215 if (((IRelation<?>) p).isMultipolygon()) {
216 return multipolygonRules;
217 } else {
218 return relationRules;
219 }
220 } else {
221 throw new IllegalArgumentException("Unsupported type: " + p);
222 }
223 }
224
225 /**
226 * return the TagCheck for which the given indexed rule was created.
227 * @param rule an indexed rule
228 * @return the original TagCheck
229 */
230 public TagCheck getCheck(MapCSSRule rule) {
231 return ruleToCheckMap.get(rule);
232 }
233 }
234
235 /**
236 * A grouped MapCSSRule with multiple selectors for a single declaration.
237 * @see MapCSSRule
238 */
239 public static class GroupedMapCSSRule {
240 /** MapCSS selectors **/
241 public final List<Selector> selectors;
242 /** MapCSS declaration **/
243 public final Declaration declaration;
244
245 /**
246 * Constructs a new {@code GroupedMapCSSRule}.
247 * @param selectors MapCSS selectors
248 * @param declaration MapCSS declaration
249 */
250 public GroupedMapCSSRule(List<Selector> selectors, Declaration declaration) {
251 this.selectors = selectors;
252 this.declaration = declaration;
253 }
254
255 @Override
256 public int hashCode() {
257 return Objects.hash(selectors, declaration);
258 }
259
260 @Override
261 public boolean equals(Object obj) {
262 if (this == obj) return true;
263 if (obj == null || getClass() != obj.getClass()) return false;
264 GroupedMapCSSRule that = (GroupedMapCSSRule) obj;
265 return Objects.equals(selectors, that.selectors) &&
266 Objects.equals(declaration, that.declaration);
267 }
268
269 @Override
270 public String toString() {
271 return "GroupedMapCSSRule [selectors=" + selectors + ", declaration=" + declaration + ']';
272 }
273 }
274
275 /**
276 * The preference key for tag checker source entries.
277 * @since 6670
278 */
279 public static final String ENTRIES_PREF_KEY = "validator." + MapCSSTagChecker.class.getName() + ".entries";
280
281 /**
282 * Constructs a new {@code MapCSSTagChecker}.
283 */
284 public MapCSSTagChecker() {
285 super(tr("Tag checker (MapCSS based)"), tr("This test checks for errors in tag keys and values."));
286 }
287
288 /**
289 * Represents a fix to a validation test. The fixing {@link Command} can be obtained by {@link #createCommand(OsmPrimitive, Selector)}.
290 */
291 @FunctionalInterface
292 interface FixCommand {
293 /**
294 * Creates the fixing {@link Command} for the given primitive. The {@code matchingSelector} is used to evaluate placeholders
295 * (cf. {@link MapCSSTagChecker.TagCheck#insertArguments(Selector, String, OsmPrimitive)}).
296 * @param p OSM primitive
297 * @param matchingSelector matching selector
298 * @return fix command
299 */
300 Command createCommand(OsmPrimitive p, Selector matchingSelector);
301
302 /**
303 * Checks that object is either an {@link Expression} or a {@link String}.
304 * @param obj object to check
305 * @throws IllegalArgumentException if object is not an {@code Expression} or a {@code String}
306 */
307 static void checkObject(final Object obj) {
308 CheckParameterUtil.ensureThat(obj instanceof Expression || obj instanceof String,
309 () -> "instance of Exception or String expected, but got " + obj);
310 }
311
312 /**
313 * Evaluates given object as {@link Expression} or {@link String} on the matched {@link OsmPrimitive} and {@code matchingSelector}.
314 * @param obj object to evaluate ({@link Expression} or {@link String})
315 * @param p OSM primitive
316 * @param matchingSelector matching selector
317 * @return result string
318 */
319 static String evaluateObject(final Object obj, final OsmPrimitive p, final Selector matchingSelector) {
320 final String s;
321 if (obj instanceof Expression) {
322 s = (String) ((Expression) obj).evaluate(new Environment(p));
323 } else if (obj instanceof String) {
324 s = (String) obj;
325 } else {
326 return null;
327 }
328 return TagCheck.insertArguments(matchingSelector, s, p);
329 }
330
331 /**
332 * Creates a fixing command which executes a {@link ChangePropertyCommand} on the specified tag.
333 * @param obj object to evaluate ({@link Expression} or {@link String})
334 * @return created fix command
335 */
336 static FixCommand fixAdd(final Object obj) {
337 checkObject(obj);
338 return new FixCommand() {
339 @Override
340 public Command createCommand(OsmPrimitive p, Selector matchingSelector) {
341 final Tag tag = Tag.ofString(evaluateObject(obj, p, matchingSelector));
342 return new ChangePropertyCommand(p, tag.getKey(), tag.getValue());
343 }
344
345 @Override
346 public String toString() {
347 return "fixAdd: " + obj;
348 }
349 };
350 }
351
352 /**
353 * Creates a fixing command which executes a {@link ChangePropertyCommand} to delete the specified key.
354 * @param obj object to evaluate ({@link Expression} or {@link String})
355 * @return created fix command
356 */
357 static FixCommand fixRemove(final Object obj) {
358 checkObject(obj);
359 return new FixCommand() {
360 @Override
361 public Command createCommand(OsmPrimitive p, Selector matchingSelector) {
362 final String key = evaluateObject(obj, p, matchingSelector);
363 return new ChangePropertyCommand(p, key, "");
364 }
365
366 @Override
367 public String toString() {
368 return "fixRemove: " + obj;
369 }
370 };
371 }
372
373 /**
374 * Creates a fixing command which executes a {@link ChangePropertyKeyCommand} on the specified keys.
375 * @param oldKey old key
376 * @param newKey new key
377 * @return created fix command
378 */
379 static FixCommand fixChangeKey(final String oldKey, final String newKey) {
380 return new FixCommand() {
381 @Override
382 public Command createCommand(OsmPrimitive p, Selector matchingSelector) {
383 return new ChangePropertyKeyCommand(p,
384 TagCheck.insertArguments(matchingSelector, oldKey, p),
385 TagCheck.insertArguments(matchingSelector, newKey, p));
386 }
387
388 @Override
389 public String toString() {
390 return "fixChangeKey: " + oldKey + " => " + newKey;
391 }
392 };
393 }
394 }
395
396 final MultiMap<String, TagCheck> checks = new MultiMap<>();
397
398 /**
399 * Result of {@link TagCheck#readMapCSS}
400 * @since 8936
401 */
402 public static class ParseResult {
403 /** Checks successfully parsed */
404 public final List<TagCheck> parseChecks;
405 /** Errors that occurred during parsing */
406 public final Collection<Throwable> parseErrors;
407
408 /**
409 * Constructs a new {@code ParseResult}.
410 * @param parseChecks Checks successfully parsed
411 * @param parseErrors Errors that occurred during parsing
412 */
413 public ParseResult(List<TagCheck> parseChecks, Collection<Throwable> parseErrors) {
414 this.parseChecks = parseChecks;
415 this.parseErrors = parseErrors;
416 }
417 }
418
419 /**
420 * Tag check.
421 */
422 public static class TagCheck implements Predicate<OsmPrimitive> {
423 /** The selector of this {@code TagCheck} */
424 protected final GroupedMapCSSRule rule;
425 /** Commands to apply in order to fix a matching primitive */
426 protected final List<FixCommand> fixCommands = new ArrayList<>();
427 /** Tags (or arbitraty strings) of alternatives to be presented to the user */
428 protected final List<String> alternatives = new ArrayList<>();
429 /** An {@link org.openstreetmap.josm.gui.mappaint.mapcss.Instruction.AssignmentInstruction}-{@link Severity} pair.
430 * Is evaluated on the matching primitive to give the error message. Map is checked to contain exactly one element. */
431 protected final Map<Instruction.AssignmentInstruction, Severity> errors = new HashMap<>();
432 /** Unit tests */
433 protected final Map<String, Boolean> assertions = new HashMap<>();
434 /** MapCSS Classes to set on matching primitives */
435 protected final Set<String> setClassExpressions = new HashSet<>();
436 /** Denotes whether the object should be deleted for fixing it */
437 protected boolean deletion;
438 /** A string used to group similar tests */
439 protected String group;
440
441 TagCheck(GroupedMapCSSRule rule) {
442 this.rule = rule;
443 }
444
445 private static final String POSSIBLE_THROWS = possibleThrows();
446
447 static final String possibleThrows() {
448 StringBuilder sb = new StringBuilder();
449 for (Severity s : Severity.values()) {
450 if (sb.length() > 0) {
451 sb.append('/');
452 }
453 sb.append("throw")
454 .append(s.name().charAt(0))
455 .append(s.name().substring(1).toLowerCase(Locale.ENGLISH));
456 }
457 return sb.toString();
458 }
459
460 static TagCheck ofMapCSSRule(final GroupedMapCSSRule rule) throws IllegalDataException {
461 final TagCheck check = new TagCheck(rule);
462 for (Instruction i : rule.declaration.instructions) {
463 if (i instanceof Instruction.AssignmentInstruction) {
464 final Instruction.AssignmentInstruction ai = (Instruction.AssignmentInstruction) i;
465 if (ai.isSetInstruction) {
466 check.setClassExpressions.add(ai.key);
467 continue;
468 }
469 try {
470 final String val = ai.val instanceof Expression
471 ? Optional.of(((Expression) ai.val).evaluate(new Environment())).map(Object::toString).orElse(null)
472 : ai.val instanceof String
473 ? (String) ai.val
474 : ai.val instanceof Keyword
475 ? ((Keyword) ai.val).val
476 : null;
477 if (ai.key.startsWith("throw")) {
478 try {
479 check.errors.put(ai, Severity.valueOf(ai.key.substring("throw".length()).toUpperCase(Locale.ENGLISH)));
480 } catch (IllegalArgumentException e) {
481 Logging.log(Logging.LEVEL_WARN,
482 "Unsupported "+ai.key+" instruction. Allowed instructions are "+POSSIBLE_THROWS+'.', e);
483 }
484 } else if ("fixAdd".equals(ai.key)) {
485 check.fixCommands.add(FixCommand.fixAdd(ai.val));
486 } else if ("fixRemove".equals(ai.key)) {
487 CheckParameterUtil.ensureThat(!(ai.val instanceof String) || !(val != null && val.contains("=")),
488 "Unexpected '='. Please only specify the key to remove in: " + ai);
489 check.fixCommands.add(FixCommand.fixRemove(ai.val));
490 } else if (val != null && "fixChangeKey".equals(ai.key)) {
491 CheckParameterUtil.ensureThat(val.contains("=>"), "Separate old from new key by '=>'!");
492 final String[] x = val.split("=>", 2);
493 check.fixCommands.add(FixCommand.fixChangeKey(Utils.removeWhiteSpaces(x[0]), Utils.removeWhiteSpaces(x[1])));
494 } else if (val != null && "fixDeleteObject".equals(ai.key)) {
495 CheckParameterUtil.ensureThat("this".equals(val), "fixDeleteObject must be followed by 'this'");
496 check.deletion = true;
497 } else if (val != null && "suggestAlternative".equals(ai.key)) {
498 check.alternatives.add(val);
499 } else if (val != null && "assertMatch".equals(ai.key)) {
500 check.assertions.put(val, Boolean.TRUE);
501 } else if (val != null && "assertNoMatch".equals(ai.key)) {
502 check.assertions.put(val, Boolean.FALSE);
503 } else if (val != null && "group".equals(ai.key)) {
504 check.group = val;
505 } else if (ai.key.startsWith("-")) {
506 Logging.debug("Ignoring extension instruction: " + ai.key + ": " + ai.val);
507 } else {
508 throw new IllegalDataException("Cannot add instruction " + ai.key + ": " + ai.val + '!');
509 }
510 } catch (IllegalArgumentException e) {
511 throw new IllegalDataException(e);
512 }
513 }
514 }
515 if (check.errors.isEmpty() && check.setClassExpressions.isEmpty()) {
516 throw new IllegalDataException(
517 "No "+POSSIBLE_THROWS+" given! You should specify a validation error message for " + rule.selectors);
518 } else if (check.errors.size() > 1) {
519 throw new IllegalDataException(
520 "More than one "+POSSIBLE_THROWS+" given! You should specify a single validation error message for "
521 + rule.selectors);
522 }
523 return check;
524 }
525
526 static ParseResult readMapCSS(Reader css) throws ParseException {
527 CheckParameterUtil.ensureParameterNotNull(css, "css");
528
529 final MapCSSStyleSource source = new MapCSSStyleSource("");
530 final MapCSSParser preprocessor = new MapCSSParser(css, MapCSSParser.LexicalState.PREPROCESSOR);
531 final StringReader mapcss = new StringReader(preprocessor.pp_root(source));
532 final MapCSSParser parser = new MapCSSParser(mapcss, MapCSSParser.LexicalState.DEFAULT);
533 parser.sheet(source);
534 // Ignore "meta" rule(s) from external rules of JOSM wiki
535 source.removeMetaRules();
536 // group rules with common declaration block
537 Map<Declaration, List<Selector>> g = new LinkedHashMap<>();
538 for (MapCSSRule rule : source.rules) {
539 if (!g.containsKey(rule.declaration)) {
540 List<Selector> sels = new ArrayList<>();
541 sels.add(rule.selector);
542 g.put(rule.declaration, sels);
543 } else {
544 g.get(rule.declaration).add(rule.selector);
545 }
546 }
547 List<TagCheck> parseChecks = new ArrayList<>();
548 for (Map.Entry<Declaration, List<Selector>> map : g.entrySet()) {
549 try {
550 parseChecks.add(TagCheck.ofMapCSSRule(
551 new GroupedMapCSSRule(map.getValue(), map.getKey())));
552 } catch (IllegalDataException e) {
553 Logging.error("Cannot add MapCss rule: "+e.getMessage());
554 source.logError(e);
555 }
556 }
557 return new ParseResult(parseChecks, source.getErrors());
558 }
559
560 @Override
561 public boolean test(OsmPrimitive primitive) {
562 // Tests whether the primitive contains a deprecated tag which is represented by this MapCSSTagChecker.
563 return whichSelectorMatchesPrimitive(primitive) != null;
564 }
565
566 Selector whichSelectorMatchesPrimitive(OsmPrimitive primitive) {
567 return whichSelectorMatchesEnvironment(new Environment(primitive));
568 }
569
570 Selector whichSelectorMatchesEnvironment(Environment env) {
571 for (Selector i : rule.selectors) {
572 env.clearSelectorMatchingInformation();
573 if (i.matches(env)) {
574 return i;
575 }
576 }
577 return null;
578 }
579
580 /**
581 * Determines the {@code index}-th key/value/tag (depending on {@code type}) of the
582 * {@link org.openstreetmap.josm.gui.mappaint.mapcss.Selector.GeneralSelector}.
583 * @param matchingSelector matching selector
584 * @param index index
585 * @param type selector type ("key", "value" or "tag")
586 * @param p OSM primitive
587 * @return argument value, can be {@code null}
588 */
589 static String determineArgument(OptimizedGeneralSelector matchingSelector, int index, String type, OsmPrimitive p) {
590 try {
591 final Condition c = matchingSelector.getConditions().get(index);
592 final Tag tag = c instanceof Condition.ToTagConvertable
593 ? ((Condition.ToTagConvertable) c).asTag(p)
594 : null;
595 if (tag == null) {
596 return null;
597 } else if ("key".equals(type)) {
598 return tag.getKey();
599 } else if ("value".equals(type)) {
600 return tag.getValue();
601 } else if ("tag".equals(type)) {
602 return tag.toString();
603 }
604 } catch (IndexOutOfBoundsException ignore) {
605 Logging.debug(ignore);
606 }
607 return null;
608 }
609
610 /**
611 * Replaces occurrences of <code>{i.key}</code>, <code>{i.value}</code>, <code>{i.tag}</code> in {@code s} by the corresponding
612 * key/value/tag of the {@code index}-th {@link Condition} of {@code matchingSelector}.
613 * @param matchingSelector matching selector
614 * @param s any string
615 * @param p OSM primitive
616 * @return string with arguments inserted
617 */
618 static String insertArguments(Selector matchingSelector, String s, OsmPrimitive p) {
619 if (s != null && matchingSelector instanceof Selector.ChildOrParentSelector) {
620 return insertArguments(((Selector.ChildOrParentSelector) matchingSelector).right, s, p);
621 } else if (s == null || !(matchingSelector instanceof Selector.OptimizedGeneralSelector)) {
622 return s;
623 }
624 final Matcher m = Pattern.compile("\\{(\\d+)\\.(key|value|tag)\\}").matcher(s);
625 final StringBuffer sb = new StringBuffer();
626 while (m.find()) {
627 final String argument = determineArgument((Selector.OptimizedGeneralSelector) matchingSelector,
628 Integer.parseInt(m.group(1)), m.group(2), p);
629 try {
630 // Perform replacement with null-safe + regex-safe handling
631 m.appendReplacement(sb, String.valueOf(argument).replace("^(", "").replace(")$", ""));
632 } catch (IndexOutOfBoundsException | IllegalArgumentException e) {
633 Logging.log(Logging.LEVEL_ERROR, tr("Unable to replace argument {0} in {1}: {2}", argument, sb, e.getMessage()), e);
634 }
635 }
636 m.appendTail(sb);
637 return sb.toString();
638 }
639
640 /**
641 * Constructs a fix in terms of a {@link org.openstreetmap.josm.command.Command} for the {@link OsmPrimitive}
642 * if the error is fixable, or {@code null} otherwise.
643 *
644 * @param p the primitive to construct the fix for
645 * @return the fix or {@code null}
646 */
647 Command fixPrimitive(OsmPrimitive p) {
648 if (fixCommands.isEmpty() && !deletion) {
649 return null;
650 }
651 try {
652 final Selector matchingSelector = whichSelectorMatchesPrimitive(p);
653 Collection<Command> cmds = new LinkedList<>();
654 for (FixCommand fixCommand : fixCommands) {
655 cmds.add(fixCommand.createCommand(p, matchingSelector));
656 }
657 if (deletion && !p.isDeleted()) {
658 cmds.add(new DeleteCommand(p));
659 }
660 return new SequenceCommand(tr("Fix of {0}", getDescriptionForMatchingSelector(p, matchingSelector)), cmds);
661 } catch (IllegalArgumentException e) {
662 Logging.error(e);
663 return null;
664 }
665 }
666
667 /**
668 * Constructs a (localized) message for this deprecation check.
669 * @param p OSM primitive
670 *
671 * @return a message
672 */
673 String getMessage(OsmPrimitive p) {
674 if (errors.isEmpty()) {
675 // Return something to avoid NPEs
676 return rule.declaration.toString();
677 } else {
678 final Object val = errors.keySet().iterator().next().val;
679 return String.valueOf(
680 val instanceof Expression
681 ? ((Expression) val).evaluate(new Environment(p))
682 : val
683 );
684 }
685 }
686
687 /**
688 * Constructs a (localized) description for this deprecation check.
689 * @param p OSM primitive
690 *
691 * @return a description (possibly with alternative suggestions)
692 * @see #getDescriptionForMatchingSelector
693 */
694 String getDescription(OsmPrimitive p) {
695 if (alternatives.isEmpty()) {
696 return getMessage(p);
697 } else {
698 /* I18N: {0} is the test error message and {1} is an alternative */
699 return tr("{0}, use {1} instead", getMessage(p), Utils.join(tr(" or "), alternatives));
700 }
701 }
702
703 /**
704 * Constructs a (localized) description for this deprecation check
705 * where any placeholders are replaced by values of the matched selector.
706 *
707 * @param matchingSelector matching selector
708 * @param p OSM primitive
709 * @return a description (possibly with alternative suggestions)
710 */
711 String getDescriptionForMatchingSelector(OsmPrimitive p, Selector matchingSelector) {
712 return insertArguments(matchingSelector, getDescription(p), p);
713 }
714
715 Severity getSeverity() {
716 return errors.isEmpty() ? null : errors.values().iterator().next();
717 }
718
719 @Override
720 public String toString() {
721 return getDescription(null);
722 }
723
724 /**
725 * Constructs a {@link TestError} for the given primitive, or returns null if the primitive does not give rise to an error.
726 *
727 * @param p the primitive to construct the error for
728 * @return an instance of {@link TestError}, or returns null if the primitive does not give rise to an error.
729 */
730 TestError getErrorForPrimitive(OsmPrimitive p) {
731 final Environment env = new Environment(p);
732 return getErrorForPrimitive(p, whichSelectorMatchesEnvironment(env), env, null);
733 }
734
735 TestError getErrorForPrimitive(OsmPrimitive p, Selector matchingSelector, Environment env, Test tester) {
736 if (matchingSelector != null && !errors.isEmpty()) {
737 final Command fix = fixPrimitive(p);
738 final String description = getDescriptionForMatchingSelector(p, matchingSelector);
739 final String description1 = group == null ? description : group;
740 final String description2 = group == null ? null : description;
741 final List<OsmPrimitive> primitives;
742 if (env.child instanceof OsmPrimitive) {
743 primitives = Arrays.asList(p, (OsmPrimitive) env.child);
744 } else {
745 primitives = Collections.singletonList(p);
746 }
747 final TestError.Builder error = TestError.builder(tester, getSeverity(), 3000)
748 .messageWithManuallyTranslatedDescription(description1, description2, matchingSelector.toString())
749 .primitives(primitives);
750 if (fix != null) {
751 return error.fix(() -> fix).build();
752 } else {
753 return error.build();
754 }
755 } else {
756 return null;
757 }
758 }
759
760 /**
761 * Returns the set of tagchecks on which this check depends on.
762 * @param schecks the collection of tagcheks to search in
763 * @return the set of tagchecks on which this check depends on
764 * @since 7881
765 */
766 public Set<TagCheck> getTagCheckDependencies(Collection<TagCheck> schecks) {
767 Set<TagCheck> result = new HashSet<>();
768 Set<String> classes = getClassesIds();
769 if (schecks != null && !classes.isEmpty()) {
770 for (TagCheck tc : schecks) {
771 if (this.equals(tc)) {
772 continue;
773 }
774 for (String id : tc.setClassExpressions) {
775 if (classes.contains(id)) {
776 result.add(tc);
777 break;
778 }
779 }
780 }
781 }
782 return result;
783 }
784
785 /**
786 * Returns the list of ids of all MapCSS classes referenced in the rule selectors.
787 * @return the list of ids of all MapCSS classes referenced in the rule selectors
788 * @since 7881
789 */
790 public Set<String> getClassesIds() {
791 Set<String> result = new HashSet<>();
792 for (Selector s : rule.selectors) {
793 if (s instanceof AbstractSelector) {
794 for (Condition c : ((AbstractSelector) s).getConditions()) {
795 if (c instanceof ClassCondition) {
796 result.add(((ClassCondition) c).id);
797 }
798 }
799 }
800 }
801 return result;
802 }
803 }
804
805 static class MapCSSTagCheckerAndRule extends MapCSSTagChecker {
806 public final GroupedMapCSSRule rule;
807
808 MapCSSTagCheckerAndRule(GroupedMapCSSRule rule) {
809 this.rule = rule;
810 }
811
812 @Override
813 public synchronized boolean equals(Object obj) {
814 return super.equals(obj)
815 || (obj instanceof TagCheck && rule.equals(((TagCheck) obj).rule))
816 || (obj instanceof GroupedMapCSSRule && rule.equals(obj));
817 }
818
819 @Override
820 public synchronized int hashCode() {
821 return Objects.hash(super.hashCode(), rule);
822 }
823
824 @Override
825 public String toString() {
826 return "MapCSSTagCheckerAndRule [rule=" + rule + ']';
827 }
828 }
829
830 /**
831 * Obtains all {@link TestError}s for the {@link OsmPrimitive} {@code p}.
832 * @param p The OSM primitive
833 * @param includeOtherSeverity if {@code true}, errors of severity {@link Severity#OTHER} (info) will also be returned
834 * @return all errors for the given primitive, with or without those of "info" severity
835 */
836 public synchronized Collection<TestError> getErrorsForPrimitive(OsmPrimitive p, boolean includeOtherSeverity) {
837 final List<TestError> res = new ArrayList<>();
838 if (indexData == null)
839 indexData = new IndexData(checks);
840
841 MapCSSRuleIndex matchingRuleIndex = indexData.get(p);
842
843 Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null);
844 // the declaration indices are sorted, so it suffices to save the last used index
845 Declaration lastDeclUsed = null;
846
847 Iterator<MapCSSRule> candidates = matchingRuleIndex.getRuleCandidates(p);
848 while (candidates.hasNext()) {
849 MapCSSRule r = candidates.next();
850 env.clearSelectorMatchingInformation();
851 if (r.selector.matches(env)) { // as side effect env.parent will be set (if s is a child selector)
852 TagCheck check = indexData.getCheck(r);
853 if (check != null) {
854 boolean ignoreError = Severity.OTHER == check.getSeverity() && !includeOtherSeverity;
855 // Do not run "information" level checks if not wanted, unless they also set a MapCSS class
856 if (ignoreError && check.setClassExpressions.isEmpty()) {
857 continue;
858 }
859 if (r.declaration == lastDeclUsed)
860 continue; // don't apply one declaration more than once
861 lastDeclUsed = r.declaration;
862
863 r.declaration.execute(env);
864 if (!ignoreError && !check.errors.isEmpty()) {
865 final TestError error = check.getErrorForPrimitive(p, r.selector, env, new MapCSSTagCheckerAndRule(check.rule));
866 if (error != null) {
867 res.add(error);
868 }
869 }
870
871 }
872 }
873 }
874 return res;
875 }
876
877 private static Collection<TestError> getErrorsForPrimitive(OsmPrimitive p, boolean includeOtherSeverity,
878 Collection<Set<TagCheck>> checksCol) {
879 final List<TestError> r = new ArrayList<>();
880 final Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null);
881 for (Set<TagCheck> schecks : checksCol) {
882 for (TagCheck check : schecks) {
883 boolean ignoreError = Severity.OTHER == check.getSeverity() && !includeOtherSeverity;
884 // Do not run "information" level checks if not wanted, unless they also set a MapCSS class
885 if (ignoreError && check.setClassExpressions.isEmpty()) {
886 continue;
887 }
888 final Selector selector = check.whichSelectorMatchesEnvironment(env);
889 if (selector != null) {
890 check.rule.declaration.execute(env);
891 if (!ignoreError && !check.errors.isEmpty()) {
892 final TestError error = check.getErrorForPrimitive(p, selector, env, new MapCSSTagCheckerAndRule(check.rule));
893 if (error != null) {
894 r.add(error);
895 }
896 }
897 }
898 }
899 }
900 return r;
901 }
902
903 /**
904 * Visiting call for primitives.
905 *
906 * @param p The primitive to inspect.
907 */
908 @Override
909 public void check(OsmPrimitive p) {
910 errors.addAll(getErrorsForPrimitive(p, ValidatorPrefHelper.PREF_OTHER.get()));
911 }
912
913 /**
914 * Adds a new MapCSS config file from the given URL.
915 * @param url The unique URL of the MapCSS config file
916 * @return List of tag checks and parsing errors, or null
917 * @throws ParseException if the config file does not match MapCSS syntax
918 * @throws IOException if any I/O error occurs
919 * @since 7275
920 */
921 public synchronized ParseResult addMapCSS(String url) throws ParseException, IOException {
922 CheckParameterUtil.ensureParameterNotNull(url, "url");
923 ParseResult result;
924 try (CachedFile cache = new CachedFile(url);
925 InputStream zip = cache.findZipEntryInputStream("validator.mapcss", "");
926 InputStream s = zip != null ? zip : cache.getInputStream();
927 Reader reader = new BufferedReader(UTFInputStreamReader.create(s))) {
928 if (zip != null)
929 I18n.addTexts(cache.getFile());
930 result = TagCheck.readMapCSS(reader);
931 checks.remove(url);
932 checks.putAll(url, result.parseChecks);
933 indexData = null;
934 // Check assertions, useful for development of local files
935 if (Config.getPref().getBoolean("validator.check_assert_local_rules", false) && Utils.isLocalUrl(url)) {
936 for (String msg : checkAsserts(result.parseChecks)) {
937 Logging.warn(msg);
938 }
939 }
940 }
941 return result;
942 }
943
944 @Override
945 public synchronized void initialize() throws Exception {
946 checks.clear();
947 indexData = null;
948 for (SourceEntry source : new ValidatorPrefHelper().get()) {
949 if (!source.active) {
950 continue;
951 }
952 String i = source.url;
953 try {
954 if (!i.startsWith("resource:")) {
955 Logging.info(tr("Adding {0} to tag checker", i));
956 } else if (Logging.isDebugEnabled()) {
957 Logging.debug(tr("Adding {0} to tag checker", i));
958 }
959 addMapCSS(i);
960 if (Config.getPref().getBoolean("validator.auto_reload_local_rules", true) && source.isLocal()) {
961 FileWatcher.getDefaultInstance().registerSource(source);
962 }
963 } catch (IOException | IllegalStateException | IllegalArgumentException ex) {
964 Logging.warn(tr("Failed to add {0} to tag checker", i));
965 Logging.log(Logging.LEVEL_WARN, ex);
966 } catch (ParseException | TokenMgrError ex) {
967 Logging.warn(tr("Failed to add {0} to tag checker", i));
968 Logging.warn(ex);
969 }
970 }
971 }
972
973 private static Method getFunctionMethod(String method) {
974 try {
975 return Functions.class.getDeclaredMethod(method, Environment.class, String.class);
976 } catch (NoSuchMethodException | SecurityException e) {
977 Logging.error(e);
978 return null;
979 }
980 }
981
982 private static Optional<String> getFirstInsideCountry(TagCheck check, Method insideMethod) {
983 return check.rule.selectors.stream()
984 .filter(s -> s instanceof GeneralSelector)
985 .flatMap(s -> ((GeneralSelector) s).getConditions().stream())
986 .filter(c -> c instanceof ExpressionCondition)
987 .map(c -> ((ExpressionCondition) c).getExpression())
988 .filter(c -> c instanceof ParameterFunction)
989 .map(c -> (ParameterFunction) c)
990 .filter(c -> c.getMethod().equals(insideMethod))
991 .flatMap(c -> c.getArgs().stream())
992 .filter(e -> e instanceof LiteralExpression)
993 .map(e -> ((LiteralExpression) e).getLiteral())
994 .filter(l -> l instanceof String)
995 .map(l -> (String) l)
996 .findFirst();
997 }
998
999 private static LatLon getLocation(TagCheck check, Method insideMethod) {
1000 Optional<String> inside = getFirstInsideCountry(check, insideMethod);
1001 if (inside.isPresent()) {
1002 GeoPropertyIndex<Boolean> index = Territories.getGeoPropertyIndex(inside.get());
1003 if (index != null) {
1004 GeoProperty<Boolean> prop = index.getGeoProperty();
1005 if (prop instanceof DefaultGeoProperty) {
1006 Rectangle bounds = ((DefaultGeoProperty) prop).getArea().getBounds();
1007 return new LatLon(bounds.getCenterY(), bounds.getCenterX());
1008 }
1009 }
1010 }
1011 return LatLon.ZERO;
1012 }
1013
1014 /**
1015 * Checks that rule assertions are met for the given set of TagChecks.
1016 * @param schecks The TagChecks for which assertions have to be checked
1017 * @return A set of error messages, empty if all assertions are met
1018 * @since 7356
1019 */
1020 public Set<String> checkAsserts(final Collection<TagCheck> schecks) {
1021 Set<String> assertionErrors = new LinkedHashSet<>();
1022 final Method insideMethod = getFunctionMethod("inside");
1023 final DataSet ds = new DataSet();
1024 for (final TagCheck check : schecks) {
1025 Logging.debug("Check: {0}", check);
1026 for (final Map.Entry<String, Boolean> i : check.assertions.entrySet()) {
1027 Logging.debug("- Assertion: {0}", i);
1028 final OsmPrimitive p = OsmUtils.createPrimitive(i.getKey(), getLocation(check, insideMethod), true);
1029 // Build minimal ordered list of checks to run to test the assertion
1030 List<Set<TagCheck>> checksToRun = new ArrayList<>();
1031 Set<TagCheck> checkDependencies = check.getTagCheckDependencies(schecks);
1032 if (!checkDependencies.isEmpty()) {
1033 checksToRun.add(checkDependencies);
1034 }
1035 checksToRun.add(Collections.singleton(check));
1036 // Add primitive to dataset to avoid DataIntegrityProblemException when evaluating selectors
1037 addPrimitive(ds, p);
1038 final Collection<TestError> pErrors = getErrorsForPrimitive(p, true, checksToRun);
1039 Logging.debug("- Errors: {0}", pErrors);
1040 @SuppressWarnings({"EqualsBetweenInconvertibleTypes", "EqualsIncompatibleType"})
1041 final boolean isError = pErrors.stream().anyMatch(e -> e.getTester().equals(check.rule));
1042 if (isError != i.getValue()) {
1043 final String error = MessageFormat.format("Expecting test ''{0}'' (i.e., {1}) to {2} {3} (i.e., {4})",
1044 check.getMessage(p), check.rule.selectors, i.getValue() ? "match" : "not match", i.getKey(), p.getKeys());
1045 assertionErrors.add(error);
1046 }
1047 ds.removePrimitive(p);
1048 }
1049 }
1050 return assertionErrors;
1051 }
1052
1053 private static void addPrimitive(DataSet ds, OsmPrimitive p) {
1054 if (p instanceof Way) {
1055 ((Way) p).getNodes().forEach(n -> addPrimitive(ds, n));
1056 } else if (p instanceof Relation) {
1057 ((Relation) p).getMembers().forEach(m -> addPrimitive(ds, m.getMember()));
1058 }
1059 ds.addPrimitive(p);
1060 }
1061
1062 @Override
1063 public synchronized int hashCode() {
1064 return Objects.hash(super.hashCode(), checks);
1065 }
1066
1067 @Override
1068 public synchronized boolean equals(Object obj) {
1069 if (this == obj) return true;
1070 if (obj == null || getClass() != obj.getClass()) return false;
1071 if (!super.equals(obj)) return false;
1072 MapCSSTagChecker that = (MapCSSTagChecker) obj;
1073 return Objects.equals(checks, that.checks);
1074 }
1075
1076 /**
1077 * Reload tagchecker rule.
1078 * @param rule tagchecker rule to reload
1079 * @since 12825
1080 */
1081 public static void reloadRule(SourceEntry rule) {
1082 MapCSSTagChecker tagChecker = OsmValidator.getTest(MapCSSTagChecker.class);
1083 if (tagChecker != null) {
1084 try {
1085 tagChecker.addMapCSS(rule.url);
1086 } catch (IOException | ParseException | TokenMgrError e) {
1087 Logging.warn(e);
1088 }
1089 }
1090 }
1091
1092 @Override
1093 public void startTest(ProgressMonitor progressMonitor) {
1094 super.startTest(progressMonitor);
1095 if (indexData == null) {
1096 indexData = new IndexData(checks);
1097 }
1098 }
1099
1100 @Override
1101 public void endTest() {
1102 super.endTest();
1103 // no need to keep the index, it is quickly build and doubles the memory needs
1104 indexData = null;
1105 }
1106
1107}
Note: See TracBrowser for help on using the repository browser.