source: josm/trunk/src/org/openstreetmap/josm/data/validation/tests/MapCSSTagChecker.java@ 7276

Last change on this file since 7276 was 7276, checked in by Don-vip, 10 years ago

fix #9518 - Automatically reload MapCSS tagchecker validator rules (configurable with validator.auto_reload_local_rules property), improve javadoc
File size: 26.0 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.validation.tests;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.BufferedReader;
7import java.io.IOException;
8import java.io.InputStream;
9import java.io.Reader;
10import java.util.ArrayList;
11import java.util.Arrays;
12import java.util.Collection;
13import java.util.Collections;
14import java.util.HashMap;
15import java.util.Iterator;
16import java.util.LinkedHashMap;
17import java.util.LinkedList;
18import java.util.List;
19import java.util.Map;
20import java.util.Set;
21import java.util.regex.Matcher;
22import java.util.regex.Pattern;
23
24import org.openstreetmap.josm.Main;
25import org.openstreetmap.josm.command.ChangePropertyCommand;
26import org.openstreetmap.josm.command.ChangePropertyKeyCommand;
27import org.openstreetmap.josm.command.Command;
28import org.openstreetmap.josm.command.SequenceCommand;
29import org.openstreetmap.josm.data.osm.OsmPrimitive;
30import org.openstreetmap.josm.data.osm.Tag;
31import org.openstreetmap.josm.data.validation.FixableTestError;
32import org.openstreetmap.josm.data.validation.Severity;
33import org.openstreetmap.josm.data.validation.Test;
34import org.openstreetmap.josm.data.validation.TestError;
35import org.openstreetmap.josm.gui.mappaint.Environment;
36import org.openstreetmap.josm.gui.mappaint.MultiCascade;
37import org.openstreetmap.josm.gui.mappaint.mapcss.Condition;
38import org.openstreetmap.josm.gui.mappaint.mapcss.Expression;
39import org.openstreetmap.josm.gui.mappaint.mapcss.Instruction;
40import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule;
41import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule.Declaration;
42import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleSource;
43import org.openstreetmap.josm.gui.mappaint.mapcss.Selector;
44import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.GeneralSelector;
45import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.MapCSSParser;
46import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.ParseException;
47import org.openstreetmap.josm.gui.preferences.SourceEntry;
48import org.openstreetmap.josm.gui.preferences.validator.ValidatorPreference;
49import org.openstreetmap.josm.gui.preferences.validator.ValidatorTagCheckerRulesPreference;
50import org.openstreetmap.josm.io.CachedFile;
51import org.openstreetmap.josm.io.UTFInputStreamReader;
52import org.openstreetmap.josm.tools.CheckParameterUtil;
53import org.openstreetmap.josm.tools.MultiMap;
54import org.openstreetmap.josm.tools.Predicate;
55import org.openstreetmap.josm.tools.Utils;
56
57/**
58 * MapCSS-based tag checker/fixer.
59 * @since 6506
60 */
61public class MapCSSTagChecker extends Test.TagTest {
62
63 /**
64 * A grouped MapCSSRule with multiple selectors for a single declaration.
65 * @see MapCSSRule
66 */
67 public static class GroupedMapCSSRule {
68 /** MapCSS selectors **/
69 final public List<Selector> selectors;
70 /** MapCSS declaration **/
71 final public Declaration declaration;
72
73 /**
74 * Constructs a new {@code GroupedMapCSSRule}.
75 * @param selectors MapCSS selectors
76 * @param declaration MapCSS declaration
77 */
78 public GroupedMapCSSRule(List<Selector> selectors, Declaration declaration) {
79 this.selectors = selectors;
80 this.declaration = declaration;
81 }
82
83 @Override
84 public int hashCode() {
85 final int prime = 31;
86 int result = 1;
87 result = prime * result + ((declaration == null) ? 0 : declaration.hashCode());
88 result = prime * result + ((selectors == null) ? 0 : selectors.hashCode());
89 return result;
90 }
91
92 @Override
93 public boolean equals(Object obj) {
94 if (this == obj)
95 return true;
96 if (obj == null)
97 return false;
98 if (!(obj instanceof GroupedMapCSSRule))
99 return false;
100 GroupedMapCSSRule other = (GroupedMapCSSRule) obj;
101 if (declaration == null) {
102 if (other.declaration != null)
103 return false;
104 } else if (!declaration.equals(other.declaration))
105 return false;
106 if (selectors == null) {
107 if (other.selectors != null)
108 return false;
109 } else if (!selectors.equals(other.selectors))
110 return false;
111 return true;
112 }
113 }
114
115 /**
116 * The preference key for tag checker source entries.
117 * @since 6670
118 */
119 public static final String ENTRIES_PREF_KEY = "validator." + MapCSSTagChecker.class.getName() + ".entries";
120
121 /**
122 * Constructs a new {@code MapCSSTagChecker}.
123 */
124 public MapCSSTagChecker() {
125 super(tr("Tag checker (MapCSS based)"), tr("This test checks for errors in tag keys and values."));
126 }
127
128 final MultiMap<String, TagCheck> checks = new MultiMap<>();
129
130 static class TagCheck implements Predicate<OsmPrimitive> {
131 protected final GroupedMapCSSRule rule;
132 protected final List<PrimitiveToTag> change = new ArrayList<>();
133 protected final Map<String, String> keyChange = new LinkedHashMap<>();
134 protected final List<String> alternatives = new ArrayList<>();
135 protected final Map<Instruction.AssignmentInstruction, Severity> errors = new HashMap<>();
136 protected final Map<String, Boolean> assertions = new HashMap<>();
137
138 TagCheck(GroupedMapCSSRule rule) {
139 this.rule = rule;
140 }
141
142 /**
143 * A function mapping the matched {@link OsmPrimitive} to a {@link Tag}.
144 */
145 abstract static class PrimitiveToTag implements Utils.Function<OsmPrimitive, Tag> {
146
147 private PrimitiveToTag() {
148 // Hide implicit public constructor for utility class
149 }
150
151 /**
152 * Creates a new mapping from an {@code MapCSS} object.
153 * In case of an {@link Expression}, that is evaluated on the matched {@link OsmPrimitive}.
154 * In case of a {@link String}, that is "compiled" to a {@link Tag} instance.
155 */
156 static PrimitiveToTag ofMapCSSObject(final Object obj, final boolean keyOnly) {
157 if (obj instanceof Expression) {
158 return new PrimitiveToTag() {
159 @Override
160 public Tag apply(OsmPrimitive p) {
161 final String s = (String) ((Expression) obj).evaluate(new Environment().withPrimitive(p));
162 return keyOnly? new Tag(s) : Tag.ofString(s);
163 }
164 };
165 } else if (obj instanceof String) {
166 final Tag tag = keyOnly ? new Tag((String) obj) : Tag.ofString((String) obj);
167 return new PrimitiveToTag() {
168 @Override
169 public Tag apply(OsmPrimitive ignore) {
170 return tag;
171 }
172 };
173 } else {
174 return null;
175 }
176 }
177 }
178
179 static final String POSSIBLE_THROWS = possibleThrows();
180
181 static final String possibleThrows() {
182 StringBuffer sb = new StringBuffer();
183 for (Severity s : Severity.values()) {
184 if (sb.length() > 0) {
185 sb.append('/');
186 }
187 sb.append("throw")
188 .append(s.name().charAt(0))
189 .append(s.name().substring(1).toLowerCase());
190 }
191 return sb.toString();
192 }
193
194 static TagCheck ofMapCSSRule(final GroupedMapCSSRule rule) {
195 final TagCheck check = new TagCheck(rule);
196 boolean containsSetClassExpression = false;
197 for (Instruction i : rule.declaration.instructions) {
198 if (i instanceof Instruction.AssignmentInstruction) {
199 final Instruction.AssignmentInstruction ai = (Instruction.AssignmentInstruction) i;
200 if (ai.isSetInstruction) {
201 containsSetClassExpression = true;
202 continue;
203 }
204 final String val = ai.val instanceof Expression
205 ? (String) ((Expression) ai.val).evaluate(new Environment())
206 : ai.val instanceof String
207 ? (String) ai.val
208 : null;
209 if (ai.key.startsWith("throw")) {
210 try {
211 final Severity severity = Severity.valueOf(ai.key.substring("throw".length()).toUpperCase());
212 check.errors.put(ai, severity);
213 } catch (IllegalArgumentException e) {
214 Main.warn("Unsupported "+ai.key+" instruction. Allowed instructions are "+POSSIBLE_THROWS);
215 }
216 } else if ("fixAdd".equals(ai.key)) {
217 final PrimitiveToTag toTag = PrimitiveToTag.ofMapCSSObject(ai.val, false);
218 check.change.add(toTag);
219 } else if ("fixRemove".equals(ai.key)) {
220 CheckParameterUtil.ensureThat(!(ai.val instanceof String) || !(val != null && val.contains("=")),
221 "Unexpected '='. Please only specify the key to remove!");
222 final PrimitiveToTag toTag = PrimitiveToTag.ofMapCSSObject(ai.val, true);
223 check.change.add(toTag);
224 } else if ("fixChangeKey".equals(ai.key) && val != null) {
225 CheckParameterUtil.ensureThat(val.contains("=>"), "Separate old from new key by '=>'!");
226 final String[] x = val.split("=>", 2);
227 check.keyChange.put(Tag.removeWhiteSpaces(x[0]), Tag.removeWhiteSpaces(x[1]));
228 } else if ("suggestAlternative".equals(ai.key) && val != null) {
229 check.alternatives.add(val);
230 } else if ("assertMatch".equals(ai.key) && val != null) {
231 check.assertions.put(val, true);
232 } else if ("assertNoMatch".equals(ai.key) && val != null) {
233 check.assertions.put(val, false);
234 } else {
235 throw new RuntimeException("Cannot add instruction " + ai.key + ": " + ai.val + "!");
236 }
237 }
238 }
239 if (check.errors.isEmpty() && !containsSetClassExpression) {
240 throw new RuntimeException("No "+POSSIBLE_THROWS+" given! You should specify a validation error message for " + rule.selectors);
241 } else if (check.errors.size() > 1) {
242 throw new RuntimeException("More than one "+POSSIBLE_THROWS+" given! You should specify a single validation error message for " + rule.selectors);
243 }
244 return check;
245 }
246
247 static List<TagCheck> readMapCSS(Reader css) throws ParseException {
248 CheckParameterUtil.ensureParameterNotNull(css, "css");
249 return readMapCSS(new MapCSSParser(css));
250 }
251
252 static List<TagCheck> readMapCSS(MapCSSParser css) throws ParseException {
253 CheckParameterUtil.ensureParameterNotNull(css, "css");
254 final MapCSSStyleSource source = new MapCSSStyleSource("");
255 css.sheet(source);
256 assert source.getErrors().isEmpty();
257 // Ignore "meta" rule(s) from external rules of JOSM wiki
258 removeMetaRules(source);
259 // group rules with common declaration block
260 Map<Declaration, List<Selector>> g = new LinkedHashMap<>();
261 for (MapCSSRule rule : source.rules) {
262 if (!g.containsKey(rule.declaration)) {
263 List<Selector> sels = new ArrayList<>();
264 sels.add(rule.selector);
265 g.put(rule.declaration, sels);
266 } else {
267 g.get(rule.declaration).add(rule.selector);
268 }
269 }
270 List<TagCheck> result = new ArrayList<>();
271 for (Map.Entry<Declaration, List<Selector>> map : g.entrySet()) {
272 result.add(TagCheck.ofMapCSSRule(
273 new GroupedMapCSSRule(map.getValue(), map.getKey())));
274 }
275 return result;
276 }
277
278 private static void removeMetaRules(MapCSSStyleSource source) {
279 for (Iterator<MapCSSRule> it = source.rules.iterator(); it.hasNext(); ) {
280 MapCSSRule x = it.next();
281 if (x.selector instanceof GeneralSelector) {
282 GeneralSelector gs = (GeneralSelector) x.selector;
283 if ("meta".equals(gs.base) && gs.getConditions().isEmpty()) {
284 it.remove();
285 }
286 }
287 }
288 }
289
290 @Override
291 public boolean evaluate(OsmPrimitive primitive) {
292 // Tests whether the primitive contains a deprecated tag which is represented by this MapCSSTagChecker.
293 return whichSelectorMatchesPrimitive(primitive) != null;
294 }
295
296 Selector whichSelectorMatchesPrimitive(OsmPrimitive primitive) {
297 return whichSelectorMatchesEnvironment(new Environment().withPrimitive(primitive));
298 }
299
300 Selector whichSelectorMatchesEnvironment(Environment env) {
301 for (Selector i : rule.selectors) {
302 env.clearSelectorMatchingInformation();
303 if (i.matches(env)) {
304 return i;
305 }
306 }
307 return null;
308 }
309
310 /**
311 * Determines the {@code index}-th key/value/tag (depending on {@code type}) of the
312 * {@link org.openstreetmap.josm.gui.mappaint.mapcss.Selector.GeneralSelector}.
313 */
314 static String determineArgument(Selector.GeneralSelector matchingSelector, int index, String type) {
315 try {
316 final Condition c = matchingSelector.getConditions().get(index);
317 final Tag tag = c instanceof Condition.KeyCondition
318 ? ((Condition.KeyCondition) c).asTag()
319 : c instanceof Condition.SimpleKeyValueCondition
320 ? ((Condition.SimpleKeyValueCondition) c).asTag()
321 : c instanceof Condition.KeyValueCondition
322 ? ((Condition.KeyValueCondition) c).asTag()
323 : null;
324 if (tag == null) {
325 return null;
326 } else if ("key".equals(type)) {
327 return tag.getKey();
328 } else if ("value".equals(type)) {
329 return tag.getValue();
330 } else if ("tag".equals(type)) {
331 return tag.toString();
332 }
333 } catch (IndexOutOfBoundsException ignore) {
334 Main.debug(ignore.getMessage());
335 }
336 return null;
337 }
338
339 /**
340 * Replaces occurrences of <code>{i.key}</code>, <code>{i.value}</code>, <code>{i.tag}</code> in {@code s} by the corresponding
341 * key/value/tag of the {@code index}-th {@link Condition} of {@code matchingSelector}.
342 */
343 static String insertArguments(Selector matchingSelector, String s) {
344 if (s != null && matchingSelector instanceof Selector.ChildOrParentSelector) {
345 return insertArguments(((Selector.ChildOrParentSelector)matchingSelector).right, s);
346 } else if (s == null || !(matchingSelector instanceof GeneralSelector)) {
347 return s;
348 }
349 final Matcher m = Pattern.compile("\\{(\\d+)\\.(key|value|tag)\\}").matcher(s);
350 final StringBuffer sb = new StringBuffer();
351 while (m.find()) {
352 final String argument = determineArgument((Selector.GeneralSelector) matchingSelector, Integer.parseInt(m.group(1)), m.group(2));
353 try {
354 // Perform replacement with null-safe + regex-safe handling
355 m.appendReplacement(sb, String.valueOf(argument).replace("^(", "").replace(")$", ""));
356 } catch (IndexOutOfBoundsException | IllegalArgumentException e) {
357 Main.error(tr("Unable to replace argument {0} in {1}: {2}", argument, sb, e.getMessage()));
358 }
359 }
360 m.appendTail(sb);
361 return sb.toString();
362 }
363
364 /**
365 * Constructs a fix in terms of a {@link org.openstreetmap.josm.command.Command} for the {@link OsmPrimitive}
366 * if the error is fixable, or {@code null} otherwise.
367 *
368 * @param p the primitive to construct the fix for
369 * @return the fix or {@code null}
370 */
371 Command fixPrimitive(OsmPrimitive p) {
372 if (change.isEmpty() && keyChange.isEmpty()) {
373 return null;
374 }
375 final Selector matchingSelector = whichSelectorMatchesPrimitive(p);
376 Collection<Command> cmds = new LinkedList<>();
377 for (PrimitiveToTag toTag : change) {
378 final Tag tag = toTag.apply(p);
379 final String key = insertArguments(matchingSelector, tag.getKey());
380 final String value = insertArguments(matchingSelector, tag.getValue());
381 cmds.add(new ChangePropertyCommand(p, key, value));
382 }
383 for (Map.Entry<String, String> i : keyChange.entrySet()) {
384 final String oldKey = insertArguments(matchingSelector, i.getKey());
385 final String newKey = insertArguments(matchingSelector, i.getValue());
386 cmds.add(new ChangePropertyKeyCommand(p, oldKey, newKey));
387 }
388 return new SequenceCommand(tr("Fix of {0}", getDescriptionForMatchingSelector(p, matchingSelector)), cmds);
389 }
390
391 /**
392 * Constructs a (localized) message for this deprecation check.
393 *
394 * @return a message
395 */
396 String getMessage(OsmPrimitive p) {
397 if (errors.isEmpty()) {
398 // Return something to avoid NPEs
399 return rule.declaration.toString();
400 } else {
401 final Object val = errors.keySet().iterator().next().val;
402 return String.valueOf(
403 val instanceof Expression
404 ? ((Expression) val).evaluate(new Environment().withPrimitive(p))
405 : val
406 );
407 }
408 }
409
410 /**
411 * Constructs a (localized) description for this deprecation check.
412 *
413 * @return a description (possibly with alternative suggestions)
414 * @see #getDescriptionForMatchingSelector
415 */
416 String getDescription(OsmPrimitive p) {
417 if (alternatives.isEmpty()) {
418 return getMessage(p);
419 } else {
420 /* I18N: {0} is the test error message and {1} is an alternative */
421 return tr("{0}, use {1} instead", getMessage(p), Utils.join(tr(" or "), alternatives));
422 }
423 }
424
425 /**
426 * Constructs a (localized) description for this deprecation check
427 * where any placeholders are replaced by values of the matched selector.
428 *
429 * @return a description (possibly with alternative suggestions)
430 */
431 String getDescriptionForMatchingSelector(OsmPrimitive p, Selector matchingSelector) {
432 return insertArguments(matchingSelector, getDescription(p));
433 }
434
435 Severity getSeverity() {
436 return errors.isEmpty() ? null : errors.values().iterator().next();
437 }
438
439 @Override
440 public String toString() {
441 return getDescription(null);
442 }
443
444 /**
445 * Constructs a {@link TestError} for the given primitive, or returns null if the primitive does not give rise to an error.
446 *
447 * @param p the primitive to construct the error for
448 * @return an instance of {@link TestError}, or returns null if the primitive does not give rise to an error.
449 */
450 TestError getErrorForPrimitive(OsmPrimitive p) {
451 final Environment env = new Environment().withPrimitive(p);
452 return getErrorForPrimitive(p, whichSelectorMatchesEnvironment(env), env);
453 }
454
455 TestError getErrorForPrimitive(OsmPrimitive p, Selector matchingSelector, Environment env) {
456 if (matchingSelector != null && !errors.isEmpty()) {
457 final Command fix = fixPrimitive(p);
458 final String description = getDescriptionForMatchingSelector(p, matchingSelector);
459 final List<OsmPrimitive> primitives;
460 if (env.child != null) {
461 primitives = Arrays.asList(p, env.child);
462 } else {
463 primitives = Collections.singletonList(p);
464 }
465 if (fix != null) {
466 return new FixableTestError(null, getSeverity(), description, null, matchingSelector.toString(), 3000, primitives, fix);
467 } else {
468 return new TestError(null, getSeverity(), description, null, matchingSelector.toString(), 3000, primitives);
469 }
470 } else {
471 return null;
472 }
473 }
474 }
475
476 static class MapCSSTagCheckerAndRule extends MapCSSTagChecker {
477 public final GroupedMapCSSRule rule;
478
479 MapCSSTagCheckerAndRule(GroupedMapCSSRule rule) {
480 this.rule = rule;
481 }
482
483 @Override
484 public boolean equals(Object obj) {
485 return super.equals(obj)
486 || (obj instanceof TagCheck && rule.equals(((TagCheck) obj).rule))
487 || (obj instanceof GroupedMapCSSRule && rule.equals(obj));
488 }
489
490 @Override
491 public int hashCode() {
492 final int prime = 31;
493 int result = super.hashCode();
494 result = prime * result + ((rule == null) ? 0 : rule.hashCode());
495 return result;
496 }
497 }
498
499 /**
500 * Obtains all {@link TestError}s for the {@link OsmPrimitive} {@code p}.
501 * @param p The OSM primitive
502 * @param includeOtherSeverity if {@code true}, errors of severity {@link Severity#OTHER} (info) will also be returned
503 * @return all errors for the given primitive, with or without those of "info" severity
504 */
505 public Collection<TestError> getErrorsForPrimitive(OsmPrimitive p, boolean includeOtherSeverity) {
506 final ArrayList<TestError> r = new ArrayList<>();
507 final Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null);
508 for (Set<TagCheck> schecks : checks.values()) {
509 for (TagCheck check : schecks) {
510 if (Severity.OTHER.equals(check.getSeverity()) && !includeOtherSeverity) {
511 continue;
512 }
513 final Selector selector = check.whichSelectorMatchesEnvironment(env);
514 if (selector != null) {
515 check.rule.declaration.execute(env);
516 final TestError error = check.getErrorForPrimitive(p, selector, env);
517 if (error != null) {
518 error.setTester(new MapCSSTagCheckerAndRule(check.rule));
519 r.add(error);
520 }
521 }
522 }
523 }
524 return r;
525 }
526
527 /**
528 * Visiting call for primitives.
529 *
530 * @param p The primitive to inspect.
531 */
532 @Override
533 public void check(OsmPrimitive p) {
534 errors.addAll(getErrorsForPrimitive(p, ValidatorPreference.PREF_OTHER.get()));
535 }
536
537 /**
538 * Adds a new MapCSS config file from the given URL.
539 * @param url The unique URL of the MapCSS config file
540 * @throws ParseException if the config file does not match MapCSS syntax
541 * @throws IOException if any I/O error occurs
542 * @since 7275
543 */
544 public synchronized void addMapCSS(String url) throws ParseException, IOException {
545 CheckParameterUtil.ensureParameterNotNull(url, "url");
546 CachedFile cache = new CachedFile(url);
547 try (InputStream s = cache.getInputStream()) {
548 List<TagCheck> tagchecks = TagCheck.readMapCSS(new BufferedReader(UTFInputStreamReader.create(s)));
549 checks.remove(url);
550 checks.putAll(url, tagchecks);
551 }
552 }
553
554 @Override
555 public synchronized void initialize() throws Exception {
556 checks.clear();
557 for (SourceEntry source : new ValidatorTagCheckerRulesPreference.RulePrefHelper().get()) {
558 if (!source.active) {
559 continue;
560 }
561 String i = source.url;
562 try {
563 if (i.startsWith("resource:")) {
564 Main.debug(tr("Adding {0} to tag checker", i));
565 } else {
566 Main.info(tr("Adding {0} to tag checker", i));
567 }
568 addMapCSS(i);
569 if (Main.pref.getBoolean("validator.auto_reload_local_rules", true) && source.isLocal()) {
570 try {
571 Main.fileWatcher.registerValidatorRule(source);
572 } catch (IOException e) {
573 Main.error(e);
574 }
575 }
576 } catch (IOException ex) {
577 Main.warn(tr("Failed to add {0} to tag checker", i));
578 Main.warn(ex, false);
579 } catch (Exception ex) {
580 Main.warn(tr("Failed to add {0} to tag checker", i));
581 Main.warn(ex);
582 }
583 }
584 }
585
586 @Override
587 public int hashCode() {
588 final int prime = 31;
589 int result = super.hashCode();
590 result = prime * result + ((checks == null) ? 0 : checks.hashCode());
591 return result;
592 }
593
594 @Override
595 public boolean equals(Object obj) {
596 if (this == obj)
597 return true;
598 if (!super.equals(obj))
599 return false;
600 if (!(obj instanceof MapCSSTagChecker))
601 return false;
602 MapCSSTagChecker other = (MapCSSTagChecker) obj;
603 if (checks == null) {
604 if (other.checks != null)
605 return false;
606 } else if (!checks.equals(other.checks))
607 return false;
608 return true;
609 }
610}
Note: See TracBrowser for help on using the repository browser.