source: josm/trunk/src/org/openstreetmap/josm/data/validation/tests/MapCSSTagChecker.java@ 7234

Last change on this file since 7234 was 7234, checked in by Don-vip, 10 years ago

see #10118 - robustness in mapcss tagchecker (exception occurs with java8, not with java7, to investigate)
File size: 24.3 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.validation.tests;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.BufferedReader;
7import java.io.IOException;
8import java.io.Reader;
9import java.util.ArrayList;
10import java.util.Arrays;
11import java.util.Collection;
12import java.util.Collections;
13import java.util.HashMap;
14import java.util.Iterator;
15import java.util.LinkedHashMap;
16import java.util.LinkedList;
17import java.util.List;
18import java.util.Map;
19import java.util.regex.Matcher;
20import java.util.regex.Pattern;
21
22import org.openstreetmap.josm.Main;
23import org.openstreetmap.josm.command.ChangePropertyCommand;
24import org.openstreetmap.josm.command.ChangePropertyKeyCommand;
25import org.openstreetmap.josm.command.Command;
26import org.openstreetmap.josm.command.SequenceCommand;
27import org.openstreetmap.josm.data.osm.OsmPrimitive;
28import org.openstreetmap.josm.data.osm.Tag;
29import org.openstreetmap.josm.data.validation.FixableTestError;
30import org.openstreetmap.josm.data.validation.Severity;
31import org.openstreetmap.josm.data.validation.Test;
32import org.openstreetmap.josm.data.validation.TestError;
33import org.openstreetmap.josm.gui.mappaint.Environment;
34import org.openstreetmap.josm.gui.mappaint.MultiCascade;
35import org.openstreetmap.josm.gui.mappaint.mapcss.Condition;
36import org.openstreetmap.josm.gui.mappaint.mapcss.Expression;
37import org.openstreetmap.josm.gui.mappaint.mapcss.Instruction;
38import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule;
39import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule.Declaration;
40import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleSource;
41import org.openstreetmap.josm.gui.mappaint.mapcss.Selector;
42import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.GeneralSelector;
43import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.MapCSSParser;
44import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.ParseException;
45import org.openstreetmap.josm.gui.preferences.validator.ValidatorPreference;
46import org.openstreetmap.josm.gui.preferences.validator.ValidatorTagCheckerRulesPreference;
47import org.openstreetmap.josm.io.MirroredInputStream;
48import org.openstreetmap.josm.io.UTFInputStreamReader;
49import org.openstreetmap.josm.tools.CheckParameterUtil;
50import org.openstreetmap.josm.tools.Predicate;
51import org.openstreetmap.josm.tools.Utils;
52
53/**
54 * MapCSS-based tag checker/fixer.
55 * @since 6506
56 */
57public class MapCSSTagChecker extends Test.TagTest {
58
59 public static class GroupedMapCSSRule {
60 final public List<Selector> selectors;
61 final public Declaration declaration;
62
63 public GroupedMapCSSRule(List<Selector> selectors, Declaration declaration) {
64 this.selectors = selectors;
65 this.declaration = declaration;
66 }
67
68 @Override
69 public int hashCode() {
70 final int prime = 31;
71 int result = 1;
72 result = prime * result + ((declaration == null) ? 0 : declaration.hashCode());
73 result = prime * result + ((selectors == null) ? 0 : selectors.hashCode());
74 return result;
75 }
76
77 @Override
78 public boolean equals(Object obj) {
79 if (this == obj)
80 return true;
81 if (obj == null)
82 return false;
83 if (!(obj instanceof GroupedMapCSSRule))
84 return false;
85 GroupedMapCSSRule other = (GroupedMapCSSRule) obj;
86 if (declaration == null) {
87 if (other.declaration != null)
88 return false;
89 } else if (!declaration.equals(other.declaration))
90 return false;
91 if (selectors == null) {
92 if (other.selectors != null)
93 return false;
94 } else if (!selectors.equals(other.selectors))
95 return false;
96 return true;
97 }
98 }
99
100 /**
101 * The preference key for tag checker source entries.
102 * @since 6670
103 */
104 public static final String ENTRIES_PREF_KEY = "validator." + MapCSSTagChecker.class.getName() + ".entries";
105
106 /**
107 * Constructs a new {@code MapCSSTagChecker}.
108 */
109 public MapCSSTagChecker() {
110 super(tr("Tag checker (MapCSS based)"), tr("This test checks for errors in tag keys and values."));
111 }
112
113 final List<TagCheck> checks = new ArrayList<>();
114
115 static class TagCheck implements Predicate<OsmPrimitive> {
116 protected final GroupedMapCSSRule rule;
117 protected final List<PrimitiveToTag> change = new ArrayList<>();
118 protected final Map<String, String> keyChange = new LinkedHashMap<>();
119 protected final List<String> alternatives = new ArrayList<>();
120 protected final Map<Instruction.AssignmentInstruction, Severity> errors = new HashMap<>();
121 protected final Map<String, Boolean> assertions = new HashMap<>();
122
123 TagCheck(GroupedMapCSSRule rule) {
124 this.rule = rule;
125 }
126
127 /**
128 * A function mapping the matched {@link OsmPrimitive} to a {@link Tag}.
129 */
130 abstract static class PrimitiveToTag implements Utils.Function<OsmPrimitive, Tag> {
131
132 private PrimitiveToTag() {
133 // Hide implicit public constructor for utility class
134 }
135
136 /**
137 * Creates a new mapping from an {@code MapCSS} object.
138 * In case of an {@link Expression}, that is evaluated on the matched {@link OsmPrimitive}.
139 * In case of a {@link String}, that is "compiled" to a {@link Tag} instance.
140 */
141 static PrimitiveToTag ofMapCSSObject(final Object obj, final boolean keyOnly) {
142 if (obj instanceof Expression) {
143 return new PrimitiveToTag() {
144 @Override
145 public Tag apply(OsmPrimitive p) {
146 final String s = (String) ((Expression) obj).evaluate(new Environment().withPrimitive(p));
147 return keyOnly? new Tag(s) : Tag.ofString(s);
148 }
149 };
150 } else if (obj instanceof String) {
151 final Tag tag = keyOnly ? new Tag((String) obj) : Tag.ofString((String) obj);
152 return new PrimitiveToTag() {
153 @Override
154 public Tag apply(OsmPrimitive ignore) {
155 return tag;
156 }
157 };
158 } else {
159 return null;
160 }
161 }
162 }
163
164 static final String POSSIBLE_THROWS = possibleThrows();
165
166 static final String possibleThrows() {
167 StringBuffer sb = new StringBuffer();
168 for (Severity s : Severity.values()) {
169 if (sb.length() > 0) {
170 sb.append('/');
171 }
172 sb.append("throw")
173 .append(s.name().charAt(0))
174 .append(s.name().substring(1).toLowerCase());
175 }
176 return sb.toString();
177 }
178
179 static TagCheck ofMapCSSRule(final GroupedMapCSSRule rule) {
180 final TagCheck check = new TagCheck(rule);
181 boolean containsSetClassExpression = false;
182 for (Instruction i : rule.declaration.instructions) {
183 if (i instanceof Instruction.AssignmentInstruction) {
184 final Instruction.AssignmentInstruction ai = (Instruction.AssignmentInstruction) i;
185 if (ai.isSetInstruction) {
186 containsSetClassExpression = true;
187 continue;
188 }
189 final String val = ai.val instanceof Expression
190 ? (String) ((Expression) ai.val).evaluate(new Environment())
191 : ai.val instanceof String
192 ? (String) ai.val
193 : null;
194 if (ai.key.startsWith("throw")) {
195 try {
196 final Severity severity = Severity.valueOf(ai.key.substring("throw".length()).toUpperCase());
197 check.errors.put(ai, severity);
198 } catch (IllegalArgumentException e) {
199 Main.warn("Unsupported "+ai.key+" instruction. Allowed instructions are "+POSSIBLE_THROWS);
200 }
201 } else if ("fixAdd".equals(ai.key)) {
202 final PrimitiveToTag toTag = PrimitiveToTag.ofMapCSSObject(ai.val, false);
203 check.change.add(toTag);
204 } else if ("fixRemove".equals(ai.key)) {
205 CheckParameterUtil.ensureThat(!(ai.val instanceof String) || !val.contains("="), "Unexpected '='. Please only specify the key to remove!");
206 final PrimitiveToTag toTag = PrimitiveToTag.ofMapCSSObject(ai.val, true);
207 check.change.add(toTag);
208 } else if ("fixChangeKey".equals(ai.key) && val != null) {
209 CheckParameterUtil.ensureThat(val.contains("=>"), "Separate old from new key by '=>'!");
210 final String[] x = val.split("=>", 2);
211 check.keyChange.put(Tag.removeWhiteSpaces(x[0]), Tag.removeWhiteSpaces(x[1]));
212 } else if ("suggestAlternative".equals(ai.key) && val != null) {
213 check.alternatives.add(val);
214 } else if ("assertMatch".equals(ai.key) && val != null) {
215 check.assertions.put(val, true);
216 } else if ("assertNoMatch".equals(ai.key) && val != null) {
217 check.assertions.put(val, false);
218 } else {
219 throw new RuntimeException("Cannot add instruction " + ai.key + ": " + ai.val + "!");
220 }
221 }
222 }
223 if (check.errors.isEmpty() && !containsSetClassExpression) {
224 throw new RuntimeException("No "+POSSIBLE_THROWS+" given! You should specify a validation error message for " + rule.selectors);
225 } else if (check.errors.size() > 1) {
226 throw new RuntimeException("More than one "+POSSIBLE_THROWS+" given! You should specify a single validation error message for " + rule.selectors);
227 }
228 return check;
229 }
230
231 static List<TagCheck> readMapCSS(Reader css) throws ParseException {
232 CheckParameterUtil.ensureParameterNotNull(css, "css");
233 return readMapCSS(new MapCSSParser(css));
234 }
235
236 static List<TagCheck> readMapCSS(MapCSSParser css) throws ParseException {
237 CheckParameterUtil.ensureParameterNotNull(css, "css");
238 final MapCSSStyleSource source = new MapCSSStyleSource("");
239 css.sheet(source);
240 assert source.getErrors().isEmpty();
241 // Ignore "meta" rule(s) from external rules of JOSM wiki
242 removeMetaRules(source);
243 // group rules with common declaration block
244 Map<Declaration, List<Selector>> g = new LinkedHashMap<>();
245 for (MapCSSRule rule : source.rules) {
246 if (!g.containsKey(rule.declaration)) {
247 List<Selector> sels = new ArrayList<>();
248 sels.add(rule.selector);
249 g.put(rule.declaration, sels);
250 } else {
251 g.get(rule.declaration).add(rule.selector);
252 }
253 }
254 List<TagCheck> result = new ArrayList<>();
255 for (Map.Entry<Declaration, List<Selector>> map : g.entrySet()) {
256 result.add(TagCheck.ofMapCSSRule(
257 new GroupedMapCSSRule(map.getValue(), map.getKey())));
258 }
259 return result;
260 }
261
262 private static void removeMetaRules(MapCSSStyleSource source) {
263 for (Iterator<MapCSSRule> it = source.rules.iterator(); it.hasNext(); ) {
264 MapCSSRule x = it.next();
265 if (x.selector instanceof GeneralSelector) {
266 GeneralSelector gs = (GeneralSelector) x.selector;
267 if ("meta".equals(gs.base) && gs.getConditions().isEmpty()) {
268 it.remove();
269 }
270 }
271 }
272 }
273
274 @Override
275 public boolean evaluate(OsmPrimitive primitive) {
276 // Tests whether the primitive contains a deprecated tag which is represented by this MapCSSTagChecker.
277 return whichSelectorMatchesPrimitive(primitive) != null;
278 }
279
280 Selector whichSelectorMatchesPrimitive(OsmPrimitive primitive) {
281 return whichSelectorMatchesEnvironment(new Environment().withPrimitive(primitive));
282 }
283
284 Selector whichSelectorMatchesEnvironment(Environment env) {
285 for (Selector i : rule.selectors) {
286 env.clearSelectorMatchingInformation();
287 if (i.matches(env)) {
288 return i;
289 }
290 }
291 return null;
292 }
293
294 /**
295 * Determines the {@code index}-th key/value/tag (depending on {@code type}) of the
296 * {@link org.openstreetmap.josm.gui.mappaint.mapcss.Selector.GeneralSelector}.
297 */
298 static String determineArgument(Selector.GeneralSelector matchingSelector, int index, String type) {
299 try {
300 final Condition c = matchingSelector.getConditions().get(index);
301 final Tag tag = c instanceof Condition.KeyCondition
302 ? ((Condition.KeyCondition) c).asTag()
303 : c instanceof Condition.SimpleKeyValueCondition
304 ? ((Condition.SimpleKeyValueCondition) c).asTag()
305 : c instanceof Condition.KeyValueCondition
306 ? ((Condition.KeyValueCondition) c).asTag()
307 : null;
308 if (tag == null) {
309 return null;
310 } else if ("key".equals(type)) {
311 return tag.getKey();
312 } else if ("value".equals(type)) {
313 return tag.getValue();
314 } else if ("tag".equals(type)) {
315 return tag.toString();
316 }
317 } catch (IndexOutOfBoundsException ignore) {
318 Main.debug(ignore.getMessage());
319 }
320 return null;
321 }
322
323 /**
324 * Replaces occurrences of {@code {i.key}}, {@code {i.value}}, {@code {i.tag}} in {@code s} by the corresponding
325 * key/value/tag of the {@code index}-th {@link Condition} of {@code matchingSelector}.
326 */
327 static String insertArguments(Selector matchingSelector, String s) {
328 if (s != null && matchingSelector instanceof Selector.ChildOrParentSelector) {
329 return insertArguments(((Selector.ChildOrParentSelector)matchingSelector).right, s);
330 } else if (s == null || !(matchingSelector instanceof GeneralSelector)) {
331 return s;
332 }
333 final Matcher m = Pattern.compile("\\{(\\d+)\\.(key|value|tag)\\}").matcher(s);
334 final StringBuffer sb = new StringBuffer();
335 while (m.find()) {
336 final String argument = determineArgument((Selector.GeneralSelector) matchingSelector, Integer.parseInt(m.group(1)), m.group(2));
337 try {
338 m.appendReplacement(sb, String.valueOf(argument));
339 } catch (IndexOutOfBoundsException | IllegalArgumentException e) {
340 Main.error(tr("Unable to replace argument {0} in {1}: {2}", argument, sb, e.getMessage()));
341 }
342 }
343 m.appendTail(sb);
344 return sb.toString();
345 }
346
347 /**
348 * Constructs a fix in terms of a {@link org.openstreetmap.josm.command.Command} for the {@link OsmPrimitive}
349 * if the error is fixable, or {@code null} otherwise.
350 *
351 * @param p the primitive to construct the fix for
352 * @return the fix or {@code null}
353 */
354 Command fixPrimitive(OsmPrimitive p) {
355 if (change.isEmpty() && keyChange.isEmpty()) {
356 return null;
357 }
358 final Selector matchingSelector = whichSelectorMatchesPrimitive(p);
359 Collection<Command> cmds = new LinkedList<>();
360 for (PrimitiveToTag toTag : change) {
361 final Tag tag = toTag.apply(p);
362 final String key = insertArguments(matchingSelector, tag.getKey());
363 final String value = insertArguments(matchingSelector, tag.getValue());
364 cmds.add(new ChangePropertyCommand(p, key, value));
365 }
366 for (Map.Entry<String, String> i : keyChange.entrySet()) {
367 final String oldKey = insertArguments(matchingSelector, i.getKey());
368 final String newKey = insertArguments(matchingSelector, i.getValue());
369 cmds.add(new ChangePropertyKeyCommand(p, oldKey, newKey));
370 }
371 return new SequenceCommand(tr("Fix of {0}", getDescriptionForMatchingSelector(p, matchingSelector)), cmds);
372 }
373
374 /**
375 * Constructs a (localized) message for this deprecation check.
376 *
377 * @return a message
378 */
379 String getMessage(OsmPrimitive p) {
380 if (errors.isEmpty()) {
381 // Return something to avoid NPEs
382 return rule.declaration.toString();
383 } else {
384 final Object val = errors.keySet().iterator().next().val;
385 return String.valueOf(
386 val instanceof Expression
387 ? ((Expression) val).evaluate(new Environment().withPrimitive(p))
388 : val
389 );
390 }
391 }
392
393 /**
394 * Constructs a (localized) description for this deprecation check.
395 *
396 * @return a description (possibly with alternative suggestions)
397 * @see #getDescriptionForMatchingSelector
398 */
399 String getDescription(OsmPrimitive p) {
400 if (alternatives.isEmpty()) {
401 return getMessage(p);
402 } else {
403 /* I18N: {0} is the test error message and {1} is an alternative */
404 return tr("{0}, use {1} instead", getMessage(p), Utils.join(tr(" or "), alternatives));
405 }
406 }
407
408 /**
409 * Constructs a (localized) description for this deprecation check
410 * where any placeholders are replaced by values of the matched selector.
411 *
412 * @return a description (possibly with alternative suggestions)
413 */
414 String getDescriptionForMatchingSelector(OsmPrimitive p, Selector matchingSelector) {
415 return insertArguments(matchingSelector, getDescription(p));
416 }
417
418 Severity getSeverity() {
419 return errors.isEmpty() ? null : errors.values().iterator().next();
420 }
421
422 @Override
423 public String toString() {
424 return getDescription(null);
425 }
426
427 /**
428 * Constructs a {@link TestError} for the given primitive, or returns null if the primitive does not give rise to an error.
429 *
430 * @param p the primitive to construct the error for
431 * @return an instance of {@link TestError}, or returns null if the primitive does not give rise to an error.
432 */
433 TestError getErrorForPrimitive(OsmPrimitive p) {
434 final Environment env = new Environment().withPrimitive(p);
435 return getErrorForPrimitive(p, whichSelectorMatchesEnvironment(env), env);
436 }
437
438 TestError getErrorForPrimitive(OsmPrimitive p, Selector matchingSelector, Environment env) {
439 if (matchingSelector != null && !errors.isEmpty()) {
440 final Command fix = fixPrimitive(p);
441 final String description = getDescriptionForMatchingSelector(p, matchingSelector);
442 final List<OsmPrimitive> primitives;
443 if (env.child != null) {
444 primitives = Arrays.asList(p, env.child);
445 } else {
446 primitives = Collections.singletonList(p);
447 }
448 if (fix != null) {
449 return new FixableTestError(null, getSeverity(), description, null, matchingSelector.toString(), 3000, primitives, fix);
450 } else {
451 return new TestError(null, getSeverity(), description, null, matchingSelector.toString(), 3000, primitives);
452 }
453 } else {
454 return null;
455 }
456 }
457 }
458
459 static class MapCSSTagCheckerAndRule extends MapCSSTagChecker {
460 public final GroupedMapCSSRule rule;
461
462 MapCSSTagCheckerAndRule(GroupedMapCSSRule rule) {
463 this.rule = rule;
464 }
465
466 @Override
467 public boolean equals(Object obj) {
468 return super.equals(obj)
469 || (obj instanceof TagCheck && rule.equals(((TagCheck) obj).rule))
470 || (obj instanceof GroupedMapCSSRule && rule.equals(obj));
471 }
472
473 @Override
474 public int hashCode() {
475 final int prime = 31;
476 int result = super.hashCode();
477 result = prime * result + ((rule == null) ? 0 : rule.hashCode());
478 return result;
479 }
480 }
481
482 /**
483 * Obtains all {@link TestError}s for the {@link OsmPrimitive} {@code p}.
484 */
485 public Collection<TestError> getErrorsForPrimitive(OsmPrimitive p, boolean includeOtherSeverity) {
486 final ArrayList<TestError> r = new ArrayList<>();
487 final Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null);
488 for (TagCheck check : checks) {
489 if (Severity.OTHER.equals(check.getSeverity()) && !includeOtherSeverity) {
490 continue;
491 }
492 final Selector selector = check.whichSelectorMatchesEnvironment(env);
493 if (selector != null) {
494 check.rule.declaration.execute(env);
495 final TestError error = check.getErrorForPrimitive(p, selector, env);
496 if (error != null) {
497 error.setTester(new MapCSSTagCheckerAndRule(check.rule));
498 r.add(error);
499 }
500 }
501 }
502 return r;
503 }
504
505 /**
506 * Visiting call for primitives.
507 *
508 * @param p The primitive to inspect.
509 */
510 @Override
511 public void check(OsmPrimitive p) {
512 errors.addAll(getErrorsForPrimitive(p, ValidatorPreference.PREF_OTHER.get()));
513 }
514
515 /**
516 * Adds a new MapCSS config file from the given {@code Reader}.
517 * @param css The reader
518 * @throws ParseException if the config file does not match MapCSS syntax
519 */
520 public void addMapCSS(Reader css) throws ParseException {
521 checks.addAll(TagCheck.readMapCSS(css));
522 }
523
524 @Override
525 public synchronized void initialize() throws Exception {
526 checks.clear();
527 for (String i : new ValidatorTagCheckerRulesPreference.RulePrefHelper().getActiveUrls()) {
528 try {
529 if (i.startsWith("resource:")) {
530 Main.debug(tr("Adding {0} to tag checker", i));
531 } else {
532 Main.info(tr("Adding {0} to tag checker", i));
533 }
534 try (MirroredInputStream s = new MirroredInputStream(i)) {
535 addMapCSS(new BufferedReader(UTFInputStreamReader.create(s)));
536 }
537 } catch (IOException ex) {
538 Main.warn(tr("Failed to add {0} to tag checker", i));
539 Main.warn(ex, false);
540 } catch (Exception ex) {
541 Main.warn(tr("Failed to add {0} to tag checker", i));
542 Main.warn(ex);
543 }
544 }
545 }
546
547 @Override
548 public int hashCode() {
549 final int prime = 31;
550 int result = super.hashCode();
551 result = prime * result + ((checks == null) ? 0 : checks.hashCode());
552 return result;
553 }
554
555 @Override
556 public boolean equals(Object obj) {
557 if (this == obj)
558 return true;
559 if (!super.equals(obj))
560 return false;
561 if (!(obj instanceof MapCSSTagChecker))
562 return false;
563 MapCSSTagChecker other = (MapCSSTagChecker) obj;
564 if (checks == null) {
565 if (other.checks != null)
566 return false;
567 } else if (!checks.equals(other.checks))
568 return false;
569 return true;
570 }
571}
Note: See TracBrowser for help on using the repository browser.