source: josm/trunk/src/org/openstreetmap/josm/data/validation/tests/TagChecker.java@ 14706

Last change on this file since 14706 was 14706, checked in by GerdP, 5 years ago

fix more recent sonar issues
  • Property svn:eol-style set to native
File size: 44.6 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.validation.tests;
3
4import static org.openstreetmap.josm.tools.I18n.marktr;
5import static org.openstreetmap.josm.tools.I18n.tr;
6
7import java.awt.GridBagConstraints;
8import java.awt.event.ActionListener;
9import java.io.BufferedReader;
10import java.io.IOException;
11import java.util.ArrayList;
12import java.util.Arrays;
13import java.util.Collection;
14import java.util.Collections;
15import java.util.HashMap;
16import java.util.HashSet;
17import java.util.List;
18import java.util.Locale;
19import java.util.Map;
20import java.util.Map.Entry;
21import java.util.Set;
22import java.util.regex.Matcher;
23import java.util.regex.Pattern;
24import java.util.regex.PatternSyntaxException;
25
26import javax.swing.JCheckBox;
27import javax.swing.JLabel;
28import javax.swing.JPanel;
29
30import org.openstreetmap.josm.command.ChangePropertyCommand;
31import org.openstreetmap.josm.command.ChangePropertyKeyCommand;
32import org.openstreetmap.josm.command.Command;
33import org.openstreetmap.josm.command.SequenceCommand;
34import org.openstreetmap.josm.data.osm.AbstractPrimitive;
35import org.openstreetmap.josm.data.osm.OsmPrimitive;
36import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
37import org.openstreetmap.josm.data.osm.OsmUtils;
38import org.openstreetmap.josm.data.osm.Tag;
39import org.openstreetmap.josm.data.osm.Tagged;
40import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper;
41import org.openstreetmap.josm.data.validation.Severity;
42import org.openstreetmap.josm.data.validation.Test.TagTest;
43import org.openstreetmap.josm.data.validation.TestError;
44import org.openstreetmap.josm.data.validation.util.Entities;
45import org.openstreetmap.josm.gui.progress.ProgressMonitor;
46import org.openstreetmap.josm.gui.tagging.presets.TaggingPreset;
47import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetItem;
48import org.openstreetmap.josm.gui.tagging.presets.TaggingPresets;
49import org.openstreetmap.josm.gui.tagging.presets.items.Check;
50import org.openstreetmap.josm.gui.tagging.presets.items.CheckGroup;
51import org.openstreetmap.josm.gui.tagging.presets.items.KeyedItem;
52import org.openstreetmap.josm.gui.widgets.EditableList;
53import org.openstreetmap.josm.io.CachedFile;
54import org.openstreetmap.josm.spi.preferences.Config;
55import org.openstreetmap.josm.tools.GBC;
56import org.openstreetmap.josm.tools.Logging;
57import org.openstreetmap.josm.tools.MultiMap;
58import org.openstreetmap.josm.tools.Utils;
59
60/**
61 * Check for misspelled or wrong tags
62 *
63 * @author frsantos
64 * @since 3669
65 */
66public class TagChecker extends TagTest {
67
68 /** The config file of ignored tags */
69 public static final String IGNORE_FILE = "resource://data/validator/ignoretags.cfg";
70 /** The config file of dictionary words */
71 public static final String SPELL_FILE = "resource://data/validator/words.cfg";
72
73 /** Normalized keys: the key should be substituted by the value if the key was not found in presets */
74 private static final Map<String, String> harmonizedKeys = new HashMap<>();
75 /** The spell check preset values which are not stored in TaggingPresets */
76 private static volatile HashSet<String> additionalPresetsValueData;
77 /** often used tags which are not in presets */
78 private static volatile MultiMap<String, String> oftenUsedTags = new MultiMap<>();
79
80 /** The TagChecker data */
81 private static final List<CheckerData> checkerData = new ArrayList<>();
82 private static final List<String> ignoreDataStartsWith = new ArrayList<>();
83 private static final Set<String> ignoreDataEquals = new HashSet<>();
84 private static final List<String> ignoreDataEndsWith = new ArrayList<>();
85 private static final List<Tag> ignoreDataTag = new ArrayList<>();
86 /** tag keys that have only numerical values in the presets */
87 private static final Set<String> ignoreForLevenshtein = new HashSet<>();
88
89 /** The preferences prefix */
90 protected static final String PREFIX = ValidatorPrefHelper.PREFIX + "." + TagChecker.class.getSimpleName();
91
92 /**
93 * The preference key to check values
94 */
95 public static final String PREF_CHECK_VALUES = PREFIX + ".checkValues";
96 /**
97 * The preference key to check keys
98 */
99 public static final String PREF_CHECK_KEYS = PREFIX + ".checkKeys";
100 /**
101 * The preference key to enable complex checks
102 */
103 public static final String PREF_CHECK_COMPLEX = PREFIX + ".checkComplex";
104 /**
105 * The preference key to search for fixme tags
106 */
107 public static final String PREF_CHECK_FIXMES = PREFIX + ".checkFixmes";
108
109 /**
110 * The preference key for source files
111 * @see #DEFAULT_SOURCES
112 */
113 public static final String PREF_SOURCES = PREFIX + ".source";
114
115 private static final String BEFORE_UPLOAD = "BeforeUpload";
116 /**
117 * The preference key to check keys - used before upload
118 */
119 public static final String PREF_CHECK_KEYS_BEFORE_UPLOAD = PREF_CHECK_KEYS + BEFORE_UPLOAD;
120 /**
121 * The preference key to check values - used before upload
122 */
123 public static final String PREF_CHECK_VALUES_BEFORE_UPLOAD = PREF_CHECK_VALUES + BEFORE_UPLOAD;
124 /**
125 * The preference key to run complex tests - used before upload
126 */
127 public static final String PREF_CHECK_COMPLEX_BEFORE_UPLOAD = PREF_CHECK_COMPLEX + BEFORE_UPLOAD;
128 /**
129 * The preference key to search for fixmes - used before upload
130 */
131 public static final String PREF_CHECK_FIXMES_BEFORE_UPLOAD = PREF_CHECK_FIXMES + BEFORE_UPLOAD;
132
133 private static final int MAX_LEVENSHTEIN_DISTANCE = 2;
134
135 protected boolean checkKeys;
136 protected boolean checkValues;
137 protected boolean checkComplex;
138 protected boolean checkFixmes;
139
140 protected JCheckBox prefCheckKeys;
141 protected JCheckBox prefCheckValues;
142 protected JCheckBox prefCheckComplex;
143 protected JCheckBox prefCheckFixmes;
144 protected JCheckBox prefCheckPaint;
145
146 protected JCheckBox prefCheckKeysBeforeUpload;
147 protected JCheckBox prefCheckValuesBeforeUpload;
148 protected JCheckBox prefCheckComplexBeforeUpload;
149 protected JCheckBox prefCheckFixmesBeforeUpload;
150 protected JCheckBox prefCheckPaintBeforeUpload;
151
152 // CHECKSTYLE.OFF: SingleSpaceSeparator
153 protected static final int EMPTY_VALUES = 1200;
154 protected static final int INVALID_KEY = 1201;
155 protected static final int INVALID_VALUE = 1202;
156 protected static final int FIXME = 1203;
157 protected static final int INVALID_SPACE = 1204;
158 protected static final int INVALID_KEY_SPACE = 1205;
159 protected static final int INVALID_HTML = 1206; /* 1207 was PAINT */
160 protected static final int LONG_VALUE = 1208;
161 protected static final int LONG_KEY = 1209;
162 protected static final int LOW_CHAR_VALUE = 1210;
163 protected static final int LOW_CHAR_KEY = 1211;
164 protected static final int MISSPELLED_VALUE = 1212;
165 protected static final int MISSPELLED_KEY = 1213;
166 protected static final int MULTIPLE_SPACES = 1214;
167 protected static final int MISSPELLED_VALUE_NO_FIX = 1215;
168 // CHECKSTYLE.ON: SingleSpaceSeparator
169 // 1250 and up is used by TagCheckLevel in class CheckerData
170
171 protected EditableList sourcesList;
172
173 private static final List<String> DEFAULT_SOURCES = Arrays.asList(/*DATA_FILE, */IGNORE_FILE, SPELL_FILE);
174
175 /**
176 * Constructor
177 */
178 public TagChecker() {
179 super(tr("Tag checker"), tr("This test checks for errors in tag keys and values."));
180 }
181
182 @Override
183 public void initialize() throws IOException {
184 initializeData();
185 initializePresets();
186 analysePresets();
187 }
188
189 /**
190 * Add presets that contain only numerical values to the ignore list
191 */
192 private static void analysePresets() {
193 for (String key : TaggingPresets.getPresetKeys()) {
194 if (isKeyIgnored(key))
195 continue;
196 boolean allNumerical = true;
197 Set<String> values = TaggingPresets.getPresetValues(key);
198 if (values.isEmpty())
199 allNumerical = false;
200 for (String val : values) {
201 if (!isNum(val)) {
202 allNumerical = false;
203 break;
204 }
205 }
206 if (allNumerical) {
207 ignoreForLevenshtein.add(key);
208 }
209 }
210 }
211
212 /**
213 * Reads the spell-check file into a HashMap.
214 * The data file is a list of words, beginning with +/-. If it starts with +,
215 * the word is valid, but if it starts with -, the word should be replaced
216 * by the nearest + word before this.
217 *
218 * @throws IOException if any I/O error occurs
219 */
220 private static void initializeData() throws IOException {
221 checkerData.clear();
222 ignoreDataStartsWith.clear();
223 ignoreDataEquals.clear();
224 ignoreDataEndsWith.clear();
225 ignoreDataTag.clear();
226 harmonizedKeys.clear();
227 ignoreForLevenshtein.clear();
228 oftenUsedTags.clear();
229
230 StringBuilder errorSources = new StringBuilder();
231 for (String source : Config.getPref().getList(PREF_SOURCES, DEFAULT_SOURCES)) {
232 try (
233 CachedFile cf = new CachedFile(source);
234 BufferedReader reader = cf.getContentReader()
235 ) {
236 String okValue = null;
237 boolean tagcheckerfile = false;
238 boolean ignorefile = false;
239 boolean isFirstLine = true;
240 String line;
241 while ((line = reader.readLine()) != null && (tagcheckerfile || !line.isEmpty())) {
242 if (line.startsWith("#")) {
243 if (line.startsWith("# JOSM TagChecker")) {
244 tagcheckerfile = true;
245 if (!DEFAULT_SOURCES.contains(source)) {
246 Logging.info(tr("Adding {0} to tag checker", source));
247 }
248 } else
249 if (line.startsWith("# JOSM IgnoreTags")) {
250 ignorefile = true;
251 if (!DEFAULT_SOURCES.contains(source)) {
252 Logging.info(tr("Adding {0} to ignore tags", source));
253 }
254 }
255 } else if (ignorefile) {
256 parseIgnoreFileLine(source, line);
257 } else if (tagcheckerfile) {
258 if (!line.isEmpty()) {
259 CheckerData d = new CheckerData();
260 String err = d.getData(line);
261
262 if (err == null) {
263 checkerData.add(d);
264 } else {
265 Logging.error(tr("Invalid tagchecker line - {0}: {1}", err, line));
266 }
267 }
268 } else if (line.charAt(0) == '+') {
269 okValue = line.substring(1);
270 } else if (line.charAt(0) == '-' && okValue != null) {
271 String hk = harmonizeKey(line.substring(1));
272 if (!okValue.equals(hk) && harmonizedKeys.put(hk, okValue) != null) {
273 Logging.debug(tr("Line was ignored: {0}", line));
274 }
275 } else {
276 Logging.error(tr("Invalid spellcheck line: {0}", line));
277 }
278 if (isFirstLine) {
279 isFirstLine = false;
280 if (!(tagcheckerfile || ignorefile) && !DEFAULT_SOURCES.contains(source)) {
281 Logging.info(tr("Adding {0} to spellchecker", source));
282 }
283 }
284 }
285 } catch (IOException e) {
286 Logging.error(e);
287 errorSources.append(source).append('\n');
288 }
289 }
290
291 if (errorSources.length() > 0)
292 throw new IOException(tr("Could not access data file(s):\n{0}", errorSources));
293 }
294
295 /**
296 * Parse a line found in a configuration file
297 * @param source name of configuration file
298 * @param line the line to parse
299 */
300 private static void parseIgnoreFileLine(String source, String line) {
301 line = line.trim();
302 if (line.length() < 4) {
303 return;
304 }
305 try {
306 String key = line.substring(0, 2);
307 line = line.substring(2);
308
309 switch (key) {
310 case "S:":
311 ignoreDataStartsWith.add(line);
312 break;
313 case "E:":
314 ignoreDataEquals.add(line);
315 break;
316 case "F:":
317 ignoreDataEndsWith.add(line);
318 break;
319 case "K:":
320 Tag tag = Tag.ofString(line);
321 ignoreDataTag.add(tag);
322 oftenUsedTags.put(tag.getKey(), tag.getValue());
323 break;
324 default:
325 if (!key.startsWith(";")) {
326 Logging.warn("Unsupported TagChecker key: " + key);
327 }
328 }
329 } catch (IllegalArgumentException e) {
330 Logging.error("Invalid line in {0} : {1}", source, e.getMessage());
331 Logging.trace(e);
332 }
333 }
334
335 /**
336 * Reads the presets data.
337 *
338 */
339 public static void initializePresets() {
340
341 if (!Config.getPref().getBoolean(PREF_CHECK_VALUES, true))
342 return;
343
344 Collection<TaggingPreset> presets = TaggingPresets.getTaggingPresets();
345 if (!presets.isEmpty()) {
346 initAdditionalPresetsValueData();
347 for (TaggingPreset p : presets) {
348 for (TaggingPresetItem i : p.data) {
349 if (i instanceof KeyedItem) {
350 addPresetValue((KeyedItem) i);
351 } else if (i instanceof CheckGroup) {
352 for (Check c : ((CheckGroup) i).checks) {
353 addPresetValue(c);
354 }
355 }
356 }
357 }
358 }
359 }
360
361 private static void initAdditionalPresetsValueData() {
362 additionalPresetsValueData = new HashSet<>();
363 for (String a : AbstractPrimitive.getUninterestingKeys()) {
364 additionalPresetsValueData.add(a);
365 }
366 for (String a : Config.getPref().getList(ValidatorPrefHelper.PREFIX + ".knownkeys",
367 Arrays.asList("is_in", "int_ref", "fixme", "population"))) {
368 additionalPresetsValueData.add(a);
369 }
370 }
371
372 private static void addPresetValue(KeyedItem ky) {
373 if (ky.key != null && ky.getValues() != null) {
374 String hk = harmonizeKey(ky.key);
375 if (!ky.key.equals(hk)) {
376 harmonizedKeys.put(hk, ky.key);
377 }
378 }
379 }
380
381 /**
382 * Checks given string (key or value) if it contains characters with code below 0x20 (either newline or some other special characters)
383 * @param s string to check
384 * @return {@code true} if {@code s} contains characters with code below 0x20
385 */
386 private static boolean containsLow(String s) {
387 if (s == null)
388 return false;
389 for (int i = 0; i < s.length(); i++) {
390 if (s.charAt(i) < 0x20)
391 return true;
392 }
393 return false;
394 }
395
396 private static Set<String> getPresetValues(String key) {
397 Set<String> res = TaggingPresets.getPresetValues(key);
398 if (res != null)
399 return res;
400 if (additionalPresetsValueData.contains(key))
401 return Collections.emptySet();
402 return null;
403 }
404
405 /**
406 * Determines if the given key is in internal presets.
407 * @param key key
408 * @return {@code true} if the given key is in internal presets
409 * @since 9023
410 */
411 public static boolean isKeyInPresets(String key) {
412 return TaggingPresets.getPresetValues(key) != null;
413 }
414
415 /**
416 * Determines if the given tag is in internal presets.
417 * @param key key
418 * @param value value
419 * @return {@code true} if the given tag is in internal presets
420 * @since 9023
421 */
422 public static boolean isTagInPresets(String key, String value) {
423 final Set<String> values = getPresetValues(key);
424 return values != null && values.contains(value);
425 }
426
427 /**
428 * Returns the list of ignored tags.
429 * @return the list of ignored tags
430 * @since 9023
431 */
432 public static List<Tag> getIgnoredTags() {
433 return new ArrayList<>(ignoreDataTag);
434 }
435
436 /**
437 * Determines if the given tag key is ignored for checks "key/tag not in presets".
438 * @param key key
439 * @return true if the given key is ignored
440 */
441 private static boolean isKeyIgnored(String key) {
442 if (ignoreDataEquals.contains(key)) {
443 return true;
444 }
445 for (String a : ignoreDataStartsWith) {
446 if (key.startsWith(a)) {
447 return true;
448 }
449 }
450 for (String a : ignoreDataEndsWith) {
451 if (key.endsWith(a)) {
452 return true;
453 }
454 }
455 return false;
456 }
457
458 /**
459 * Determines if the given tag is ignored for checks "key/tag not in presets".
460 * @param key key
461 * @param value value
462 * @return {@code true} if the given tag is ignored
463 * @since 9023
464 */
465 public static boolean isTagIgnored(String key, String value) {
466 if (isKeyIgnored(key))
467 return true;
468 final Set<String> values = getPresetValues(key);
469 if (values != null && values.isEmpty())
470 return true;
471 if (!isTagInPresets(key, value)) {
472 for (Tag a : ignoreDataTag) {
473 if (key.equals(a.getKey()) && value.equals(a.getValue())) {
474 return true;
475 }
476 }
477 }
478 return false;
479 }
480
481 /**
482 * Checks the primitive tags
483 * @param p The primitive to check
484 */
485 @Override
486 public void check(OsmPrimitive p) {
487 if (!p.isTagged())
488 return;
489
490 // Just a collection to know if a primitive has been already marked with error
491 MultiMap<OsmPrimitive, String> withErrors = new MultiMap<>();
492
493 if (checkComplex) {
494 Map<String, String> keys = p.getKeys();
495 for (CheckerData d : checkerData) {
496 if (d.match(p, keys)) {
497 errors.add(TestError.builder(this, d.getSeverity(), d.getCode())
498 .message(tr("Suspicious tag/value combinations"), d.getDescription())
499 .primitives(p)
500 .build());
501 withErrors.put(p, "TC");
502 }
503 }
504 }
505
506 for (Entry<String, String> prop : p.getKeys().entrySet()) {
507 String s = marktr("Tag ''{0}'' invalid.");
508 String key = prop.getKey();
509 String value = prop.getValue();
510
511 if (checkKeys) {
512 checkSingleTagKeySimple(withErrors, p, s, key);
513 }
514 if (checkValues) {
515 checkSingleTagValueSimple(withErrors, p, s, key, value);
516 checkSingleTagComplex(withErrors, p, key, value);
517 }
518 if (checkFixmes && key != null && value != null && !value.isEmpty() && isFixme(key, value) && !withErrors.contains(p, "FIXME")) {
519 errors.add(TestError.builder(this, Severity.OTHER, FIXME)
520 .message(tr("FIXMES"))
521 .primitives(p)
522 .build());
523 withErrors.put(p, "FIXME");
524 }
525 }
526 }
527
528 private void checkSingleTagValueSimple(MultiMap<OsmPrimitive, String> withErrors, OsmPrimitive p, String s, String key, String value) {
529 if (!checkValues || value == null)
530 return;
531 if ((containsLow(value)) && !withErrors.contains(p, "ICV")) {
532 errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_VALUE)
533 .message(tr("Tag value contains character with code less than 0x20"), s, key)
534 .primitives(p)
535 .build());
536 withErrors.put(p, "ICV");
537 }
538 if ((value.length() > Tagged.MAX_TAG_LENGTH) && !withErrors.contains(p, "LV")) {
539 errors.add(TestError.builder(this, Severity.ERROR, LONG_VALUE)
540 .message(tr("Tag value longer than {0} characters ({1} characters)", Tagged.MAX_TAG_LENGTH, value.length()), s, key)
541 .primitives(p)
542 .build());
543 withErrors.put(p, "LV");
544 }
545 if ((value.trim().isEmpty()) && !withErrors.contains(p, "EV")) {
546 errors.add(TestError.builder(this, Severity.WARNING, EMPTY_VALUES)
547 .message(tr("Tags with empty values"), s, key)
548 .primitives(p)
549 .build());
550 withErrors.put(p, "EV");
551 }
552 final String ERR_TYPE_SPACE = "SPACE";
553 if ((value.startsWith(" ") || value.endsWith(" ")) && !withErrors.contains(p, ERR_TYPE_SPACE)) {
554 errors.add(TestError.builder(this, Severity.WARNING, INVALID_SPACE)
555 .message(tr("Property values start or end with white space"), s, key)
556 .primitives(p)
557 .build());
558 withErrors.put(p, ERR_TYPE_SPACE);
559 }
560 if (value.contains(" ") && !withErrors.contains(p, ERR_TYPE_SPACE)) {
561 errors.add(TestError.builder(this, Severity.WARNING, MULTIPLE_SPACES)
562 .message(tr("Property values contain multiple white spaces"), s, key)
563 .primitives(p)
564 .build());
565 withErrors.put(p, ERR_TYPE_SPACE);
566 }
567 if (!value.equals(Entities.unescape(value)) && !withErrors.contains(p, "HTML")) {
568 errors.add(TestError.builder(this, Severity.OTHER, INVALID_HTML)
569 .message(tr("Property values contain HTML entity"), s, key)
570 .primitives(p)
571 .build());
572 withErrors.put(p, "HTML");
573 }
574 }
575
576 private void checkSingleTagKeySimple(MultiMap<OsmPrimitive, String> withErrors, OsmPrimitive p, String s, String key) {
577 if (!checkKeys || key == null)
578 return;
579 if ((containsLow(key)) && !withErrors.contains(p, "ICK")) {
580 errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_KEY)
581 .message(tr("Tag key contains character with code less than 0x20"), s, key)
582 .primitives(p)
583 .build());
584 withErrors.put(p, "ICK");
585 }
586 if (key.length() > Tagged.MAX_TAG_LENGTH && !withErrors.contains(p, "LK")) {
587 errors.add(TestError.builder(this, Severity.ERROR, LONG_KEY)
588 .message(tr("Tag key longer than {0} characters ({1} characters)", Tagged.MAX_TAG_LENGTH, key.length()), s, key)
589 .primitives(p)
590 .build());
591 withErrors.put(p, "LK");
592 }
593 if (key.indexOf(' ') >= 0 && !withErrors.contains(p, "IPK")) {
594 errors.add(TestError.builder(this, Severity.WARNING, INVALID_KEY_SPACE)
595 .message(tr("Invalid white space in property key"), s, key)
596 .primitives(p)
597 .build());
598 withErrors.put(p, "IPK");
599 }
600 }
601
602 private void checkSingleTagComplex(MultiMap<OsmPrimitive, String> withErrors, OsmPrimitive p, String key, String value) {
603 if (!checkValues || key == null || value == null || value.isEmpty())
604 return;
605 if (additionalPresetsValueData != null && !isTagIgnored(key, value)) {
606 if (!isKeyInPresets(key)) {
607 spellCheckKey(withErrors, p, key);
608 } else if (!isTagInPresets(key, value)) {
609 if (oftenUsedTags.contains(key, value)) {
610 // tag is quite often used but not in presets
611 errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE)
612 .message(tr("Presets do not contain property value"),
613 marktr("Value ''{0}'' for key ''{1}'' not in presets, but is known."), value, key)
614 .primitives(p)
615 .build());
616 withErrors.put(p, "UPV");
617 } else {
618 tryGuess(p, key, value, withErrors);
619 }
620 }
621 }
622 }
623
624 private void spellCheckKey(MultiMap<OsmPrimitive, String> withErrors, OsmPrimitive p, String key) {
625 String prettifiedKey = harmonizeKey(key);
626 String fixedKey = isKeyInPresets(prettifiedKey) ? prettifiedKey : harmonizedKeys.get(prettifiedKey);
627 if (fixedKey != null && !"".equals(fixedKey) && !fixedKey.equals(key)) {
628 // misspelled preset key
629 final TestError.Builder error = TestError.builder(this, Severity.WARNING, MISSPELLED_KEY)
630 .message(tr("Misspelled property key"), marktr("Key ''{0}'' looks like ''{1}''."), key, fixedKey)
631 .primitives(p);
632 if (p.hasKey(fixedKey)) {
633 errors.add(error.build());
634 } else {
635 errors.add(error.fix(() -> new ChangePropertyKeyCommand(p, key, fixedKey)).build());
636 }
637 withErrors.put(p, "WPK");
638 } else {
639 errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE)
640 .message(tr("Presets do not contain property key"), marktr("Key ''{0}'' not in presets."), key)
641 .primitives(p)
642 .build());
643 withErrors.put(p, "UPK");
644 }
645 }
646
647 private void tryGuess(OsmPrimitive p, String key, String value, MultiMap<OsmPrimitive, String> withErrors) {
648 // try to fix common typos and check again if value is still unknown
649 final String harmonizedValue = harmonizeValue(value);
650 if (harmonizedValue == null || harmonizedValue.isEmpty())
651 return;
652 String fixedValue = null;
653 Set<String> presetValues = getPresetValues(key);
654 Set<String> oftenUsedValues = oftenUsedTags.get(key);
655 for (Set<String> possibleValues: Arrays.asList(presetValues, oftenUsedValues)) {
656 if (possibleValues != null && possibleValues.contains(harmonizedValue)) {
657 fixedValue = harmonizedValue;
658 break;
659 }
660 }
661 if (fixedValue == null && !ignoreForLevenshtein.contains(key)) {
662 int maxPresetValueLen = 0;
663 List<String> fixVals = new ArrayList<>();
664 // use Levenshtein distance to find typical typos
665 int minDist = MAX_LEVENSHTEIN_DISTANCE + 1;
666 String closest = null;
667 for (Set<String> possibleValues: Arrays.asList(presetValues, oftenUsedValues)) {
668 if (possibleValues == null)
669 continue;
670 for (String possibleVal : possibleValues) {
671 if (possibleVal.isEmpty())
672 continue;
673 maxPresetValueLen = Math.max(maxPresetValueLen, possibleVal.length());
674 if (harmonizedValue.length() < 3 && possibleVal.length() >= harmonizedValue.length() + MAX_LEVENSHTEIN_DISTANCE) {
675 // don't suggest fix value when given value is short and lengths are too different
676 // for example surface=u would result in surface=mud
677 continue;
678 }
679 int dist = Utils.getLevenshteinDistance(possibleVal, harmonizedValue);
680 if (dist >= harmonizedValue.length()) {
681 // short value, all characters are different. Don't warn, might say Value '10' for key 'fee' looks like 'no'.
682 continue;
683 }
684 if (dist < minDist) {
685 closest = possibleVal;
686 minDist = dist;
687 fixVals.clear();
688 fixVals.add(possibleVal);
689 } else if (dist == minDist) {
690 fixVals.add(possibleVal);
691 }
692 }
693 }
694
695 if (minDist <= MAX_LEVENSHTEIN_DISTANCE && maxPresetValueLen > MAX_LEVENSHTEIN_DISTANCE
696 && (harmonizedValue.length() > 3 || minDist < MAX_LEVENSHTEIN_DISTANCE)) {
697 if (fixVals.size() < 2) {
698 fixedValue = closest;
699 } else {
700 Collections.sort(fixVals);
701 // misspelled preset value with multiple good alternatives
702 errors.add(TestError.builder(this, Severity.WARNING, MISSPELLED_VALUE_NO_FIX)
703 .message(tr("Unknown property value"),
704 marktr("Value ''{0}'' for key ''{1}'' is unknown, maybe one of {2} is meant?"),
705 value, key, fixVals)
706 .primitives(p).build());
707 withErrors.put(p, "WPV");
708 return;
709 }
710 }
711 }
712 if (fixedValue != null && !fixedValue.equals(value)) {
713 final String newValue = fixedValue;
714 // misspelled preset value
715 errors.add(TestError.builder(this, Severity.WARNING, MISSPELLED_VALUE)
716 .message(tr("Unknown property value"),
717 marktr("Value ''{0}'' for key ''{1}'' is unknown, maybe ''{2}'' is meant?"), value, key, newValue)
718 .primitives(p)
719 .build());
720 withErrors.put(p, "WPV");
721 } else {
722 // unknown preset value
723 errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE)
724 .message(tr("Presets do not contain property value"),
725 marktr("Value ''{0}'' for key ''{1}'' not in presets."), value, key)
726 .primitives(p)
727 .build());
728 withErrors.put(p, "UPV");
729 }
730 }
731
732 private static boolean isNum(String harmonizedValue) {
733 try {
734 Double.parseDouble(harmonizedValue);
735 return true;
736 } catch (NumberFormatException e) {
737 return false;
738 }
739 }
740
741 private static boolean isFixme(String key, String value) {
742 return key.toLowerCase(Locale.ENGLISH).contains("fixme") || key.contains("todo")
743 || value.toLowerCase(Locale.ENGLISH).contains("fixme") || value.contains("check and delete");
744 }
745
746 private static String harmonizeKey(String key) {
747 return Utils.strip(key.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(':', '_').replace(' ', '_'), "-_;:,");
748 }
749
750 private static String harmonizeValue(String value) {
751 return Utils.strip(value.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(' ', '_'), "-_;:,");
752 }
753
754 @Override
755 public void startTest(ProgressMonitor monitor) {
756 super.startTest(monitor);
757 checkKeys = Config.getPref().getBoolean(PREF_CHECK_KEYS, true);
758 if (isBeforeUpload) {
759 checkKeys = checkKeys && Config.getPref().getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true);
760 }
761
762 checkValues = Config.getPref().getBoolean(PREF_CHECK_VALUES, true);
763 if (isBeforeUpload) {
764 checkValues = checkValues && Config.getPref().getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true);
765 }
766
767 checkComplex = Config.getPref().getBoolean(PREF_CHECK_COMPLEX, true) && !checkerData.isEmpty();
768 if (isBeforeUpload) {
769 checkComplex = checkComplex && Config.getPref().getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true);
770 }
771
772 checkFixmes = Config.getPref().getBoolean(PREF_CHECK_FIXMES, true);
773 if (isBeforeUpload) {
774 checkFixmes = checkFixmes && Config.getPref().getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true);
775 }
776 }
777
778 @Override
779 public void visit(Collection<OsmPrimitive> selection) {
780 if (checkKeys || checkValues || checkComplex || checkFixmes) {
781 super.visit(selection);
782 }
783 }
784
785 @Override
786 public void addGui(JPanel testPanel) {
787 GBC a = GBC.eol();
788 a.anchor = GridBagConstraints.EAST;
789
790 testPanel.add(new JLabel(name+" :"), GBC.eol().insets(3, 0, 0, 0));
791
792 prefCheckKeys = new JCheckBox(tr("Check property keys."), Config.getPref().getBoolean(PREF_CHECK_KEYS, true));
793 prefCheckKeys.setToolTipText(tr("Validate that property keys are valid checking against list of words."));
794 testPanel.add(prefCheckKeys, GBC.std().insets(20, 0, 0, 0));
795
796 prefCheckKeysBeforeUpload = new JCheckBox();
797 prefCheckKeysBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true));
798 testPanel.add(prefCheckKeysBeforeUpload, a);
799
800 prefCheckComplex = new JCheckBox(tr("Use complex property checker."), Config.getPref().getBoolean(PREF_CHECK_COMPLEX, true));
801 prefCheckComplex.setToolTipText(tr("Validate property values and tags using complex rules."));
802 testPanel.add(prefCheckComplex, GBC.std().insets(20, 0, 0, 0));
803
804 prefCheckComplexBeforeUpload = new JCheckBox();
805 prefCheckComplexBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true));
806 testPanel.add(prefCheckComplexBeforeUpload, a);
807
808 final Collection<String> sources = Config.getPref().getList(PREF_SOURCES, DEFAULT_SOURCES);
809 sourcesList = new EditableList(tr("TagChecker source"));
810 sourcesList.setItems(sources);
811 testPanel.add(new JLabel(tr("Data sources ({0})", "*.cfg")), GBC.eol().insets(23, 0, 0, 0));
812 testPanel.add(sourcesList, GBC.eol().fill(GridBagConstraints.HORIZONTAL).insets(23, 0, 0, 0));
813
814 ActionListener disableCheckActionListener = e -> handlePrefEnable();
815 prefCheckKeys.addActionListener(disableCheckActionListener);
816 prefCheckKeysBeforeUpload.addActionListener(disableCheckActionListener);
817 prefCheckComplex.addActionListener(disableCheckActionListener);
818 prefCheckComplexBeforeUpload.addActionListener(disableCheckActionListener);
819
820 handlePrefEnable();
821
822 prefCheckValues = new JCheckBox(tr("Check property values."), Config.getPref().getBoolean(PREF_CHECK_VALUES, true));
823 prefCheckValues.setToolTipText(tr("Validate that property values are valid checking against presets."));
824 testPanel.add(prefCheckValues, GBC.std().insets(20, 0, 0, 0));
825
826 prefCheckValuesBeforeUpload = new JCheckBox();
827 prefCheckValuesBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true));
828 testPanel.add(prefCheckValuesBeforeUpload, a);
829
830 prefCheckFixmes = new JCheckBox(tr("Check for FIXMES."), Config.getPref().getBoolean(PREF_CHECK_FIXMES, true));
831 prefCheckFixmes.setToolTipText(tr("Looks for nodes or ways with FIXME in any property value."));
832 testPanel.add(prefCheckFixmes, GBC.std().insets(20, 0, 0, 0));
833
834 prefCheckFixmesBeforeUpload = new JCheckBox();
835 prefCheckFixmesBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true));
836 testPanel.add(prefCheckFixmesBeforeUpload, a);
837 }
838
839 /**
840 * Enables/disables the source list field
841 */
842 public void handlePrefEnable() {
843 boolean selected = prefCheckKeys.isSelected() || prefCheckKeysBeforeUpload.isSelected()
844 || prefCheckComplex.isSelected() || prefCheckComplexBeforeUpload.isSelected();
845 sourcesList.setEnabled(selected);
846 }
847
848 @Override
849 public boolean ok() {
850 enabled = prefCheckKeys.isSelected() || prefCheckValues.isSelected() || prefCheckComplex.isSelected() || prefCheckFixmes.isSelected();
851 testBeforeUpload = prefCheckKeysBeforeUpload.isSelected() || prefCheckValuesBeforeUpload.isSelected()
852 || prefCheckFixmesBeforeUpload.isSelected() || prefCheckComplexBeforeUpload.isSelected();
853
854 Config.getPref().putBoolean(PREF_CHECK_VALUES, prefCheckValues.isSelected());
855 Config.getPref().putBoolean(PREF_CHECK_COMPLEX, prefCheckComplex.isSelected());
856 Config.getPref().putBoolean(PREF_CHECK_KEYS, prefCheckKeys.isSelected());
857 Config.getPref().putBoolean(PREF_CHECK_FIXMES, prefCheckFixmes.isSelected());
858 Config.getPref().putBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, prefCheckValuesBeforeUpload.isSelected());
859 Config.getPref().putBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, prefCheckComplexBeforeUpload.isSelected());
860 Config.getPref().putBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, prefCheckKeysBeforeUpload.isSelected());
861 Config.getPref().putBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, prefCheckFixmesBeforeUpload.isSelected());
862 return Config.getPref().putList(PREF_SOURCES, sourcesList.getItems());
863 }
864
865 @Override
866 public Command fixError(TestError testError) {
867 List<Command> commands = new ArrayList<>(50);
868
869 Collection<? extends OsmPrimitive> primitives = testError.getPrimitives();
870 for (OsmPrimitive p : primitives) {
871 Map<String, String> tags = p.getKeys();
872 if (tags.isEmpty()) {
873 continue;
874 }
875
876 for (Entry<String, String> prop: tags.entrySet()) {
877 String key = prop.getKey();
878 String value = prop.getValue();
879 if (value == null || value.trim().isEmpty()) {
880 commands.add(new ChangePropertyCommand(p, key, null));
881 } else if (value.startsWith(" ") || value.endsWith(" ") || value.contains(" ")) {
882 commands.add(new ChangePropertyCommand(p, key, Utils.removeWhiteSpaces(value)));
883 } else if (key.startsWith(" ") || key.endsWith(" ") || key.contains(" ")) {
884 commands.add(new ChangePropertyKeyCommand(p, key, Utils.removeWhiteSpaces(key)));
885 } else {
886 String evalue = Entities.unescape(value);
887 if (!evalue.equals(value)) {
888 commands.add(new ChangePropertyCommand(p, key, evalue));
889 }
890 }
891 }
892 }
893
894 if (commands.isEmpty())
895 return null;
896 if (commands.size() == 1)
897 return commands.get(0);
898
899 return new SequenceCommand(tr("Fix tags"), commands);
900 }
901
902 @Override
903 public boolean isFixable(TestError testError) {
904 if (testError.getTester() instanceof TagChecker) {
905 int code = testError.getCode();
906 return code == INVALID_KEY || code == EMPTY_VALUES || code == INVALID_SPACE ||
907 code == INVALID_KEY_SPACE || code == INVALID_HTML ||
908 code == MULTIPLE_SPACES;
909 }
910
911 return false;
912 }
913
914 protected static class CheckerData {
915 private String description;
916 protected List<CheckerElement> data = new ArrayList<>();
917 private OsmPrimitiveType type;
918 private TagCheckLevel level;
919 protected Severity severity;
920
921 private enum TagCheckLevel {
922 TAG_CHECK_ERROR(1250),
923 TAG_CHECK_WARN(1260),
924 TAG_CHECK_INFO(1270);
925
926 final int code;
927
928 TagCheckLevel(int code) {
929 this.code = code;
930 }
931 }
932
933 protected static class CheckerElement {
934 Object tag;
935 Object value;
936 boolean noMatch;
937 boolean tagAll;
938 boolean valueAll;
939 boolean valueBool;
940
941 private static Pattern getPattern(String str) {
942 if (str.endsWith("/i"))
943 return Pattern.compile(str.substring(1, str.length()-2), Pattern.CASE_INSENSITIVE);
944 if (str.endsWith("/"))
945 return Pattern.compile(str.substring(1, str.length()-1));
946
947 throw new IllegalStateException();
948 }
949
950 CheckerElement(String exp) {
951 Matcher m = Pattern.compile("(.+)([!=]=)(.+)").matcher(exp);
952 m.matches();
953
954 String n = m.group(1).trim();
955
956 if ("*".equals(n)) {
957 tagAll = true;
958 } else {
959 tag = n.startsWith("/") ? getPattern(n) : n;
960 noMatch = "!=".equals(m.group(2));
961 n = m.group(3).trim();
962 switch (n) {
963 case "*": valueAll = true; break;
964 case "BOOLEAN_TRUE":
965 valueBool = true;
966 value = OsmUtils.TRUE_VALUE;
967 break;
968 case "BOOLEAN_FALSE":
969 valueBool = true;
970 value = OsmUtils.FALSE_VALUE;
971 break;
972 default:
973 value = n.startsWith("/") ? getPattern(n) : n;
974 }
975 }
976 }
977
978 boolean match(Map<String, String> keys) {
979 for (Entry<String, String> prop: keys.entrySet()) {
980 String key = prop.getKey();
981 String val = valueBool ? OsmUtils.getNamedOsmBoolean(prop.getValue()) : prop.getValue();
982 if ((tagAll || (tag instanceof Pattern ? ((Pattern) tag).matcher(key).matches() : key.equals(tag)))
983 && (valueAll || (value instanceof Pattern ? ((Pattern) value).matcher(val).matches() : val.equals(value))))
984 return !noMatch;
985 }
986 return noMatch;
987 }
988 }
989
990 private static final Pattern CLEAN_STR_PATTERN = Pattern.compile(" *# *([^#]+) *$");
991 private static final Pattern SPLIT_TRIMMED_PATTERN = Pattern.compile(" *: *");
992 private static final Pattern SPLIT_ELEMENTS_PATTERN = Pattern.compile(" *&& *");
993
994 public String getData(final String str) {
995 Matcher m = CLEAN_STR_PATTERN.matcher(str);
996 String trimmed = m.replaceFirst("").trim();
997 try {
998 description = m.group(1);
999 if (description != null && description.isEmpty()) {
1000 description = null;
1001 }
1002 } catch (IllegalStateException e) {
1003 Logging.error(e);
1004 description = null;
1005 }
1006 String[] n = SPLIT_TRIMMED_PATTERN.split(trimmed, 3);
1007 switch (n[0]) {
1008 case "way":
1009 type = OsmPrimitiveType.WAY;
1010 break;
1011 case "node":
1012 type = OsmPrimitiveType.NODE;
1013 break;
1014 case "relation":
1015 type = OsmPrimitiveType.RELATION;
1016 break;
1017 case "*":
1018 type = null;
1019 break;
1020 default:
1021 return tr("Could not find element type");
1022 }
1023 if (n.length != 3)
1024 return tr("Incorrect number of parameters");
1025
1026 switch (n[1]) {
1027 case "W":
1028 severity = Severity.WARNING;
1029 level = TagCheckLevel.TAG_CHECK_WARN;
1030 break;
1031 case "E":
1032 severity = Severity.ERROR;
1033 level = TagCheckLevel.TAG_CHECK_ERROR;
1034 break;
1035 case "I":
1036 severity = Severity.OTHER;
1037 level = TagCheckLevel.TAG_CHECK_INFO;
1038 break;
1039 default:
1040 return tr("Could not find warning level");
1041 }
1042 for (String exp: SPLIT_ELEMENTS_PATTERN.split(n[2])) {
1043 try {
1044 data.add(new CheckerElement(exp));
1045 } catch (IllegalStateException e) {
1046 Logging.trace(e);
1047 return tr("Illegal expression ''{0}''", exp);
1048 } catch (PatternSyntaxException e) {
1049 Logging.trace(e);
1050 return tr("Illegal regular expression ''{0}''", exp);
1051 }
1052 }
1053 return null;
1054 }
1055
1056 public boolean match(OsmPrimitive osm, Map<String, String> keys) {
1057 if (type != null && OsmPrimitiveType.from(osm) != type)
1058 return false;
1059
1060 for (CheckerElement ce : data) {
1061 if (!ce.match(keys))
1062 return false;
1063 }
1064 return true;
1065 }
1066
1067 /**
1068 * Returns the error description.
1069 * @return the error description
1070 */
1071 public String getDescription() {
1072 return description;
1073 }
1074
1075 /**
1076 * Returns the error severity.
1077 * @return the error severity
1078 */
1079 public Severity getSeverity() {
1080 return severity;
1081 }
1082
1083 /**
1084 * Returns the error code.
1085 * @return the error code
1086 */
1087 public int getCode() {
1088 if (type == null)
1089 return level.code;
1090
1091 return level.code + type.ordinal() + 1;
1092 }
1093 }
1094}
Note: See TracBrowser for help on using the repository browser.