source: josm/trunk/src/org/openstreetmap/josm/data/validation/tests/TagChecker.java@ 14859

Last change on this file since 14859 was 14859, checked in by GerdP, 5 years ago

fix #17438: Use constant INVALID_KEY for error "Presets do not contain property key"
instead of INVALID_VALUE (very old typo)
  • Property svn:eol-style set to native
File size: 37.4 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.validation.tests;
3
4import static org.openstreetmap.josm.tools.I18n.marktr;
5import static org.openstreetmap.josm.tools.I18n.tr;
6
7import java.awt.GridBagConstraints;
8import java.awt.event.ActionListener;
9import java.io.BufferedReader;
10import java.io.IOException;
11import java.util.ArrayList;
12import java.util.Arrays;
13import java.util.Collection;
14import java.util.Collections;
15import java.util.HashMap;
16import java.util.HashSet;
17import java.util.List;
18import java.util.Locale;
19import java.util.Map;
20import java.util.Map.Entry;
21import java.util.Set;
22
23import javax.swing.JCheckBox;
24import javax.swing.JLabel;
25import javax.swing.JPanel;
26
27import org.openstreetmap.josm.command.ChangePropertyCommand;
28import org.openstreetmap.josm.command.ChangePropertyKeyCommand;
29import org.openstreetmap.josm.command.Command;
30import org.openstreetmap.josm.command.SequenceCommand;
31import org.openstreetmap.josm.data.osm.AbstractPrimitive;
32import org.openstreetmap.josm.data.osm.OsmPrimitive;
33import org.openstreetmap.josm.data.osm.Tag;
34import org.openstreetmap.josm.data.osm.Tagged;
35import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper;
36import org.openstreetmap.josm.data.validation.Severity;
37import org.openstreetmap.josm.data.validation.Test.TagTest;
38import org.openstreetmap.josm.data.validation.TestError;
39import org.openstreetmap.josm.data.validation.util.Entities;
40import org.openstreetmap.josm.gui.progress.ProgressMonitor;
41import org.openstreetmap.josm.gui.tagging.presets.TaggingPreset;
42import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetItem;
43import org.openstreetmap.josm.gui.tagging.presets.TaggingPresets;
44import org.openstreetmap.josm.gui.tagging.presets.items.Check;
45import org.openstreetmap.josm.gui.tagging.presets.items.CheckGroup;
46import org.openstreetmap.josm.gui.tagging.presets.items.KeyedItem;
47import org.openstreetmap.josm.gui.widgets.EditableList;
48import org.openstreetmap.josm.io.CachedFile;
49import org.openstreetmap.josm.spi.preferences.Config;
50import org.openstreetmap.josm.tools.GBC;
51import org.openstreetmap.josm.tools.Logging;
52import org.openstreetmap.josm.tools.MultiMap;
53import org.openstreetmap.josm.tools.Utils;
54
55/**
56 * Check for misspelled or wrong tags
57 *
58 * @author frsantos
59 * @since 3669
60 */
61public class TagChecker extends TagTest {
62
63 /** The config file of ignored tags */
64 public static final String IGNORE_FILE = "resource://data/validator/ignoretags.cfg";
65 /** The config file of dictionary words */
66 public static final String SPELL_FILE = "resource://data/validator/words.cfg";
67
68 /** Normalized keys: the key should be substituted by the value if the key was not found in presets */
69 private static final Map<String, String> harmonizedKeys = new HashMap<>();
70 /** The spell check preset values which are not stored in TaggingPresets */
71 private static volatile HashSet<String> additionalPresetsValueData;
72 /** often used tags which are not in presets */
73 private static volatile MultiMap<String, String> oftenUsedTags = new MultiMap<>();
74
75 /** The TagChecker data */
76 private static final List<String> ignoreDataStartsWith = new ArrayList<>();
77 private static final Set<String> ignoreDataEquals = new HashSet<>();
78 private static final List<String> ignoreDataEndsWith = new ArrayList<>();
79 private static final List<Tag> ignoreDataTag = new ArrayList<>();
80 /** tag keys that have only numerical values in the presets */
81 private static final Set<String> ignoreForLevenshtein = new HashSet<>();
82
83 /** The preferences prefix */
84 protected static final String PREFIX = ValidatorPrefHelper.PREFIX + "." + TagChecker.class.getSimpleName();
85
86 /**
87 * The preference key to check values
88 */
89 public static final String PREF_CHECK_VALUES = PREFIX + ".checkValues";
90 /**
91 * The preference key to check keys
92 */
93 public static final String PREF_CHECK_KEYS = PREFIX + ".checkKeys";
94 /**
95 * The preference key to enable complex checks
96 */
97 public static final String PREF_CHECK_COMPLEX = PREFIX + ".checkComplex";
98 /**
99 * The preference key to search for fixme tags
100 */
101 public static final String PREF_CHECK_FIXMES = PREFIX + ".checkFixmes";
102
103 /**
104 * The preference key for source files
105 * @see #DEFAULT_SOURCES
106 */
107 public static final String PREF_SOURCES = PREFIX + ".source";
108
109 private static final String BEFORE_UPLOAD = "BeforeUpload";
110 /**
111 * The preference key to check keys - used before upload
112 */
113 public static final String PREF_CHECK_KEYS_BEFORE_UPLOAD = PREF_CHECK_KEYS + BEFORE_UPLOAD;
114 /**
115 * The preference key to check values - used before upload
116 */
117 public static final String PREF_CHECK_VALUES_BEFORE_UPLOAD = PREF_CHECK_VALUES + BEFORE_UPLOAD;
118 /**
119 * The preference key to run complex tests - used before upload
120 */
121 public static final String PREF_CHECK_COMPLEX_BEFORE_UPLOAD = PREF_CHECK_COMPLEX + BEFORE_UPLOAD;
122 /**
123 * The preference key to search for fixmes - used before upload
124 */
125 public static final String PREF_CHECK_FIXMES_BEFORE_UPLOAD = PREF_CHECK_FIXMES + BEFORE_UPLOAD;
126
127 private static final int MAX_LEVENSHTEIN_DISTANCE = 2;
128
129 protected boolean checkKeys;
130 protected boolean checkValues;
131 /** Was used for special configuration file, might be used to disable value spell checker. */
132 protected boolean checkComplex;
133 protected boolean checkFixmes;
134
135 protected JCheckBox prefCheckKeys;
136 protected JCheckBox prefCheckValues;
137 protected JCheckBox prefCheckComplex;
138 protected JCheckBox prefCheckFixmes;
139 protected JCheckBox prefCheckPaint;
140
141 protected JCheckBox prefCheckKeysBeforeUpload;
142 protected JCheckBox prefCheckValuesBeforeUpload;
143 protected JCheckBox prefCheckComplexBeforeUpload;
144 protected JCheckBox prefCheckFixmesBeforeUpload;
145 protected JCheckBox prefCheckPaintBeforeUpload;
146
147 // CHECKSTYLE.OFF: SingleSpaceSeparator
148 protected static final int EMPTY_VALUES = 1200;
149 protected static final int INVALID_KEY = 1201;
150 protected static final int INVALID_VALUE = 1202;
151 protected static final int FIXME = 1203;
152 protected static final int INVALID_SPACE = 1204;
153 protected static final int INVALID_KEY_SPACE = 1205;
154 protected static final int INVALID_HTML = 1206; /* 1207 was PAINT */
155 protected static final int LONG_VALUE = 1208;
156 protected static final int LONG_KEY = 1209;
157 protected static final int LOW_CHAR_VALUE = 1210;
158 protected static final int LOW_CHAR_KEY = 1211;
159 protected static final int MISSPELLED_VALUE = 1212;
160 protected static final int MISSPELLED_KEY = 1213;
161 protected static final int MULTIPLE_SPACES = 1214;
162 protected static final int MISSPELLED_VALUE_NO_FIX = 1215;
163 // CHECKSTYLE.ON: SingleSpaceSeparator
164
165 protected EditableList sourcesList;
166
167 private static final List<String> DEFAULT_SOURCES = Arrays.asList(IGNORE_FILE, SPELL_FILE);
168
169 /**
170 * Constructor
171 */
172 public TagChecker() {
173 super(tr("Tag checker"), tr("This test checks for errors in tag keys and values."));
174 }
175
176 @Override
177 public void initialize() throws IOException {
178 initializeData();
179 initializePresets();
180 analysePresets();
181 }
182
183 /**
184 * Add presets that contain only numerical values to the ignore list
185 */
186 private static void analysePresets() {
187 for (String key : TaggingPresets.getPresetKeys()) {
188 if (isKeyIgnored(key))
189 continue;
190 boolean allNumerical = true;
191 Set<String> values = TaggingPresets.getPresetValues(key);
192 if (values.isEmpty())
193 allNumerical = false;
194 for (String val : values) {
195 if (!isNum(val)) {
196 allNumerical = false;
197 break;
198 }
199 }
200 if (allNumerical) {
201 ignoreForLevenshtein.add(key);
202 }
203 }
204 }
205
206 /**
207 * Reads the spell-check file into a HashMap.
208 * The data file is a list of words, beginning with +/-. If it starts with +,
209 * the word is valid, but if it starts with -, the word should be replaced
210 * by the nearest + word before this.
211 *
212 * @throws IOException if any I/O error occurs
213 */
214 private static void initializeData() throws IOException {
215 ignoreDataStartsWith.clear();
216 ignoreDataEquals.clear();
217 ignoreDataEndsWith.clear();
218 ignoreDataTag.clear();
219 harmonizedKeys.clear();
220 ignoreForLevenshtein.clear();
221 oftenUsedTags.clear();
222
223 StringBuilder errorSources = new StringBuilder();
224 for (String source : Config.getPref().getList(PREF_SOURCES, DEFAULT_SOURCES)) {
225 try (
226 CachedFile cf = new CachedFile(source);
227 BufferedReader reader = cf.getContentReader()
228 ) {
229 String okValue = null;
230 boolean tagcheckerfile = false;
231 boolean ignorefile = false;
232 boolean isFirstLine = true;
233 String line;
234 while ((line = reader.readLine()) != null) {
235 if (line.isEmpty()) {
236 // ignore
237 } else if (line.startsWith("#")) {
238 if (line.startsWith("# JOSM TagChecker")) {
239 tagcheckerfile = true;
240 Logging.error(tr("Ignoring {0}. Support was dropped", source));
241 } else
242 if (line.startsWith("# JOSM IgnoreTags")) {
243 ignorefile = true;
244 if (!DEFAULT_SOURCES.contains(source)) {
245 Logging.info(tr("Adding {0} to ignore tags", source));
246 }
247 }
248 } else if (ignorefile) {
249 parseIgnoreFileLine(source, line);
250 } else if (tagcheckerfile) {
251 // ignore
252 } else if (line.charAt(0) == '+') {
253 okValue = line.substring(1);
254 } else if (line.charAt(0) == '-' && okValue != null) {
255 String hk = harmonizeKey(line.substring(1));
256 if (!okValue.equals(hk) && harmonizedKeys.put(hk, okValue) != null) {
257 Logging.debug(tr("Line was ignored: {0}", line));
258 }
259 } else {
260 Logging.error(tr("Invalid spellcheck line: {0}", line));
261 }
262 if (isFirstLine) {
263 isFirstLine = false;
264 if (!(tagcheckerfile || ignorefile) && !DEFAULT_SOURCES.contains(source)) {
265 Logging.info(tr("Adding {0} to spellchecker", source));
266 }
267 }
268 }
269 } catch (IOException e) {
270 Logging.error(e);
271 errorSources.append(source).append('\n');
272 }
273 }
274
275 if (errorSources.length() > 0)
276 throw new IOException(tr("Could not access data file(s):\n{0}", errorSources));
277 }
278
279 /**
280 * Parse a line found in a configuration file
281 * @param source name of configuration file
282 * @param line the line to parse
283 */
284 private static void parseIgnoreFileLine(String source, String line) {
285 line = line.trim();
286 if (line.length() < 4) {
287 return;
288 }
289 try {
290 String key = line.substring(0, 2);
291 line = line.substring(2);
292
293 switch (key) {
294 case "S:":
295 ignoreDataStartsWith.add(line);
296 break;
297 case "E:":
298 ignoreDataEquals.add(line);
299 break;
300 case "F:":
301 ignoreDataEndsWith.add(line);
302 break;
303 case "K:":
304 Tag tag = Tag.ofString(line);
305 ignoreDataTag.add(tag);
306 oftenUsedTags.put(tag.getKey(), tag.getValue());
307 break;
308 default:
309 if (!key.startsWith(";")) {
310 Logging.warn("Unsupported TagChecker key: " + key);
311 }
312 }
313 } catch (IllegalArgumentException e) {
314 Logging.error("Invalid line in {0} : {1}", source, e.getMessage());
315 Logging.trace(e);
316 }
317 }
318
319 /**
320 * Reads the presets data.
321 *
322 */
323 public static void initializePresets() {
324
325 if (!Config.getPref().getBoolean(PREF_CHECK_VALUES, true))
326 return;
327
328 Collection<TaggingPreset> presets = TaggingPresets.getTaggingPresets();
329 if (!presets.isEmpty()) {
330 initAdditionalPresetsValueData();
331 for (TaggingPreset p : presets) {
332 for (TaggingPresetItem i : p.data) {
333 if (i instanceof KeyedItem) {
334 addPresetValue((KeyedItem) i);
335 } else if (i instanceof CheckGroup) {
336 for (Check c : ((CheckGroup) i).checks) {
337 addPresetValue(c);
338 }
339 }
340 }
341 }
342 }
343 }
344
345 private static void initAdditionalPresetsValueData() {
346 additionalPresetsValueData = new HashSet<>();
347 for (String a : AbstractPrimitive.getUninterestingKeys()) {
348 additionalPresetsValueData.add(a);
349 }
350 for (String a : Config.getPref().getList(ValidatorPrefHelper.PREFIX + ".knownkeys",
351 Arrays.asList("is_in", "int_ref", "fixme", "population"))) {
352 additionalPresetsValueData.add(a);
353 }
354 }
355
356 private static void addPresetValue(KeyedItem ky) {
357 if (ky.key != null && ky.getValues() != null) {
358 String hk = harmonizeKey(ky.key);
359 if (!ky.key.equals(hk)) {
360 harmonizedKeys.put(hk, ky.key);
361 }
362 }
363 }
364
365 /**
366 * Checks given string (key or value) if it contains characters with code below 0x20 (either newline or some other special characters)
367 * @param s string to check
368 * @return {@code true} if {@code s} contains characters with code below 0x20
369 */
370 private static boolean containsLow(String s) {
371 if (s == null)
372 return false;
373 for (int i = 0; i < s.length(); i++) {
374 if (s.charAt(i) < 0x20)
375 return true;
376 }
377 return false;
378 }
379
380 /**
381 * Get set of preset values for the given key.
382 * @param key the key
383 * @return null if key is not in presets or in additionalPresetsValueData,
384 * else a set which might be empty.
385 */
386 private static Set<String> getPresetValues(String key) {
387 Set<String> res = TaggingPresets.getPresetValues(key);
388 if (res != null)
389 return res;
390 if (additionalPresetsValueData.contains(key))
391 return Collections.emptySet();
392 // null means key is not known
393 return null;
394 }
395
396 /**
397 * Determines if the given key is in internal presets.
398 * @param key key
399 * @return {@code true} if the given key is in internal presets
400 * @since 9023
401 */
402 public static boolean isKeyInPresets(String key) {
403 return TaggingPresets.getPresetValues(key) != null;
404 }
405
406 /**
407 * Determines if the given tag is in internal presets.
408 * @param key key
409 * @param value value
410 * @return {@code true} if the given tag is in internal presets
411 * @since 9023
412 */
413 public static boolean isTagInPresets(String key, String value) {
414 final Set<String> values = getPresetValues(key);
415 return values != null && values.contains(value);
416 }
417
418 /**
419 * Returns the list of ignored tags.
420 * @return the list of ignored tags
421 * @since 9023
422 */
423 public static List<Tag> getIgnoredTags() {
424 return new ArrayList<>(ignoreDataTag);
425 }
426
427 /**
428 * Determines if the given tag key is ignored for checks "key/tag not in presets".
429 * @param key key
430 * @return true if the given key is ignored
431 */
432 private static boolean isKeyIgnored(String key) {
433 if (ignoreDataEquals.contains(key)) {
434 return true;
435 }
436 for (String a : ignoreDataStartsWith) {
437 if (key.startsWith(a)) {
438 return true;
439 }
440 }
441 for (String a : ignoreDataEndsWith) {
442 if (key.endsWith(a)) {
443 return true;
444 }
445 }
446 return false;
447 }
448
449 /**
450 * Determines if the given tag is ignored for checks "key/tag not in presets".
451 * @param key key
452 * @param value value
453 * @return {@code true} if the given tag is ignored
454 * @since 9023
455 */
456 public static boolean isTagIgnored(String key, String value) {
457 if (isKeyIgnored(key))
458 return true;
459 final Set<String> values = getPresetValues(key);
460 if (values != null && values.isEmpty())
461 return true;
462 if (!isTagInPresets(key, value)) {
463 for (Tag a : ignoreDataTag) {
464 if (key.equals(a.getKey()) && value.equals(a.getValue())) {
465 return true;
466 }
467 }
468 }
469 return false;
470 }
471
472 /**
473 * Checks the primitive tags
474 * @param p The primitive to check
475 */
476 @Override
477 public void check(OsmPrimitive p) {
478 if (!p.isTagged())
479 return;
480
481 // Just a collection to know if a primitive has been already marked with error
482 MultiMap<OsmPrimitive, String> withErrors = new MultiMap<>();
483
484 for (Entry<String, String> prop : p.getKeys().entrySet()) {
485 String s = marktr("Tag ''{0}'' invalid.");
486 String key = prop.getKey();
487 String value = prop.getValue();
488
489 if (checkKeys) {
490 checkSingleTagKeySimple(withErrors, p, s, key);
491 }
492 if (checkValues) {
493 checkSingleTagValueSimple(withErrors, p, s, key, value);
494 checkSingleTagComplex(withErrors, p, key, value);
495 }
496 if (checkFixmes && key != null && value != null && !value.isEmpty() && isFixme(key, value) && !withErrors.contains(p, "FIXME")) {
497 errors.add(TestError.builder(this, Severity.OTHER, FIXME)
498 .message(tr("FIXMES"))
499 .primitives(p)
500 .build());
501 withErrors.put(p, "FIXME");
502 }
503 }
504 }
505
506 private void checkSingleTagValueSimple(MultiMap<OsmPrimitive, String> withErrors, OsmPrimitive p, String s, String key, String value) {
507 if (!checkValues || value == null)
508 return;
509 if ((containsLow(value)) && !withErrors.contains(p, "ICV")) {
510 errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_VALUE)
511 .message(tr("Tag value contains character with code less than 0x20"), s, key)
512 .primitives(p)
513 .build());
514 withErrors.put(p, "ICV");
515 }
516 if ((value.length() > Tagged.MAX_TAG_LENGTH) && !withErrors.contains(p, "LV")) {
517 errors.add(TestError.builder(this, Severity.ERROR, LONG_VALUE)
518 .message(tr("Tag value longer than {0} characters ({1} characters)", Tagged.MAX_TAG_LENGTH, value.length()), s, key)
519 .primitives(p)
520 .build());
521 withErrors.put(p, "LV");
522 }
523 if ((value.trim().isEmpty()) && !withErrors.contains(p, "EV")) {
524 errors.add(TestError.builder(this, Severity.WARNING, EMPTY_VALUES)
525 .message(tr("Tags with empty values"), s, key)
526 .primitives(p)
527 .build());
528 withErrors.put(p, "EV");
529 }
530 final String errTypeSpace = "SPACE";
531 if ((value.startsWith(" ") || value.endsWith(" ")) && !withErrors.contains(p, errTypeSpace)) {
532 errors.add(TestError.builder(this, Severity.WARNING, INVALID_SPACE)
533 .message(tr("Property values start or end with white space"), s, key)
534 .primitives(p)
535 .build());
536 withErrors.put(p, errTypeSpace);
537 }
538 if (value.contains(" ") && !withErrors.contains(p, errTypeSpace)) {
539 errors.add(TestError.builder(this, Severity.WARNING, MULTIPLE_SPACES)
540 .message(tr("Property values contain multiple white spaces"), s, key)
541 .primitives(p)
542 .build());
543 withErrors.put(p, errTypeSpace);
544 }
545 if (!value.equals(Entities.unescape(value)) && !withErrors.contains(p, "HTML")) {
546 errors.add(TestError.builder(this, Severity.OTHER, INVALID_HTML)
547 .message(tr("Property values contain HTML entity"), s, key)
548 .primitives(p)
549 .build());
550 withErrors.put(p, "HTML");
551 }
552 }
553
554 private void checkSingleTagKeySimple(MultiMap<OsmPrimitive, String> withErrors, OsmPrimitive p, String s, String key) {
555 if (!checkKeys || key == null)
556 return;
557 if ((containsLow(key)) && !withErrors.contains(p, "ICK")) {
558 errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_KEY)
559 .message(tr("Tag key contains character with code less than 0x20"), s, key)
560 .primitives(p)
561 .build());
562 withErrors.put(p, "ICK");
563 }
564 if (key.length() > Tagged.MAX_TAG_LENGTH && !withErrors.contains(p, "LK")) {
565 errors.add(TestError.builder(this, Severity.ERROR, LONG_KEY)
566 .message(tr("Tag key longer than {0} characters ({1} characters)", Tagged.MAX_TAG_LENGTH, key.length()), s, key)
567 .primitives(p)
568 .build());
569 withErrors.put(p, "LK");
570 }
571 if (key.indexOf(' ') >= 0 && !withErrors.contains(p, "IPK")) {
572 errors.add(TestError.builder(this, Severity.WARNING, INVALID_KEY_SPACE)
573 .message(tr("Invalid white space in property key"), s, key)
574 .primitives(p)
575 .build());
576 withErrors.put(p, "IPK");
577 }
578 }
579
580 private void checkSingleTagComplex(MultiMap<OsmPrimitive, String> withErrors, OsmPrimitive p, String key, String value) {
581 if (!checkValues || key == null || value == null || value.isEmpty())
582 return;
583 if (additionalPresetsValueData != null && !isTagIgnored(key, value)) {
584 if (!isKeyInPresets(key)) {
585 spellCheckKey(withErrors, p, key);
586 } else if (!isTagInPresets(key, value)) {
587 if (oftenUsedTags.contains(key, value)) {
588 // tag is quite often used but not in presets
589 errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE)
590 .message(tr("Presets do not contain property value"),
591 marktr("Value ''{0}'' for key ''{1}'' not in presets, but is known."), value, key)
592 .primitives(p)
593 .build());
594 withErrors.put(p, "UPV");
595 } else {
596 tryGuess(p, key, value, withErrors);
597 }
598 }
599 }
600 }
601
602 private void spellCheckKey(MultiMap<OsmPrimitive, String> withErrors, OsmPrimitive p, String key) {
603 String prettifiedKey = harmonizeKey(key);
604 String fixedKey = isKeyInPresets(prettifiedKey) ? prettifiedKey : harmonizedKeys.get(prettifiedKey);
605 if (fixedKey != null && !"".equals(fixedKey) && !fixedKey.equals(key)) {
606 // misspelled preset key
607 final TestError.Builder error = TestError.builder(this, Severity.WARNING, MISSPELLED_KEY)
608 .message(tr("Misspelled property key"), marktr("Key ''{0}'' looks like ''{1}''."), key, fixedKey)
609 .primitives(p);
610 if (p.hasKey(fixedKey)) {
611 errors.add(error.build());
612 } else {
613 errors.add(error.fix(() -> new ChangePropertyKeyCommand(p, key, fixedKey)).build());
614 }
615 withErrors.put(p, "WPK");
616 } else {
617 errors.add(TestError.builder(this, Severity.OTHER, INVALID_KEY)
618 .message(tr("Presets do not contain property key"), marktr("Key ''{0}'' not in presets."), key)
619 .primitives(p)
620 .build());
621 withErrors.put(p, "UPK");
622 }
623 }
624
625 private void tryGuess(OsmPrimitive p, String key, String value, MultiMap<OsmPrimitive, String> withErrors) {
626 // try to fix common typos and check again if value is still unknown
627 final String harmonizedValue = harmonizeValue(value);
628 if (harmonizedValue == null || harmonizedValue.isEmpty())
629 return;
630 String fixedValue = null;
631 List<Set<String>> sets = new ArrayList<>();
632 Set<String> presetValues = getPresetValues(key);
633 if (presetValues != null)
634 sets.add(presetValues);
635 Set<String> usedValues = oftenUsedTags.get(key);
636 if (usedValues != null)
637 sets.add(usedValues);
638 for (Set<String> possibleValues: sets) {
639 if (possibleValues.contains(harmonizedValue)) {
640 fixedValue = harmonizedValue;
641 break;
642 }
643 }
644 if (fixedValue == null && !ignoreForLevenshtein.contains(key)) {
645 int maxPresetValueLen = 0;
646 List<String> fixVals = new ArrayList<>();
647 // use Levenshtein distance to find typical typos
648 int minDist = MAX_LEVENSHTEIN_DISTANCE + 1;
649 String closest = null;
650 for (Set<String> possibleValues: sets) {
651 for (String possibleVal : possibleValues) {
652 if (possibleVal.isEmpty())
653 continue;
654 maxPresetValueLen = Math.max(maxPresetValueLen, possibleVal.length());
655 if (harmonizedValue.length() < 3 && possibleVal.length() >= harmonizedValue.length() + MAX_LEVENSHTEIN_DISTANCE) {
656 // don't suggest fix value when given value is short and lengths are too different
657 // for example surface=u would result in surface=mud
658 continue;
659 }
660 int dist = Utils.getLevenshteinDistance(possibleVal, harmonizedValue);
661 if (dist >= harmonizedValue.length()) {
662 // short value, all characters are different. Don't warn, might say Value '10' for key 'fee' looks like 'no'.
663 continue;
664 }
665 if (dist < minDist) {
666 closest = possibleVal;
667 minDist = dist;
668 fixVals.clear();
669 fixVals.add(possibleVal);
670 } else if (dist == minDist) {
671 fixVals.add(possibleVal);
672 }
673 }
674 }
675
676 if (minDist <= MAX_LEVENSHTEIN_DISTANCE && maxPresetValueLen > MAX_LEVENSHTEIN_DISTANCE
677 && (harmonizedValue.length() > 3 || minDist < MAX_LEVENSHTEIN_DISTANCE)) {
678 if (fixVals.size() < 2) {
679 fixedValue = closest;
680 } else {
681 Collections.sort(fixVals);
682 // misspelled preset value with multiple good alternatives
683 errors.add(TestError.builder(this, Severity.WARNING, MISSPELLED_VALUE_NO_FIX)
684 .message(tr("Unknown property value"),
685 marktr("Value ''{0}'' for key ''{1}'' is unknown, maybe one of {2} is meant?"),
686 value, key, fixVals)
687 .primitives(p).build());
688 withErrors.put(p, "WPV");
689 return;
690 }
691 }
692 }
693 if (fixedValue != null && !fixedValue.equals(value)) {
694 final String newValue = fixedValue;
695 // misspelled preset value
696 errors.add(TestError.builder(this, Severity.WARNING, MISSPELLED_VALUE)
697 .message(tr("Unknown property value"),
698 marktr("Value ''{0}'' for key ''{1}'' is unknown, maybe ''{2}'' is meant?"), value, key, newValue)
699 .primitives(p)
700 .build());
701 withErrors.put(p, "WPV");
702 } else {
703 // unknown preset value
704 errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE)
705 .message(tr("Presets do not contain property value"),
706 marktr("Value ''{0}'' for key ''{1}'' not in presets."), value, key)
707 .primitives(p)
708 .build());
709 withErrors.put(p, "UPV");
710 }
711 }
712
713 private static boolean isNum(String harmonizedValue) {
714 try {
715 Double.parseDouble(harmonizedValue);
716 return true;
717 } catch (NumberFormatException e) {
718 return false;
719 }
720 }
721
722 private static boolean isFixme(String key, String value) {
723 return key.toLowerCase(Locale.ENGLISH).contains("fixme") || key.contains("todo")
724 || value.toLowerCase(Locale.ENGLISH).contains("fixme") || value.contains("check and delete");
725 }
726
727 private static String harmonizeKey(String key) {
728 return Utils.strip(key.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(':', '_').replace(' ', '_'), "-_;:,");
729 }
730
731 private static String harmonizeValue(String value) {
732 return Utils.strip(value.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(' ', '_'), "-_;:,");
733 }
734
735 @Override
736 public void startTest(ProgressMonitor monitor) {
737 super.startTest(monitor);
738 checkKeys = Config.getPref().getBoolean(PREF_CHECK_KEYS, true);
739 if (isBeforeUpload) {
740 checkKeys = checkKeys && Config.getPref().getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true);
741 }
742
743 checkValues = Config.getPref().getBoolean(PREF_CHECK_VALUES, true);
744 if (isBeforeUpload) {
745 checkValues = checkValues && Config.getPref().getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true);
746 }
747
748 checkComplex = Config.getPref().getBoolean(PREF_CHECK_COMPLEX, true);
749 if (isBeforeUpload) {
750 checkComplex = checkComplex && Config.getPref().getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true);
751 }
752
753 checkFixmes = Config.getPref().getBoolean(PREF_CHECK_FIXMES, true);
754 if (isBeforeUpload) {
755 checkFixmes = checkFixmes && Config.getPref().getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true);
756 }
757 }
758
759 @Override
760 public void visit(Collection<OsmPrimitive> selection) {
761 if (checkKeys || checkValues || checkComplex || checkFixmes) {
762 super.visit(selection);
763 }
764 }
765
766 @Override
767 public void addGui(JPanel testPanel) {
768 GBC a = GBC.eol();
769 a.anchor = GridBagConstraints.EAST;
770
771 testPanel.add(new JLabel(name+" :"), GBC.eol().insets(3, 0, 0, 0));
772
773 prefCheckKeys = new JCheckBox(tr("Check property keys."), Config.getPref().getBoolean(PREF_CHECK_KEYS, true));
774 prefCheckKeys.setToolTipText(tr("Validate that property keys are valid checking against list of words."));
775 testPanel.add(prefCheckKeys, GBC.std().insets(20, 0, 0, 0));
776
777 prefCheckKeysBeforeUpload = new JCheckBox();
778 prefCheckKeysBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true));
779 testPanel.add(prefCheckKeysBeforeUpload, a);
780
781 prefCheckComplex = new JCheckBox(tr("Use complex property checker."), Config.getPref().getBoolean(PREF_CHECK_COMPLEX, true));
782 prefCheckComplex.setToolTipText(tr("Validate property values and tags using complex rules."));
783 testPanel.add(prefCheckComplex, GBC.std().insets(20, 0, 0, 0));
784
785 prefCheckComplexBeforeUpload = new JCheckBox();
786 prefCheckComplexBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true));
787 testPanel.add(prefCheckComplexBeforeUpload, a);
788
789 final Collection<String> sources = Config.getPref().getList(PREF_SOURCES, DEFAULT_SOURCES);
790 sourcesList = new EditableList(tr("TagChecker source"));
791 sourcesList.setItems(sources);
792 testPanel.add(new JLabel(tr("Data sources ({0})", "*.cfg")), GBC.eol().insets(23, 0, 0, 0));
793 testPanel.add(sourcesList, GBC.eol().fill(GridBagConstraints.HORIZONTAL).insets(23, 0, 0, 0));
794
795 ActionListener disableCheckActionListener = e -> handlePrefEnable();
796 prefCheckKeys.addActionListener(disableCheckActionListener);
797 prefCheckKeysBeforeUpload.addActionListener(disableCheckActionListener);
798 prefCheckComplex.addActionListener(disableCheckActionListener);
799 prefCheckComplexBeforeUpload.addActionListener(disableCheckActionListener);
800
801 handlePrefEnable();
802
803 prefCheckValues = new JCheckBox(tr("Check property values."), Config.getPref().getBoolean(PREF_CHECK_VALUES, true));
804 prefCheckValues.setToolTipText(tr("Validate that property values are valid checking against presets."));
805 testPanel.add(prefCheckValues, GBC.std().insets(20, 0, 0, 0));
806
807 prefCheckValuesBeforeUpload = new JCheckBox();
808 prefCheckValuesBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true));
809 testPanel.add(prefCheckValuesBeforeUpload, a);
810
811 prefCheckFixmes = new JCheckBox(tr("Check for FIXMES."), Config.getPref().getBoolean(PREF_CHECK_FIXMES, true));
812 prefCheckFixmes.setToolTipText(tr("Looks for nodes or ways with FIXME in any property value."));
813 testPanel.add(prefCheckFixmes, GBC.std().insets(20, 0, 0, 0));
814
815 prefCheckFixmesBeforeUpload = new JCheckBox();
816 prefCheckFixmesBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true));
817 testPanel.add(prefCheckFixmesBeforeUpload, a);
818 }
819
820 /**
821 * Enables/disables the source list field
822 */
823 public void handlePrefEnable() {
824 boolean selected = prefCheckKeys.isSelected() || prefCheckKeysBeforeUpload.isSelected()
825 || prefCheckComplex.isSelected() || prefCheckComplexBeforeUpload.isSelected();
826 sourcesList.setEnabled(selected);
827 }
828
829 @Override
830 public boolean ok() {
831 enabled = prefCheckKeys.isSelected() || prefCheckValues.isSelected() || prefCheckComplex.isSelected() || prefCheckFixmes.isSelected();
832 testBeforeUpload = prefCheckKeysBeforeUpload.isSelected() || prefCheckValuesBeforeUpload.isSelected()
833 || prefCheckFixmesBeforeUpload.isSelected() || prefCheckComplexBeforeUpload.isSelected();
834
835 Config.getPref().putBoolean(PREF_CHECK_VALUES, prefCheckValues.isSelected());
836 Config.getPref().putBoolean(PREF_CHECK_COMPLEX, prefCheckComplex.isSelected());
837 Config.getPref().putBoolean(PREF_CHECK_KEYS, prefCheckKeys.isSelected());
838 Config.getPref().putBoolean(PREF_CHECK_FIXMES, prefCheckFixmes.isSelected());
839 Config.getPref().putBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, prefCheckValuesBeforeUpload.isSelected());
840 Config.getPref().putBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, prefCheckComplexBeforeUpload.isSelected());
841 Config.getPref().putBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, prefCheckKeysBeforeUpload.isSelected());
842 Config.getPref().putBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, prefCheckFixmesBeforeUpload.isSelected());
843 return Config.getPref().putList(PREF_SOURCES, sourcesList.getItems());
844 }
845
846 @Override
847 public Command fixError(TestError testError) {
848 List<Command> commands = new ArrayList<>(50);
849
850 Collection<? extends OsmPrimitive> primitives = testError.getPrimitives();
851 for (OsmPrimitive p : primitives) {
852 Map<String, String> tags = p.getKeys();
853 if (tags.isEmpty()) {
854 continue;
855 }
856
857 for (Entry<String, String> prop: tags.entrySet()) {
858 String key = prop.getKey();
859 String value = prop.getValue();
860 if (value == null || value.trim().isEmpty()) {
861 commands.add(new ChangePropertyCommand(p, key, null));
862 } else if (value.startsWith(" ") || value.endsWith(" ") || value.contains(" ")) {
863 commands.add(new ChangePropertyCommand(p, key, Utils.removeWhiteSpaces(value)));
864 } else if (key.startsWith(" ") || key.endsWith(" ") || key.contains(" ")) {
865 commands.add(new ChangePropertyKeyCommand(p, key, Utils.removeWhiteSpaces(key)));
866 } else {
867 String evalue = Entities.unescape(value);
868 if (!evalue.equals(value)) {
869 commands.add(new ChangePropertyCommand(p, key, evalue));
870 }
871 }
872 }
873 }
874
875 if (commands.isEmpty())
876 return null;
877 if (commands.size() == 1)
878 return commands.get(0);
879
880 return new SequenceCommand(tr("Fix tags"), commands);
881 }
882
883 @Override
884 public boolean isFixable(TestError testError) {
885 if (testError.getTester() instanceof TagChecker) {
886 int code = testError.getCode();
887 return code == INVALID_KEY || code == EMPTY_VALUES || code == INVALID_SPACE ||
888 code == INVALID_KEY_SPACE || code == INVALID_HTML ||
889 code == MULTIPLE_SPACES;
890 }
891
892 return false;
893 }
894}
Note: See TracBrowser for help on using the repository browser.