source: josm/trunk/src/org/openstreetmap/josm/data/validation/tests/TagChecker.java@ 14696

Last change on this file since 14696 was 14696, checked in by GerdP, 5 years ago

fix #17199

  • Don't suggest different spelling when all preset values for a key are numerical
  • Add layer to ignore list, is checked in MapCSSTagChecker
  • Add start_date end end_date to ignore list
  • apply modified version of 17055-use-ignore.patch: + the lines from ignoretags.cfg starting with K: are treated as "often used". + instead of "... is not in the presets ..." use "... is unknown ..." in messages
  • Property svn:eol-style set to native
File size: 43.7 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.validation.tests;
3
4import static org.openstreetmap.josm.tools.I18n.marktr;
5import static org.openstreetmap.josm.tools.I18n.tr;
6
7import java.awt.GridBagConstraints;
8import java.awt.event.ActionListener;
9import java.io.BufferedReader;
10import java.io.IOException;
11import java.util.ArrayList;
12import java.util.Arrays;
13import java.util.Collection;
14import java.util.Collections;
15import java.util.HashMap;
16import java.util.HashSet;
17import java.util.List;
18import java.util.Locale;
19import java.util.Map;
20import java.util.Map.Entry;
21import java.util.Set;
22import java.util.regex.Matcher;
23import java.util.regex.Pattern;
24import java.util.regex.PatternSyntaxException;
25
26import javax.swing.JCheckBox;
27import javax.swing.JLabel;
28import javax.swing.JPanel;
29
30import org.openstreetmap.josm.command.ChangePropertyCommand;
31import org.openstreetmap.josm.command.ChangePropertyKeyCommand;
32import org.openstreetmap.josm.command.Command;
33import org.openstreetmap.josm.command.SequenceCommand;
34import org.openstreetmap.josm.data.osm.AbstractPrimitive;
35import org.openstreetmap.josm.data.osm.OsmPrimitive;
36import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
37import org.openstreetmap.josm.data.osm.OsmUtils;
38import org.openstreetmap.josm.data.osm.Tag;
39import org.openstreetmap.josm.data.osm.Tagged;
40import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper;
41import org.openstreetmap.josm.data.validation.Severity;
42import org.openstreetmap.josm.data.validation.Test.TagTest;
43import org.openstreetmap.josm.data.validation.TestError;
44import org.openstreetmap.josm.data.validation.util.Entities;
45import org.openstreetmap.josm.gui.progress.ProgressMonitor;
46import org.openstreetmap.josm.gui.tagging.presets.TaggingPreset;
47import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetItem;
48import org.openstreetmap.josm.gui.tagging.presets.TaggingPresets;
49import org.openstreetmap.josm.gui.tagging.presets.items.Check;
50import org.openstreetmap.josm.gui.tagging.presets.items.CheckGroup;
51import org.openstreetmap.josm.gui.tagging.presets.items.KeyedItem;
52import org.openstreetmap.josm.gui.widgets.EditableList;
53import org.openstreetmap.josm.io.CachedFile;
54import org.openstreetmap.josm.spi.preferences.Config;
55import org.openstreetmap.josm.tools.GBC;
56import org.openstreetmap.josm.tools.Logging;
57import org.openstreetmap.josm.tools.MultiMap;
58import org.openstreetmap.josm.tools.Utils;
59
60/**
61 * Check for misspelled or wrong tags
62 *
63 * @author frsantos
64 * @since 3669
65 */
66public class TagChecker extends TagTest {
67
68 /** The config file of ignored tags */
69 public static final String IGNORE_FILE = "resource://data/validator/ignoretags.cfg";
70 /** The config file of dictionary words */
71 public static final String SPELL_FILE = "resource://data/validator/words.cfg";
72
73 /** Normalized keys: the key should be substituted by the value if the key was not found in presets */
74 private static final Map<String, String> harmonizedKeys = new HashMap<>();
75 /** The spell check preset values which are not stored in TaggingPresets */
76 private static volatile MultiMap<String, String> additionalPresetsValueData;
77 /** The spell check preset values which are not stored in TaggingPresets */
78 private static volatile MultiMap<String, String> oftenUsedValueData = new MultiMap<>();
79
80 /** The TagChecker data */
81 private static final List<CheckerData> checkerData = new ArrayList<>();
82 private static final List<String> ignoreDataStartsWith = new ArrayList<>();
83 private static final Set<String> ignoreDataEquals = new HashSet<>();
84 private static final List<String> ignoreDataEndsWith = new ArrayList<>();
85 private static final List<Tag> ignoreDataTag = new ArrayList<>();
86 /** tag keys that have only numerical values in the presets */
87 private static final Set<String> ignoreForLevenshtein = new HashSet<>();
88
89 /** The preferences prefix */
90 protected static final String PREFIX = ValidatorPrefHelper.PREFIX + "." + TagChecker.class.getSimpleName();
91
92 /**
93 * The preference key to check values
94 */
95 public static final String PREF_CHECK_VALUES = PREFIX + ".checkValues";
96 /**
97 * The preference key to check keys
98 */
99 public static final String PREF_CHECK_KEYS = PREFIX + ".checkKeys";
100 /**
101 * The preference key to enable complex checks
102 */
103 public static final String PREF_CHECK_COMPLEX = PREFIX + ".checkComplex";
104 /**
105 * The preference key to search for fixme tags
106 */
107 public static final String PREF_CHECK_FIXMES = PREFIX + ".checkFixmes";
108
109 /**
110 * The preference key for source files
111 * @see #DEFAULT_SOURCES
112 */
113 public static final String PREF_SOURCES = PREFIX + ".source";
114
115 /**
116 * The preference key to check keys - used before upload
117 */
118 public static final String PREF_CHECK_KEYS_BEFORE_UPLOAD = PREF_CHECK_KEYS + "BeforeUpload";
119 /**
120 * The preference key to check values - used before upload
121 */
122 public static final String PREF_CHECK_VALUES_BEFORE_UPLOAD = PREF_CHECK_VALUES + "BeforeUpload";
123 /**
124 * The preference key to run complex tests - used before upload
125 */
126 public static final String PREF_CHECK_COMPLEX_BEFORE_UPLOAD = PREF_CHECK_COMPLEX + "BeforeUpload";
127 /**
128 * The preference key to search for fixmes - used before upload
129 */
130 public static final String PREF_CHECK_FIXMES_BEFORE_UPLOAD = PREF_CHECK_FIXMES + "BeforeUpload";
131
132 private static final int MAX_LEVENSHTEIN_DISTANCE = 2;
133
134 protected boolean checkKeys;
135 protected boolean checkValues;
136 protected boolean checkComplex;
137 protected boolean checkFixmes;
138
139 protected JCheckBox prefCheckKeys;
140 protected JCheckBox prefCheckValues;
141 protected JCheckBox prefCheckComplex;
142 protected JCheckBox prefCheckFixmes;
143 protected JCheckBox prefCheckPaint;
144
145 protected JCheckBox prefCheckKeysBeforeUpload;
146 protected JCheckBox prefCheckValuesBeforeUpload;
147 protected JCheckBox prefCheckComplexBeforeUpload;
148 protected JCheckBox prefCheckFixmesBeforeUpload;
149 protected JCheckBox prefCheckPaintBeforeUpload;
150
151 // CHECKSTYLE.OFF: SingleSpaceSeparator
152 protected static final int EMPTY_VALUES = 1200;
153 protected static final int INVALID_KEY = 1201;
154 protected static final int INVALID_VALUE = 1202;
155 protected static final int FIXME = 1203;
156 protected static final int INVALID_SPACE = 1204;
157 protected static final int INVALID_KEY_SPACE = 1205;
158 protected static final int INVALID_HTML = 1206; /* 1207 was PAINT */
159 protected static final int LONG_VALUE = 1208;
160 protected static final int LONG_KEY = 1209;
161 protected static final int LOW_CHAR_VALUE = 1210;
162 protected static final int LOW_CHAR_KEY = 1211;
163 protected static final int MISSPELLED_VALUE = 1212;
164 protected static final int MISSPELLED_KEY = 1213;
165 protected static final int MULTIPLE_SPACES = 1214;
166 protected static final int MISSPELLED_VALUE_NO_FIX = 1215;
167 // CHECKSTYLE.ON: SingleSpaceSeparator
168 // 1250 and up is used by tagcheck
169
170 protected EditableList sourcesList;
171
172 private static final List<String> DEFAULT_SOURCES = Arrays.asList(/*DATA_FILE, */IGNORE_FILE, SPELL_FILE);
173
174 /**
175 * Constructor
176 */
177 public TagChecker() {
178 super(tr("Tag checker"), tr("This test checks for errors in tag keys and values."));
179 }
180
181 @Override
182 public void initialize() throws IOException {
183 initializeData();
184 initializePresets();
185 analysePresets();
186 }
187
188 /**
189 * Add presets that contain only numerical values to the ignore list
190 */
191 private void analysePresets() {
192 for (String key : TaggingPresets.getPresetKeys()) {
193 if (isKeyIgnored(key))
194 continue;
195 boolean allNumerical = true;
196 Set<String> values = TaggingPresets.getPresetValues(key);
197 if (values.isEmpty())
198 allNumerical = false;
199 for (String val : values) {
200 if (!isNum(val)) {
201 allNumerical = false;
202 break;
203 }
204 }
205 if (allNumerical) {
206 ignoreForLevenshtein.add(key);
207 }
208 }
209 }
210
211 /**
212 * Reads the spellcheck file into a HashMap.
213 * The data file is a list of words, beginning with +/-. If it starts with +,
214 * the word is valid, but if it starts with -, the word should be replaced
215 * by the nearest + word before this.
216 *
217 * @throws IOException if any I/O error occurs
218 */
219 private static void initializeData() throws IOException {
220 checkerData.clear();
221 ignoreDataStartsWith.clear();
222 ignoreDataEquals.clear();
223 ignoreDataEndsWith.clear();
224 ignoreDataTag.clear();
225 harmonizedKeys.clear();
226 ignoreForLevenshtein.clear();
227
228 StringBuilder errorSources = new StringBuilder();
229 for (String source : Config.getPref().getList(PREF_SOURCES, DEFAULT_SOURCES)) {
230 try (
231 CachedFile cf = new CachedFile(source);
232 BufferedReader reader = cf.getContentReader()
233 ) {
234 String okValue = null;
235 boolean tagcheckerfile = false;
236 boolean ignorefile = false;
237 boolean isFirstLine = true;
238 String line;
239 while ((line = reader.readLine()) != null && (tagcheckerfile || !line.isEmpty())) {
240 if (line.startsWith("#")) {
241 if (line.startsWith("# JOSM TagChecker")) {
242 tagcheckerfile = true;
243 if (!DEFAULT_SOURCES.contains(source)) {
244 Logging.info(tr("Adding {0} to tag checker", source));
245 }
246 } else
247 if (line.startsWith("# JOSM IgnoreTags")) {
248 ignorefile = true;
249 if (!DEFAULT_SOURCES.contains(source)) {
250 Logging.info(tr("Adding {0} to ignore tags", source));
251 }
252 }
253 } else if (ignorefile) {
254 line = line.trim();
255 if (line.length() < 4) {
256 continue;
257 }
258 try {
259 String key = line.substring(0, 2);
260 line = line.substring(2);
261
262 switch (key) {
263 case "S:":
264 ignoreDataStartsWith.add(line);
265 break;
266 case "E:":
267 ignoreDataEquals.add(line);
268 break;
269 case "F:":
270 ignoreDataEndsWith.add(line);
271 break;
272 case "K:":
273 Tag tag = Tag.ofString(line);
274 ignoreDataTag.add(tag);
275 oftenUsedValueData.put(tag.getKey(), tag.getValue());
276 break;
277 default:
278 if (!key.startsWith(";")) {
279 Logging.warn("Unsupported TagChecker key: " + key);
280 }
281 }
282 } catch (IllegalArgumentException e) {
283 Logging.error("Invalid line in {0} : {1}", source, e.getMessage());
284 Logging.trace(e);
285 }
286 } else if (tagcheckerfile) {
287 if (!line.isEmpty()) {
288 CheckerData d = new CheckerData();
289 String err = d.getData(line);
290
291 if (err == null) {
292 checkerData.add(d);
293 } else {
294 Logging.error(tr("Invalid tagchecker line - {0}: {1}", err, line));
295 }
296 }
297 } else if (line.charAt(0) == '+') {
298 okValue = line.substring(1);
299 } else if (line.charAt(0) == '-' && okValue != null) {
300 String hk = harmonizeKey(line.substring(1));
301 if (!okValue.equals(hk)) {
302 if (harmonizedKeys.put(hk, okValue) != null) {
303 Logging.debug(tr("Line was ignored: {0}", line));
304 }
305 }
306 } else {
307 Logging.error(tr("Invalid spellcheck line: {0}", line));
308 }
309 if (isFirstLine) {
310 isFirstLine = false;
311 if (!(tagcheckerfile || ignorefile) && !DEFAULT_SOURCES.contains(source)) {
312 Logging.info(tr("Adding {0} to spellchecker", source));
313 }
314 }
315 }
316 } catch (IOException e) {
317 Logging.error(e);
318 errorSources.append(source).append('\n');
319 }
320 }
321
322 if (errorSources.length() > 0)
323 throw new IOException(tr("Could not access data file(s):\n{0}", errorSources));
324 }
325
326 /**
327 * Reads the presets data.
328 *
329 */
330 public static void initializePresets() {
331
332 if (!Config.getPref().getBoolean(PREF_CHECK_VALUES, true))
333 return;
334
335 Collection<TaggingPreset> presets = TaggingPresets.getTaggingPresets();
336 if (!presets.isEmpty()) {
337 additionalPresetsValueData = new MultiMap<>();
338 for (String a : AbstractPrimitive.getUninterestingKeys()) {
339 additionalPresetsValueData.putVoid(a);
340 }
341 // TODO directionKeys are no longer in OsmPrimitive (search pattern is used instead)
342 for (String a : Config.getPref().getList(ValidatorPrefHelper.PREFIX + ".knownkeys",
343 Arrays.asList("is_in", "int_ref", "fixme", "population"))) {
344 additionalPresetsValueData.putVoid(a);
345 }
346 for (TaggingPreset p : presets) {
347 for (TaggingPresetItem i : p.data) {
348 if (i instanceof KeyedItem) {
349 addPresetValue((KeyedItem) i);
350 } else if (i instanceof CheckGroup) {
351 for (Check c : ((CheckGroup) i).checks) {
352 addPresetValue(c);
353 }
354 }
355 }
356 }
357 }
358 }
359
360 private static void addPresetValue(KeyedItem ky) {
361 if (ky.key != null && ky.getValues() != null) {
362 String hk = harmonizeKey(ky.key);
363 if (!ky.key.equals(hk)) {
364 harmonizedKeys.put(hk, ky.key);
365 }
366 }
367 }
368
369 /**
370 * Checks given string (key or value) if it contains characters with code below 0x20 (either newline or some other special characters)
371 * @param s string to check
372 * @return {@code true} if {@code s} contains characters with code below 0x20
373 */
374 private static boolean containsLow(String s) {
375 if (s == null)
376 return false;
377 for (int i = 0; i < s.length(); i++) {
378 if (s.charAt(i) < 0x20)
379 return true;
380 }
381 return false;
382 }
383
384 private static Set<String> getPresetValues(String key) {
385 Set<String> res = TaggingPresets.getPresetValues(key);
386 if (res != null)
387 return res;
388 return additionalPresetsValueData.get(key);
389 }
390
391 /**
392 * Determines if the given key is in internal presets.
393 * @param key key
394 * @return {@code true} if the given key is in internal presets
395 * @since 9023
396 */
397 public static boolean isKeyInPresets(String key) {
398 return getPresetValues(key) != null;
399 }
400
401 /**
402 * Determines if the given tag is in internal presets.
403 * @param key key
404 * @param value value
405 * @return {@code true} if the given tag is in internal presets
406 * @since 9023
407 */
408 public static boolean isTagInPresets(String key, String value) {
409 final Set<String> values = getPresetValues(key);
410 return values != null && (values.isEmpty() || values.contains(value));
411 }
412
413 /**
414 * Returns the list of ignored tags.
415 * @return the list of ignored tags
416 * @since 9023
417 */
418 public static List<Tag> getIgnoredTags() {
419 return new ArrayList<>(ignoreDataTag);
420 }
421
422 /**
423 * Determines if the given tag key is ignored for checks "key/tag not in presets".
424 * @param key key
425 * @return true if the given key is ignored
426 */
427 private static boolean isKeyIgnored(String key) {
428 if (ignoreDataEquals.contains(key)) {
429 return true;
430 }
431 for (String a : ignoreDataStartsWith) {
432 if (key.startsWith(a)) {
433 return true;
434 }
435 }
436 for (String a : ignoreDataEndsWith) {
437 if (key.endsWith(a)) {
438 return true;
439 }
440 }
441 return false;
442 }
443
444 /**
445 * Determines if the given tag is ignored for checks "key/tag not in presets".
446 * @param key key
447 * @param value value
448 * @return {@code true} if the given tag is ignored
449 * @since 9023
450 */
451 public static boolean isTagIgnored(String key, String value) {
452 if (isKeyIgnored(key))
453 return true;
454 if (!isTagInPresets(key, value)) {
455 for (Tag a : ignoreDataTag) {
456 if (key.equals(a.getKey()) && value.equals(a.getValue())) {
457 return true;
458 }
459 }
460 }
461 return false;
462 }
463
464 /**
465 * Checks the primitive tags
466 * @param p The primitive to check
467 */
468 @Override
469 public void check(OsmPrimitive p) {
470 if (!p.isTagged())
471 return;
472
473 // Just a collection to know if a primitive has been already marked with error
474 MultiMap<OsmPrimitive, String> withErrors = new MultiMap<>();
475
476 if (checkComplex) {
477 Map<String, String> keys = p.getKeys();
478 for (CheckerData d : checkerData) {
479 if (d.match(p, keys)) {
480 errors.add(TestError.builder(this, d.getSeverity(), d.getCode())
481 .message(tr("Suspicious tag/value combinations"), d.getDescription())
482 .primitives(p)
483 .build());
484 withErrors.put(p, "TC");
485 }
486 }
487 }
488
489 for (Entry<String, String> prop : p.getKeys().entrySet()) {
490 String s = marktr("Tag ''{0}'' invalid.");
491 String key = prop.getKey();
492 String value = prop.getValue();
493 if (checkValues && (containsLow(value)) && !withErrors.contains(p, "ICV")) {
494 errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_VALUE)
495 .message(tr("Tag value contains character with code less than 0x20"), s, key)
496 .primitives(p)
497 .build());
498 withErrors.put(p, "ICV");
499 }
500 if (checkKeys && (containsLow(key)) && !withErrors.contains(p, "ICK")) {
501 errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_KEY)
502 .message(tr("Tag key contains character with code less than 0x20"), s, key)
503 .primitives(p)
504 .build());
505 withErrors.put(p, "ICK");
506 }
507 if (checkValues && (value != null && value.length() > Tagged.MAX_TAG_LENGTH) && !withErrors.contains(p, "LV")) {
508 errors.add(TestError.builder(this, Severity.ERROR, LONG_VALUE)
509 .message(tr("Tag value longer than {0} characters ({1} characters)", Tagged.MAX_TAG_LENGTH, value.length()), s, key)
510 .primitives(p)
511 .build());
512 withErrors.put(p, "LV");
513 }
514 if (checkKeys && (key != null && key.length() > Tagged.MAX_TAG_LENGTH) && !withErrors.contains(p, "LK")) {
515 errors.add(TestError.builder(this, Severity.ERROR, LONG_KEY)
516 .message(tr("Tag key longer than {0} characters ({1} characters)", Tagged.MAX_TAG_LENGTH, key.length()), s, key)
517 .primitives(p)
518 .build());
519 withErrors.put(p, "LK");
520 }
521 if (checkValues && (value == null || value.trim().isEmpty()) && !withErrors.contains(p, "EV")) {
522 errors.add(TestError.builder(this, Severity.WARNING, EMPTY_VALUES)
523 .message(tr("Tags with empty values"), s, key)
524 .primitives(p)
525 .build());
526 withErrors.put(p, "EV");
527 }
528 if (checkKeys && key != null && key.indexOf(' ') >= 0 && !withErrors.contains(p, "IPK")) {
529 errors.add(TestError.builder(this, Severity.WARNING, INVALID_KEY_SPACE)
530 .message(tr("Invalid white space in property key"), s, key)
531 .primitives(p)
532 .build());
533 withErrors.put(p, "IPK");
534 }
535 if (checkValues && value != null && (value.startsWith(" ") || value.endsWith(" ")) && !withErrors.contains(p, "SPACE")) {
536 errors.add(TestError.builder(this, Severity.WARNING, INVALID_SPACE)
537 .message(tr("Property values start or end with white space"), s, key)
538 .primitives(p)
539 .build());
540 withErrors.put(p, "SPACE");
541 }
542 if (checkValues && value != null && value.contains(" ") && !withErrors.contains(p, "SPACE")) {
543 errors.add(TestError.builder(this, Severity.WARNING, MULTIPLE_SPACES)
544 .message(tr("Property values contain multiple white spaces"), s, key)
545 .primitives(p)
546 .build());
547 withErrors.put(p, "SPACE");
548 }
549 if (checkValues && value != null && !value.equals(Entities.unescape(value)) && !withErrors.contains(p, "HTML")) {
550 errors.add(TestError.builder(this, Severity.OTHER, INVALID_HTML)
551 .message(tr("Property values contain HTML entity"), s, key)
552 .primitives(p)
553 .build());
554 withErrors.put(p, "HTML");
555 }
556 if (checkValues && key != null && value != null && !value.isEmpty() && additionalPresetsValueData != null
557 && !isTagIgnored(key, value)) {
558 if (!isKeyInPresets(key)) {
559 String prettifiedKey = harmonizeKey(key);
560 String fixedKey = isKeyInPresets(prettifiedKey) ? prettifiedKey : harmonizedKeys.get(prettifiedKey);
561 if (fixedKey != null && !"".equals(fixedKey) && !fixedKey.equals(key)) {
562 // misspelled preset key
563 final TestError.Builder error = TestError.builder(this, Severity.WARNING, MISSPELLED_KEY)
564 .message(tr("Misspelled property key"), marktr("Key ''{0}'' looks like ''{1}''."), key, fixedKey)
565 .primitives(p);
566 if (p.hasKey(fixedKey)) {
567 errors.add(error.build());
568 } else {
569 errors.add(error.fix(() -> new ChangePropertyKeyCommand(p, key, fixedKey)).build());
570 }
571 withErrors.put(p, "WPK");
572 } else {
573 errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE)
574 .message(tr("Presets do not contain property key"), marktr("Key ''{0}'' not in presets."), key)
575 .primitives(p)
576 .build());
577 withErrors.put(p, "UPK");
578 }
579 } else if (!isTagInPresets(key, value)) {
580 tryGuess(p, key, value, withErrors);
581 }
582 }
583 if (checkFixmes && key != null && value != null && !value.isEmpty() && isFixme(key, value) && !withErrors.contains(p, "FIXME")) {
584 errors.add(TestError.builder(this, Severity.OTHER, FIXME)
585 .message(tr("FIXMES"))
586 .primitives(p)
587 .build());
588 withErrors.put(p, "FIXME");
589 }
590 }
591 }
592
593 private void tryGuess(OsmPrimitive p, String key, String value, MultiMap<OsmPrimitive, String> withErrors) {
594 // try to fix common typos and check again if value is still unknown
595 final String harmonizedValue = harmonizeValue(value);
596 String fixedValue = null;
597 Set<String> presetValues = getPresetValues(key);
598 Set<String> oftenUsedValues = oftenUsedValueData.get(key);
599 for (Set<String> possibleValues: Arrays.asList(presetValues, oftenUsedValues)) {
600 if (possibleValues != null && possibleValues.contains(harmonizedValue)) {
601 fixedValue = harmonizedValue;
602 break;
603 }
604 }
605 if (fixedValue == null && !ignoreForLevenshtein.contains(key)) {
606 int maxPresetValueLen = 0;
607 List<String> fixVals = new ArrayList<>();
608 // use Levenshtein distance to find typical typos
609 int minDist = MAX_LEVENSHTEIN_DISTANCE + 1;
610 String closest = null;
611 for (Set<String> possibleValues: Arrays.asList(presetValues, oftenUsedValues)) {
612 if (possibleValues == null)
613 continue;
614 for (String possibleVal : possibleValues) {
615 if (possibleVal.isEmpty())
616 continue;
617 maxPresetValueLen = Math.max(maxPresetValueLen, possibleVal.length());
618 if (harmonizedValue.length() < 3 && possibleVal.length() >= harmonizedValue.length() + MAX_LEVENSHTEIN_DISTANCE) {
619 // don't suggest fix value when given value is short and lengths are too different
620 // for example surface=u would result in surface=mud
621 continue;
622 }
623 int dist = Utils.getLevenshteinDistance(possibleVal, harmonizedValue);
624 if (dist >= harmonizedValue.length()) {
625 // short value, all characters are different. Don't warn, might say Value '10' for key 'fee' looks like 'no'.
626 continue;
627 }
628 if (dist < minDist) {
629 closest = possibleVal;
630 minDist = dist;
631 fixVals.clear();
632 fixVals.add(possibleVal);
633 } else if (dist == minDist) {
634 fixVals.add(possibleVal);
635 }
636 }
637 }
638
639 if (minDist <= MAX_LEVENSHTEIN_DISTANCE && maxPresetValueLen > MAX_LEVENSHTEIN_DISTANCE
640 && (harmonizedValue.length() > 3 || minDist < MAX_LEVENSHTEIN_DISTANCE)) {
641 if (fixVals.size() < 2) {
642 fixedValue = closest;
643 } else {
644 Collections.sort(fixVals);
645 // misspelled preset value with multiple good alternatives
646 errors.add(TestError.builder(this, Severity.WARNING, MISSPELLED_VALUE_NO_FIX)
647 .message(tr("Unknown property value"),
648 marktr("Value ''{0}'' for key ''{1}'' is unknown, maybe one of {2} is meant?"),
649 value, key, fixVals)
650 .primitives(p).build());
651 withErrors.put(p, "WPV");
652 return;
653 }
654 }
655 }
656 if (fixedValue != null) {
657 final String newValue = fixedValue;
658 // misspelled preset value
659 errors.add(TestError.builder(this, Severity.WARNING, MISSPELLED_VALUE)
660 .message(tr("Unknown property value"),
661 marktr("Value ''{0}'' for key ''{1}'' is unknown, maybe ''{2}'' is meant?"), value, key, newValue)
662 .primitives(p)
663 .build());
664 withErrors.put(p, "WPV");
665 } else {
666 // unknown preset value
667 errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE)
668 .message(tr("Presets do not contain property value"),
669 marktr("Value ''{0}'' for key ''{1}'' not in presets."), value, key)
670 .primitives(p)
671 .build());
672 withErrors.put(p, "UPV");
673 }
674 }
675
676 private boolean isNum(String harmonizedValue) {
677 try {
678 Double.parseDouble(harmonizedValue);
679 return true;
680 } catch (NumberFormatException e) {
681 return false;
682 }
683 }
684
685 private static boolean isFixme(String key, String value) {
686 return key.toLowerCase(Locale.ENGLISH).contains("fixme") || key.contains("todo")
687 || value.toLowerCase(Locale.ENGLISH).contains("fixme") || value.contains("check and delete");
688 }
689
690 private static String harmonizeKey(String key) {
691 return Utils.strip(key.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(':', '_').replace(' ', '_'), "-_;:,");
692 }
693
694 private static String harmonizeValue(String value) {
695 return Utils.strip(value.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(' ', '_'), "-_;:,");
696 }
697
698 @Override
699 public void startTest(ProgressMonitor monitor) {
700 super.startTest(monitor);
701 checkKeys = Config.getPref().getBoolean(PREF_CHECK_KEYS, true);
702 if (isBeforeUpload) {
703 checkKeys = checkKeys && Config.getPref().getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true);
704 }
705
706 checkValues = Config.getPref().getBoolean(PREF_CHECK_VALUES, true);
707 if (isBeforeUpload) {
708 checkValues = checkValues && Config.getPref().getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true);
709 }
710
711 checkComplex = Config.getPref().getBoolean(PREF_CHECK_COMPLEX, true) && !checkerData.isEmpty();
712 if (isBeforeUpload) {
713 checkComplex = checkComplex && Config.getPref().getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true);
714 }
715
716 checkFixmes = Config.getPref().getBoolean(PREF_CHECK_FIXMES, true);
717 if (isBeforeUpload) {
718 checkFixmes = checkFixmes && Config.getPref().getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true);
719 }
720 }
721
722 @Override
723 public void visit(Collection<OsmPrimitive> selection) {
724 if (checkKeys || checkValues || checkComplex || checkFixmes) {
725 super.visit(selection);
726 }
727 }
728
729 @Override
730 public void addGui(JPanel testPanel) {
731 GBC a = GBC.eol();
732 a.anchor = GridBagConstraints.EAST;
733
734 testPanel.add(new JLabel(name+" :"), GBC.eol().insets(3, 0, 0, 0));
735
736 prefCheckKeys = new JCheckBox(tr("Check property keys."), Config.getPref().getBoolean(PREF_CHECK_KEYS, true));
737 prefCheckKeys.setToolTipText(tr("Validate that property keys are valid checking against list of words."));
738 testPanel.add(prefCheckKeys, GBC.std().insets(20, 0, 0, 0));
739
740 prefCheckKeysBeforeUpload = new JCheckBox();
741 prefCheckKeysBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true));
742 testPanel.add(prefCheckKeysBeforeUpload, a);
743
744 prefCheckComplex = new JCheckBox(tr("Use complex property checker."), Config.getPref().getBoolean(PREF_CHECK_COMPLEX, true));
745 prefCheckComplex.setToolTipText(tr("Validate property values and tags using complex rules."));
746 testPanel.add(prefCheckComplex, GBC.std().insets(20, 0, 0, 0));
747
748 prefCheckComplexBeforeUpload = new JCheckBox();
749 prefCheckComplexBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true));
750 testPanel.add(prefCheckComplexBeforeUpload, a);
751
752 final Collection<String> sources = Config.getPref().getList(PREF_SOURCES, DEFAULT_SOURCES);
753 sourcesList = new EditableList(tr("TagChecker source"));
754 sourcesList.setItems(sources);
755 testPanel.add(new JLabel(tr("Data sources ({0})", "*.cfg")), GBC.eol().insets(23, 0, 0, 0));
756 testPanel.add(sourcesList, GBC.eol().fill(GridBagConstraints.HORIZONTAL).insets(23, 0, 0, 0));
757
758 ActionListener disableCheckActionListener = e -> handlePrefEnable();
759 prefCheckKeys.addActionListener(disableCheckActionListener);
760 prefCheckKeysBeforeUpload.addActionListener(disableCheckActionListener);
761 prefCheckComplex.addActionListener(disableCheckActionListener);
762 prefCheckComplexBeforeUpload.addActionListener(disableCheckActionListener);
763
764 handlePrefEnable();
765
766 prefCheckValues = new JCheckBox(tr("Check property values."), Config.getPref().getBoolean(PREF_CHECK_VALUES, true));
767 prefCheckValues.setToolTipText(tr("Validate that property values are valid checking against presets."));
768 testPanel.add(prefCheckValues, GBC.std().insets(20, 0, 0, 0));
769
770 prefCheckValuesBeforeUpload = new JCheckBox();
771 prefCheckValuesBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true));
772 testPanel.add(prefCheckValuesBeforeUpload, a);
773
774 prefCheckFixmes = new JCheckBox(tr("Check for FIXMES."), Config.getPref().getBoolean(PREF_CHECK_FIXMES, true));
775 prefCheckFixmes.setToolTipText(tr("Looks for nodes or ways with FIXME in any property value."));
776 testPanel.add(prefCheckFixmes, GBC.std().insets(20, 0, 0, 0));
777
778 prefCheckFixmesBeforeUpload = new JCheckBox();
779 prefCheckFixmesBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true));
780 testPanel.add(prefCheckFixmesBeforeUpload, a);
781 }
782
783 /**
784 * Enables/disables the source list field
785 */
786 public void handlePrefEnable() {
787 boolean selected = prefCheckKeys.isSelected() || prefCheckKeysBeforeUpload.isSelected()
788 || prefCheckComplex.isSelected() || prefCheckComplexBeforeUpload.isSelected();
789 sourcesList.setEnabled(selected);
790 }
791
792 @Override
793 public boolean ok() {
794 enabled = prefCheckKeys.isSelected() || prefCheckValues.isSelected() || prefCheckComplex.isSelected() || prefCheckFixmes.isSelected();
795 testBeforeUpload = prefCheckKeysBeforeUpload.isSelected() || prefCheckValuesBeforeUpload.isSelected()
796 || prefCheckFixmesBeforeUpload.isSelected() || prefCheckComplexBeforeUpload.isSelected();
797
798 Config.getPref().putBoolean(PREF_CHECK_VALUES, prefCheckValues.isSelected());
799 Config.getPref().putBoolean(PREF_CHECK_COMPLEX, prefCheckComplex.isSelected());
800 Config.getPref().putBoolean(PREF_CHECK_KEYS, prefCheckKeys.isSelected());
801 Config.getPref().putBoolean(PREF_CHECK_FIXMES, prefCheckFixmes.isSelected());
802 Config.getPref().putBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, prefCheckValuesBeforeUpload.isSelected());
803 Config.getPref().putBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, prefCheckComplexBeforeUpload.isSelected());
804 Config.getPref().putBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, prefCheckKeysBeforeUpload.isSelected());
805 Config.getPref().putBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, prefCheckFixmesBeforeUpload.isSelected());
806 return Config.getPref().putList(PREF_SOURCES, sourcesList.getItems());
807 }
808
809 @Override
810 public Command fixError(TestError testError) {
811 List<Command> commands = new ArrayList<>(50);
812
813 Collection<? extends OsmPrimitive> primitives = testError.getPrimitives();
814 for (OsmPrimitive p : primitives) {
815 Map<String, String> tags = p.getKeys();
816 if (tags.isEmpty()) {
817 continue;
818 }
819
820 for (Entry<String, String> prop: tags.entrySet()) {
821 String key = prop.getKey();
822 String value = prop.getValue();
823 if (value == null || value.trim().isEmpty()) {
824 commands.add(new ChangePropertyCommand(p, key, null));
825 } else if (value.startsWith(" ") || value.endsWith(" ") || value.contains(" ")) {
826 commands.add(new ChangePropertyCommand(p, key, Utils.removeWhiteSpaces(value)));
827 } else if (key.startsWith(" ") || key.endsWith(" ") || key.contains(" ")) {
828 commands.add(new ChangePropertyKeyCommand(p, key, Utils.removeWhiteSpaces(key)));
829 } else {
830 String evalue = Entities.unescape(value);
831 if (!evalue.equals(value)) {
832 commands.add(new ChangePropertyCommand(p, key, evalue));
833 }
834 }
835 }
836 }
837
838 if (commands.isEmpty())
839 return null;
840 if (commands.size() == 1)
841 return commands.get(0);
842
843 return new SequenceCommand(tr("Fix tags"), commands);
844 }
845
846 @Override
847 public boolean isFixable(TestError testError) {
848 if (testError.getTester() instanceof TagChecker) {
849 int code = testError.getCode();
850 return code == INVALID_KEY || code == EMPTY_VALUES || code == INVALID_SPACE ||
851 code == INVALID_KEY_SPACE || code == INVALID_HTML ||
852 code == MULTIPLE_SPACES;
853 }
854
855 return false;
856 }
857
858 protected static class CheckerData {
859 private String description;
860 protected List<CheckerElement> data = new ArrayList<>();
861 private OsmPrimitiveType type;
862 private TagCheckLevel level;
863 protected Severity severity;
864
865 private enum TagCheckLevel {
866 TAG_CHECK_ERROR(1250),
867 TAG_CHECK_WARN(1260),
868 TAG_CHECK_INFO(1270);
869
870 final int code;
871
872 TagCheckLevel(int code) {
873 this.code = code;
874 }
875 }
876
877 protected static class CheckerElement {
878 public Object tag;
879 public Object value;
880 public boolean noMatch;
881 public boolean tagAll;
882 public boolean valueAll;
883 public boolean valueBool;
884
885 private static Pattern getPattern(String str) {
886 if (str.endsWith("/i"))
887 return Pattern.compile(str.substring(1, str.length()-2), Pattern.CASE_INSENSITIVE);
888 if (str.endsWith("/"))
889 return Pattern.compile(str.substring(1, str.length()-1));
890
891 throw new IllegalStateException();
892 }
893
894 public CheckerElement(String exp) {
895 Matcher m = Pattern.compile("(.+)([!=]=)(.+)").matcher(exp);
896 m.matches();
897
898 String n = m.group(1).trim();
899
900 if ("*".equals(n)) {
901 tagAll = true;
902 } else {
903 tag = n.startsWith("/") ? getPattern(n) : n;
904 noMatch = "!=".equals(m.group(2));
905 n = m.group(3).trim();
906 if ("*".equals(n)) {
907 valueAll = true;
908 } else if ("BOOLEAN_TRUE".equals(n)) {
909 valueBool = true;
910 value = OsmUtils.TRUE_VALUE;
911 } else if ("BOOLEAN_FALSE".equals(n)) {
912 valueBool = true;
913 value = OsmUtils.FALSE_VALUE;
914 } else {
915 value = n.startsWith("/") ? getPattern(n) : n;
916 }
917 }
918 }
919
920 public boolean match(Map<String, String> keys) {
921 for (Entry<String, String> prop: keys.entrySet()) {
922 String key = prop.getKey();
923 String val = valueBool ? OsmUtils.getNamedOsmBoolean(prop.getValue()) : prop.getValue();
924 if ((tagAll || (tag instanceof Pattern ? ((Pattern) tag).matcher(key).matches() : key.equals(tag)))
925 && (valueAll || (value instanceof Pattern ? ((Pattern) value).matcher(val).matches() : val.equals(value))))
926 return !noMatch;
927 }
928 return noMatch;
929 }
930 }
931
932 private static final Pattern CLEAN_STR_PATTERN = Pattern.compile(" *# *([^#]+) *$");
933 private static final Pattern SPLIT_TRIMMED_PATTERN = Pattern.compile(" *: *");
934 private static final Pattern SPLIT_ELEMENTS_PATTERN = Pattern.compile(" *&& *");
935
936 public String getData(final String str) {
937 Matcher m = CLEAN_STR_PATTERN.matcher(str);
938 String trimmed = m.replaceFirst("").trim();
939 try {
940 description = m.group(1);
941 if (description != null && description.isEmpty()) {
942 description = null;
943 }
944 } catch (IllegalStateException e) {
945 Logging.error(e);
946 description = null;
947 }
948 String[] n = SPLIT_TRIMMED_PATTERN.split(trimmed, 3);
949 switch (n[0]) {
950 case "way":
951 type = OsmPrimitiveType.WAY;
952 break;
953 case "node":
954 type = OsmPrimitiveType.NODE;
955 break;
956 case "relation":
957 type = OsmPrimitiveType.RELATION;
958 break;
959 case "*":
960 type = null;
961 break;
962 default:
963 return tr("Could not find element type");
964 }
965 if (n.length != 3)
966 return tr("Incorrect number of parameters");
967
968 switch (n[1]) {
969 case "W":
970 severity = Severity.WARNING;
971 level = TagCheckLevel.TAG_CHECK_WARN;
972 break;
973 case "E":
974 severity = Severity.ERROR;
975 level = TagCheckLevel.TAG_CHECK_ERROR;
976 break;
977 case "I":
978 severity = Severity.OTHER;
979 level = TagCheckLevel.TAG_CHECK_INFO;
980 break;
981 default:
982 return tr("Could not find warning level");
983 }
984 for (String exp: SPLIT_ELEMENTS_PATTERN.split(n[2])) {
985 try {
986 data.add(new CheckerElement(exp));
987 } catch (IllegalStateException e) {
988 Logging.trace(e);
989 return tr("Illegal expression ''{0}''", exp);
990 } catch (PatternSyntaxException e) {
991 Logging.trace(e);
992 return tr("Illegal regular expression ''{0}''", exp);
993 }
994 }
995 return null;
996 }
997
998 public boolean match(OsmPrimitive osm, Map<String, String> keys) {
999 if (type != null && OsmPrimitiveType.from(osm) != type)
1000 return false;
1001
1002 for (CheckerElement ce : data) {
1003 if (!ce.match(keys))
1004 return false;
1005 }
1006 return true;
1007 }
1008
1009 /**
1010 * Returns the error description.
1011 * @return the error description
1012 */
1013 public String getDescription() {
1014 return description;
1015 }
1016
1017 /**
1018 * Returns the error severity.
1019 * @return the error severity
1020 */
1021 public Severity getSeverity() {
1022 return severity;
1023 }
1024
1025 /**
1026 * Returns the error code.
1027 * @return the error code
1028 */
1029 public int getCode() {
1030 if (type == null)
1031 return level.code;
1032
1033 return level.code + type.ordinal() + 1;
1034 }
1035 }
1036}
Note: See TracBrowser for help on using the repository browser.