source: josm/trunk/src/org/openstreetmap/josm/data/validation/tests/TagChecker.java@ 14501

Last change on this file since 14501 was 14490, checked in by GerdP, 5 years ago

fix #17055 17055-v2.patch with one empty line removed
  • Property svn:eol-style set to native
File size: 40.9 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.validation.tests;
3
4import static org.openstreetmap.josm.tools.I18n.marktr;
5import static org.openstreetmap.josm.tools.I18n.tr;
6
7import java.awt.GridBagConstraints;
8import java.awt.event.ActionListener;
9import java.io.BufferedReader;
10import java.io.IOException;
11import java.util.ArrayList;
12import java.util.Arrays;
13import java.util.Collection;
14import java.util.Collections;
15import java.util.HashMap;
16import java.util.List;
17import java.util.Locale;
18import java.util.Map;
19import java.util.Map.Entry;
20import java.util.Set;
21import java.util.regex.Matcher;
22import java.util.regex.Pattern;
23import java.util.regex.PatternSyntaxException;
24
25import javax.swing.JCheckBox;
26import javax.swing.JLabel;
27import javax.swing.JPanel;
28
29import org.openstreetmap.josm.command.ChangePropertyCommand;
30import org.openstreetmap.josm.command.ChangePropertyKeyCommand;
31import org.openstreetmap.josm.command.Command;
32import org.openstreetmap.josm.command.SequenceCommand;
33import org.openstreetmap.josm.data.osm.AbstractPrimitive;
34import org.openstreetmap.josm.data.osm.OsmPrimitive;
35import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
36import org.openstreetmap.josm.data.osm.OsmUtils;
37import org.openstreetmap.josm.data.osm.Tag;
38import org.openstreetmap.josm.data.osm.Tagged;
39import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper;
40import org.openstreetmap.josm.data.validation.Severity;
41import org.openstreetmap.josm.data.validation.Test.TagTest;
42import org.openstreetmap.josm.data.validation.TestError;
43import org.openstreetmap.josm.data.validation.util.Entities;
44import org.openstreetmap.josm.gui.progress.ProgressMonitor;
45import org.openstreetmap.josm.gui.tagging.presets.TaggingPreset;
46import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetItem;
47import org.openstreetmap.josm.gui.tagging.presets.TaggingPresets;
48import org.openstreetmap.josm.gui.tagging.presets.items.Check;
49import org.openstreetmap.josm.gui.tagging.presets.items.CheckGroup;
50import org.openstreetmap.josm.gui.tagging.presets.items.KeyedItem;
51import org.openstreetmap.josm.gui.widgets.EditableList;
52import org.openstreetmap.josm.io.CachedFile;
53import org.openstreetmap.josm.spi.preferences.Config;
54import org.openstreetmap.josm.tools.GBC;
55import org.openstreetmap.josm.tools.Logging;
56import org.openstreetmap.josm.tools.MultiMap;
57import org.openstreetmap.josm.tools.Utils;
58
59/**
60 * Check for misspelled or wrong tags
61 *
62 * @author frsantos
63 * @since 3669
64 */
65public class TagChecker extends TagTest {
66
67 /** The config file of ignored tags */
68 public static final String IGNORE_FILE = "resource://data/validator/ignoretags.cfg";
69 /** The config file of dictionary words */
70 public static final String SPELL_FILE = "resource://data/validator/words.cfg";
71
72 /** Normalized keys: the key should be substituted by the value if the key was not found in presets */
73 private static final Map<String, String> harmonizedKeys = new HashMap<>();
74 /** The spell check preset values which are not stored in TaggingPresets */
75 private static volatile MultiMap<String, String> additionalPresetsValueData;
76 /** The TagChecker data */
77 private static final List<CheckerData> checkerData = new ArrayList<>();
78 private static final List<String> ignoreDataStartsWith = new ArrayList<>();
79 private static final List<String> ignoreDataEquals = new ArrayList<>();
80 private static final List<String> ignoreDataEndsWith = new ArrayList<>();
81 private static final List<Tag> ignoreDataTag = new ArrayList<>();
82
83 /** The preferences prefix */
84 protected static final String PREFIX = ValidatorPrefHelper.PREFIX + "." + TagChecker.class.getSimpleName();
85
86 /**
87 * The preference key to check values
88 */
89 public static final String PREF_CHECK_VALUES = PREFIX + ".checkValues";
90 /**
91 * The preference key to check keys
92 */
93 public static final String PREF_CHECK_KEYS = PREFIX + ".checkKeys";
94 /**
95 * The preference key to enable complex checks
96 */
97 public static final String PREF_CHECK_COMPLEX = PREFIX + ".checkComplex";
98 /**
99 * The preference key to search for fixme tags
100 */
101 public static final String PREF_CHECK_FIXMES = PREFIX + ".checkFixmes";
102
103 /**
104 * The preference key for source files
105 * @see #DEFAULT_SOURCES
106 */
107 public static final String PREF_SOURCES = PREFIX + ".source";
108
109 /**
110 * The preference key to check keys - used before upload
111 */
112 public static final String PREF_CHECK_KEYS_BEFORE_UPLOAD = PREF_CHECK_KEYS + "BeforeUpload";
113 /**
114 * The preference key to check values - used before upload
115 */
116 public static final String PREF_CHECK_VALUES_BEFORE_UPLOAD = PREF_CHECK_VALUES + "BeforeUpload";
117 /**
118 * The preference key to run complex tests - used before upload
119 */
120 public static final String PREF_CHECK_COMPLEX_BEFORE_UPLOAD = PREF_CHECK_COMPLEX + "BeforeUpload";
121 /**
122 * The preference key to search for fixmes - used before upload
123 */
124 public static final String PREF_CHECK_FIXMES_BEFORE_UPLOAD = PREF_CHECK_FIXMES + "BeforeUpload";
125
126 protected boolean checkKeys;
127 protected boolean checkValues;
128 protected boolean checkComplex;
129 protected boolean checkFixmes;
130
131 protected JCheckBox prefCheckKeys;
132 protected JCheckBox prefCheckValues;
133 protected JCheckBox prefCheckComplex;
134 protected JCheckBox prefCheckFixmes;
135 protected JCheckBox prefCheckPaint;
136
137 protected JCheckBox prefCheckKeysBeforeUpload;
138 protected JCheckBox prefCheckValuesBeforeUpload;
139 protected JCheckBox prefCheckComplexBeforeUpload;
140 protected JCheckBox prefCheckFixmesBeforeUpload;
141 protected JCheckBox prefCheckPaintBeforeUpload;
142
143 // CHECKSTYLE.OFF: SingleSpaceSeparator
144 protected static final int EMPTY_VALUES = 1200;
145 protected static final int INVALID_KEY = 1201;
146 protected static final int INVALID_VALUE = 1202;
147 protected static final int FIXME = 1203;
148 protected static final int INVALID_SPACE = 1204;
149 protected static final int INVALID_KEY_SPACE = 1205;
150 protected static final int INVALID_HTML = 1206; /* 1207 was PAINT */
151 protected static final int LONG_VALUE = 1208;
152 protected static final int LONG_KEY = 1209;
153 protected static final int LOW_CHAR_VALUE = 1210;
154 protected static final int LOW_CHAR_KEY = 1211;
155 protected static final int MISSPELLED_VALUE = 1212;
156 protected static final int MISSPELLED_KEY = 1213;
157 protected static final int MULTIPLE_SPACES = 1214;
158 // CHECKSTYLE.ON: SingleSpaceSeparator
159 // 1250 and up is used by tagcheck
160
161 protected EditableList sourcesList;
162
163 private static final List<String> DEFAULT_SOURCES = Arrays.asList(/*DATA_FILE, */IGNORE_FILE, SPELL_FILE);
164
165 /**
166 * Constructor
167 */
168 public TagChecker() {
169 super(tr("Tag checker"), tr("This test checks for errors in tag keys and values."));
170 }
171
172 @Override
173 public void initialize() throws IOException {
174 initializeData();
175 initializePresets();
176 }
177
178 /**
179 * Reads the spellcheck file into a HashMap.
180 * The data file is a list of words, beginning with +/-. If it starts with +,
181 * the word is valid, but if it starts with -, the word should be replaced
182 * by the nearest + word before this.
183 *
184 * @throws IOException if any I/O error occurs
185 */
186 private static void initializeData() throws IOException {
187 checkerData.clear();
188 ignoreDataStartsWith.clear();
189 ignoreDataEquals.clear();
190 ignoreDataEndsWith.clear();
191 ignoreDataTag.clear();
192 harmonizedKeys.clear();
193
194 StringBuilder errorSources = new StringBuilder();
195 for (String source : Config.getPref().getList(PREF_SOURCES, DEFAULT_SOURCES)) {
196 try (
197 CachedFile cf = new CachedFile(source);
198 BufferedReader reader = cf.getContentReader()
199 ) {
200 String okValue = null;
201 boolean tagcheckerfile = false;
202 boolean ignorefile = false;
203 boolean isFirstLine = true;
204 String line;
205 while ((line = reader.readLine()) != null && (tagcheckerfile || !line.isEmpty())) {
206 if (line.startsWith("#")) {
207 if (line.startsWith("# JOSM TagChecker")) {
208 tagcheckerfile = true;
209 if (!DEFAULT_SOURCES.contains(source)) {
210 Logging.info(tr("Adding {0} to tag checker", source));
211 }
212 } else
213 if (line.startsWith("# JOSM IgnoreTags")) {
214 ignorefile = true;
215 if (!DEFAULT_SOURCES.contains(source)) {
216 Logging.info(tr("Adding {0} to ignore tags", source));
217 }
218 }
219 } else if (ignorefile) {
220 line = line.trim();
221 if (line.length() < 4) {
222 continue;
223 }
224
225 String key = line.substring(0, 2);
226 line = line.substring(2);
227
228 switch (key) {
229 case "S:":
230 ignoreDataStartsWith.add(line);
231 break;
232 case "E:":
233 ignoreDataEquals.add(line);
234 break;
235 case "F:":
236 ignoreDataEndsWith.add(line);
237 break;
238 case "K:":
239 ignoreDataTag.add(Tag.ofString(line));
240 break;
241 default:
242 if (!key.startsWith(";")) {
243 Logging.warn("Unsupported TagChecker key: " + key);
244 }
245 }
246 } else if (tagcheckerfile) {
247 if (!line.isEmpty()) {
248 CheckerData d = new CheckerData();
249 String err = d.getData(line);
250
251 if (err == null) {
252 checkerData.add(d);
253 } else {
254 Logging.error(tr("Invalid tagchecker line - {0}: {1}", err, line));
255 }
256 }
257 } else if (line.charAt(0) == '+') {
258 okValue = line.substring(1);
259 } else if (line.charAt(0) == '-' && okValue != null) {
260 String hk = harmonizeKey(line.substring(1));
261 if (!okValue.equals(hk)) {
262 if (harmonizedKeys.put(hk, okValue) != null) {
263 Logging.debug(tr("Line was ignored: {0}", line));
264 }
265 }
266 } else {
267 Logging.error(tr("Invalid spellcheck line: {0}", line));
268 }
269 if (isFirstLine) {
270 isFirstLine = false;
271 if (!(tagcheckerfile || ignorefile) && !DEFAULT_SOURCES.contains(source)) {
272 Logging.info(tr("Adding {0} to spellchecker", source));
273 }
274 }
275 }
276 } catch (IOException e) {
277 Logging.error(e);
278 errorSources.append(source).append('\n');
279 }
280 }
281
282 if (errorSources.length() > 0)
283 throw new IOException(tr("Could not access data file(s):\n{0}", errorSources));
284 }
285
286 /**
287 * Reads the presets data.
288 *
289 */
290 public static void initializePresets() {
291
292 if (!Config.getPref().getBoolean(PREF_CHECK_VALUES, true))
293 return;
294
295 Collection<TaggingPreset> presets = TaggingPresets.getTaggingPresets();
296 if (!presets.isEmpty()) {
297 additionalPresetsValueData = new MultiMap<>();
298 for (String a : AbstractPrimitive.getUninterestingKeys()) {
299 additionalPresetsValueData.putVoid(a);
300 }
301 // TODO directionKeys are no longer in OsmPrimitive (search pattern is used instead)
302 for (String a : Config.getPref().getList(ValidatorPrefHelper.PREFIX + ".knownkeys",
303 Arrays.asList("is_in", "int_ref", "fixme", "population"))) {
304 additionalPresetsValueData.putVoid(a);
305 }
306 for (TaggingPreset p : presets) {
307 for (TaggingPresetItem i : p.data) {
308 if (i instanceof KeyedItem) {
309 addPresetValue((KeyedItem) i);
310 } else if (i instanceof CheckGroup) {
311 for (Check c : ((CheckGroup) i).checks) {
312 addPresetValue(c);
313 }
314 }
315 }
316 }
317 }
318 }
319
320 private static void addPresetValue(KeyedItem ky) {
321 if (ky.key != null && ky.getValues() != null) {
322 String hk = harmonizeKey(ky.key);
323 if (!ky.key.equals(hk)) {
324 harmonizedKeys.put(hk, ky.key);
325 }
326 }
327 }
328
329 /**
330 * Checks given string (key or value) if it contains characters with code below 0x20 (either newline or some other special characters)
331 * @param s string to check
332 * @return {@code true} if {@code s} contains characters with code below 0x20
333 */
334 private static boolean containsLow(String s) {
335 if (s == null)
336 return false;
337 for (int i = 0; i < s.length(); i++) {
338 if (s.charAt(i) < 0x20)
339 return true;
340 }
341 return false;
342 }
343
344 private static Set<String> getPresetValues(String key) {
345 Set<String> res = TaggingPresets.getPresetValues(key);
346 if (res != null)
347 return res;
348 return additionalPresetsValueData.get(key);
349 }
350
351 /**
352 * Determines if the given key is in internal presets.
353 * @param key key
354 * @return {@code true} if the given key is in internal presets
355 * @since 9023
356 */
357 public static boolean isKeyInPresets(String key) {
358 return getPresetValues(key) != null;
359 }
360
361 /**
362 * Determines if the given tag is in internal presets.
363 * @param key key
364 * @param value value
365 * @return {@code true} if the given tag is in internal presets
366 * @since 9023
367 */
368 public static boolean isTagInPresets(String key, String value) {
369 final Set<String> values = getPresetValues(key);
370 return values != null && (values.isEmpty() || values.contains(value));
371 }
372
373 /**
374 * Returns the list of ignored tags.
375 * @return the list of ignored tags
376 * @since 9023
377 */
378 public static List<Tag> getIgnoredTags() {
379 return new ArrayList<>(ignoreDataTag);
380 }
381
382 /**
383 * Determines if the given tag is ignored for checks "key/tag not in presets".
384 * @param key key
385 * @param value value
386 * @return {@code true} if the given tag is ignored
387 * @since 9023
388 */
389 public static boolean isTagIgnored(String key, String value) {
390 boolean tagInPresets = isTagInPresets(key, value);
391 boolean ignore = false;
392
393 for (String a : ignoreDataStartsWith) {
394 if (key.startsWith(a)) {
395 ignore = true;
396 }
397 }
398 for (String a : ignoreDataEquals) {
399 if (key.equals(a)) {
400 ignore = true;
401 }
402 }
403 for (String a : ignoreDataEndsWith) {
404 if (key.endsWith(a)) {
405 ignore = true;
406 }
407 }
408
409 if (!tagInPresets) {
410 for (Tag a : ignoreDataTag) {
411 if (key.equals(a.getKey()) && value.equals(a.getValue())) {
412 ignore = true;
413 }
414 }
415 }
416 return ignore;
417 }
418
419 /**
420 * Checks the primitive tags
421 * @param p The primitive to check
422 */
423 @Override
424 public void check(OsmPrimitive p) {
425 if (!p.isTagged())
426 return;
427
428 // Just a collection to know if a primitive has been already marked with error
429 MultiMap<OsmPrimitive, String> withErrors = new MultiMap<>();
430
431 if (checkComplex) {
432 Map<String, String> keys = p.getKeys();
433 for (CheckerData d : checkerData) {
434 if (d.match(p, keys)) {
435 errors.add(TestError.builder(this, d.getSeverity(), d.getCode())
436 .message(tr("Suspicious tag/value combinations"), d.getDescription())
437 .primitives(p)
438 .build());
439 withErrors.put(p, "TC");
440 }
441 }
442 }
443
444 for (Entry<String, String> prop : p.getKeys().entrySet()) {
445 String s = marktr("Tag ''{0}'' invalid.");
446 String key = prop.getKey();
447 String value = prop.getValue();
448 if (checkValues && (containsLow(value)) && !withErrors.contains(p, "ICV")) {
449 errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_VALUE)
450 .message(tr("Tag value contains character with code less than 0x20"), s, key)
451 .primitives(p)
452 .build());
453 withErrors.put(p, "ICV");
454 }
455 if (checkKeys && (containsLow(key)) && !withErrors.contains(p, "ICK")) {
456 errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_KEY)
457 .message(tr("Tag key contains character with code less than 0x20"), s, key)
458 .primitives(p)
459 .build());
460 withErrors.put(p, "ICK");
461 }
462 if (checkValues && (value != null && value.length() > Tagged.MAX_TAG_LENGTH) && !withErrors.contains(p, "LV")) {
463 errors.add(TestError.builder(this, Severity.ERROR, LONG_VALUE)
464 .message(tr("Tag value longer than {0} characters ({1} characters)", Tagged.MAX_TAG_LENGTH, value.length()), s, key)
465 .primitives(p)
466 .build());
467 withErrors.put(p, "LV");
468 }
469 if (checkKeys && (key != null && key.length() > Tagged.MAX_TAG_LENGTH) && !withErrors.contains(p, "LK")) {
470 errors.add(TestError.builder(this, Severity.ERROR, LONG_KEY)
471 .message(tr("Tag key longer than {0} characters ({1} characters)", Tagged.MAX_TAG_LENGTH, key.length()), s, key)
472 .primitives(p)
473 .build());
474 withErrors.put(p, "LK");
475 }
476 if (checkValues && (value == null || value.trim().isEmpty()) && !withErrors.contains(p, "EV")) {
477 errors.add(TestError.builder(this, Severity.WARNING, EMPTY_VALUES)
478 .message(tr("Tags with empty values"), s, key)
479 .primitives(p)
480 .build());
481 withErrors.put(p, "EV");
482 }
483 if (checkKeys && key != null && key.indexOf(' ') >= 0 && !withErrors.contains(p, "IPK")) {
484 errors.add(TestError.builder(this, Severity.WARNING, INVALID_KEY_SPACE)
485 .message(tr("Invalid white space in property key"), s, key)
486 .primitives(p)
487 .build());
488 withErrors.put(p, "IPK");
489 }
490 if (checkValues && value != null && (value.startsWith(" ") || value.endsWith(" ")) && !withErrors.contains(p, "SPACE")) {
491 errors.add(TestError.builder(this, Severity.WARNING, INVALID_SPACE)
492 .message(tr("Property values start or end with white space"), s, key)
493 .primitives(p)
494 .build());
495 withErrors.put(p, "SPACE");
496 }
497 if (checkValues && value != null && value.contains(" ") && !withErrors.contains(p, "SPACE")) {
498 errors.add(TestError.builder(this, Severity.WARNING, MULTIPLE_SPACES)
499 .message(tr("Property values contain multiple white spaces"), s, key)
500 .primitives(p)
501 .build());
502 withErrors.put(p, "SPACE");
503 }
504 if (checkValues && value != null && !value.equals(Entities.unescape(value)) && !withErrors.contains(p, "HTML")) {
505 errors.add(TestError.builder(this, Severity.OTHER, INVALID_HTML)
506 .message(tr("Property values contain HTML entity"), s, key)
507 .primitives(p)
508 .build());
509 withErrors.put(p, "HTML");
510 }
511 if (checkValues && key != null && value != null && !value.isEmpty() && additionalPresetsValueData != null
512 && !isTagIgnored(key, value)) {
513 if (!isKeyInPresets(key)) {
514 String prettifiedKey = harmonizeKey(key);
515 String fixedKey = isKeyInPresets(prettifiedKey) ? prettifiedKey : harmonizedKeys.get(prettifiedKey);
516 if (fixedKey != null && !"".equals(fixedKey) && !fixedKey.equals(key)) {
517 // misspelled preset key
518 final TestError.Builder error = TestError.builder(this, Severity.WARNING, MISSPELLED_KEY)
519 .message(tr("Misspelled property key"), marktr("Key ''{0}'' looks like ''{1}''."), key, fixedKey)
520 .primitives(p);
521 if (p.hasKey(fixedKey)) {
522 errors.add(error.build());
523 } else {
524 errors.add(error.fix(() -> new ChangePropertyKeyCommand(p, key, fixedKey)).build());
525 }
526 withErrors.put(p, "WPK");
527 } else {
528 errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE)
529 .message(tr("Presets do not contain property key"), marktr("Key ''{0}'' not in presets."), key)
530 .primitives(p)
531 .build());
532 withErrors.put(p, "UPK");
533 }
534 } else if (!isTagInPresets(key, value)) {
535 // try to fix common typos and check again if value is still unknown
536 String fixedValue = harmonizeValue(prop.getValue());
537 Map<String, String> possibleValues = getPossibleValues(getPresetValues(key));
538 List<String> fixVals = new ArrayList<>();
539 if (!possibleValues.containsKey(fixedValue)) {
540 int minDist = 2;
541 String closest = null;
542 for (String possibleVal : possibleValues.keySet()) {
543 int dist = Utils.getLevenshteinDistance(possibleVal, fixedValue);
544 if (dist < minDist) {
545 closest = possibleVal;
546 minDist = dist;
547 fixVals.clear();
548 fixVals.add(possibleVal);
549 } else if (dist == minDist) {
550 fixVals.add(possibleVal);
551 }
552 }
553 if (minDist <= 1) {
554 if (fixVals.size() < 2) {
555 fixedValue = closest;
556 } else {
557 Collections.sort(fixVals);
558 // misspelled preset value with multiple good alternatives
559 errors.add(TestError.builder(this, Severity.WARNING, MISSPELLED_VALUE)
560 .message(tr("Misspelled property value"),
561 marktr("Value ''{0}'' for key ''{1}'' looks like one of {2}."), prop.getValue(), key, fixVals)
562 .primitives(p)
563 .build());
564 withErrors.put(p, "WPV");
565 continue;
566 }
567 }
568 }
569 if (possibleValues.containsKey(fixedValue)) {
570 final String newValue = possibleValues.get(fixedValue);
571 // misspelled preset value
572 errors.add(TestError.builder(this, Severity.WARNING, MISSPELLED_VALUE)
573 .message(tr("Misspelled property value"),
574 marktr("Value ''{0}'' for key ''{1}'' looks like ''{2}''."), prop.getValue(), key, newValue)
575 .primitives(p)
576 .fix(() -> new ChangePropertyCommand(p, key, newValue))
577 .build());
578 withErrors.put(p, "WPV");
579 } else {
580 // unknown preset value
581 errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE)
582 .message(tr("Presets do not contain property value"),
583 marktr("Value ''{0}'' for key ''{1}'' not in presets."), prop.getValue(), key)
584 .primitives(p)
585 .build());
586 withErrors.put(p, "UPV");
587 }
588 }
589 }
590 if (checkFixmes && key != null && value != null && !value.isEmpty() && isFixme(key, value) && !withErrors.contains(p, "FIXME")) {
591 errors.add(TestError.builder(this, Severity.OTHER, FIXME)
592 .message(tr("FIXMES"))
593 .primitives(p)
594 .build());
595 withErrors.put(p, "FIXME");
596 }
597 }
598 }
599
600 private static boolean isFixme(String key, String value) {
601 return key.toLowerCase(Locale.ENGLISH).contains("fixme") || key.contains("todo")
602 || value.toLowerCase(Locale.ENGLISH).contains("fixme") || value.contains("check and delete");
603 }
604
605 private static Map<String, String> getPossibleValues(Set<String> values) {
606 // generate a map with common typos
607 Map<String, String> map = new HashMap<>();
608 if (values != null) {
609 for (String value : values) {
610 map.put(value, value);
611 if (value.contains("_")) {
612 map.put(value.replace("_", ""), value);
613 }
614 }
615 }
616 return map;
617 }
618
619 private static String harmonizeKey(String key) {
620 return Utils.strip(key.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(':', '_').replace(' ', '_'), "-_;:,");
621 }
622
623 private static String harmonizeValue(String value) {
624 return Utils.strip(value.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(' ', '_'), "-_;:,");
625 }
626
627 @Override
628 public void startTest(ProgressMonitor monitor) {
629 super.startTest(monitor);
630 checkKeys = Config.getPref().getBoolean(PREF_CHECK_KEYS, true);
631 if (isBeforeUpload) {
632 checkKeys = checkKeys && Config.getPref().getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true);
633 }
634
635 checkValues = Config.getPref().getBoolean(PREF_CHECK_VALUES, true);
636 if (isBeforeUpload) {
637 checkValues = checkValues && Config.getPref().getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true);
638 }
639
640 checkComplex = Config.getPref().getBoolean(PREF_CHECK_COMPLEX, true) && !checkerData.isEmpty();
641 if (isBeforeUpload) {
642 checkComplex = checkComplex && Config.getPref().getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true);
643 }
644
645 checkFixmes = Config.getPref().getBoolean(PREF_CHECK_FIXMES, true);
646 if (isBeforeUpload) {
647 checkFixmes = checkFixmes && Config.getPref().getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true);
648 }
649 }
650
651 @Override
652 public void visit(Collection<OsmPrimitive> selection) {
653 if (checkKeys || checkValues || checkComplex || checkFixmes) {
654 super.visit(selection);
655 }
656 }
657
658 @Override
659 public void addGui(JPanel testPanel) {
660 GBC a = GBC.eol();
661 a.anchor = GridBagConstraints.EAST;
662
663 testPanel.add(new JLabel(name+" :"), GBC.eol().insets(3, 0, 0, 0));
664
665 prefCheckKeys = new JCheckBox(tr("Check property keys."), Config.getPref().getBoolean(PREF_CHECK_KEYS, true));
666 prefCheckKeys.setToolTipText(tr("Validate that property keys are valid checking against list of words."));
667 testPanel.add(prefCheckKeys, GBC.std().insets(20, 0, 0, 0));
668
669 prefCheckKeysBeforeUpload = new JCheckBox();
670 prefCheckKeysBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true));
671 testPanel.add(prefCheckKeysBeforeUpload, a);
672
673 prefCheckComplex = new JCheckBox(tr("Use complex property checker."), Config.getPref().getBoolean(PREF_CHECK_COMPLEX, true));
674 prefCheckComplex.setToolTipText(tr("Validate property values and tags using complex rules."));
675 testPanel.add(prefCheckComplex, GBC.std().insets(20, 0, 0, 0));
676
677 prefCheckComplexBeforeUpload = new JCheckBox();
678 prefCheckComplexBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true));
679 testPanel.add(prefCheckComplexBeforeUpload, a);
680
681 final Collection<String> sources = Config.getPref().getList(PREF_SOURCES, DEFAULT_SOURCES);
682 sourcesList = new EditableList(tr("TagChecker source"));
683 sourcesList.setItems(sources);
684 testPanel.add(new JLabel(tr("Data sources ({0})", "*.cfg")), GBC.eol().insets(23, 0, 0, 0));
685 testPanel.add(sourcesList, GBC.eol().fill(GridBagConstraints.HORIZONTAL).insets(23, 0, 0, 0));
686
687 ActionListener disableCheckActionListener = e -> handlePrefEnable();
688 prefCheckKeys.addActionListener(disableCheckActionListener);
689 prefCheckKeysBeforeUpload.addActionListener(disableCheckActionListener);
690 prefCheckComplex.addActionListener(disableCheckActionListener);
691 prefCheckComplexBeforeUpload.addActionListener(disableCheckActionListener);
692
693 handlePrefEnable();
694
695 prefCheckValues = new JCheckBox(tr("Check property values."), Config.getPref().getBoolean(PREF_CHECK_VALUES, true));
696 prefCheckValues.setToolTipText(tr("Validate that property values are valid checking against presets."));
697 testPanel.add(prefCheckValues, GBC.std().insets(20, 0, 0, 0));
698
699 prefCheckValuesBeforeUpload = new JCheckBox();
700 prefCheckValuesBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true));
701 testPanel.add(prefCheckValuesBeforeUpload, a);
702
703 prefCheckFixmes = new JCheckBox(tr("Check for FIXMES."), Config.getPref().getBoolean(PREF_CHECK_FIXMES, true));
704 prefCheckFixmes.setToolTipText(tr("Looks for nodes or ways with FIXME in any property value."));
705 testPanel.add(prefCheckFixmes, GBC.std().insets(20, 0, 0, 0));
706
707 prefCheckFixmesBeforeUpload = new JCheckBox();
708 prefCheckFixmesBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true));
709 testPanel.add(prefCheckFixmesBeforeUpload, a);
710 }
711
712 /**
713 * Enables/disables the source list field
714 */
715 public void handlePrefEnable() {
716 boolean selected = prefCheckKeys.isSelected() || prefCheckKeysBeforeUpload.isSelected()
717 || prefCheckComplex.isSelected() || prefCheckComplexBeforeUpload.isSelected();
718 sourcesList.setEnabled(selected);
719 }
720
721 @Override
722 public boolean ok() {
723 enabled = prefCheckKeys.isSelected() || prefCheckValues.isSelected() || prefCheckComplex.isSelected() || prefCheckFixmes.isSelected();
724 testBeforeUpload = prefCheckKeysBeforeUpload.isSelected() || prefCheckValuesBeforeUpload.isSelected()
725 || prefCheckFixmesBeforeUpload.isSelected() || prefCheckComplexBeforeUpload.isSelected();
726
727 Config.getPref().putBoolean(PREF_CHECK_VALUES, prefCheckValues.isSelected());
728 Config.getPref().putBoolean(PREF_CHECK_COMPLEX, prefCheckComplex.isSelected());
729 Config.getPref().putBoolean(PREF_CHECK_KEYS, prefCheckKeys.isSelected());
730 Config.getPref().putBoolean(PREF_CHECK_FIXMES, prefCheckFixmes.isSelected());
731 Config.getPref().putBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, prefCheckValuesBeforeUpload.isSelected());
732 Config.getPref().putBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, prefCheckComplexBeforeUpload.isSelected());
733 Config.getPref().putBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, prefCheckKeysBeforeUpload.isSelected());
734 Config.getPref().putBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, prefCheckFixmesBeforeUpload.isSelected());
735 return Config.getPref().putList(PREF_SOURCES, sourcesList.getItems());
736 }
737
738 @Override
739 public Command fixError(TestError testError) {
740 List<Command> commands = new ArrayList<>(50);
741
742 Collection<? extends OsmPrimitive> primitives = testError.getPrimitives();
743 for (OsmPrimitive p : primitives) {
744 Map<String, String> tags = p.getKeys();
745 if (tags.isEmpty()) {
746 continue;
747 }
748
749 for (Entry<String, String> prop: tags.entrySet()) {
750 String key = prop.getKey();
751 String value = prop.getValue();
752 if (value == null || value.trim().isEmpty()) {
753 commands.add(new ChangePropertyCommand(p, key, null));
754 } else if (value.startsWith(" ") || value.endsWith(" ") || value.contains(" ")) {
755 commands.add(new ChangePropertyCommand(p, key, Utils.removeWhiteSpaces(value)));
756 } else if (key.startsWith(" ") || key.endsWith(" ") || key.contains(" ")) {
757 commands.add(new ChangePropertyKeyCommand(p, key, Utils.removeWhiteSpaces(key)));
758 } else {
759 String evalue = Entities.unescape(value);
760 if (!evalue.equals(value)) {
761 commands.add(new ChangePropertyCommand(p, key, evalue));
762 }
763 }
764 }
765 }
766
767 if (commands.isEmpty())
768 return null;
769 if (commands.size() == 1)
770 return commands.get(0);
771
772 return new SequenceCommand(tr("Fix tags"), commands);
773 }
774
775 @Override
776 public boolean isFixable(TestError testError) {
777 if (testError.getTester() instanceof TagChecker) {
778 int code = testError.getCode();
779 return code == INVALID_KEY || code == EMPTY_VALUES || code == INVALID_SPACE ||
780 code == INVALID_KEY_SPACE || code == INVALID_HTML || code == MISSPELLED_VALUE ||
781 code == MULTIPLE_SPACES;
782 }
783
784 return false;
785 }
786
787 protected static class CheckerData {
788 private String description;
789 protected List<CheckerElement> data = new ArrayList<>();
790 private OsmPrimitiveType type;
791 private TagCheckLevel level;
792 protected Severity severity;
793
794 private enum TagCheckLevel {
795 TAG_CHECK_ERROR(1250),
796 TAG_CHECK_WARN(1260),
797 TAG_CHECK_INFO(1270);
798
799 final int code;
800
801 TagCheckLevel(int code) {
802 this.code = code;
803 }
804 }
805
806 protected static class CheckerElement {
807 public Object tag;
808 public Object value;
809 public boolean noMatch;
810 public boolean tagAll;
811 public boolean valueAll;
812 public boolean valueBool;
813
814 private static Pattern getPattern(String str) {
815 if (str.endsWith("/i"))
816 return Pattern.compile(str.substring(1, str.length()-2), Pattern.CASE_INSENSITIVE);
817 if (str.endsWith("/"))
818 return Pattern.compile(str.substring(1, str.length()-1));
819
820 throw new IllegalStateException();
821 }
822
823 public CheckerElement(String exp) {
824 Matcher m = Pattern.compile("(.+)([!=]=)(.+)").matcher(exp);
825 m.matches();
826
827 String n = m.group(1).trim();
828
829 if ("*".equals(n)) {
830 tagAll = true;
831 } else {
832 tag = n.startsWith("/") ? getPattern(n) : n;
833 noMatch = "!=".equals(m.group(2));
834 n = m.group(3).trim();
835 if ("*".equals(n)) {
836 valueAll = true;
837 } else if ("BOOLEAN_TRUE".equals(n)) {
838 valueBool = true;
839 value = OsmUtils.TRUE_VALUE;
840 } else if ("BOOLEAN_FALSE".equals(n)) {
841 valueBool = true;
842 value = OsmUtils.FALSE_VALUE;
843 } else {
844 value = n.startsWith("/") ? getPattern(n) : n;
845 }
846 }
847 }
848
849 public boolean match(Map<String, String> keys) {
850 for (Entry<String, String> prop: keys.entrySet()) {
851 String key = prop.getKey();
852 String val = valueBool ? OsmUtils.getNamedOsmBoolean(prop.getValue()) : prop.getValue();
853 if ((tagAll || (tag instanceof Pattern ? ((Pattern) tag).matcher(key).matches() : key.equals(tag)))
854 && (valueAll || (value instanceof Pattern ? ((Pattern) value).matcher(val).matches() : val.equals(value))))
855 return !noMatch;
856 }
857 return noMatch;
858 }
859 }
860
861 private static final Pattern CLEAN_STR_PATTERN = Pattern.compile(" *# *([^#]+) *$");
862 private static final Pattern SPLIT_TRIMMED_PATTERN = Pattern.compile(" *: *");
863 private static final Pattern SPLIT_ELEMENTS_PATTERN = Pattern.compile(" *&& *");
864
865 public String getData(final String str) {
866 Matcher m = CLEAN_STR_PATTERN.matcher(str);
867 String trimmed = m.replaceFirst("").trim();
868 try {
869 description = m.group(1);
870 if (description != null && description.isEmpty()) {
871 description = null;
872 }
873 } catch (IllegalStateException e) {
874 Logging.error(e);
875 description = null;
876 }
877 String[] n = SPLIT_TRIMMED_PATTERN.split(trimmed, 3);
878 switch (n[0]) {
879 case "way":
880 type = OsmPrimitiveType.WAY;
881 break;
882 case "node":
883 type = OsmPrimitiveType.NODE;
884 break;
885 case "relation":
886 type = OsmPrimitiveType.RELATION;
887 break;
888 case "*":
889 type = null;
890 break;
891 default:
892 return tr("Could not find element type");
893 }
894 if (n.length != 3)
895 return tr("Incorrect number of parameters");
896
897 switch (n[1]) {
898 case "W":
899 severity = Severity.WARNING;
900 level = TagCheckLevel.TAG_CHECK_WARN;
901 break;
902 case "E":
903 severity = Severity.ERROR;
904 level = TagCheckLevel.TAG_CHECK_ERROR;
905 break;
906 case "I":
907 severity = Severity.OTHER;
908 level = TagCheckLevel.TAG_CHECK_INFO;
909 break;
910 default:
911 return tr("Could not find warning level");
912 }
913 for (String exp: SPLIT_ELEMENTS_PATTERN.split(n[2])) {
914 try {
915 data.add(new CheckerElement(exp));
916 } catch (IllegalStateException e) {
917 Logging.trace(e);
918 return tr("Illegal expression ''{0}''", exp);
919 } catch (PatternSyntaxException e) {
920 Logging.trace(e);
921 return tr("Illegal regular expression ''{0}''", exp);
922 }
923 }
924 return null;
925 }
926
927 public boolean match(OsmPrimitive osm, Map<String, String> keys) {
928 if (type != null && OsmPrimitiveType.from(osm) != type)
929 return false;
930
931 for (CheckerElement ce : data) {
932 if (!ce.match(keys))
933 return false;
934 }
935 return true;
936 }
937
938 /**
939 * Returns the error description.
940 * @return the error description
941 */
942 public String getDescription() {
943 return description;
944 }
945
946 /**
947 * Returns the error severity.
948 * @return the error severity
949 */
950 public Severity getSeverity() {
951 return severity;
952 }
953
954 /**
955 * Returns the error code.
956 * @return the error code
957 */
958 public int getCode() {
959 if (type == null)
960 return level.code;
961
962 return level.code + type.ordinal() + 1;
963 }
964 }
965}
Note: See TracBrowser for help on using the repository browser.