source: josm/trunk/src/org/openstreetmap/josm/data/validation/tests/TagChecker.java@ 12841

Last change on this file since 12841 was 12841, checked in by bastiK, 7 years ago

see #15229 - fix deprecations caused by [12840]
  • Property svn:eol-style set to native
File size: 38.2 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.validation.tests;
3
4import static org.openstreetmap.josm.tools.I18n.marktr;
5import static org.openstreetmap.josm.tools.I18n.tr;
6
7import java.awt.GridBagConstraints;
8import java.awt.event.ActionListener;
9import java.io.BufferedReader;
10import java.io.IOException;
11import java.util.ArrayList;
12import java.util.Arrays;
13import java.util.Collection;
14import java.util.HashMap;
15import java.util.List;
16import java.util.Locale;
17import java.util.Map;
18import java.util.Map.Entry;
19import java.util.Set;
20import java.util.regex.Matcher;
21import java.util.regex.Pattern;
22import java.util.regex.PatternSyntaxException;
23
24import javax.swing.JCheckBox;
25import javax.swing.JLabel;
26import javax.swing.JPanel;
27
28import org.openstreetmap.josm.Main;
29import org.openstreetmap.josm.command.ChangePropertyCommand;
30import org.openstreetmap.josm.command.ChangePropertyKeyCommand;
31import org.openstreetmap.josm.command.Command;
32import org.openstreetmap.josm.command.SequenceCommand;
33import org.openstreetmap.josm.data.osm.OsmPrimitive;
34import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
35import org.openstreetmap.josm.data.osm.OsmUtils;
36import org.openstreetmap.josm.data.osm.Tag;
37import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper;
38import org.openstreetmap.josm.data.validation.Severity;
39import org.openstreetmap.josm.data.validation.Test.TagTest;
40import org.openstreetmap.josm.data.validation.TestError;
41import org.openstreetmap.josm.data.validation.util.Entities;
42import org.openstreetmap.josm.gui.progress.ProgressMonitor;
43import org.openstreetmap.josm.gui.tagging.presets.TaggingPreset;
44import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetItem;
45import org.openstreetmap.josm.gui.tagging.presets.TaggingPresets;
46import org.openstreetmap.josm.gui.tagging.presets.items.Check;
47import org.openstreetmap.josm.gui.tagging.presets.items.CheckGroup;
48import org.openstreetmap.josm.gui.tagging.presets.items.KeyedItem;
49import org.openstreetmap.josm.gui.widgets.EditableList;
50import org.openstreetmap.josm.io.CachedFile;
51import org.openstreetmap.josm.tools.GBC;
52import org.openstreetmap.josm.tools.Logging;
53import org.openstreetmap.josm.tools.MultiMap;
54import org.openstreetmap.josm.tools.Utils;
55
56/**
57 * Check for misspelled or wrong tags
58 *
59 * @author frsantos
60 * @since 3669
61 */
62public class TagChecker extends TagTest {
63
64 /** The config file of ignored tags */
65 public static final String IGNORE_FILE = "resource://data/validator/ignoretags.cfg";
66 /** The config file of dictionary words */
67 public static final String SPELL_FILE = "resource://data/validator/words.cfg";
68
69 /** Normalized keys: the key should be substituted by the value if the key was not found in presets */
70 private static final Map<String, String> harmonizedKeys = new HashMap<>();
71 /** The spell check preset values which are not stored in TaggingPresets */
72 private static volatile MultiMap<String, String> additionalPresetsValueData;
73 /** The TagChecker data */
74 private static final List<CheckerData> checkerData = new ArrayList<>();
75 private static final List<String> ignoreDataStartsWith = new ArrayList<>();
76 private static final List<String> ignoreDataEquals = new ArrayList<>();
77 private static final List<String> ignoreDataEndsWith = new ArrayList<>();
78 private static final List<Tag> ignoreDataTag = new ArrayList<>();
79
80 /** The preferences prefix */
81 protected static final String PREFIX = ValidatorPrefHelper.PREFIX + "." + TagChecker.class.getSimpleName();
82
83 /**
84 * The preference key to check values
85 */
86 public static final String PREF_CHECK_VALUES = PREFIX + ".checkValues";
87 /**
88 * The preference key to check keys
89 */
90 public static final String PREF_CHECK_KEYS = PREFIX + ".checkKeys";
91 /**
92 * The preference key to enable complex checks
93 */
94 public static final String PREF_CHECK_COMPLEX = PREFIX + ".checkComplex";
95 /**
96 * The preference key to search for fixme tags
97 */
98 public static final String PREF_CHECK_FIXMES = PREFIX + ".checkFixmes";
99
100 /**
101 * The preference key for source files
102 * @see #DEFAULT_SOURCES
103 */
104 public static final String PREF_SOURCES = PREFIX + ".source";
105
106 /**
107 * The preference key to check keys - used before upload
108 */
109 public static final String PREF_CHECK_KEYS_BEFORE_UPLOAD = PREF_CHECK_KEYS + "BeforeUpload";
110 /**
111 * The preference key to check values - used before upload
112 */
113 public static final String PREF_CHECK_VALUES_BEFORE_UPLOAD = PREF_CHECK_VALUES + "BeforeUpload";
114 /**
115 * The preference key to run complex tests - used before upload
116 */
117 public static final String PREF_CHECK_COMPLEX_BEFORE_UPLOAD = PREF_CHECK_COMPLEX + "BeforeUpload";
118 /**
119 * The preference key to search for fixmes - used before upload
120 */
121 public static final String PREF_CHECK_FIXMES_BEFORE_UPLOAD = PREF_CHECK_FIXMES + "BeforeUpload";
122
123 protected boolean checkKeys;
124 protected boolean checkValues;
125 protected boolean checkComplex;
126 protected boolean checkFixmes;
127
128 protected JCheckBox prefCheckKeys;
129 protected JCheckBox prefCheckValues;
130 protected JCheckBox prefCheckComplex;
131 protected JCheckBox prefCheckFixmes;
132 protected JCheckBox prefCheckPaint;
133
134 protected JCheckBox prefCheckKeysBeforeUpload;
135 protected JCheckBox prefCheckValuesBeforeUpload;
136 protected JCheckBox prefCheckComplexBeforeUpload;
137 protected JCheckBox prefCheckFixmesBeforeUpload;
138 protected JCheckBox prefCheckPaintBeforeUpload;
139
140 // CHECKSTYLE.OFF: SingleSpaceSeparator
141 protected static final int EMPTY_VALUES = 1200;
142 protected static final int INVALID_KEY = 1201;
143 protected static final int INVALID_VALUE = 1202;
144 protected static final int FIXME = 1203;
145 protected static final int INVALID_SPACE = 1204;
146 protected static final int INVALID_KEY_SPACE = 1205;
147 protected static final int INVALID_HTML = 1206; /* 1207 was PAINT */
148 protected static final int LONG_VALUE = 1208;
149 protected static final int LONG_KEY = 1209;
150 protected static final int LOW_CHAR_VALUE = 1210;
151 protected static final int LOW_CHAR_KEY = 1211;
152 protected static final int MISSPELLED_VALUE = 1212;
153 protected static final int MISSPELLED_KEY = 1213;
154 protected static final int MULTIPLE_SPACES = 1214;
155 // CHECKSTYLE.ON: SingleSpaceSeparator
156 // 1250 and up is used by tagcheck
157
158 protected EditableList sourcesList;
159
160 private static final List<String> DEFAULT_SOURCES = Arrays.asList(/*DATA_FILE, */IGNORE_FILE, SPELL_FILE);
161
162 /**
163 * Constructor
164 */
165 public TagChecker() {
166 super(tr("Tag checker"), tr("This test checks for errors in tag keys and values."));
167 }
168
169 @Override
170 public void initialize() throws IOException {
171 initializeData();
172 initializePresets();
173 }
174
175 /**
176 * Reads the spellcheck file into a HashMap.
177 * The data file is a list of words, beginning with +/-. If it starts with +,
178 * the word is valid, but if it starts with -, the word should be replaced
179 * by the nearest + word before this.
180 *
181 * @throws IOException if any I/O error occurs
182 */
183 private static void initializeData() throws IOException {
184 checkerData.clear();
185 ignoreDataStartsWith.clear();
186 ignoreDataEquals.clear();
187 ignoreDataEndsWith.clear();
188 ignoreDataTag.clear();
189 harmonizedKeys.clear();
190
191 StringBuilder errorSources = new StringBuilder();
192 for (String source : Main.pref.getList(PREF_SOURCES, DEFAULT_SOURCES)) {
193 try (
194 CachedFile cf = new CachedFile(source);
195 BufferedReader reader = cf.getContentReader()
196 ) {
197 String okValue = null;
198 boolean tagcheckerfile = false;
199 boolean ignorefile = false;
200 boolean isFirstLine = true;
201 String line;
202 while ((line = reader.readLine()) != null && (tagcheckerfile || !line.isEmpty())) {
203 if (line.startsWith("#")) {
204 if (line.startsWith("# JOSM TagChecker")) {
205 tagcheckerfile = true;
206 if (!DEFAULT_SOURCES.contains(source)) {
207 Logging.info(tr("Adding {0} to tag checker", source));
208 }
209 } else
210 if (line.startsWith("# JOSM IgnoreTags")) {
211 ignorefile = true;
212 if (!DEFAULT_SOURCES.contains(source)) {
213 Logging.info(tr("Adding {0} to ignore tags", source));
214 }
215 }
216 } else if (ignorefile) {
217 line = line.trim();
218 if (line.length() < 4) {
219 continue;
220 }
221
222 String key = line.substring(0, 2);
223 line = line.substring(2);
224
225 switch (key) {
226 case "S:":
227 ignoreDataStartsWith.add(line);
228 break;
229 case "E:":
230 ignoreDataEquals.add(line);
231 break;
232 case "F:":
233 ignoreDataEndsWith.add(line);
234 break;
235 case "K:":
236 ignoreDataTag.add(Tag.ofString(line));
237 break;
238 default:
239 if (!key.startsWith(";")) {
240 Logging.warn("Unsupported TagChecker key: " + key);
241 }
242 }
243 } else if (tagcheckerfile) {
244 if (!line.isEmpty()) {
245 CheckerData d = new CheckerData();
246 String err = d.getData(line);
247
248 if (err == null) {
249 checkerData.add(d);
250 } else {
251 Logging.error(tr("Invalid tagchecker line - {0}: {1}", err, line));
252 }
253 }
254 } else if (line.charAt(0) == '+') {
255 okValue = line.substring(1);
256 } else if (line.charAt(0) == '-' && okValue != null) {
257 harmonizedKeys.put(harmonizeKey(line.substring(1)), okValue);
258 } else {
259 Logging.error(tr("Invalid spellcheck line: {0}", line));
260 }
261 if (isFirstLine) {
262 isFirstLine = false;
263 if (!(tagcheckerfile || ignorefile) && !DEFAULT_SOURCES.contains(source)) {
264 Logging.info(tr("Adding {0} to spellchecker", source));
265 }
266 }
267 }
268 } catch (IOException e) {
269 Logging.error(e);
270 errorSources.append(source).append('\n');
271 }
272 }
273
274 if (errorSources.length() > 0)
275 throw new IOException(tr("Could not access data file(s):\n{0}", errorSources));
276 }
277
278 /**
279 * Reads the presets data.
280 *
281 */
282 public static void initializePresets() {
283
284 if (!Main.pref.getBoolean(PREF_CHECK_VALUES, true))
285 return;
286
287 Collection<TaggingPreset> presets = TaggingPresets.getTaggingPresets();
288 if (!presets.isEmpty()) {
289 additionalPresetsValueData = new MultiMap<>();
290 for (String a : OsmPrimitive.getUninterestingKeys()) {
291 additionalPresetsValueData.putVoid(a);
292 }
293 // TODO directionKeys are no longer in OsmPrimitive (search pattern is used instead)
294 for (String a : Main.pref.getList(ValidatorPrefHelper.PREFIX + ".knownkeys",
295 Arrays.asList("is_in", "int_ref", "fixme", "population"))) {
296 additionalPresetsValueData.putVoid(a);
297 }
298 for (TaggingPreset p : presets) {
299 for (TaggingPresetItem i : p.data) {
300 if (i instanceof KeyedItem) {
301 addPresetValue((KeyedItem) i);
302 } else if (i instanceof CheckGroup) {
303 for (Check c : ((CheckGroup) i).checks) {
304 addPresetValue(c);
305 }
306 }
307 }
308 }
309 }
310 }
311
312 private static void addPresetValue(KeyedItem ky) {
313 Collection<String> values = ky.getValues();
314 if (ky.key != null && values != null) {
315 harmonizedKeys.put(harmonizeKey(ky.key), ky.key);
316 }
317 }
318
319 /**
320 * Checks given string (key or value) if it contains characters with code below 0x20 (either newline or some other special characters)
321 * @param s string to check
322 * @return {@code true} if {@code s} contains characters with code below 0x20
323 */
324 private static boolean containsLow(String s) {
325 if (s == null)
326 return false;
327 for (int i = 0; i < s.length(); i++) {
328 if (s.charAt(i) < 0x20)
329 return true;
330 }
331 return false;
332 }
333
334 private static Set<String> getPresetValues(String key) {
335 Set<String> res = TaggingPresets.getPresetValues(key);
336 if (res != null)
337 return res;
338 return additionalPresetsValueData.get(key);
339 }
340
341 /**
342 * Determines if the given key is in internal presets.
343 * @param key key
344 * @return {@code true} if the given key is in internal presets
345 * @since 9023
346 */
347 public static boolean isKeyInPresets(String key) {
348 return getPresetValues(key) != null;
349 }
350
351 /**
352 * Determines if the given tag is in internal presets.
353 * @param key key
354 * @param value value
355 * @return {@code true} if the given tag is in internal presets
356 * @since 9023
357 */
358 public static boolean isTagInPresets(String key, String value) {
359 final Set<String> values = getPresetValues(key);
360 return values != null && (values.isEmpty() || values.contains(value));
361 }
362
363 /**
364 * Returns the list of ignored tags.
365 * @return the list of ignored tags
366 * @since 9023
367 */
368 public static List<Tag> getIgnoredTags() {
369 return new ArrayList<>(ignoreDataTag);
370 }
371
372 /**
373 * Determines if the given tag is ignored for checks "key/tag not in presets".
374 * @param key key
375 * @param value value
376 * @return {@code true} if the given tag is ignored
377 * @since 9023
378 */
379 public static boolean isTagIgnored(String key, String value) {
380 boolean tagInPresets = isTagInPresets(key, value);
381 boolean ignore = false;
382
383 for (String a : ignoreDataStartsWith) {
384 if (key.startsWith(a)) {
385 ignore = true;
386 }
387 }
388 for (String a : ignoreDataEquals) {
389 if (key.equals(a)) {
390 ignore = true;
391 }
392 }
393 for (String a : ignoreDataEndsWith) {
394 if (key.endsWith(a)) {
395 ignore = true;
396 }
397 }
398
399 if (!tagInPresets) {
400 for (Tag a : ignoreDataTag) {
401 if (key.equals(a.getKey()) && value.equals(a.getValue())) {
402 ignore = true;
403 }
404 }
405 }
406 return ignore;
407 }
408
409 /**
410 * Checks the primitive tags
411 * @param p The primitive to check
412 */
413 @Override
414 public void check(OsmPrimitive p) {
415 // Just a collection to know if a primitive has been already marked with error
416 MultiMap<OsmPrimitive, String> withErrors = new MultiMap<>();
417
418 if (checkComplex) {
419 Map<String, String> keys = p.getKeys();
420 for (CheckerData d : checkerData) {
421 if (d.match(p, keys)) {
422 errors.add(TestError.builder(this, d.getSeverity(), d.getCode())
423 .message(tr("Suspicious tag/value combinations"), d.getDescription())
424 .primitives(p)
425 .build());
426 withErrors.put(p, "TC");
427 }
428 }
429 }
430
431 for (Entry<String, String> prop : p.getKeys().entrySet()) {
432 String s = marktr("Key ''{0}'' invalid.");
433 String key = prop.getKey();
434 String value = prop.getValue();
435 if (checkValues && (containsLow(value)) && !withErrors.contains(p, "ICV")) {
436 errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_VALUE)
437 .message(tr("Tag value contains character with code less than 0x20"), s, key)
438 .primitives(p)
439 .build());
440 withErrors.put(p, "ICV");
441 }
442 if (checkKeys && (containsLow(key)) && !withErrors.contains(p, "ICK")) {
443 errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_KEY)
444 .message(tr("Tag key contains character with code less than 0x20"), s, key)
445 .primitives(p)
446 .build());
447 withErrors.put(p, "ICK");
448 }
449 if (checkValues && (value != null && value.length() > 255) && !withErrors.contains(p, "LV")) {
450 errors.add(TestError.builder(this, Severity.ERROR, LONG_VALUE)
451 .message(tr("Tag value longer than allowed"), s, key)
452 .primitives(p)
453 .build());
454 withErrors.put(p, "LV");
455 }
456 if (checkKeys && (key != null && key.length() > 255) && !withErrors.contains(p, "LK")) {
457 errors.add(TestError.builder(this, Severity.ERROR, LONG_KEY)
458 .message(tr("Tag key longer than allowed"), s, key)
459 .primitives(p)
460 .build());
461 withErrors.put(p, "LK");
462 }
463 if (checkValues && (value == null || value.trim().isEmpty()) && !withErrors.contains(p, "EV")) {
464 errors.add(TestError.builder(this, Severity.WARNING, EMPTY_VALUES)
465 .message(tr("Tags with empty values"), s, key)
466 .primitives(p)
467 .build());
468 withErrors.put(p, "EV");
469 }
470 if (checkKeys && key != null && key.indexOf(' ') >= 0 && !withErrors.contains(p, "IPK")) {
471 errors.add(TestError.builder(this, Severity.WARNING, INVALID_KEY_SPACE)
472 .message(tr("Invalid white space in property key"), s, key)
473 .primitives(p)
474 .build());
475 withErrors.put(p, "IPK");
476 }
477 if (checkValues && value != null && (value.startsWith(" ") || value.endsWith(" ")) && !withErrors.contains(p, "SPACE")) {
478 errors.add(TestError.builder(this, Severity.WARNING, INVALID_SPACE)
479 .message(tr("Property values start or end with white space"), s, key)
480 .primitives(p)
481 .build());
482 withErrors.put(p, "SPACE");
483 }
484 if (checkValues && value != null && value.contains(" ") && !withErrors.contains(p, "SPACE")) {
485 errors.add(TestError.builder(this, Severity.WARNING, MULTIPLE_SPACES)
486 .message(tr("Property values contain multiple white spaces"), s, key)
487 .primitives(p)
488 .build());
489 withErrors.put(p, "SPACE");
490 }
491 if (checkValues && value != null && !value.equals(Entities.unescape(value)) && !withErrors.contains(p, "HTML")) {
492 errors.add(TestError.builder(this, Severity.OTHER, INVALID_HTML)
493 .message(tr("Property values contain HTML entity"), s, key)
494 .primitives(p)
495 .build());
496 withErrors.put(p, "HTML");
497 }
498 if (checkValues && key != null && value != null && !value.isEmpty() && additionalPresetsValueData != null
499 && !isTagIgnored(key, value)) {
500 if (!isKeyInPresets(key)) {
501 String prettifiedKey = harmonizeKey(key);
502 String fixedKey = harmonizedKeys.get(prettifiedKey);
503 if (fixedKey != null && !"".equals(fixedKey) && !fixedKey.equals(key)) {
504 // misspelled preset key
505 final TestError.Builder error = TestError.builder(this, Severity.WARNING, MISSPELLED_KEY)
506 .message(tr("Misspelled property key"), marktr("Key ''{0}'' looks like ''{1}''."), key, fixedKey)
507 .primitives(p);
508 if (p.hasKey(fixedKey)) {
509 errors.add(error.build());
510 } else {
511 errors.add(error.fix(() -> new ChangePropertyKeyCommand(p, key, fixedKey)).build());
512 }
513 withErrors.put(p, "WPK");
514 } else {
515 errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE)
516 .message(tr("Presets do not contain property key"), marktr("Key ''{0}'' not in presets."), key)
517 .primitives(p)
518 .build());
519 withErrors.put(p, "UPK");
520 }
521 } else if (!isTagInPresets(key, value)) {
522 // try to fix common typos and check again if value is still unknown
523 String fixedValue = harmonizeValue(prop.getValue());
524 Map<String, String> possibleValues = getPossibleValues(getPresetValues(key));
525 if (possibleValues.containsKey(fixedValue)) {
526 final String newKey = possibleValues.get(fixedValue);
527 // misspelled preset value
528 errors.add(TestError.builder(this, Severity.WARNING, MISSPELLED_VALUE)
529 .message(tr("Misspelled property value"),
530 marktr("Value ''{0}'' for key ''{1}'' looks like ''{2}''."), prop.getValue(), key, fixedValue)
531 .primitives(p)
532 .fix(() -> new ChangePropertyCommand(p, key, newKey))
533 .build());
534 withErrors.put(p, "WPV");
535 } else {
536 // unknown preset value
537 errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE)
538 .message(tr("Presets do not contain property value"),
539 marktr("Value ''{0}'' for key ''{1}'' not in presets."), prop.getValue(), key)
540 .primitives(p)
541 .build());
542 withErrors.put(p, "UPV");
543 }
544 }
545 }
546 if (checkFixmes && key != null && value != null && !value.isEmpty() && isFixme(key, value) && !withErrors.contains(p, "FIXME")) {
547 errors.add(TestError.builder(this, Severity.OTHER, FIXME)
548 .message(tr("FIXMES"))
549 .primitives(p)
550 .build());
551 withErrors.put(p, "FIXME");
552 }
553 }
554 }
555
556 private static boolean isFixme(String key, String value) {
557 return key.toLowerCase(Locale.ENGLISH).contains("fixme") || key.contains("todo")
558 || value.toLowerCase(Locale.ENGLISH).contains("fixme") || value.contains("check and delete");
559 }
560
561 private static Map<String, String> getPossibleValues(Set<String> values) {
562 // generate a map with common typos
563 Map<String, String> map = new HashMap<>();
564 if (values != null) {
565 for (String value : values) {
566 map.put(value, value);
567 if (value.contains("_")) {
568 map.put(value.replace("_", ""), value);
569 }
570 }
571 }
572 return map;
573 }
574
575 private static String harmonizeKey(String key) {
576 return Utils.strip(key.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(':', '_').replace(' ', '_'), "-_;:,");
577 }
578
579 private static String harmonizeValue(String value) {
580 return Utils.strip(value.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(' ', '_'), "-_;:,");
581 }
582
583 @Override
584 public void startTest(ProgressMonitor monitor) {
585 super.startTest(monitor);
586 checkKeys = Main.pref.getBoolean(PREF_CHECK_KEYS, true);
587 if (isBeforeUpload) {
588 checkKeys = checkKeys && Main.pref.getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true);
589 }
590
591 checkValues = Main.pref.getBoolean(PREF_CHECK_VALUES, true);
592 if (isBeforeUpload) {
593 checkValues = checkValues && Main.pref.getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true);
594 }
595
596 checkComplex = Main.pref.getBoolean(PREF_CHECK_COMPLEX, true);
597 if (isBeforeUpload) {
598 checkComplex = checkComplex && Main.pref.getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true);
599 }
600
601 checkFixmes = Main.pref.getBoolean(PREF_CHECK_FIXMES, true);
602 if (isBeforeUpload) {
603 checkFixmes = checkFixmes && Main.pref.getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true);
604 }
605 }
606
607 @Override
608 public void visit(Collection<OsmPrimitive> selection) {
609 if (checkKeys || checkValues || checkComplex || checkFixmes) {
610 super.visit(selection);
611 }
612 }
613
614 @Override
615 public void addGui(JPanel testPanel) {
616 GBC a = GBC.eol();
617 a.anchor = GridBagConstraints.EAST;
618
619 testPanel.add(new JLabel(name+" :"), GBC.eol().insets(3, 0, 0, 0));
620
621 prefCheckKeys = new JCheckBox(tr("Check property keys."), Main.pref.getBoolean(PREF_CHECK_KEYS, true));
622 prefCheckKeys.setToolTipText(tr("Validate that property keys are valid checking against list of words."));
623 testPanel.add(prefCheckKeys, GBC.std().insets(20, 0, 0, 0));
624
625 prefCheckKeysBeforeUpload = new JCheckBox();
626 prefCheckKeysBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true));
627 testPanel.add(prefCheckKeysBeforeUpload, a);
628
629 prefCheckComplex = new JCheckBox(tr("Use complex property checker."), Main.pref.getBoolean(PREF_CHECK_COMPLEX, true));
630 prefCheckComplex.setToolTipText(tr("Validate property values and tags using complex rules."));
631 testPanel.add(prefCheckComplex, GBC.std().insets(20, 0, 0, 0));
632
633 prefCheckComplexBeforeUpload = new JCheckBox();
634 prefCheckComplexBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true));
635 testPanel.add(prefCheckComplexBeforeUpload, a);
636
637 final Collection<String> sources = Main.pref.getList(PREF_SOURCES, DEFAULT_SOURCES);
638 sourcesList = new EditableList(tr("TagChecker source"));
639 sourcesList.setItems(sources);
640 testPanel.add(new JLabel(tr("Data sources ({0})", "*.cfg")), GBC.eol().insets(23, 0, 0, 0));
641 testPanel.add(sourcesList, GBC.eol().fill(GridBagConstraints.HORIZONTAL).insets(23, 0, 0, 0));
642
643 ActionListener disableCheckActionListener = e -> handlePrefEnable();
644 prefCheckKeys.addActionListener(disableCheckActionListener);
645 prefCheckKeysBeforeUpload.addActionListener(disableCheckActionListener);
646 prefCheckComplex.addActionListener(disableCheckActionListener);
647 prefCheckComplexBeforeUpload.addActionListener(disableCheckActionListener);
648
649 handlePrefEnable();
650
651 prefCheckValues = new JCheckBox(tr("Check property values."), Main.pref.getBoolean(PREF_CHECK_VALUES, true));
652 prefCheckValues.setToolTipText(tr("Validate that property values are valid checking against presets."));
653 testPanel.add(prefCheckValues, GBC.std().insets(20, 0, 0, 0));
654
655 prefCheckValuesBeforeUpload = new JCheckBox();
656 prefCheckValuesBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true));
657 testPanel.add(prefCheckValuesBeforeUpload, a);
658
659 prefCheckFixmes = new JCheckBox(tr("Check for FIXMES."), Main.pref.getBoolean(PREF_CHECK_FIXMES, true));
660 prefCheckFixmes.setToolTipText(tr("Looks for nodes or ways with FIXME in any property value."));
661 testPanel.add(prefCheckFixmes, GBC.std().insets(20, 0, 0, 0));
662
663 prefCheckFixmesBeforeUpload = new JCheckBox();
664 prefCheckFixmesBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true));
665 testPanel.add(prefCheckFixmesBeforeUpload, a);
666 }
667
668 /**
669 * Enables/disables the source list field
670 */
671 public void handlePrefEnable() {
672 boolean selected = prefCheckKeys.isSelected() || prefCheckKeysBeforeUpload.isSelected()
673 || prefCheckComplex.isSelected() || prefCheckComplexBeforeUpload.isSelected();
674 sourcesList.setEnabled(selected);
675 }
676
677 @Override
678 public boolean ok() {
679 enabled = prefCheckKeys.isSelected() || prefCheckValues.isSelected() || prefCheckComplex.isSelected() || prefCheckFixmes.isSelected();
680 testBeforeUpload = prefCheckKeysBeforeUpload.isSelected() || prefCheckValuesBeforeUpload.isSelected()
681 || prefCheckFixmesBeforeUpload.isSelected() || prefCheckComplexBeforeUpload.isSelected();
682
683 Main.pref.putBoolean(PREF_CHECK_VALUES, prefCheckValues.isSelected());
684 Main.pref.putBoolean(PREF_CHECK_COMPLEX, prefCheckComplex.isSelected());
685 Main.pref.putBoolean(PREF_CHECK_KEYS, prefCheckKeys.isSelected());
686 Main.pref.putBoolean(PREF_CHECK_FIXMES, prefCheckFixmes.isSelected());
687 Main.pref.putBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, prefCheckValuesBeforeUpload.isSelected());
688 Main.pref.putBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, prefCheckComplexBeforeUpload.isSelected());
689 Main.pref.putBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, prefCheckKeysBeforeUpload.isSelected());
690 Main.pref.putBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, prefCheckFixmesBeforeUpload.isSelected());
691 return Main.pref.putList(PREF_SOURCES, sourcesList.getItems());
692 }
693
694 @Override
695 public Command fixError(TestError testError) {
696 List<Command> commands = new ArrayList<>(50);
697
698 Collection<? extends OsmPrimitive> primitives = testError.getPrimitives();
699 for (OsmPrimitive p : primitives) {
700 Map<String, String> tags = p.getKeys();
701 if (tags.isEmpty()) {
702 continue;
703 }
704
705 for (Entry<String, String> prop: tags.entrySet()) {
706 String key = prop.getKey();
707 String value = prop.getValue();
708 if (value == null || value.trim().isEmpty()) {
709 commands.add(new ChangePropertyCommand(p, key, null));
710 } else if (value.startsWith(" ") || value.endsWith(" ") || value.contains(" ")) {
711 commands.add(new ChangePropertyCommand(p, key, Tag.removeWhiteSpaces(value)));
712 } else if (key.startsWith(" ") || key.endsWith(" ") || key.contains(" ")) {
713 commands.add(new ChangePropertyKeyCommand(p, key, Tag.removeWhiteSpaces(key)));
714 } else {
715 String evalue = Entities.unescape(value);
716 if (!evalue.equals(value)) {
717 commands.add(new ChangePropertyCommand(p, key, evalue));
718 }
719 }
720 }
721 }
722
723 if (commands.isEmpty())
724 return null;
725 if (commands.size() == 1)
726 return commands.get(0);
727
728 return new SequenceCommand(tr("Fix tags"), commands);
729 }
730
731 @Override
732 public boolean isFixable(TestError testError) {
733 if (testError.getTester() instanceof TagChecker) {
734 int code = testError.getCode();
735 return code == INVALID_KEY || code == EMPTY_VALUES || code == INVALID_SPACE ||
736 code == INVALID_KEY_SPACE || code == INVALID_HTML || code == MISSPELLED_VALUE ||
737 code == MULTIPLE_SPACES;
738 }
739
740 return false;
741 }
742
743 protected static class CheckerData {
744 private String description;
745 protected List<CheckerElement> data = new ArrayList<>();
746 private OsmPrimitiveType type;
747 private TagCheckLevel level;
748 protected Severity severity;
749
750 private enum TagCheckLevel {
751 TAG_CHECK_ERROR(1250),
752 TAG_CHECK_WARN(1260),
753 TAG_CHECK_INFO(1270);
754
755 final int code;
756
757 TagCheckLevel(int code) {
758 this.code = code;
759 }
760 }
761
762 protected static class CheckerElement {
763 public Object tag;
764 public Object value;
765 public boolean noMatch;
766 public boolean tagAll;
767 public boolean valueAll;
768 public boolean valueBool;
769
770 private static Pattern getPattern(String str) {
771 if (str.endsWith("/i"))
772 return Pattern.compile(str.substring(1, str.length()-2), Pattern.CASE_INSENSITIVE);
773 if (str.endsWith("/"))
774 return Pattern.compile(str.substring(1, str.length()-1));
775
776 throw new IllegalStateException();
777 }
778
779 public CheckerElement(String exp) {
780 Matcher m = Pattern.compile("(.+)([!=]=)(.+)").matcher(exp);
781 m.matches();
782
783 String n = m.group(1).trim();
784
785 if ("*".equals(n)) {
786 tagAll = true;
787 } else {
788 tag = n.startsWith("/") ? getPattern(n) : n;
789 noMatch = "!=".equals(m.group(2));
790 n = m.group(3).trim();
791 if ("*".equals(n)) {
792 valueAll = true;
793 } else if ("BOOLEAN_TRUE".equals(n)) {
794 valueBool = true;
795 value = OsmUtils.TRUE_VALUE;
796 } else if ("BOOLEAN_FALSE".equals(n)) {
797 valueBool = true;
798 value = OsmUtils.FALSE_VALUE;
799 } else {
800 value = n.startsWith("/") ? getPattern(n) : n;
801 }
802 }
803 }
804
805 public boolean match(Map<String, String> keys) {
806 for (Entry<String, String> prop: keys.entrySet()) {
807 String key = prop.getKey();
808 String val = valueBool ? OsmUtils.getNamedOsmBoolean(prop.getValue()) : prop.getValue();
809 if ((tagAll || (tag instanceof Pattern ? ((Pattern) tag).matcher(key).matches() : key.equals(tag)))
810 && (valueAll || (value instanceof Pattern ? ((Pattern) value).matcher(val).matches() : val.equals(value))))
811 return !noMatch;
812 }
813 return noMatch;
814 }
815 }
816
817 private static final Pattern CLEAN_STR_PATTERN = Pattern.compile(" *# *([^#]+) *$");
818 private static final Pattern SPLIT_TRIMMED_PATTERN = Pattern.compile(" *: *");
819 private static final Pattern SPLIT_ELEMENTS_PATTERN = Pattern.compile(" *&& *");
820
821 public String getData(final String str) {
822 Matcher m = CLEAN_STR_PATTERN.matcher(str);
823 String trimmed = m.replaceFirst("").trim();
824 try {
825 description = m.group(1);
826 if (description != null && description.isEmpty()) {
827 description = null;
828 }
829 } catch (IllegalStateException e) {
830 Logging.error(e);
831 description = null;
832 }
833 String[] n = SPLIT_TRIMMED_PATTERN.split(trimmed, 3);
834 switch (n[0]) {
835 case "way":
836 type = OsmPrimitiveType.WAY;
837 break;
838 case "node":
839 type = OsmPrimitiveType.NODE;
840 break;
841 case "relation":
842 type = OsmPrimitiveType.RELATION;
843 break;
844 case "*":
845 type = null;
846 break;
847 default:
848 return tr("Could not find element type");
849 }
850 if (n.length != 3)
851 return tr("Incorrect number of parameters");
852
853 switch (n[1]) {
854 case "W":
855 severity = Severity.WARNING;
856 level = TagCheckLevel.TAG_CHECK_WARN;
857 break;
858 case "E":
859 severity = Severity.ERROR;
860 level = TagCheckLevel.TAG_CHECK_ERROR;
861 break;
862 case "I":
863 severity = Severity.OTHER;
864 level = TagCheckLevel.TAG_CHECK_INFO;
865 break;
866 default:
867 return tr("Could not find warning level");
868 }
869 for (String exp: SPLIT_ELEMENTS_PATTERN.split(n[2])) {
870 try {
871 data.add(new CheckerElement(exp));
872 } catch (IllegalStateException e) {
873 Logging.trace(e);
874 return tr("Illegal expression ''{0}''", exp);
875 } catch (PatternSyntaxException e) {
876 Logging.trace(e);
877 return tr("Illegal regular expression ''{0}''", exp);
878 }
879 }
880 return null;
881 }
882
883 public boolean match(OsmPrimitive osm, Map<String, String> keys) {
884 if (type != null && OsmPrimitiveType.from(osm) != type)
885 return false;
886
887 for (CheckerElement ce : data) {
888 if (!ce.match(keys))
889 return false;
890 }
891 return true;
892 }
893
894 /**
895 * Returns the error description.
896 * @return the error description
897 */
898 public String getDescription() {
899 return description;
900 }
901
902 /**
903 * Returns the error severity.
904 * @return the error severity
905 */
906 public Severity getSeverity() {
907 return severity;
908 }
909
910 /**
911 * Returns the error code.
912 * @return the error code
913 */
914 public int getCode() {
915 if (type == null)
916 return level.code;
917
918 return level.code + type.ordinal() + 1;
919 }
920 }
921}
Note: See TracBrowser for help on using the repository browser.