source: josm/trunk/src/org/openstreetmap/josm/data/validation/tests/TagChecker.java@ 12649

Last change on this file since 12649 was 12649, checked in by Don-vip, 7 years ago

see #15182 - code refactoring to avoid dependence on GUI packages from Preferences
  • Property svn:eol-style set to native
File size: 38.2 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.validation.tests;
3
4import static org.openstreetmap.josm.tools.I18n.marktr;
5import static org.openstreetmap.josm.tools.I18n.tr;
6
7import java.awt.GridBagConstraints;
8import java.awt.event.ActionListener;
9import java.io.BufferedReader;
10import java.io.IOException;
11import java.util.ArrayList;
12import java.util.Arrays;
13import java.util.Collection;
14import java.util.HashMap;
15import java.util.HashSet;
16import java.util.List;
17import java.util.Locale;
18import java.util.Map;
19import java.util.Map.Entry;
20import java.util.Set;
21import java.util.regex.Matcher;
22import java.util.regex.Pattern;
23import java.util.regex.PatternSyntaxException;
24
25import javax.swing.JCheckBox;
26import javax.swing.JLabel;
27import javax.swing.JPanel;
28
29import org.openstreetmap.josm.Main;
30import org.openstreetmap.josm.command.ChangePropertyCommand;
31import org.openstreetmap.josm.command.ChangePropertyKeyCommand;
32import org.openstreetmap.josm.command.Command;
33import org.openstreetmap.josm.command.SequenceCommand;
34import org.openstreetmap.josm.data.osm.OsmPrimitive;
35import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
36import org.openstreetmap.josm.data.osm.OsmUtils;
37import org.openstreetmap.josm.data.osm.Tag;
38import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper;
39import org.openstreetmap.josm.data.validation.Severity;
40import org.openstreetmap.josm.data.validation.Test.TagTest;
41import org.openstreetmap.josm.data.validation.TestError;
42import org.openstreetmap.josm.data.validation.util.Entities;
43import org.openstreetmap.josm.gui.progress.ProgressMonitor;
44import org.openstreetmap.josm.gui.tagging.presets.TaggingPreset;
45import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetItem;
46import org.openstreetmap.josm.gui.tagging.presets.TaggingPresets;
47import org.openstreetmap.josm.gui.tagging.presets.items.Check;
48import org.openstreetmap.josm.gui.tagging.presets.items.CheckGroup;
49import org.openstreetmap.josm.gui.tagging.presets.items.KeyedItem;
50import org.openstreetmap.josm.gui.widgets.EditableList;
51import org.openstreetmap.josm.io.CachedFile;
52import org.openstreetmap.josm.tools.GBC;
53import org.openstreetmap.josm.tools.Logging;
54import org.openstreetmap.josm.tools.MultiMap;
55import org.openstreetmap.josm.tools.Utils;
56
57/**
58 * Check for misspelled or wrong tags
59 *
60 * @author frsantos
61 * @since 3669
62 */
63public class TagChecker extends TagTest {
64
65 /** The config file of ignored tags */
66 public static final String IGNORE_FILE = "resource://data/validator/ignoretags.cfg";
67 /** The config file of dictionary words */
68 public static final String SPELL_FILE = "resource://data/validator/words.cfg";
69
70 /** Normalized keys: the key should be substituted by the value if the key was not found in presets */
71 private static final Map<String, String> harmonizedKeys = new HashMap<>();
72 /** The spell check preset values which are not stored in TaggingPresets */
73 private static volatile MultiMap<String, String> additionalPresetsValueData;
74 /** The TagChecker data */
75 private static final List<CheckerData> checkerData = new ArrayList<>();
76 private static final List<String> ignoreDataStartsWith = new ArrayList<>();
77 private static final List<String> ignoreDataEquals = new ArrayList<>();
78 private static final List<String> ignoreDataEndsWith = new ArrayList<>();
79 private static final List<Tag> ignoreDataTag = new ArrayList<>();
80
81 /** The preferences prefix */
82 protected static final String PREFIX = ValidatorPrefHelper.PREFIX + "." + TagChecker.class.getSimpleName();
83
84 /**
85 * The preference key to check values
86 */
87 public static final String PREF_CHECK_VALUES = PREFIX + ".checkValues";
88 /**
89 * The preference key to check keys
90 */
91 public static final String PREF_CHECK_KEYS = PREFIX + ".checkKeys";
92 /**
93 * The preference key to enable complex checks
94 */
95 public static final String PREF_CHECK_COMPLEX = PREFIX + ".checkComplex";
96 /**
97 * The preference key to search for fixme tags
98 */
99 public static final String PREF_CHECK_FIXMES = PREFIX + ".checkFixmes";
100
101 /**
102 * The preference key for source files
103 * @see #DEFAULT_SOURCES
104 */
105 public static final String PREF_SOURCES = PREFIX + ".source";
106
107 /**
108 * The preference key to check keys - used before upload
109 */
110 public static final String PREF_CHECK_KEYS_BEFORE_UPLOAD = PREF_CHECK_KEYS + "BeforeUpload";
111 /**
112 * The preference key to check values - used before upload
113 */
114 public static final String PREF_CHECK_VALUES_BEFORE_UPLOAD = PREF_CHECK_VALUES + "BeforeUpload";
115 /**
116 * The preference key to run complex tests - used before upload
117 */
118 public static final String PREF_CHECK_COMPLEX_BEFORE_UPLOAD = PREF_CHECK_COMPLEX + "BeforeUpload";
119 /**
120 * The preference key to search for fixmes - used before upload
121 */
122 public static final String PREF_CHECK_FIXMES_BEFORE_UPLOAD = PREF_CHECK_FIXMES + "BeforeUpload";
123
124 protected boolean checkKeys;
125 protected boolean checkValues;
126 protected boolean checkComplex;
127 protected boolean checkFixmes;
128
129 protected JCheckBox prefCheckKeys;
130 protected JCheckBox prefCheckValues;
131 protected JCheckBox prefCheckComplex;
132 protected JCheckBox prefCheckFixmes;
133 protected JCheckBox prefCheckPaint;
134
135 protected JCheckBox prefCheckKeysBeforeUpload;
136 protected JCheckBox prefCheckValuesBeforeUpload;
137 protected JCheckBox prefCheckComplexBeforeUpload;
138 protected JCheckBox prefCheckFixmesBeforeUpload;
139 protected JCheckBox prefCheckPaintBeforeUpload;
140
141 // CHECKSTYLE.OFF: SingleSpaceSeparator
142 protected static final int EMPTY_VALUES = 1200;
143 protected static final int INVALID_KEY = 1201;
144 protected static final int INVALID_VALUE = 1202;
145 protected static final int FIXME = 1203;
146 protected static final int INVALID_SPACE = 1204;
147 protected static final int INVALID_KEY_SPACE = 1205;
148 protected static final int INVALID_HTML = 1206; /* 1207 was PAINT */
149 protected static final int LONG_VALUE = 1208;
150 protected static final int LONG_KEY = 1209;
151 protected static final int LOW_CHAR_VALUE = 1210;
152 protected static final int LOW_CHAR_KEY = 1211;
153 protected static final int MISSPELLED_VALUE = 1212;
154 protected static final int MISSPELLED_KEY = 1213;
155 protected static final int MULTIPLE_SPACES = 1214;
156 // CHECKSTYLE.ON: SingleSpaceSeparator
157 // 1250 and up is used by tagcheck
158
159 protected EditableList sourcesList;
160
161 private static final Set<String> DEFAULT_SOURCES = new HashSet<>(Arrays.asList(/*DATA_FILE, */IGNORE_FILE, SPELL_FILE));
162
163 /**
164 * Constructor
165 */
166 public TagChecker() {
167 super(tr("Tag checker"), tr("This test checks for errors in tag keys and values."));
168 }
169
170 @Override
171 public void initialize() throws IOException {
172 initializeData();
173 initializePresets();
174 }
175
176 /**
177 * Reads the spellcheck file into a HashMap.
178 * The data file is a list of words, beginning with +/-. If it starts with +,
179 * the word is valid, but if it starts with -, the word should be replaced
180 * by the nearest + word before this.
181 *
182 * @throws IOException if any I/O error occurs
183 */
184 private static void initializeData() throws IOException {
185 checkerData.clear();
186 ignoreDataStartsWith.clear();
187 ignoreDataEquals.clear();
188 ignoreDataEndsWith.clear();
189 ignoreDataTag.clear();
190 harmonizedKeys.clear();
191
192 StringBuilder errorSources = new StringBuilder();
193 for (String source : Main.pref.getCollection(PREF_SOURCES, DEFAULT_SOURCES)) {
194 try (
195 CachedFile cf = new CachedFile(source);
196 BufferedReader reader = cf.getContentReader()
197 ) {
198 String okValue = null;
199 boolean tagcheckerfile = false;
200 boolean ignorefile = false;
201 boolean isFirstLine = true;
202 String line;
203 while ((line = reader.readLine()) != null && (tagcheckerfile || !line.isEmpty())) {
204 if (line.startsWith("#")) {
205 if (line.startsWith("# JOSM TagChecker")) {
206 tagcheckerfile = true;
207 if (!DEFAULT_SOURCES.contains(source)) {
208 Logging.info(tr("Adding {0} to tag checker", source));
209 }
210 } else
211 if (line.startsWith("# JOSM IgnoreTags")) {
212 ignorefile = true;
213 if (!DEFAULT_SOURCES.contains(source)) {
214 Logging.info(tr("Adding {0} to ignore tags", source));
215 }
216 }
217 } else if (ignorefile) {
218 line = line.trim();
219 if (line.length() < 4) {
220 continue;
221 }
222
223 String key = line.substring(0, 2);
224 line = line.substring(2);
225
226 switch (key) {
227 case "S:":
228 ignoreDataStartsWith.add(line);
229 break;
230 case "E:":
231 ignoreDataEquals.add(line);
232 break;
233 case "F:":
234 ignoreDataEndsWith.add(line);
235 break;
236 case "K:":
237 ignoreDataTag.add(Tag.ofString(line));
238 break;
239 default:
240 if (!key.startsWith(";")) {
241 Logging.warn("Unsupported TagChecker key: " + key);
242 }
243 }
244 } else if (tagcheckerfile) {
245 if (!line.isEmpty()) {
246 CheckerData d = new CheckerData();
247 String err = d.getData(line);
248
249 if (err == null) {
250 checkerData.add(d);
251 } else {
252 Logging.error(tr("Invalid tagchecker line - {0}: {1}", err, line));
253 }
254 }
255 } else if (line.charAt(0) == '+') {
256 okValue = line.substring(1);
257 } else if (line.charAt(0) == '-' && okValue != null) {
258 harmonizedKeys.put(harmonizeKey(line.substring(1)), okValue);
259 } else {
260 Logging.error(tr("Invalid spellcheck line: {0}", line));
261 }
262 if (isFirstLine) {
263 isFirstLine = false;
264 if (!(tagcheckerfile || ignorefile) && !DEFAULT_SOURCES.contains(source)) {
265 Logging.info(tr("Adding {0} to spellchecker", source));
266 }
267 }
268 }
269 } catch (IOException e) {
270 Logging.error(e);
271 errorSources.append(source).append('\n');
272 }
273 }
274
275 if (errorSources.length() > 0)
276 throw new IOException(tr("Could not access data file(s):\n{0}", errorSources));
277 }
278
279 /**
280 * Reads the presets data.
281 *
282 */
283 public static void initializePresets() {
284
285 if (!Main.pref.getBoolean(PREF_CHECK_VALUES, true))
286 return;
287
288 Collection<TaggingPreset> presets = TaggingPresets.getTaggingPresets();
289 if (!presets.isEmpty()) {
290 additionalPresetsValueData = new MultiMap<>();
291 for (String a : OsmPrimitive.getUninterestingKeys()) {
292 additionalPresetsValueData.putVoid(a);
293 }
294 // TODO directionKeys are no longer in OsmPrimitive (search pattern is used instead)
295 for (String a : Main.pref.getCollection(ValidatorPrefHelper.PREFIX + ".knownkeys",
296 Arrays.asList("is_in", "int_ref", "fixme", "population"))) {
297 additionalPresetsValueData.putVoid(a);
298 }
299 for (TaggingPreset p : presets) {
300 for (TaggingPresetItem i : p.data) {
301 if (i instanceof KeyedItem) {
302 addPresetValue((KeyedItem) i);
303 } else if (i instanceof CheckGroup) {
304 for (Check c : ((CheckGroup) i).checks) {
305 addPresetValue(c);
306 }
307 }
308 }
309 }
310 }
311 }
312
313 private static void addPresetValue(KeyedItem ky) {
314 Collection<String> values = ky.getValues();
315 if (ky.key != null && values != null) {
316 harmonizedKeys.put(harmonizeKey(ky.key), ky.key);
317 }
318 }
319
320 /**
321 * Checks given string (key or value) if it contains characters with code below 0x20 (either newline or some other special characters)
322 * @param s string to check
323 * @return {@code true} if {@code s} contains characters with code below 0x20
324 */
325 private static boolean containsLow(String s) {
326 if (s == null)
327 return false;
328 for (int i = 0; i < s.length(); i++) {
329 if (s.charAt(i) < 0x20)
330 return true;
331 }
332 return false;
333 }
334
335 private static Set<String> getPresetValues(String key) {
336 Set<String> res = TaggingPresets.getPresetValues(key);
337 if (res != null)
338 return res;
339 return additionalPresetsValueData.get(key);
340 }
341
342 /**
343 * Determines if the given key is in internal presets.
344 * @param key key
345 * @return {@code true} if the given key is in internal presets
346 * @since 9023
347 */
348 public static boolean isKeyInPresets(String key) {
349 return getPresetValues(key) != null;
350 }
351
352 /**
353 * Determines if the given tag is in internal presets.
354 * @param key key
355 * @param value value
356 * @return {@code true} if the given tag is in internal presets
357 * @since 9023
358 */
359 public static boolean isTagInPresets(String key, String value) {
360 final Set<String> values = getPresetValues(key);
361 return values != null && (values.isEmpty() || values.contains(value));
362 }
363
364 /**
365 * Returns the list of ignored tags.
366 * @return the list of ignored tags
367 * @since 9023
368 */
369 public static List<Tag> getIgnoredTags() {
370 return new ArrayList<>(ignoreDataTag);
371 }
372
373 /**
374 * Determines if the given tag is ignored for checks "key/tag not in presets".
375 * @param key key
376 * @param value value
377 * @return {@code true} if the given tag is ignored
378 * @since 9023
379 */
380 public static boolean isTagIgnored(String key, String value) {
381 boolean tagInPresets = isTagInPresets(key, value);
382 boolean ignore = false;
383
384 for (String a : ignoreDataStartsWith) {
385 if (key.startsWith(a)) {
386 ignore = true;
387 }
388 }
389 for (String a : ignoreDataEquals) {
390 if (key.equals(a)) {
391 ignore = true;
392 }
393 }
394 for (String a : ignoreDataEndsWith) {
395 if (key.endsWith(a)) {
396 ignore = true;
397 }
398 }
399
400 if (!tagInPresets) {
401 for (Tag a : ignoreDataTag) {
402 if (key.equals(a.getKey()) && value.equals(a.getValue())) {
403 ignore = true;
404 }
405 }
406 }
407 return ignore;
408 }
409
410 /**
411 * Checks the primitive tags
412 * @param p The primitive to check
413 */
414 @Override
415 public void check(OsmPrimitive p) {
416 // Just a collection to know if a primitive has been already marked with error
417 MultiMap<OsmPrimitive, String> withErrors = new MultiMap<>();
418
419 if (checkComplex) {
420 Map<String, String> keys = p.getKeys();
421 for (CheckerData d : checkerData) {
422 if (d.match(p, keys)) {
423 errors.add(TestError.builder(this, d.getSeverity(), d.getCode())
424 .message(tr("Suspicious tag/value combinations"), d.getDescription())
425 .primitives(p)
426 .build());
427 withErrors.put(p, "TC");
428 }
429 }
430 }
431
432 for (Entry<String, String> prop : p.getKeys().entrySet()) {
433 String s = marktr("Key ''{0}'' invalid.");
434 String key = prop.getKey();
435 String value = prop.getValue();
436 if (checkValues && (containsLow(value)) && !withErrors.contains(p, "ICV")) {
437 errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_VALUE)
438 .message(tr("Tag value contains character with code less than 0x20"), s, key)
439 .primitives(p)
440 .build());
441 withErrors.put(p, "ICV");
442 }
443 if (checkKeys && (containsLow(key)) && !withErrors.contains(p, "ICK")) {
444 errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_KEY)
445 .message(tr("Tag key contains character with code less than 0x20"), s, key)
446 .primitives(p)
447 .build());
448 withErrors.put(p, "ICK");
449 }
450 if (checkValues && (value != null && value.length() > 255) && !withErrors.contains(p, "LV")) {
451 errors.add(TestError.builder(this, Severity.ERROR, LONG_VALUE)
452 .message(tr("Tag value longer than allowed"), s, key)
453 .primitives(p)
454 .build());
455 withErrors.put(p, "LV");
456 }
457 if (checkKeys && (key != null && key.length() > 255) && !withErrors.contains(p, "LK")) {
458 errors.add(TestError.builder(this, Severity.ERROR, LONG_KEY)
459 .message(tr("Tag key longer than allowed"), s, key)
460 .primitives(p)
461 .build());
462 withErrors.put(p, "LK");
463 }
464 if (checkValues && (value == null || value.trim().isEmpty()) && !withErrors.contains(p, "EV")) {
465 errors.add(TestError.builder(this, Severity.WARNING, EMPTY_VALUES)
466 .message(tr("Tags with empty values"), s, key)
467 .primitives(p)
468 .build());
469 withErrors.put(p, "EV");
470 }
471 if (checkKeys && key != null && key.indexOf(' ') >= 0 && !withErrors.contains(p, "IPK")) {
472 errors.add(TestError.builder(this, Severity.WARNING, INVALID_KEY_SPACE)
473 .message(tr("Invalid white space in property key"), s, key)
474 .primitives(p)
475 .build());
476 withErrors.put(p, "IPK");
477 }
478 if (checkValues && value != null && (value.startsWith(" ") || value.endsWith(" ")) && !withErrors.contains(p, "SPACE")) {
479 errors.add(TestError.builder(this, Severity.WARNING, INVALID_SPACE)
480 .message(tr("Property values start or end with white space"), s, key)
481 .primitives(p)
482 .build());
483 withErrors.put(p, "SPACE");
484 }
485 if (checkValues && value != null && value.contains(" ") && !withErrors.contains(p, "SPACE")) {
486 errors.add(TestError.builder(this, Severity.WARNING, MULTIPLE_SPACES)
487 .message(tr("Property values contain multiple white spaces"), s, key)
488 .primitives(p)
489 .build());
490 withErrors.put(p, "SPACE");
491 }
492 if (checkValues && value != null && !value.equals(Entities.unescape(value)) && !withErrors.contains(p, "HTML")) {
493 errors.add(TestError.builder(this, Severity.OTHER, INVALID_HTML)
494 .message(tr("Property values contain HTML entity"), s, key)
495 .primitives(p)
496 .build());
497 withErrors.put(p, "HTML");
498 }
499 if (checkValues && key != null && value != null && !value.isEmpty() && additionalPresetsValueData != null
500 && !isTagIgnored(key, value)) {
501 if (!isKeyInPresets(key)) {
502 String prettifiedKey = harmonizeKey(key);
503 String fixedKey = harmonizedKeys.get(prettifiedKey);
504 if (fixedKey != null && !"".equals(fixedKey) && !fixedKey.equals(key)) {
505 // misspelled preset key
506 final TestError.Builder error = TestError.builder(this, Severity.WARNING, MISSPELLED_KEY)
507 .message(tr("Misspelled property key"), marktr("Key ''{0}'' looks like ''{1}''."), key, fixedKey)
508 .primitives(p);
509 if (p.hasKey(fixedKey)) {
510 errors.add(error.build());
511 } else {
512 errors.add(error.fix(() -> new ChangePropertyKeyCommand(p, key, fixedKey)).build());
513 }
514 withErrors.put(p, "WPK");
515 } else {
516 errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE)
517 .message(tr("Presets do not contain property key"), marktr("Key ''{0}'' not in presets."), key)
518 .primitives(p)
519 .build());
520 withErrors.put(p, "UPK");
521 }
522 } else if (!isTagInPresets(key, value)) {
523 // try to fix common typos and check again if value is still unknown
524 String fixedValue = harmonizeValue(prop.getValue());
525 Map<String, String> possibleValues = getPossibleValues(getPresetValues(key));
526 if (possibleValues.containsKey(fixedValue)) {
527 final String newKey = possibleValues.get(fixedValue);
528 // misspelled preset value
529 errors.add(TestError.builder(this, Severity.WARNING, MISSPELLED_VALUE)
530 .message(tr("Misspelled property value"),
531 marktr("Value ''{0}'' for key ''{1}'' looks like ''{2}''."), prop.getValue(), key, fixedValue)
532 .primitives(p)
533 .fix(() -> new ChangePropertyCommand(p, key, newKey))
534 .build());
535 withErrors.put(p, "WPV");
536 } else {
537 // unknown preset value
538 errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE)
539 .message(tr("Presets do not contain property value"),
540 marktr("Value ''{0}'' for key ''{1}'' not in presets."), prop.getValue(), key)
541 .primitives(p)
542 .build());
543 withErrors.put(p, "UPV");
544 }
545 }
546 }
547 if (checkFixmes && key != null && value != null && !value.isEmpty() && isFixme(key, value) && !withErrors.contains(p, "FIXME")) {
548 errors.add(TestError.builder(this, Severity.OTHER, FIXME)
549 .message(tr("FIXMES"))
550 .primitives(p)
551 .build());
552 withErrors.put(p, "FIXME");
553 }
554 }
555 }
556
557 private static boolean isFixme(String key, String value) {
558 return key.toLowerCase(Locale.ENGLISH).contains("fixme") || key.contains("todo")
559 || value.toLowerCase(Locale.ENGLISH).contains("fixme") || value.contains("check and delete");
560 }
561
562 private static Map<String, String> getPossibleValues(Set<String> values) {
563 // generate a map with common typos
564 Map<String, String> map = new HashMap<>();
565 if (values != null) {
566 for (String value : values) {
567 map.put(value, value);
568 if (value.contains("_")) {
569 map.put(value.replace("_", ""), value);
570 }
571 }
572 }
573 return map;
574 }
575
576 private static String harmonizeKey(String key) {
577 return Utils.strip(key.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(':', '_').replace(' ', '_'), "-_;:,");
578 }
579
580 private static String harmonizeValue(String value) {
581 return Utils.strip(value.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(' ', '_'), "-_;:,");
582 }
583
584 @Override
585 public void startTest(ProgressMonitor monitor) {
586 super.startTest(monitor);
587 checkKeys = Main.pref.getBoolean(PREF_CHECK_KEYS, true);
588 if (isBeforeUpload) {
589 checkKeys = checkKeys && Main.pref.getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true);
590 }
591
592 checkValues = Main.pref.getBoolean(PREF_CHECK_VALUES, true);
593 if (isBeforeUpload) {
594 checkValues = checkValues && Main.pref.getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true);
595 }
596
597 checkComplex = Main.pref.getBoolean(PREF_CHECK_COMPLEX, true);
598 if (isBeforeUpload) {
599 checkComplex = checkComplex && Main.pref.getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true);
600 }
601
602 checkFixmes = Main.pref.getBoolean(PREF_CHECK_FIXMES, true);
603 if (isBeforeUpload) {
604 checkFixmes = checkFixmes && Main.pref.getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true);
605 }
606 }
607
608 @Override
609 public void visit(Collection<OsmPrimitive> selection) {
610 if (checkKeys || checkValues || checkComplex || checkFixmes) {
611 super.visit(selection);
612 }
613 }
614
615 @Override
616 public void addGui(JPanel testPanel) {
617 GBC a = GBC.eol();
618 a.anchor = GridBagConstraints.EAST;
619
620 testPanel.add(new JLabel(name+" :"), GBC.eol().insets(3, 0, 0, 0));
621
622 prefCheckKeys = new JCheckBox(tr("Check property keys."), Main.pref.getBoolean(PREF_CHECK_KEYS, true));
623 prefCheckKeys.setToolTipText(tr("Validate that property keys are valid checking against list of words."));
624 testPanel.add(prefCheckKeys, GBC.std().insets(20, 0, 0, 0));
625
626 prefCheckKeysBeforeUpload = new JCheckBox();
627 prefCheckKeysBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true));
628 testPanel.add(prefCheckKeysBeforeUpload, a);
629
630 prefCheckComplex = new JCheckBox(tr("Use complex property checker."), Main.pref.getBoolean(PREF_CHECK_COMPLEX, true));
631 prefCheckComplex.setToolTipText(tr("Validate property values and tags using complex rules."));
632 testPanel.add(prefCheckComplex, GBC.std().insets(20, 0, 0, 0));
633
634 prefCheckComplexBeforeUpload = new JCheckBox();
635 prefCheckComplexBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true));
636 testPanel.add(prefCheckComplexBeforeUpload, a);
637
638 final Collection<String> sources = Main.pref.getCollection(PREF_SOURCES, DEFAULT_SOURCES);
639 sourcesList = new EditableList(tr("TagChecker source"));
640 sourcesList.setItems(sources);
641 testPanel.add(new JLabel(tr("Data sources ({0})", "*.cfg")), GBC.eol().insets(23, 0, 0, 0));
642 testPanel.add(sourcesList, GBC.eol().fill(GridBagConstraints.HORIZONTAL).insets(23, 0, 0, 0));
643
644 ActionListener disableCheckActionListener = e -> handlePrefEnable();
645 prefCheckKeys.addActionListener(disableCheckActionListener);
646 prefCheckKeysBeforeUpload.addActionListener(disableCheckActionListener);
647 prefCheckComplex.addActionListener(disableCheckActionListener);
648 prefCheckComplexBeforeUpload.addActionListener(disableCheckActionListener);
649
650 handlePrefEnable();
651
652 prefCheckValues = new JCheckBox(tr("Check property values."), Main.pref.getBoolean(PREF_CHECK_VALUES, true));
653 prefCheckValues.setToolTipText(tr("Validate that property values are valid checking against presets."));
654 testPanel.add(prefCheckValues, GBC.std().insets(20, 0, 0, 0));
655
656 prefCheckValuesBeforeUpload = new JCheckBox();
657 prefCheckValuesBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true));
658 testPanel.add(prefCheckValuesBeforeUpload, a);
659
660 prefCheckFixmes = new JCheckBox(tr("Check for FIXMES."), Main.pref.getBoolean(PREF_CHECK_FIXMES, true));
661 prefCheckFixmes.setToolTipText(tr("Looks for nodes or ways with FIXME in any property value."));
662 testPanel.add(prefCheckFixmes, GBC.std().insets(20, 0, 0, 0));
663
664 prefCheckFixmesBeforeUpload = new JCheckBox();
665 prefCheckFixmesBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true));
666 testPanel.add(prefCheckFixmesBeforeUpload, a);
667 }
668
669 /**
670 * Enables/disables the source list field
671 */
672 public void handlePrefEnable() {
673 boolean selected = prefCheckKeys.isSelected() || prefCheckKeysBeforeUpload.isSelected()
674 || prefCheckComplex.isSelected() || prefCheckComplexBeforeUpload.isSelected();
675 sourcesList.setEnabled(selected);
676 }
677
678 @Override
679 public boolean ok() {
680 enabled = prefCheckKeys.isSelected() || prefCheckValues.isSelected() || prefCheckComplex.isSelected() || prefCheckFixmes.isSelected();
681 testBeforeUpload = prefCheckKeysBeforeUpload.isSelected() || prefCheckValuesBeforeUpload.isSelected()
682 || prefCheckFixmesBeforeUpload.isSelected() || prefCheckComplexBeforeUpload.isSelected();
683
684 Main.pref.put(PREF_CHECK_VALUES, prefCheckValues.isSelected());
685 Main.pref.put(PREF_CHECK_COMPLEX, prefCheckComplex.isSelected());
686 Main.pref.put(PREF_CHECK_KEYS, prefCheckKeys.isSelected());
687 Main.pref.put(PREF_CHECK_FIXMES, prefCheckFixmes.isSelected());
688 Main.pref.put(PREF_CHECK_VALUES_BEFORE_UPLOAD, prefCheckValuesBeforeUpload.isSelected());
689 Main.pref.put(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, prefCheckComplexBeforeUpload.isSelected());
690 Main.pref.put(PREF_CHECK_KEYS_BEFORE_UPLOAD, prefCheckKeysBeforeUpload.isSelected());
691 Main.pref.put(PREF_CHECK_FIXMES_BEFORE_UPLOAD, prefCheckFixmesBeforeUpload.isSelected());
692 return Main.pref.putCollection(PREF_SOURCES, sourcesList.getItems());
693 }
694
695 @Override
696 public Command fixError(TestError testError) {
697 List<Command> commands = new ArrayList<>(50);
698
699 Collection<? extends OsmPrimitive> primitives = testError.getPrimitives();
700 for (OsmPrimitive p : primitives) {
701 Map<String, String> tags = p.getKeys();
702 if (tags.isEmpty()) {
703 continue;
704 }
705
706 for (Entry<String, String> prop: tags.entrySet()) {
707 String key = prop.getKey();
708 String value = prop.getValue();
709 if (value == null || value.trim().isEmpty()) {
710 commands.add(new ChangePropertyCommand(p, key, null));
711 } else if (value.startsWith(" ") || value.endsWith(" ") || value.contains(" ")) {
712 commands.add(new ChangePropertyCommand(p, key, Tag.removeWhiteSpaces(value)));
713 } else if (key.startsWith(" ") || key.endsWith(" ") || key.contains(" ")) {
714 commands.add(new ChangePropertyKeyCommand(p, key, Tag.removeWhiteSpaces(key)));
715 } else {
716 String evalue = Entities.unescape(value);
717 if (!evalue.equals(value)) {
718 commands.add(new ChangePropertyCommand(p, key, evalue));
719 }
720 }
721 }
722 }
723
724 if (commands.isEmpty())
725 return null;
726 if (commands.size() == 1)
727 return commands.get(0);
728
729 return new SequenceCommand(tr("Fix tags"), commands);
730 }
731
732 @Override
733 public boolean isFixable(TestError testError) {
734 if (testError.getTester() instanceof TagChecker) {
735 int code = testError.getCode();
736 return code == INVALID_KEY || code == EMPTY_VALUES || code == INVALID_SPACE ||
737 code == INVALID_KEY_SPACE || code == INVALID_HTML || code == MISSPELLED_VALUE ||
738 code == MULTIPLE_SPACES;
739 }
740
741 return false;
742 }
743
744 protected static class CheckerData {
745 private String description;
746 protected List<CheckerElement> data = new ArrayList<>();
747 private OsmPrimitiveType type;
748 private TagCheckLevel level;
749 protected Severity severity;
750
751 private enum TagCheckLevel {
752 TAG_CHECK_ERROR(1250),
753 TAG_CHECK_WARN(1260),
754 TAG_CHECK_INFO(1270);
755
756 final int code;
757
758 TagCheckLevel(int code) {
759 this.code = code;
760 }
761 }
762
763 protected static class CheckerElement {
764 public Object tag;
765 public Object value;
766 public boolean noMatch;
767 public boolean tagAll;
768 public boolean valueAll;
769 public boolean valueBool;
770
771 private static Pattern getPattern(String str) {
772 if (str.endsWith("/i"))
773 return Pattern.compile(str.substring(1, str.length()-2), Pattern.CASE_INSENSITIVE);
774 if (str.endsWith("/"))
775 return Pattern.compile(str.substring(1, str.length()-1));
776
777 throw new IllegalStateException();
778 }
779
780 public CheckerElement(String exp) {
781 Matcher m = Pattern.compile("(.+)([!=]=)(.+)").matcher(exp);
782 m.matches();
783
784 String n = m.group(1).trim();
785
786 if ("*".equals(n)) {
787 tagAll = true;
788 } else {
789 tag = n.startsWith("/") ? getPattern(n) : n;
790 noMatch = "!=".equals(m.group(2));
791 n = m.group(3).trim();
792 if ("*".equals(n)) {
793 valueAll = true;
794 } else if ("BOOLEAN_TRUE".equals(n)) {
795 valueBool = true;
796 value = OsmUtils.TRUE_VALUE;
797 } else if ("BOOLEAN_FALSE".equals(n)) {
798 valueBool = true;
799 value = OsmUtils.FALSE_VALUE;
800 } else {
801 value = n.startsWith("/") ? getPattern(n) : n;
802 }
803 }
804 }
805
806 public boolean match(Map<String, String> keys) {
807 for (Entry<String, String> prop: keys.entrySet()) {
808 String key = prop.getKey();
809 String val = valueBool ? OsmUtils.getNamedOsmBoolean(prop.getValue()) : prop.getValue();
810 if ((tagAll || (tag instanceof Pattern ? ((Pattern) tag).matcher(key).matches() : key.equals(tag)))
811 && (valueAll || (value instanceof Pattern ? ((Pattern) value).matcher(val).matches() : val.equals(value))))
812 return !noMatch;
813 }
814 return noMatch;
815 }
816 }
817
818 private static final Pattern CLEAN_STR_PATTERN = Pattern.compile(" *# *([^#]+) *$");
819 private static final Pattern SPLIT_TRIMMED_PATTERN = Pattern.compile(" *: *");
820 private static final Pattern SPLIT_ELEMENTS_PATTERN = Pattern.compile(" *&& *");
821
822 public String getData(final String str) {
823 Matcher m = CLEAN_STR_PATTERN.matcher(str);
824 String trimmed = m.replaceFirst("").trim();
825 try {
826 description = m.group(1);
827 if (description != null && description.isEmpty()) {
828 description = null;
829 }
830 } catch (IllegalStateException e) {
831 Logging.error(e);
832 description = null;
833 }
834 String[] n = SPLIT_TRIMMED_PATTERN.split(trimmed, 3);
835 switch (n[0]) {
836 case "way":
837 type = OsmPrimitiveType.WAY;
838 break;
839 case "node":
840 type = OsmPrimitiveType.NODE;
841 break;
842 case "relation":
843 type = OsmPrimitiveType.RELATION;
844 break;
845 case "*":
846 type = null;
847 break;
848 default:
849 return tr("Could not find element type");
850 }
851 if (n.length != 3)
852 return tr("Incorrect number of parameters");
853
854 switch (n[1]) {
855 case "W":
856 severity = Severity.WARNING;
857 level = TagCheckLevel.TAG_CHECK_WARN;
858 break;
859 case "E":
860 severity = Severity.ERROR;
861 level = TagCheckLevel.TAG_CHECK_ERROR;
862 break;
863 case "I":
864 severity = Severity.OTHER;
865 level = TagCheckLevel.TAG_CHECK_INFO;
866 break;
867 default:
868 return tr("Could not find warning level");
869 }
870 for (String exp: SPLIT_ELEMENTS_PATTERN.split(n[2])) {
871 try {
872 data.add(new CheckerElement(exp));
873 } catch (IllegalStateException e) {
874 Logging.trace(e);
875 return tr("Illegal expression ''{0}''", exp);
876 } catch (PatternSyntaxException e) {
877 Logging.trace(e);
878 return tr("Illegal regular expression ''{0}''", exp);
879 }
880 }
881 return null;
882 }
883
884 public boolean match(OsmPrimitive osm, Map<String, String> keys) {
885 if (type != null && OsmPrimitiveType.from(osm) != type)
886 return false;
887
888 for (CheckerElement ce : data) {
889 if (!ce.match(keys))
890 return false;
891 }
892 return true;
893 }
894
895 /**
896 * Returns the error description.
897 * @return the error description
898 */
899 public String getDescription() {
900 return description;
901 }
902
903 /**
904 * Returns the error severity.
905 * @return the error severity
906 */
907 public Severity getSeverity() {
908 return severity;
909 }
910
911 /**
912 * Returns the error code.
913 * @return the error code
914 */
915 public int getCode() {
916 if (type == null)
917 return level.code;
918
919 return level.code + type.ordinal() + 1;
920 }
921 }
922}
Note: See TracBrowser for help on using the repository browser.