source: josm/trunk/src/org/openstreetmap/josm/data/validation/tests/TagChecker.java@ 12187

Last change on this file since 12187 was 12187, checked in by michael2402, 7 years ago

Add javadoc to OsmUtils, fix constant naming.
  • Property svn:eol-style set to native
File size: 37.5 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.validation.tests;
3
4import static org.openstreetmap.josm.tools.I18n.marktr;
5import static org.openstreetmap.josm.tools.I18n.tr;
6
7import java.awt.GridBagConstraints;
8import java.awt.event.ActionListener;
9import java.io.BufferedReader;
10import java.io.IOException;
11import java.util.ArrayList;
12import java.util.Arrays;
13import java.util.Collection;
14import java.util.HashMap;
15import java.util.HashSet;
16import java.util.List;
17import java.util.Locale;
18import java.util.Map;
19import java.util.Map.Entry;
20import java.util.Set;
21import java.util.regex.Matcher;
22import java.util.regex.Pattern;
23import java.util.regex.PatternSyntaxException;
24
25import javax.swing.JCheckBox;
26import javax.swing.JLabel;
27import javax.swing.JPanel;
28
29import org.openstreetmap.josm.Main;
30import org.openstreetmap.josm.command.ChangePropertyCommand;
31import org.openstreetmap.josm.command.ChangePropertyKeyCommand;
32import org.openstreetmap.josm.command.Command;
33import org.openstreetmap.josm.command.SequenceCommand;
34import org.openstreetmap.josm.data.osm.OsmPrimitive;
35import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
36import org.openstreetmap.josm.data.osm.OsmUtils;
37import org.openstreetmap.josm.data.osm.Tag;
38import org.openstreetmap.josm.data.validation.Severity;
39import org.openstreetmap.josm.data.validation.Test.TagTest;
40import org.openstreetmap.josm.data.validation.TestError;
41import org.openstreetmap.josm.data.validation.util.Entities;
42import org.openstreetmap.josm.gui.preferences.validator.ValidatorPreference;
43import org.openstreetmap.josm.gui.progress.ProgressMonitor;
44import org.openstreetmap.josm.gui.tagging.presets.TaggingPreset;
45import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetItem;
46import org.openstreetmap.josm.gui.tagging.presets.TaggingPresets;
47import org.openstreetmap.josm.gui.tagging.presets.items.Check;
48import org.openstreetmap.josm.gui.tagging.presets.items.CheckGroup;
49import org.openstreetmap.josm.gui.tagging.presets.items.KeyedItem;
50import org.openstreetmap.josm.gui.widgets.EditableList;
51import org.openstreetmap.josm.io.CachedFile;
52import org.openstreetmap.josm.tools.GBC;
53import org.openstreetmap.josm.tools.MultiMap;
54import org.openstreetmap.josm.tools.Utils;
55
56/**
57 * Check for misspelled or wrong tags
58 *
59 * @author frsantos
60 * @since 3669
61 */
62public class TagChecker extends TagTest {
63
64 /** The config file of ignored tags */
65 public static final String IGNORE_FILE = "resource://data/validator/ignoretags.cfg";
66 /** The config file of dictionary words */
67 public static final String SPELL_FILE = "resource://data/validator/words.cfg";
68
69 /** Normalized keys: the key should be substituted by the value if the key was not found in presets */
70 private static final Map<String, String> harmonizedKeys = new HashMap<>();
71 /** The spell check preset values which are not stored in TaggingPresets */
72 private static volatile MultiMap<String, String> additionalPresetsValueData;
73 /** The TagChecker data */
74 private static final List<CheckerData> checkerData = new ArrayList<>();
75 private static final List<String> ignoreDataStartsWith = new ArrayList<>();
76 private static final List<String> ignoreDataEquals = new ArrayList<>();
77 private static final List<String> ignoreDataEndsWith = new ArrayList<>();
78 private static final List<Tag> ignoreDataTag = new ArrayList<>();
79
80 /** The preferences prefix */
81 protected static final String PREFIX = ValidatorPreference.PREFIX + "." + TagChecker.class.getSimpleName();
82
83 public static final String PREF_CHECK_VALUES = PREFIX + ".checkValues";
84 public static final String PREF_CHECK_KEYS = PREFIX + ".checkKeys";
85 public static final String PREF_CHECK_COMPLEX = PREFIX + ".checkComplex";
86 public static final String PREF_CHECK_FIXMES = PREFIX + ".checkFixmes";
87
88 public static final String PREF_SOURCES = PREFIX + ".source";
89
90 public static final String PREF_CHECK_KEYS_BEFORE_UPLOAD = PREF_CHECK_KEYS + "BeforeUpload";
91 public static final String PREF_CHECK_VALUES_BEFORE_UPLOAD = PREF_CHECK_VALUES + "BeforeUpload";
92 public static final String PREF_CHECK_COMPLEX_BEFORE_UPLOAD = PREF_CHECK_COMPLEX + "BeforeUpload";
93 public static final String PREF_CHECK_FIXMES_BEFORE_UPLOAD = PREF_CHECK_FIXMES + "BeforeUpload";
94
95 protected boolean checkKeys;
96 protected boolean checkValues;
97 protected boolean checkComplex;
98 protected boolean checkFixmes;
99
100 protected JCheckBox prefCheckKeys;
101 protected JCheckBox prefCheckValues;
102 protected JCheckBox prefCheckComplex;
103 protected JCheckBox prefCheckFixmes;
104 protected JCheckBox prefCheckPaint;
105
106 protected JCheckBox prefCheckKeysBeforeUpload;
107 protected JCheckBox prefCheckValuesBeforeUpload;
108 protected JCheckBox prefCheckComplexBeforeUpload;
109 protected JCheckBox prefCheckFixmesBeforeUpload;
110 protected JCheckBox prefCheckPaintBeforeUpload;
111
112 // CHECKSTYLE.OFF: SingleSpaceSeparator
113 protected static final int EMPTY_VALUES = 1200;
114 protected static final int INVALID_KEY = 1201;
115 protected static final int INVALID_VALUE = 1202;
116 protected static final int FIXME = 1203;
117 protected static final int INVALID_SPACE = 1204;
118 protected static final int INVALID_KEY_SPACE = 1205;
119 protected static final int INVALID_HTML = 1206; /* 1207 was PAINT */
120 protected static final int LONG_VALUE = 1208;
121 protected static final int LONG_KEY = 1209;
122 protected static final int LOW_CHAR_VALUE = 1210;
123 protected static final int LOW_CHAR_KEY = 1211;
124 protected static final int MISSPELLED_VALUE = 1212;
125 protected static final int MISSPELLED_KEY = 1213;
126 protected static final int MULTIPLE_SPACES = 1214;
127 // CHECKSTYLE.ON: SingleSpaceSeparator
128 // 1250 and up is used by tagcheck
129
130 protected EditableList sourcesList;
131
132 private static final Set<String> DEFAULT_SOURCES = new HashSet<>(Arrays.asList(/*DATA_FILE, */IGNORE_FILE, SPELL_FILE));
133
134 /**
135 * Constructor
136 */
137 public TagChecker() {
138 super(tr("Tag checker"), tr("This test checks for errors in tag keys and values."));
139 }
140
141 @Override
142 public void initialize() throws IOException {
143 initializeData();
144 initializePresets();
145 }
146
147 /**
148 * Reads the spellcheck file into a HashMap.
149 * The data file is a list of words, beginning with +/-. If it starts with +,
150 * the word is valid, but if it starts with -, the word should be replaced
151 * by the nearest + word before this.
152 *
153 * @throws IOException if any I/O error occurs
154 */
155 private static void initializeData() throws IOException {
156 checkerData.clear();
157 ignoreDataStartsWith.clear();
158 ignoreDataEquals.clear();
159 ignoreDataEndsWith.clear();
160 ignoreDataTag.clear();
161 harmonizedKeys.clear();
162
163 StringBuilder errorSources = new StringBuilder();
164 for (String source : Main.pref.getCollection(PREF_SOURCES, DEFAULT_SOURCES)) {
165 try (
166 CachedFile cf = new CachedFile(source);
167 BufferedReader reader = cf.getContentReader()
168 ) {
169 String okValue = null;
170 boolean tagcheckerfile = false;
171 boolean ignorefile = false;
172 boolean isFirstLine = true;
173 String line;
174 while ((line = reader.readLine()) != null && (tagcheckerfile || !line.isEmpty())) {
175 if (line.startsWith("#")) {
176 if (line.startsWith("# JOSM TagChecker")) {
177 tagcheckerfile = true;
178 if (!DEFAULT_SOURCES.contains(source)) {
179 Main.info(tr("Adding {0} to tag checker", source));
180 }
181 } else
182 if (line.startsWith("# JOSM IgnoreTags")) {
183 ignorefile = true;
184 if (!DEFAULT_SOURCES.contains(source)) {
185 Main.info(tr("Adding {0} to ignore tags", source));
186 }
187 }
188 } else if (ignorefile) {
189 line = line.trim();
190 if (line.length() < 4) {
191 continue;
192 }
193
194 String key = line.substring(0, 2);
195 line = line.substring(2);
196
197 switch (key) {
198 case "S:":
199 ignoreDataStartsWith.add(line);
200 break;
201 case "E:":
202 ignoreDataEquals.add(line);
203 break;
204 case "F:":
205 ignoreDataEndsWith.add(line);
206 break;
207 case "K:":
208 ignoreDataTag.add(Tag.ofString(line));
209 break;
210 default:
211 if (!key.startsWith(";")) {
212 Main.warn("Unsupported TagChecker key: " + key);
213 }
214 }
215 } else if (tagcheckerfile) {
216 if (!line.isEmpty()) {
217 CheckerData d = new CheckerData();
218 String err = d.getData(line);
219
220 if (err == null) {
221 checkerData.add(d);
222 } else {
223 Main.error(tr("Invalid tagchecker line - {0}: {1}", err, line));
224 }
225 }
226 } else if (line.charAt(0) == '+') {
227 okValue = line.substring(1);
228 } else if (line.charAt(0) == '-' && okValue != null) {
229 harmonizedKeys.put(harmonizeKey(line.substring(1)), okValue);
230 } else {
231 Main.error(tr("Invalid spellcheck line: {0}", line));
232 }
233 if (isFirstLine) {
234 isFirstLine = false;
235 if (!(tagcheckerfile || ignorefile) && !DEFAULT_SOURCES.contains(source)) {
236 Main.info(tr("Adding {0} to spellchecker", source));
237 }
238 }
239 }
240 } catch (IOException e) {
241 Main.error(e);
242 errorSources.append(source).append('\n');
243 }
244 }
245
246 if (errorSources.length() > 0)
247 throw new IOException(tr("Could not access data file(s):\n{0}", errorSources));
248 }
249
250 /**
251 * Reads the presets data.
252 *
253 */
254 public static void initializePresets() {
255
256 if (!Main.pref.getBoolean(PREF_CHECK_VALUES, true))
257 return;
258
259 Collection<TaggingPreset> presets = TaggingPresets.getTaggingPresets();
260 if (!presets.isEmpty()) {
261 additionalPresetsValueData = new MultiMap<>();
262 for (String a : OsmPrimitive.getUninterestingKeys()) {
263 additionalPresetsValueData.putVoid(a);
264 }
265 // TODO directionKeys are no longer in OsmPrimitive (search pattern is used instead)
266 for (String a : Main.pref.getCollection(ValidatorPreference.PREFIX + ".knownkeys",
267 Arrays.asList(new String[]{"is_in", "int_ref", "fixme", "population"}))) {
268 additionalPresetsValueData.putVoid(a);
269 }
270 for (TaggingPreset p : presets) {
271 for (TaggingPresetItem i : p.data) {
272 if (i instanceof KeyedItem) {
273 addPresetValue((KeyedItem) i);
274 } else if (i instanceof CheckGroup) {
275 for (Check c : ((CheckGroup) i).checks) {
276 addPresetValue(c);
277 }
278 }
279 }
280 }
281 }
282 }
283
284 private static void addPresetValue(KeyedItem ky) {
285 Collection<String> values = ky.getValues();
286 if (ky.key != null && values != null) {
287 harmonizedKeys.put(harmonizeKey(ky.key), ky.key);
288 }
289 }
290
291 /**
292 * Checks given string (key or value) if it contains characters with code below 0x20 (either newline or some other special characters)
293 * @param s string to check
294 * @return {@code true} if {@code s} contains characters with code below 0x20
295 */
296 private static boolean containsLow(String s) {
297 if (s == null)
298 return false;
299 for (int i = 0; i < s.length(); i++) {
300 if (s.charAt(i) < 0x20)
301 return true;
302 }
303 return false;
304 }
305
306 private static Set<String> getPresetValues(String key) {
307 Set<String> res = TaggingPresets.getPresetValues(key);
308 if (res != null)
309 return res;
310 return additionalPresetsValueData.get(key);
311 }
312
313 /**
314 * Determines if the given key is in internal presets.
315 * @param key key
316 * @return {@code true} if the given key is in internal presets
317 * @since 9023
318 */
319 public static boolean isKeyInPresets(String key) {
320 return getPresetValues(key) != null;
321 }
322
323 /**
324 * Determines if the given tag is in internal presets.
325 * @param key key
326 * @param value value
327 * @return {@code true} if the given tag is in internal presets
328 * @since 9023
329 */
330 public static boolean isTagInPresets(String key, String value) {
331 final Set<String> values = getPresetValues(key);
332 return values != null && (values.isEmpty() || values.contains(value));
333 }
334
335 /**
336 * Returns the list of ignored tags.
337 * @return the list of ignored tags
338 * @since 9023
339 */
340 public static List<Tag> getIgnoredTags() {
341 return new ArrayList<>(ignoreDataTag);
342 }
343
344 /**
345 * Determines if the given tag is ignored for checks "key/tag not in presets".
346 * @param key key
347 * @param value value
348 * @return {@code true} if the given tag is ignored
349 * @since 9023
350 */
351 public static boolean isTagIgnored(String key, String value) {
352 boolean tagInPresets = isTagInPresets(key, value);
353 boolean ignore = false;
354
355 for (String a : ignoreDataStartsWith) {
356 if (key.startsWith(a)) {
357 ignore = true;
358 }
359 }
360 for (String a : ignoreDataEquals) {
361 if (key.equals(a)) {
362 ignore = true;
363 }
364 }
365 for (String a : ignoreDataEndsWith) {
366 if (key.endsWith(a)) {
367 ignore = true;
368 }
369 }
370
371 if (!tagInPresets) {
372 for (Tag a : ignoreDataTag) {
373 if (key.equals(a.getKey()) && value.equals(a.getValue())) {
374 ignore = true;
375 }
376 }
377 }
378 return ignore;
379 }
380
381 /**
382 * Checks the primitive tags
383 * @param p The primitive to check
384 */
385 @Override
386 public void check(OsmPrimitive p) {
387 // Just a collection to know if a primitive has been already marked with error
388 MultiMap<OsmPrimitive, String> withErrors = new MultiMap<>();
389
390 if (checkComplex) {
391 Map<String, String> keys = p.getKeys();
392 for (CheckerData d : checkerData) {
393 if (d.match(p, keys)) {
394 errors.add(TestError.builder(this, d.getSeverity(), d.getCode())
395 .message(tr("Suspicious tag/value combinations"), d.getDescription())
396 .primitives(p)
397 .build());
398 withErrors.put(p, "TC");
399 }
400 }
401 }
402
403 for (Entry<String, String> prop : p.getKeys().entrySet()) {
404 String s = marktr("Key ''{0}'' invalid.");
405 String key = prop.getKey();
406 String value = prop.getValue();
407 if (checkValues && (containsLow(value)) && !withErrors.contains(p, "ICV")) {
408 errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_VALUE)
409 .message(tr("Tag value contains character with code less than 0x20"), s, key)
410 .primitives(p)
411 .build());
412 withErrors.put(p, "ICV");
413 }
414 if (checkKeys && (containsLow(key)) && !withErrors.contains(p, "ICK")) {
415 errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_KEY)
416 .message(tr("Tag key contains character with code less than 0x20"), s, key)
417 .primitives(p)
418 .build());
419 withErrors.put(p, "ICK");
420 }
421 if (checkValues && (value != null && value.length() > 255) && !withErrors.contains(p, "LV")) {
422 errors.add(TestError.builder(this, Severity.ERROR, LONG_VALUE)
423 .message(tr("Tag value longer than allowed"), s, key)
424 .primitives(p)
425 .build());
426 withErrors.put(p, "LV");
427 }
428 if (checkKeys && (key != null && key.length() > 255) && !withErrors.contains(p, "LK")) {
429 errors.add(TestError.builder(this, Severity.ERROR, LONG_KEY)
430 .message(tr("Tag key longer than allowed"), s, key)
431 .primitives(p)
432 .build());
433 withErrors.put(p, "LK");
434 }
435 if (checkValues && (value == null || value.trim().isEmpty()) && !withErrors.contains(p, "EV")) {
436 errors.add(TestError.builder(this, Severity.WARNING, EMPTY_VALUES)
437 .message(tr("Tags with empty values"), s, key)
438 .primitives(p)
439 .build());
440 withErrors.put(p, "EV");
441 }
442 if (checkKeys && key != null && key.indexOf(' ') >= 0 && !withErrors.contains(p, "IPK")) {
443 errors.add(TestError.builder(this, Severity.WARNING, INVALID_KEY_SPACE)
444 .message(tr("Invalid white space in property key"), s, key)
445 .primitives(p)
446 .build());
447 withErrors.put(p, "IPK");
448 }
449 if (checkValues && value != null && (value.startsWith(" ") || value.endsWith(" ")) && !withErrors.contains(p, "SPACE")) {
450 errors.add(TestError.builder(this, Severity.WARNING, INVALID_SPACE)
451 .message(tr("Property values start or end with white space"), s, key)
452 .primitives(p)
453 .build());
454 withErrors.put(p, "SPACE");
455 }
456 if (checkValues && value != null && value.contains(" ") && !withErrors.contains(p, "SPACE")) {
457 errors.add(TestError.builder(this, Severity.WARNING, MULTIPLE_SPACES)
458 .message(tr("Property values contain multiple white spaces"), s, key)
459 .primitives(p)
460 .build());
461 withErrors.put(p, "SPACE");
462 }
463 if (checkValues && value != null && !value.equals(Entities.unescape(value)) && !withErrors.contains(p, "HTML")) {
464 errors.add(TestError.builder(this, Severity.OTHER, INVALID_HTML)
465 .message(tr("Property values contain HTML entity"), s, key)
466 .primitives(p)
467 .build());
468 withErrors.put(p, "HTML");
469 }
470 if (checkValues && key != null && value != null && !value.isEmpty() && additionalPresetsValueData != null
471 && !isTagIgnored(key, value)) {
472 if (!isKeyInPresets(key)) {
473 String prettifiedKey = harmonizeKey(key);
474 String fixedKey = harmonizedKeys.get(prettifiedKey);
475 if (fixedKey != null && !"".equals(fixedKey) && !fixedKey.equals(key)) {
476 // misspelled preset key
477 final TestError.Builder error = TestError.builder(this, Severity.WARNING, MISSPELLED_KEY)
478 .message(tr("Misspelled property key"), marktr("Key ''{0}'' looks like ''{1}''."), key, fixedKey)
479 .primitives(p);
480 if (p.hasKey(fixedKey)) {
481 errors.add(error.build());
482 } else {
483 errors.add(error.fix(() -> new ChangePropertyKeyCommand(p, key, fixedKey)).build());
484 }
485 withErrors.put(p, "WPK");
486 } else {
487 errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE)
488 .message(tr("Presets do not contain property key"), marktr("Key ''{0}'' not in presets."), key)
489 .primitives(p)
490 .build());
491 withErrors.put(p, "UPK");
492 }
493 } else if (!isTagInPresets(key, value)) {
494 // try to fix common typos and check again if value is still unknown
495 String fixedValue = harmonizeValue(prop.getValue());
496 Map<String, String> possibleValues = getPossibleValues(getPresetValues(key));
497 if (possibleValues.containsKey(fixedValue)) {
498 final String newKey = possibleValues.get(fixedValue);
499 // misspelled preset value
500 errors.add(TestError.builder(this, Severity.WARNING, MISSPELLED_VALUE)
501 .message(tr("Misspelled property value"),
502 marktr("Value ''{0}'' for key ''{1}'' looks like ''{2}''."), prop.getValue(), key, fixedValue)
503 .primitives(p)
504 .fix(() -> new ChangePropertyCommand(p, key, newKey))
505 .build());
506 withErrors.put(p, "WPV");
507 } else {
508 // unknown preset value
509 errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE)
510 .message(tr("Presets do not contain property value"),
511 marktr("Value ''{0}'' for key ''{1}'' not in presets."), prop.getValue(), key)
512 .primitives(p)
513 .build());
514 withErrors.put(p, "UPV");
515 }
516 }
517 }
518 if (checkFixmes && key != null && value != null && !value.isEmpty() && isFixme(key, value) && !withErrors.contains(p, "FIXME")) {
519 errors.add(TestError.builder(this, Severity.OTHER, FIXME)
520 .message(tr("FIXMES"))
521 .primitives(p)
522 .build());
523 withErrors.put(p, "FIXME");
524 }
525 }
526 }
527
528 private static boolean isFixme(String key, String value) {
529 return key.toLowerCase(Locale.ENGLISH).contains("fixme") || key.contains("todo")
530 || value.toLowerCase(Locale.ENGLISH).contains("fixme") || value.contains("check and delete");
531 }
532
533 private static Map<String, String> getPossibleValues(Set<String> values) {
534 // generate a map with common typos
535 Map<String, String> map = new HashMap<>();
536 if (values != null) {
537 for (String value : values) {
538 map.put(value, value);
539 if (value.contains("_")) {
540 map.put(value.replace("_", ""), value);
541 }
542 }
543 }
544 return map;
545 }
546
547 private static String harmonizeKey(String key) {
548 return Utils.strip(key.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(':', '_').replace(' ', '_'), "-_;:,");
549 }
550
551 private static String harmonizeValue(String value) {
552 return Utils.strip(value.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(' ', '_'), "-_;:,");
553 }
554
555 @Override
556 public void startTest(ProgressMonitor monitor) {
557 super.startTest(monitor);
558 checkKeys = Main.pref.getBoolean(PREF_CHECK_KEYS, true);
559 if (isBeforeUpload) {
560 checkKeys = checkKeys && Main.pref.getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true);
561 }
562
563 checkValues = Main.pref.getBoolean(PREF_CHECK_VALUES, true);
564 if (isBeforeUpload) {
565 checkValues = checkValues && Main.pref.getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true);
566 }
567
568 checkComplex = Main.pref.getBoolean(PREF_CHECK_COMPLEX, true);
569 if (isBeforeUpload) {
570 checkComplex = checkComplex && Main.pref.getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true);
571 }
572
573 checkFixmes = Main.pref.getBoolean(PREF_CHECK_FIXMES, true);
574 if (isBeforeUpload) {
575 checkFixmes = checkFixmes && Main.pref.getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true);
576 }
577 }
578
579 @Override
580 public void visit(Collection<OsmPrimitive> selection) {
581 if (checkKeys || checkValues || checkComplex || checkFixmes) {
582 super.visit(selection);
583 }
584 }
585
586 @Override
587 public void addGui(JPanel testPanel) {
588 GBC a = GBC.eol();
589 a.anchor = GridBagConstraints.EAST;
590
591 testPanel.add(new JLabel(name+" :"), GBC.eol().insets(3, 0, 0, 0));
592
593 prefCheckKeys = new JCheckBox(tr("Check property keys."), Main.pref.getBoolean(PREF_CHECK_KEYS, true));
594 prefCheckKeys.setToolTipText(tr("Validate that property keys are valid checking against list of words."));
595 testPanel.add(prefCheckKeys, GBC.std().insets(20, 0, 0, 0));
596
597 prefCheckKeysBeforeUpload = new JCheckBox();
598 prefCheckKeysBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true));
599 testPanel.add(prefCheckKeysBeforeUpload, a);
600
601 prefCheckComplex = new JCheckBox(tr("Use complex property checker."), Main.pref.getBoolean(PREF_CHECK_COMPLEX, true));
602 prefCheckComplex.setToolTipText(tr("Validate property values and tags using complex rules."));
603 testPanel.add(prefCheckComplex, GBC.std().insets(20, 0, 0, 0));
604
605 prefCheckComplexBeforeUpload = new JCheckBox();
606 prefCheckComplexBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true));
607 testPanel.add(prefCheckComplexBeforeUpload, a);
608
609 final Collection<String> sources = Main.pref.getCollection(PREF_SOURCES, DEFAULT_SOURCES);
610 sourcesList = new EditableList(tr("TagChecker source"));
611 sourcesList.setItems(sources);
612 testPanel.add(new JLabel(tr("Data sources ({0})", "*.cfg")), GBC.eol().insets(23, 0, 0, 0));
613 testPanel.add(sourcesList, GBC.eol().fill(GridBagConstraints.HORIZONTAL).insets(23, 0, 0, 0));
614
615 ActionListener disableCheckActionListener = e -> handlePrefEnable();
616 prefCheckKeys.addActionListener(disableCheckActionListener);
617 prefCheckKeysBeforeUpload.addActionListener(disableCheckActionListener);
618 prefCheckComplex.addActionListener(disableCheckActionListener);
619 prefCheckComplexBeforeUpload.addActionListener(disableCheckActionListener);
620
621 handlePrefEnable();
622
623 prefCheckValues = new JCheckBox(tr("Check property values."), Main.pref.getBoolean(PREF_CHECK_VALUES, true));
624 prefCheckValues.setToolTipText(tr("Validate that property values are valid checking against presets."));
625 testPanel.add(prefCheckValues, GBC.std().insets(20, 0, 0, 0));
626
627 prefCheckValuesBeforeUpload = new JCheckBox();
628 prefCheckValuesBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true));
629 testPanel.add(prefCheckValuesBeforeUpload, a);
630
631 prefCheckFixmes = new JCheckBox(tr("Check for FIXMES."), Main.pref.getBoolean(PREF_CHECK_FIXMES, true));
632 prefCheckFixmes.setToolTipText(tr("Looks for nodes or ways with FIXME in any property value."));
633 testPanel.add(prefCheckFixmes, GBC.std().insets(20, 0, 0, 0));
634
635 prefCheckFixmesBeforeUpload = new JCheckBox();
636 prefCheckFixmesBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true));
637 testPanel.add(prefCheckFixmesBeforeUpload, a);
638 }
639
640 public void handlePrefEnable() {
641 boolean selected = prefCheckKeys.isSelected() || prefCheckKeysBeforeUpload.isSelected()
642 || prefCheckComplex.isSelected() || prefCheckComplexBeforeUpload.isSelected();
643 sourcesList.setEnabled(selected);
644 }
645
646 @Override
647 public boolean ok() {
648 enabled = prefCheckKeys.isSelected() || prefCheckValues.isSelected() || prefCheckComplex.isSelected() || prefCheckFixmes.isSelected();
649 testBeforeUpload = prefCheckKeysBeforeUpload.isSelected() || prefCheckValuesBeforeUpload.isSelected()
650 || prefCheckFixmesBeforeUpload.isSelected() || prefCheckComplexBeforeUpload.isSelected();
651
652 Main.pref.put(PREF_CHECK_VALUES, prefCheckValues.isSelected());
653 Main.pref.put(PREF_CHECK_COMPLEX, prefCheckComplex.isSelected());
654 Main.pref.put(PREF_CHECK_KEYS, prefCheckKeys.isSelected());
655 Main.pref.put(PREF_CHECK_FIXMES, prefCheckFixmes.isSelected());
656 Main.pref.put(PREF_CHECK_VALUES_BEFORE_UPLOAD, prefCheckValuesBeforeUpload.isSelected());
657 Main.pref.put(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, prefCheckComplexBeforeUpload.isSelected());
658 Main.pref.put(PREF_CHECK_KEYS_BEFORE_UPLOAD, prefCheckKeysBeforeUpload.isSelected());
659 Main.pref.put(PREF_CHECK_FIXMES_BEFORE_UPLOAD, prefCheckFixmesBeforeUpload.isSelected());
660 return Main.pref.putCollection(PREF_SOURCES, sourcesList.getItems());
661 }
662
663 @Override
664 public Command fixError(TestError testError) {
665 List<Command> commands = new ArrayList<>(50);
666
667 Collection<? extends OsmPrimitive> primitives = testError.getPrimitives();
668 for (OsmPrimitive p : primitives) {
669 Map<String, String> tags = p.getKeys();
670 if (tags.isEmpty()) {
671 continue;
672 }
673
674 for (Entry<String, String> prop: tags.entrySet()) {
675 String key = prop.getKey();
676 String value = prop.getValue();
677 if (value == null || value.trim().isEmpty()) {
678 commands.add(new ChangePropertyCommand(p, key, null));
679 } else if (value.startsWith(" ") || value.endsWith(" ") || value.contains(" ")) {
680 commands.add(new ChangePropertyCommand(p, key, Tag.removeWhiteSpaces(value)));
681 } else if (key.startsWith(" ") || key.endsWith(" ") || key.contains(" ")) {
682 commands.add(new ChangePropertyKeyCommand(p, key, Tag.removeWhiteSpaces(key)));
683 } else {
684 String evalue = Entities.unescape(value);
685 if (!evalue.equals(value)) {
686 commands.add(new ChangePropertyCommand(p, key, evalue));
687 }
688 }
689 }
690 }
691
692 if (commands.isEmpty())
693 return null;
694 if (commands.size() == 1)
695 return commands.get(0);
696
697 return new SequenceCommand(tr("Fix tags"), commands);
698 }
699
700 @Override
701 public boolean isFixable(TestError testError) {
702 if (testError.getTester() instanceof TagChecker) {
703 int code = testError.getCode();
704 return code == INVALID_KEY || code == EMPTY_VALUES || code == INVALID_SPACE ||
705 code == INVALID_KEY_SPACE || code == INVALID_HTML || code == MISSPELLED_VALUE ||
706 code == MULTIPLE_SPACES;
707 }
708
709 return false;
710 }
711
712 protected static class CheckerData {
713 private String description;
714 protected List<CheckerElement> data = new ArrayList<>();
715 private OsmPrimitiveType type;
716 private TagCheckLevel level;
717 protected Severity severity;
718
719 private enum TagCheckLevel {
720 TAG_CHECK_ERROR(1250),
721 TAG_CHECK_WARN(1260),
722 TAG_CHECK_INFO(1270);
723
724 final int code;
725
726 TagCheckLevel(int code) {
727 this.code = code;
728 }
729 }
730
731 protected static class CheckerElement {
732 public Object tag;
733 public Object value;
734 public boolean noMatch;
735 public boolean tagAll;
736 public boolean valueAll;
737 public boolean valueBool;
738
739 private static Pattern getPattern(String str) {
740 if (str.endsWith("/i"))
741 return Pattern.compile(str.substring(1, str.length()-2), Pattern.CASE_INSENSITIVE);
742 if (str.endsWith("/"))
743 return Pattern.compile(str.substring(1, str.length()-1));
744
745 throw new IllegalStateException();
746 }
747
748 public CheckerElement(String exp) {
749 Matcher m = Pattern.compile("(.+)([!=]=)(.+)").matcher(exp);
750 m.matches();
751
752 String n = m.group(1).trim();
753
754 if ("*".equals(n)) {
755 tagAll = true;
756 } else {
757 tag = n.startsWith("/") ? getPattern(n) : n;
758 noMatch = "!=".equals(m.group(2));
759 n = m.group(3).trim();
760 if ("*".equals(n)) {
761 valueAll = true;
762 } else if ("BOOLEAN_TRUE".equals(n)) {
763 valueBool = true;
764 value = OsmUtils.TRUE_VALUE;
765 } else if ("BOOLEAN_FALSE".equals(n)) {
766 valueBool = true;
767 value = OsmUtils.FALSE_VALUE;
768 } else {
769 value = n.startsWith("/") ? getPattern(n) : n;
770 }
771 }
772 }
773
774 public boolean match(Map<String, String> keys) {
775 for (Entry<String, String> prop: keys.entrySet()) {
776 String key = prop.getKey();
777 String val = valueBool ? OsmUtils.getNamedOsmBoolean(prop.getValue()) : prop.getValue();
778 if ((tagAll || (tag instanceof Pattern ? ((Pattern) tag).matcher(key).matches() : key.equals(tag)))
779 && (valueAll || (value instanceof Pattern ? ((Pattern) value).matcher(val).matches() : val.equals(value))))
780 return !noMatch;
781 }
782 return noMatch;
783 }
784 }
785
786 private static final Pattern CLEAN_STR_PATTERN = Pattern.compile(" *# *([^#]+) *$");
787 private static final Pattern SPLIT_TRIMMED_PATTERN = Pattern.compile(" *: *");
788 private static final Pattern SPLIT_ELEMENTS_PATTERN = Pattern.compile(" *&& *");
789
790 public String getData(final String str) {
791 Matcher m = CLEAN_STR_PATTERN.matcher(str);
792 String trimmed = m.replaceFirst("").trim();
793 try {
794 description = m.group(1);
795 if (description != null && description.isEmpty()) {
796 description = null;
797 }
798 } catch (IllegalStateException e) {
799 Main.error(e);
800 description = null;
801 }
802 String[] n = SPLIT_TRIMMED_PATTERN.split(trimmed, 3);
803 switch (n[0]) {
804 case "way":
805 type = OsmPrimitiveType.WAY;
806 break;
807 case "node":
808 type = OsmPrimitiveType.NODE;
809 break;
810 case "relation":
811 type = OsmPrimitiveType.RELATION;
812 break;
813 case "*":
814 type = null;
815 break;
816 default:
817 return tr("Could not find element type");
818 }
819 if (n.length != 3)
820 return tr("Incorrect number of parameters");
821
822 switch (n[1]) {
823 case "W":
824 severity = Severity.WARNING;
825 level = TagCheckLevel.TAG_CHECK_WARN;
826 break;
827 case "E":
828 severity = Severity.ERROR;
829 level = TagCheckLevel.TAG_CHECK_ERROR;
830 break;
831 case "I":
832 severity = Severity.OTHER;
833 level = TagCheckLevel.TAG_CHECK_INFO;
834 break;
835 default:
836 return tr("Could not find warning level");
837 }
838 for (String exp: SPLIT_ELEMENTS_PATTERN.split(n[2])) {
839 try {
840 data.add(new CheckerElement(exp));
841 } catch (IllegalStateException e) {
842 Main.trace(e);
843 return tr("Illegal expression ''{0}''", exp);
844 } catch (PatternSyntaxException e) {
845 Main.trace(e);
846 return tr("Illegal regular expression ''{0}''", exp);
847 }
848 }
849 return null;
850 }
851
852 public boolean match(OsmPrimitive osm, Map<String, String> keys) {
853 if (type != null && OsmPrimitiveType.from(osm) != type)
854 return false;
855
856 for (CheckerElement ce : data) {
857 if (!ce.match(keys))
858 return false;
859 }
860 return true;
861 }
862
863 /**
864 * Returns the error description.
865 * @return the error description
866 */
867 public String getDescription() {
868 return description;
869 }
870
871 /**
872 * Returns the error severity.
873 * @return the error severity
874 */
875 public Severity getSeverity() {
876 return severity;
877 }
878
879 /**
880 * Returns the error code.
881 * @return the error code
882 */
883 public int getCode() {
884 if (type == null)
885 return level.code;
886
887 return level.code + type.ordinal() + 1;
888 }
889 }
890}
Note: See TracBrowser for help on using the repository browser.