source: josm/trunk/src/org/openstreetmap/josm/data/validation/tests/TagChecker.java@ 17113

Last change on this file since 17113 was 17113, checked in by GerdP, 4 years ago

fix #19895: TagChecker cannot cope with read-only datasets.
Use different method to create clones for special case where we test agains deprecated.mapcss rules
  • Property svn:eol-style set to native
File size: 55.3 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.validation.tests;
3
4import static org.openstreetmap.josm.tools.I18n.marktr;
5import static org.openstreetmap.josm.tools.I18n.tr;
6import static org.openstreetmap.josm.tools.I18n.trn;
7
8import java.awt.GridBagConstraints;
9import java.awt.event.ActionListener;
10import java.io.BufferedReader;
11import java.io.IOException;
12import java.lang.Character.UnicodeBlock;
13import java.util.ArrayList;
14import java.util.Arrays;
15import java.util.Collection;
16import java.util.Collections;
17import java.util.EnumSet;
18import java.util.HashMap;
19import java.util.HashSet;
20import java.util.Iterator;
21import java.util.LinkedHashMap;
22import java.util.LinkedHashSet;
23import java.util.List;
24import java.util.Locale;
25import java.util.Map;
26import java.util.Map.Entry;
27import java.util.OptionalInt;
28import java.util.Set;
29import java.util.regex.Pattern;
30import java.util.stream.Collectors;
31
32import javax.swing.JCheckBox;
33import javax.swing.JLabel;
34import javax.swing.JPanel;
35
36import org.openstreetmap.josm.command.ChangePropertyCommand;
37import org.openstreetmap.josm.command.ChangePropertyKeyCommand;
38import org.openstreetmap.josm.command.Command;
39import org.openstreetmap.josm.command.SequenceCommand;
40import org.openstreetmap.josm.data.osm.AbstractPrimitive;
41import org.openstreetmap.josm.data.osm.DataSet;
42import org.openstreetmap.josm.data.osm.OsmPrimitive;
43import org.openstreetmap.josm.data.osm.OsmUtils;
44import org.openstreetmap.josm.data.osm.Relation;
45import org.openstreetmap.josm.data.osm.Tag;
46import org.openstreetmap.josm.data.osm.TagMap;
47import org.openstreetmap.josm.data.osm.Tagged;
48import org.openstreetmap.josm.data.osm.visitor.MergeSourceBuildingVisitor;
49import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper;
50import org.openstreetmap.josm.data.validation.OsmValidator;
51import org.openstreetmap.josm.data.validation.Severity;
52import org.openstreetmap.josm.data.validation.Test.TagTest;
53import org.openstreetmap.josm.data.validation.TestError;
54import org.openstreetmap.josm.data.validation.util.Entities;
55import org.openstreetmap.josm.gui.progress.ProgressMonitor;
56import org.openstreetmap.josm.gui.tagging.presets.TaggingPreset;
57import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetItem;
58import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetListener;
59import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetType;
60import org.openstreetmap.josm.gui.tagging.presets.TaggingPresets;
61import org.openstreetmap.josm.gui.tagging.presets.items.Check;
62import org.openstreetmap.josm.gui.tagging.presets.items.CheckGroup;
63import org.openstreetmap.josm.gui.tagging.presets.items.KeyedItem;
64import org.openstreetmap.josm.gui.widgets.EditableList;
65import org.openstreetmap.josm.io.CachedFile;
66import org.openstreetmap.josm.spi.preferences.Config;
67import org.openstreetmap.josm.tools.GBC;
68import org.openstreetmap.josm.tools.Logging;
69import org.openstreetmap.josm.tools.MultiMap;
70import org.openstreetmap.josm.tools.Utils;
71
72/**
73 * Check for misspelled or wrong tags
74 *
75 * @author frsantos
76 * @since 3669
77 */
78public class TagChecker extends TagTest implements TaggingPresetListener {
79
80 /** The config file of ignored tags */
81 public static final String IGNORE_FILE = "resource://data/validator/ignoretags.cfg";
82 /** The config file of dictionary words */
83 public static final String SPELL_FILE = "resource://data/validator/words.cfg";
84
85 /** Normalized keys: the key should be substituted by the value if the key was not found in presets */
86 private static final Map<String, String> harmonizedKeys = new HashMap<>();
87 /** The spell check preset values which are not stored in TaggingPresets */
88 private static volatile HashSet<String> additionalPresetsValueData;
89 /** often used tags which are not in presets */
90 private static final MultiMap<String, String> oftenUsedTags = new MultiMap<>();
91 private static final Map<TaggingPreset, List<TaggingPresetItem>> presetIndex = new LinkedHashMap<>();
92
93 private static final Pattern UNWANTED_NON_PRINTING_CONTROL_CHARACTERS = Pattern.compile(
94 "[\\x00-\\x09\\x0B\\x0C\\x0E-\\x1F\\x7F\\u200e-\\u200f\\u202a-\\u202e]");
95
96 /** The TagChecker data */
97 private static final List<String> ignoreDataStartsWith = new ArrayList<>();
98 private static final Set<String> ignoreDataEquals = new HashSet<>();
99 private static final List<String> ignoreDataEndsWith = new ArrayList<>();
100 private static final List<Tag> ignoreDataTag = new ArrayList<>();
101 /** tag keys that have only numerical values in the presets */
102 private static final Set<String> ignoreForLevenshtein = new HashSet<>();
103
104 /** The preferences prefix */
105 protected static final String PREFIX = ValidatorPrefHelper.PREFIX + "." + TagChecker.class.getSimpleName();
106
107 MapCSSTagChecker deprecatedChecker;
108
109 /**
110 * The preference key to check values
111 */
112 public static final String PREF_CHECK_VALUES = PREFIX + ".checkValues";
113 /**
114 * The preference key to check keys
115 */
116 public static final String PREF_CHECK_KEYS = PREFIX + ".checkKeys";
117 /**
118 * The preference key to enable complex checks
119 */
120 public static final String PREF_CHECK_COMPLEX = PREFIX + ".checkComplex";
121 /**
122 * The preference key to search for fixme tags
123 */
124 public static final String PREF_CHECK_FIXMES = PREFIX + ".checkFixmes";
125 /**
126 * The preference key to check presets
127 */
128 public static final String PREF_CHECK_PRESETS_TYPES = PREFIX + ".checkPresetsTypes";
129
130 /**
131 * The preference key for source files
132 * @see #DEFAULT_SOURCES
133 */
134 public static final String PREF_SOURCES = PREFIX + ".source";
135
136 private static final String BEFORE_UPLOAD = "BeforeUpload";
137 /**
138 * The preference key to check keys - used before upload
139 */
140 public static final String PREF_CHECK_KEYS_BEFORE_UPLOAD = PREF_CHECK_KEYS + BEFORE_UPLOAD;
141 /**
142 * The preference key to check values - used before upload
143 */
144 public static final String PREF_CHECK_VALUES_BEFORE_UPLOAD = PREF_CHECK_VALUES + BEFORE_UPLOAD;
145 /**
146 * The preference key to run complex tests - used before upload
147 */
148 public static final String PREF_CHECK_COMPLEX_BEFORE_UPLOAD = PREF_CHECK_COMPLEX + BEFORE_UPLOAD;
149 /**
150 * The preference key to search for fixmes - used before upload
151 */
152 public static final String PREF_CHECK_FIXMES_BEFORE_UPLOAD = PREF_CHECK_FIXMES + BEFORE_UPLOAD;
153 /**
154 * The preference key to search for presets - used before upload
155 */
156 public static final String PREF_CHECK_PRESETS_TYPES_BEFORE_UPLOAD = PREF_CHECK_PRESETS_TYPES + BEFORE_UPLOAD;
157
158 private static final int MAX_LEVENSHTEIN_DISTANCE = 2;
159
160 protected boolean includeOtherSeverity;
161
162 protected boolean checkKeys;
163 protected boolean checkValues;
164 /** Was used for special configuration file, might be used to disable value spell checker. */
165 protected boolean checkComplex;
166 protected boolean checkFixmes;
167 protected boolean checkPresetsTypes;
168
169 protected JCheckBox prefCheckKeys;
170 protected JCheckBox prefCheckValues;
171 protected JCheckBox prefCheckComplex;
172 protected JCheckBox prefCheckFixmes;
173 protected JCheckBox prefCheckPresetsTypes;
174
175 protected JCheckBox prefCheckKeysBeforeUpload;
176 protected JCheckBox prefCheckValuesBeforeUpload;
177 protected JCheckBox prefCheckComplexBeforeUpload;
178 protected JCheckBox prefCheckFixmesBeforeUpload;
179 protected JCheckBox prefCheckPresetsTypesBeforeUpload;
180
181 // CHECKSTYLE.OFF: SingleSpaceSeparator
182 protected static final int EMPTY_VALUES = 1200;
183 protected static final int INVALID_KEY = 1201;
184 protected static final int INVALID_VALUE = 1202;
185 protected static final int FIXME = 1203;
186 protected static final int INVALID_SPACE = 1204;
187 protected static final int INVALID_KEY_SPACE = 1205;
188 protected static final int INVALID_HTML = 1206; /* 1207 was PAINT */
189 protected static final int LONG_VALUE = 1208;
190 protected static final int LONG_KEY = 1209;
191 protected static final int LOW_CHAR_VALUE = 1210;
192 protected static final int LOW_CHAR_KEY = 1211;
193 protected static final int MISSPELLED_VALUE = 1212;
194 protected static final int MISSPELLED_KEY = 1213;
195 protected static final int MULTIPLE_SPACES = 1214;
196 protected static final int MISSPELLED_VALUE_NO_FIX = 1215;
197 protected static final int UNUSUAL_UNICODE_CHAR_VALUE = 1216;
198 protected static final int INVALID_PRESETS_TYPE = 1217;
199 protected static final int MULTIPOLYGON_NO_AREA = 1218;
200 protected static final int MULTIPOLYGON_INCOMPLETE = 1219;
201 protected static final int MULTIPOLYGON_MAYBE_NO_AREA = 1220;
202 // CHECKSTYLE.ON: SingleSpaceSeparator
203
204 protected EditableList sourcesList;
205
206 private static final List<String> DEFAULT_SOURCES = Arrays.asList(IGNORE_FILE, SPELL_FILE);
207
208 /**
209 * Constructor
210 */
211 public TagChecker() {
212 super(tr("Tag checker"), tr("This test checks for errors in tag keys and values."));
213 }
214
215 @Override
216 public void initialize() throws IOException {
217 TaggingPresets.addListener(this);
218 initializeData();
219 initializePresets();
220 analysePresets();
221 }
222
223 /**
224 * Add presets that contain only numerical values to the ignore list
225 */
226 private static void analysePresets() {
227 for (String key : TaggingPresets.getPresetKeys()) {
228 if (isKeyIgnored(key))
229 continue;
230 Set<String> values = TaggingPresets.getPresetValues(key);
231 boolean allNumerical = values != null && !values.isEmpty()
232 && values.stream().allMatch(TagChecker::isNum);
233 if (allNumerical) {
234 ignoreForLevenshtein.add(key);
235 }
236 }
237 }
238
239 /**
240 * Reads the spell-check file into a HashMap.
241 * The data file is a list of words, beginning with +/-. If it starts with +,
242 * the word is valid, but if it starts with -, the word should be replaced
243 * by the nearest + word before this.
244 *
245 * @throws IOException if any I/O error occurs
246 */
247 private static void initializeData() throws IOException {
248 ignoreDataStartsWith.clear();
249 ignoreDataEquals.clear();
250 ignoreDataEndsWith.clear();
251 ignoreDataTag.clear();
252 harmonizedKeys.clear();
253 ignoreForLevenshtein.clear();
254 oftenUsedTags.clear();
255 presetIndex.clear();
256
257 StringBuilder errorSources = new StringBuilder();
258 for (String source : Config.getPref().getList(PREF_SOURCES, DEFAULT_SOURCES)) {
259 try (
260 CachedFile cf = new CachedFile(source);
261 BufferedReader reader = cf.getContentReader()
262 ) {
263 String okValue = null;
264 boolean tagcheckerfile = false;
265 boolean ignorefile = false;
266 boolean isFirstLine = true;
267 String line;
268 while ((line = reader.readLine()) != null) {
269 if (line.isEmpty()) {
270 // ignore
271 } else if (line.startsWith("#")) {
272 if (line.startsWith("# JOSM TagChecker")) {
273 tagcheckerfile = true;
274 Logging.error(tr("Ignoring {0}. Support was dropped", source));
275 } else
276 if (line.startsWith("# JOSM IgnoreTags")) {
277 ignorefile = true;
278 if (!DEFAULT_SOURCES.contains(source)) {
279 Logging.info(tr("Adding {0} to ignore tags", source));
280 }
281 }
282 } else if (ignorefile) {
283 parseIgnoreFileLine(source, line);
284 } else if (tagcheckerfile) {
285 // ignore
286 } else if (line.charAt(0) == '+') {
287 okValue = line.substring(1);
288 } else if (line.charAt(0) == '-' && okValue != null) {
289 String hk = harmonizeKey(line.substring(1));
290 if (!okValue.equals(hk) && harmonizedKeys.put(hk, okValue) != null && Logging.isDebugEnabled()) {
291 Logging.debug("Line was ignored: " + line);
292 }
293 } else {
294 Logging.error(tr("Invalid spellcheck line: {0}", line));
295 }
296 if (isFirstLine) {
297 isFirstLine = false;
298 if (!(tagcheckerfile || ignorefile) && !DEFAULT_SOURCES.contains(source)) {
299 Logging.info(tr("Adding {0} to spellchecker", source));
300 }
301 }
302 }
303 } catch (IOException e) {
304 Logging.error(e);
305 errorSources.append(source).append('\n');
306 }
307 }
308
309 if (errorSources.length() > 0)
310 throw new IOException(trn(
311 "Could not access data file:\n{0}",
312 "Could not access data files:\n{0}", errorSources.length(), errorSources));
313 }
314
315 /**
316 * Parse a line found in a configuration file
317 * @param source name of configuration file
318 * @param line the line to parse
319 */
320 private static void parseIgnoreFileLine(String source, String line) {
321 line = line.trim();
322 if (line.length() < 4) {
323 return;
324 }
325 try {
326 String key = line.substring(0, 2);
327 line = line.substring(2);
328
329 switch (key) {
330 case "S:":
331 ignoreDataStartsWith.add(line);
332 break;
333 case "E:":
334 ignoreDataEquals.add(line);
335 addToKeyDictionary(line);
336 break;
337 case "F:":
338 ignoreDataEndsWith.add(line);
339 break;
340 case "K:":
341 Tag tag = Tag.ofString(line);
342 ignoreDataTag.add(tag);
343 oftenUsedTags.put(tag.getKey(), tag.getValue());
344 addToKeyDictionary(tag.getKey());
345 break;
346 default:
347 if (!key.startsWith(";")) {
348 Logging.warn("Unsupported TagChecker key: " + key);
349 }
350 }
351 } catch (IllegalArgumentException e) {
352 Logging.error("Invalid line in {0} : {1}", source, e.getMessage());
353 Logging.trace(e);
354 }
355 }
356
357 private static void addToKeyDictionary(String key) {
358 if (key != null) {
359 String hk = harmonizeKey(key);
360 if (!key.equals(hk)) {
361 harmonizedKeys.put(hk, key);
362 }
363 }
364 }
365
366 /**
367 * Reads the presets data.
368 *
369 */
370 public static void initializePresets() {
371
372 if (!Config.getPref().getBoolean(PREF_CHECK_VALUES, true))
373 return;
374
375 Collection<TaggingPreset> presets = TaggingPresets.getTaggingPresets();
376 if (!presets.isEmpty()) {
377 initAdditionalPresetsValueData();
378 for (TaggingPreset p : presets) {
379 List<TaggingPresetItem> minData = new ArrayList<>();
380 for (TaggingPresetItem i : p.data) {
381 if (i instanceof KeyedItem) {
382 if (!"none".equals(((KeyedItem) i).match))
383 minData.add(i);
384 addPresetValue((KeyedItem) i);
385 } else if (i instanceof CheckGroup) {
386 for (Check c : ((CheckGroup) i).checks) {
387 addPresetValue(c);
388 }
389 }
390 }
391 if (!minData.isEmpty()) {
392 presetIndex .put(p, minData);
393 }
394 }
395 }
396 }
397
398 private static void initAdditionalPresetsValueData() {
399 additionalPresetsValueData = new HashSet<>();
400 additionalPresetsValueData.addAll(AbstractPrimitive.getUninterestingKeys());
401 additionalPresetsValueData.addAll(Config.getPref().getList(
402 ValidatorPrefHelper.PREFIX + ".knownkeys",
403 Arrays.asList("is_in", "int_ref", "fixme", "population")));
404 }
405
406 private static void addPresetValue(KeyedItem ky) {
407 if (ky.key != null && ky.getValues() != null) {
408 addToKeyDictionary(ky.key);
409 }
410 }
411
412 /**
413 * Checks given string (key or value) if it contains unwanted non-printing control characters (either ASCII or Unicode bidi characters)
414 * @param s string to check
415 * @return {@code true} if {@code s} contains non-printing control characters
416 */
417 static boolean containsUnwantedNonPrintingControlCharacter(String s) {
418 return s != null && !s.isEmpty() && (
419 isJoiningChar(s.charAt(0)) ||
420 isJoiningChar(s.charAt(s.length() - 1)) ||
421 s.chars().anyMatch(c -> (isAsciiControlChar(c) && !isNewLineChar(c)) || isBidiControlChar(c))
422 );
423 }
424
425 private static boolean isAsciiControlChar(int c) {
426 return c < 0x20 || c == 0x7F;
427 }
428
429 private static boolean isNewLineChar(int c) {
430 return c == 0x0a || c == 0x0d;
431 }
432
433 private static boolean isJoiningChar(int c) {
434 return c == 0x200c || c == 0x200d; // ZWNJ, ZWJ
435 }
436
437 private static boolean isBidiControlChar(int c) {
438 /* check for range 0x200e to 0x200f (LRM, RLM) or
439 0x202a to 0x202e (LRE, RLE, PDF, LRO, RLO) */
440 return (c >= 0x200e && c <= 0x200f) || (c >= 0x202a && c <= 0x202e);
441 }
442
443 static String removeUnwantedNonPrintingControlCharacters(String s) {
444 // Remove all unwanted characters
445 String result = UNWANTED_NON_PRINTING_CONTROL_CHARACTERS.matcher(s).replaceAll("");
446 // Remove joining characters located at the beginning of the string
447 while (!result.isEmpty() && isJoiningChar(result.charAt(0))) {
448 result = result.substring(1);
449 }
450 // Remove joining characters located at the end of the string
451 while (!result.isEmpty() && isJoiningChar(result.charAt(result.length() - 1))) {
452 result = result.substring(0, result.length() - 1);
453 }
454 return result;
455 }
456
457 static boolean containsUnusualUnicodeCharacter(String key, String value) {
458 return getUnusualUnicodeCharacter(key, value).isPresent();
459 }
460
461 static OptionalInt getUnusualUnicodeCharacter(String key, String value) {
462 return value == null
463 ? OptionalInt.empty()
464 : value.chars().filter(c -> isUnusualUnicodeBlock(key, c)).findFirst();
465 }
466
467 /**
468 * Detects highly suspicious Unicode characters that have been seen in OSM database.
469 * @param key tag key
470 * @param c current character code point
471 * @return {@code true} if the current unicode block is very unusual for the given key
472 */
473 private static boolean isUnusualUnicodeBlock(String key, int c) {
474 UnicodeBlock b = UnicodeBlock.of(c);
475 return isUnusualPhoneticUse(key, b, c) || isUnusualBmpUse(b) || isUnusualSmpUse(b);
476 }
477
478 private static boolean isAllowedPhoneticCharacter(String key, int c) {
479 // CHECKSTYLE.OFF: BooleanExpressionComplexity
480 return c == 0x0259 || c == 0x018F // U+0259 is paired with the capital letter U+018F in Azeri, see #18740
481 || c == 0x0254 || c == 0x0186 // U+0254 is paired with the capital letter U+0186 in several African languages, see #18740
482 || c == 0x0257 || c == 0x018A // "ɗ/Ɗ" (U+0257/U+018A), see #19760
483 || c == 0x025B || c == 0x0190 // U+025B is paired with the capital letter U+0190 in several African languages, see #18740
484 || c == 0x0263 || c == 0x0194 // "ɣ/Ɣ" (U+0263/U+0194), see #18740
485 || c == 0x0268 || c == 0x0197 // "ɨ/Ɨ" (U+0268/U+0197), see #18740
486 || c == 0x0272 || c == 0x019D // "ɲ/Ɲ" (U+0272/U+019D), see #18740
487 || c == 0x0273 || c == 0x019E // "ŋ/Ŋ" (U+0273/U+019E), see #18740
488 || (key.endsWith("ref") && 0x1D2C <= c && c <= 0x1D42); // allow uppercase superscript latin characters in *ref tags
489 }
490
491 private static boolean isUnusualPhoneticUse(String key, UnicodeBlock b, int c) {
492 return !isAllowedPhoneticCharacter(key, c)
493 && (b == UnicodeBlock.IPA_EXTENSIONS // U+0250..U+02AF
494 || b == UnicodeBlock.PHONETIC_EXTENSIONS // U+1D00..U+1D7F
495 || b == UnicodeBlock.PHONETIC_EXTENSIONS_SUPPLEMENT) // U+1D80..U+1DBF
496 && !key.endsWith(":pronunciation");
497 }
498
499 private static boolean isUnusualBmpUse(UnicodeBlock b) {
500 return b == UnicodeBlock.COMBINING_MARKS_FOR_SYMBOLS // U+20D0..U+20FF
501 || b == UnicodeBlock.MATHEMATICAL_OPERATORS // U+2200..U+22FF
502 || b == UnicodeBlock.ENCLOSED_ALPHANUMERICS // U+2460..U+24FF
503 || b == UnicodeBlock.BOX_DRAWING // U+2500..U+257F
504 || b == UnicodeBlock.GEOMETRIC_SHAPES // U+25A0..U+25FF
505 || b == UnicodeBlock.DINGBATS // U+2700..U+27BF
506 || b == UnicodeBlock.MISCELLANEOUS_SYMBOLS_AND_ARROWS // U+2B00..U+2BFF
507 || b == UnicodeBlock.GLAGOLITIC // U+2C00..U+2C5F
508 || b == UnicodeBlock.HANGUL_COMPATIBILITY_JAMO // U+3130..U+318F
509 || b == UnicodeBlock.ENCLOSED_CJK_LETTERS_AND_MONTHS // U+3200..U+32FF
510 || b == UnicodeBlock.LATIN_EXTENDED_D // U+A720..U+A7FF
511 || b == UnicodeBlock.CJK_COMPATIBILITY_IDEOGRAPHS // U+F900..U+FAFF
512 || b == UnicodeBlock.ALPHABETIC_PRESENTATION_FORMS // U+FB00..U+FB4F
513 || b == UnicodeBlock.VARIATION_SELECTORS // U+FE00..U+FE0F
514 || b == UnicodeBlock.SPECIALS; // U+FFF0..U+FFFF
515 // CHECKSTYLE.ON: BooleanExpressionComplexity
516 }
517
518 private static boolean isUnusualSmpUse(UnicodeBlock b) {
519 // UnicodeBlock.SUPPLEMENTAL_SYMBOLS_AND_PICTOGRAPHS is only defined in Java 9+
520 return b == UnicodeBlock.MUSICAL_SYMBOLS // U+1D100..U+1D1FF
521 || b == UnicodeBlock.ENCLOSED_ALPHANUMERIC_SUPPLEMENT // U+1F100..U+1F1FF
522 || b == UnicodeBlock.EMOTICONS // U+1F600..U+1F64F
523 || b == UnicodeBlock.TRANSPORT_AND_MAP_SYMBOLS; // U+1F680..U+1F6FF
524 }
525
526 /**
527 * Get set of preset values for the given key.
528 * @param key the key
529 * @return null if key is not in presets or in additionalPresetsValueData,
530 * else a set which might be empty.
531 */
532 private static Set<String> getPresetValues(String key) {
533 Set<String> res = TaggingPresets.getPresetValues(key);
534 if (res != null)
535 return res;
536 if (additionalPresetsValueData.contains(key))
537 return Collections.emptySet();
538 // null means key is not known
539 return null;
540 }
541
542 /**
543 * Determines if the given key is in internal presets.
544 * @param key key
545 * @return {@code true} if the given key is in internal presets
546 * @since 9023
547 */
548 public static boolean isKeyInPresets(String key) {
549 return TaggingPresets.getPresetValues(key) != null;
550 }
551
552 /**
553 * Determines if the given tag is in internal presets.
554 * @param key key
555 * @param value value
556 * @return {@code true} if the given tag is in internal presets
557 * @since 9023
558 */
559 public static boolean isTagInPresets(String key, String value) {
560 final Set<String> values = getPresetValues(key);
561 return values != null && values.contains(value);
562 }
563
564 /**
565 * Returns the list of ignored tags.
566 * @return the list of ignored tags
567 * @since 9023
568 */
569 public static List<Tag> getIgnoredTags() {
570 return new ArrayList<>(ignoreDataTag);
571 }
572
573 /**
574 * Determines if the given tag key is ignored for checks "key/tag not in presets".
575 * @param key key
576 * @return true if the given key is ignored
577 */
578 private static boolean isKeyIgnored(String key) {
579 return ignoreDataEquals.contains(key)
580 || ignoreDataStartsWith.stream().anyMatch(key::startsWith)
581 || ignoreDataEndsWith.stream().anyMatch(key::endsWith);
582 }
583
584 /**
585 * Determines if the given tag is ignored for checks "key/tag not in presets".
586 * @param key key
587 * @param value value
588 * @return {@code true} if the given tag is ignored
589 * @since 9023
590 */
591 public static boolean isTagIgnored(String key, String value) {
592 if (isKeyIgnored(key))
593 return true;
594 final Set<String> values = getPresetValues(key);
595 if (values != null && values.isEmpty())
596 return true;
597 if (!isTagInPresets(key, value)) {
598 return ignoreDataTag.stream()
599 .anyMatch(a -> key.equals(a.getKey()) && value.equals(a.getValue()));
600 }
601 return false;
602 }
603
604 /**
605 * Checks the primitive tags
606 * @param p The primitive to check
607 */
608 @Override
609 public void check(OsmPrimitive p) {
610 if (!p.isTagged())
611 return;
612
613 // Just a collection to know if a primitive has been already marked with error
614 MultiMap<OsmPrimitive, String> withErrors = new MultiMap<>();
615
616 for (Entry<String, String> prop : p.getKeys().entrySet()) {
617 String s = marktr("Tag ''{0}'' invalid.");
618 String key = prop.getKey();
619 String value = prop.getValue();
620
621 if (checkKeys) {
622 checkSingleTagKeySimple(withErrors, p, s, key);
623 }
624 if (checkValues) {
625 checkSingleTagValueSimple(withErrors, p, s, key, value);
626 checkSingleTagComplex(withErrors, p, key, value);
627 }
628 if (checkFixmes && key != null && value != null && !value.isEmpty() && isFixme(key, value) && !withErrors.contains(p, "FIXME")) {
629 errors.add(TestError.builder(this, Severity.OTHER, FIXME)
630 .message(tr("FIXMES"))
631 .primitives(p)
632 .build());
633 withErrors.put(p, "FIXME");
634 }
635 }
636
637 if (p instanceof Relation && p.hasTag("type", "multipolygon")) {
638 checkMultipolygonTags(p);
639 }
640
641 if (checkPresetsTypes) {
642 TagMap tags = p.getKeys();
643 TaggingPresetType presetType = TaggingPresetType.forPrimitive(p);
644 EnumSet<TaggingPresetType> presetTypes = EnumSet.of(presetType);
645
646 Collection<TaggingPreset> matchingPresets = presetIndex.entrySet().stream()
647 .filter(e -> TaggingPresetItem.matches(e.getValue(), tags))
648 .map(Entry::getKey)
649 .collect(Collectors.toCollection(LinkedHashSet::new));
650 Collection<TaggingPreset> matchingPresetsOK = matchingPresets.stream().filter(
651 tp -> tp.typeMatches(presetTypes)).collect(Collectors.toList());
652 Collection<TaggingPreset> matchingPresetsKO = matchingPresets.stream().filter(
653 tp -> !tp.typeMatches(presetTypes)).collect(Collectors.toList());
654
655 for (TaggingPreset tp : matchingPresetsKO) {
656 // Potential error, unless matching tags are all known by a supported preset
657 Map<String, String> matchingTags = tp.data.stream()
658 .filter(i -> Boolean.TRUE.equals(i.matches(tags)))
659 .filter(i -> i instanceof KeyedItem).map(i -> ((KeyedItem) i).key)
660 .collect(Collectors.toMap(k -> k, tags::get));
661 if (matchingPresetsOK.stream().noneMatch(
662 tp2 -> matchingTags.entrySet().stream().allMatch(
663 e -> tp2.data.stream().anyMatch(
664 i -> i instanceof KeyedItem && ((KeyedItem) i).key.equals(e.getKey()))))) {
665 errors.add(TestError.builder(this, Severity.OTHER, INVALID_PRESETS_TYPE)
666 .message(tr("Object type not in preset"),
667 marktr("Object type {0} is not supported by tagging preset: {1}"),
668 tr(presetType.getName()), tp.getLocaleName())
669 .primitives(p)
670 .build());
671 }
672 }
673 }
674 }
675
676 private static final Collection<String> NO_AREA_KEYS = Arrays.asList("name", "area", "ref", "access", "operator");
677
678 private void checkMultipolygonTags(OsmPrimitive p) {
679 if (p.isAnnotated() || hasAcceptedPrimaryTagForMultipolygon(p))
680 return;
681 if (p.keySet().stream().anyMatch(k -> k.matches("^(abandoned|construction|demolished|disused|planned|razed|removed|was).*")))
682 return;
683
684 TestError.Builder builder = null;
685 if (p.hasKey("surface")) {
686 // accept often used tag surface=* as area tag
687 builder = TestError.builder(this, Severity.OTHER, MULTIPOLYGON_INCOMPLETE)
688 .message(tr("Multipolygon tags"), marktr("only {0} tag"), "surface");
689 } else {
690 Map<String, String> filteredTags = p.getInterestingTags();
691 filteredTags.remove("type");
692 NO_AREA_KEYS.forEach(filteredTags::remove);
693 filteredTags.keySet().removeIf(key -> !key.matches("[a-z0-9:_]+"));
694
695 if (filteredTags.isEmpty()) {
696 builder = TestError.builder(this, Severity.ERROR, MULTIPOLYGON_NO_AREA)
697 .message(tr("Multipolygon tags"), marktr("tag describing the area is missing"), new Object());
698
699 }
700 }
701 if (builder == null) {
702 // multipolygon has either no area tag or a rarely used one
703 builder = TestError.builder(this, Severity.WARNING, MULTIPOLYGON_MAYBE_NO_AREA)
704 .message(tr("Multipolygon tags"), marktr("tag describing the area might be missing"), new Object());
705 }
706 errors.add(builder.primitives(p).build());
707 }
708
709 /**
710 * Check if a multipolygon has a main tag that describes the type of area. Accepts also some deprecated tags and typos.
711 * @param p the multipolygon
712 * @return true if the multipolygon has a main tag that (likely) describes the type of area.
713 */
714 private static boolean hasAcceptedPrimaryTagForMultipolygon(OsmPrimitive p) {
715 if (p.hasKey("landuse", "amenity", "building", "building:part", "area:highway", "shop", "place", "boundary",
716 "landform", "piste:type", "sport", "golf", "landcover", "aeroway", "office", "healthcare", "craft", "room")
717 || p.hasTagDifferent("natural", "tree", "peek", "saddle", "tree_row")
718 || p.hasTagDifferent("man_made", "survey_point", "mast", "flagpole", "manhole", "watertap")
719 || p.hasTagDifferent("highway", "crossing", "bus_stop", "turning_circle", "street_lamp",
720 "traffic_signals", "stop", "milestone", "mini_roundabout", "motorway_junction", "passing_place",
721 "speed_camera", "traffic_mirror", "trailhead", "turning_circle", "turning_loop", "toll_gantry")
722 || p.hasTagDifferent("tourism", "attraction", "artwork")
723 || p.hasTagDifferent("leisure", "picnic_table", "slipway", "firepit")
724 || p.hasTagDifferent("historic", "wayside_cross", "milestone"))
725 return true;
726 if (p.hasTag("barrier", "hedge", "retaining_wall")
727 || p.hasTag("public_transport", "platform", "station")
728 || p.hasTag("railway", "platform")
729 || p.hasTag("waterway", "riverbank", "dam", "rapids", "dock", "boatyard", "fuel")
730 || p.hasTag("indoor", "corridor", "room", "area")
731 || p.hasTag("power", "substation", "generator", "plant", "switchgear", "converter", "sub_station")
732 || p.hasTag("seamark:type", "harbour", "fairway", "anchorage", "landmark", "berth", "harbour_basin",
733 "separation_zone")
734 || (p.get("seamark:type") != null && p.get("seamark:type").matches(".*\\_(area|zone)$")))
735 return true;
736 return p.hasTag("harbour", OsmUtils.TRUE_VALUE)
737 || p.hasTag("flood_prone", OsmUtils.TRUE_VALUE)
738 || p.hasTag("bridge", OsmUtils.TRUE_VALUE)
739 || p.hasTag("ruins", OsmUtils.TRUE_VALUE)
740 || p.hasTag("junction", OsmUtils.TRUE_VALUE);
741 }
742
743 private void checkSingleTagValueSimple(MultiMap<OsmPrimitive, String> withErrors, OsmPrimitive p, String s, String key, String value) {
744 if (!checkValues || value == null)
745 return;
746 if (containsUnwantedNonPrintingControlCharacter(value) && !withErrors.contains(p, "ICV")) {
747 errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_VALUE)
748 .message(tr("Tag value contains non-printing (usually invisible) character"), s, key)
749 .primitives(p)
750 .fix(() -> new ChangePropertyCommand(p, key, removeUnwantedNonPrintingControlCharacters(value)))
751 .build());
752 withErrors.put(p, "ICV");
753 }
754 final OptionalInt unusualUnicodeCharacter = getUnusualUnicodeCharacter(key, value);
755 if (unusualUnicodeCharacter.isPresent() && !withErrors.contains(p, "UUCV")) {
756 final String codepoint = String.format(Locale.ROOT, "U+%04X", unusualUnicodeCharacter.getAsInt());
757 errors.add(TestError.builder(this, Severity.WARNING, UNUSUAL_UNICODE_CHAR_VALUE)
758 .message(tr("Tag value contains unusual Unicode character {0}", codepoint), s, key)
759 .primitives(p)
760 .build());
761 withErrors.put(p, "UUCV");
762 }
763 if ((value.length() > Tagged.MAX_TAG_LENGTH) && !withErrors.contains(p, "LV")) {
764 errors.add(TestError.builder(this, Severity.ERROR, LONG_VALUE)
765 .message(tr("Tag value longer than {0} characters ({1} characters)", Tagged.MAX_TAG_LENGTH, value.length()), s, key)
766 .primitives(p)
767 .build());
768 withErrors.put(p, "LV");
769 }
770 if (value.trim().isEmpty() && !withErrors.contains(p, "EV")) {
771 errors.add(TestError.builder(this, Severity.WARNING, EMPTY_VALUES)
772 .message(tr("Tags with empty values"), s, key)
773 .primitives(p)
774 .build());
775 withErrors.put(p, "EV");
776 }
777 final String errTypeSpace = "SPACE";
778 if ((value.startsWith(" ") || value.endsWith(" ")) && !withErrors.contains(p, errTypeSpace)) {
779 errors.add(TestError.builder(this, Severity.WARNING, INVALID_SPACE)
780 .message(tr("Property values start or end with white space"), s, key)
781 .primitives(p)
782 .build());
783 withErrors.put(p, errTypeSpace);
784 }
785 if (value.contains(" ") && !withErrors.contains(p, errTypeSpace)) {
786 errors.add(TestError.builder(this, Severity.WARNING, MULTIPLE_SPACES)
787 .message(tr("Property values contain multiple white spaces"), s, key)
788 .primitives(p)
789 .build());
790 withErrors.put(p, errTypeSpace);
791 }
792 if (includeOtherSeverity && !value.equals(Entities.unescape(value)) && !withErrors.contains(p, "HTML")) {
793 errors.add(TestError.builder(this, Severity.OTHER, INVALID_HTML)
794 .message(tr("Property values contain HTML entity"), s, key)
795 .primitives(p)
796 .build());
797 withErrors.put(p, "HTML");
798 }
799 }
800
801 private void checkSingleTagKeySimple(MultiMap<OsmPrimitive, String> withErrors, OsmPrimitive p, String s, String key) {
802 if (!checkKeys || key == null)
803 return;
804 if (containsUnwantedNonPrintingControlCharacter(key) && !withErrors.contains(p, "ICK")) {
805 errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_KEY)
806 .message(tr("Tag key contains non-printing character"), s, key)
807 .primitives(p)
808 .fix(() -> new ChangePropertyCommand(p, key, removeUnwantedNonPrintingControlCharacters(key)))
809 .build());
810 withErrors.put(p, "ICK");
811 }
812 if (key.length() > Tagged.MAX_TAG_LENGTH && !withErrors.contains(p, "LK")) {
813 errors.add(TestError.builder(this, Severity.ERROR, LONG_KEY)
814 .message(tr("Tag key longer than {0} characters ({1} characters)", Tagged.MAX_TAG_LENGTH, key.length()), s, key)
815 .primitives(p)
816 .build());
817 withErrors.put(p, "LK");
818 }
819 if (key.indexOf(' ') >= 0 && !withErrors.contains(p, "IPK")) {
820 errors.add(TestError.builder(this, Severity.WARNING, INVALID_KEY_SPACE)
821 .message(tr("Invalid white space in property key"), s, key)
822 .primitives(p)
823 .build());
824 withErrors.put(p, "IPK");
825 }
826 }
827
828 private void checkSingleTagComplex(MultiMap<OsmPrimitive, String> withErrors, OsmPrimitive p, String key, String value) {
829 if (!checkValues || key == null || value == null || value.isEmpty())
830 return;
831 if (additionalPresetsValueData != null && !isTagIgnored(key, value)) {
832 if (!isKeyInPresets(key)) {
833 spellCheckKey(withErrors, p, key);
834 } else if (!isTagInPresets(key, value)) {
835 if (oftenUsedTags.contains(key, value)) {
836 // tag is quite often used but not in presets
837 errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE)
838 .message(tr("Presets do not contain property value"),
839 marktr("Value ''{0}'' for key ''{1}'' not in presets, but is known."), value, key)
840 .primitives(p)
841 .build());
842 withErrors.put(p, "UPV");
843 } else {
844 tryGuess(p, key, value, withErrors);
845 }
846 }
847 }
848 }
849
850 private void spellCheckKey(MultiMap<OsmPrimitive, String> withErrors, OsmPrimitive p, String key) {
851 String prettifiedKey = harmonizeKey(key);
852 String fixedKey;
853 if (ignoreDataEquals.contains(prettifiedKey)) {
854 fixedKey = prettifiedKey;
855 } else {
856 fixedKey = isKeyInPresets(prettifiedKey) ? prettifiedKey : harmonizedKeys.get(prettifiedKey);
857 }
858 if (fixedKey == null && ignoreDataTag.stream().anyMatch(a -> a.getKey().equals(prettifiedKey))) {
859 fixedKey = prettifiedKey;
860 }
861
862 if (fixedKey != null && !"".equals(fixedKey) && !fixedKey.equals(key)) {
863 final String proposedKey = fixedKey;
864 // misspelled preset key
865 final TestError.Builder error = TestError.builder(this, Severity.WARNING, MISSPELLED_KEY)
866 .message(tr("Misspelled property key"), marktr("Key ''{0}'' looks like ''{1}''."), key, proposedKey)
867 .primitives(p);
868 if (p.hasKey(fixedKey)) {
869 errors.add(error.build());
870 } else {
871 errors.add(error.fix(() -> new ChangePropertyKeyCommand(p, key, proposedKey)).build());
872 }
873 withErrors.put(p, "WPK");
874 } else if (includeOtherSeverity) {
875 errors.add(TestError.builder(this, Severity.OTHER, INVALID_KEY)
876 .message(tr("Presets do not contain property key"), marktr("Key ''{0}'' not in presets."), key)
877 .primitives(p)
878 .build());
879 withErrors.put(p, "UPK");
880 }
881 }
882
883 private void tryGuess(OsmPrimitive p, String key, String value, MultiMap<OsmPrimitive, String> withErrors) {
884 // try to fix common typos and check again if value is still unknown
885 final String harmonizedValue = harmonizeValue(value);
886 if (harmonizedValue == null || harmonizedValue.isEmpty())
887 return;
888 String fixedValue;
889 List<Set<String>> sets = new ArrayList<>();
890 Set<String> presetValues = getPresetValues(key);
891 if (presetValues != null)
892 sets.add(presetValues);
893 Set<String> usedValues = oftenUsedTags.get(key);
894 if (usedValues != null)
895 sets.add(usedValues);
896 fixedValue = sets.stream().anyMatch(possibleValues -> possibleValues.contains(harmonizedValue))
897 ? harmonizedValue : null;
898 if (fixedValue == null && !ignoreForLevenshtein.contains(key)) {
899 int maxPresetValueLen = 0;
900 List<String> fixVals = new ArrayList<>();
901 // use Levenshtein distance to find typical typos
902 int minDist = MAX_LEVENSHTEIN_DISTANCE + 1;
903 for (Set<String> possibleValues: sets) {
904 for (String possibleVal : possibleValues) {
905 if (possibleVal.isEmpty())
906 continue;
907 maxPresetValueLen = Math.max(maxPresetValueLen, possibleVal.length());
908 if (harmonizedValue.length() < 3 && possibleVal.length() >= harmonizedValue.length() + MAX_LEVENSHTEIN_DISTANCE) {
909 // don't suggest fix value when given value is short and lengths are too different
910 // for example surface=u would result in surface=mud
911 continue;
912 }
913 int dist = Utils.getLevenshteinDistance(possibleVal, harmonizedValue);
914 if (dist >= harmonizedValue.length()) {
915 // short value, all characters are different. Don't warn, might say Value '10' for key 'fee' looks like 'no'.
916 continue;
917 }
918 if (dist < minDist) {
919 minDist = dist;
920 fixVals.clear();
921 fixVals.add(possibleVal);
922 } else if (dist == minDist) {
923 fixVals.add(possibleVal);
924 }
925 }
926 }
927 if (minDist <= MAX_LEVENSHTEIN_DISTANCE && maxPresetValueLen > MAX_LEVENSHTEIN_DISTANCE
928 && !fixVals.isEmpty()
929 && (harmonizedValue.length() > 3 || minDist < MAX_LEVENSHTEIN_DISTANCE)) {
930 filterDeprecatedTags(p, key, fixVals);
931 if (!fixVals.isEmpty()) {
932 if (fixVals.size() < 2) {
933 fixedValue = fixVals.get(0);
934 } else {
935 Collections.sort(fixVals);
936 // misspelled preset value with multiple good alternatives
937 errors.add(TestError.builder(this, Severity.WARNING, MISSPELLED_VALUE_NO_FIX)
938 .message(tr("Unknown property value"),
939 marktr("Value ''{0}'' for key ''{1}'' is unknown, maybe one of {2} is meant?"),
940 value, key, fixVals)
941 .primitives(p).build());
942 withErrors.put(p, "WPV");
943 return;
944 }
945 }
946 }
947 }
948 if (fixedValue != null && !fixedValue.equals(value)) {
949 final String newValue = fixedValue;
950 // misspelled preset value
951 errors.add(TestError.builder(this, Severity.WARNING, MISSPELLED_VALUE)
952 .message(tr("Unknown property value"),
953 marktr("Value ''{0}'' for key ''{1}'' is unknown, maybe ''{2}'' is meant?"), value, key, newValue)
954 .primitives(p)
955 .build());
956 withErrors.put(p, "WPV");
957 } else if (includeOtherSeverity) {
958 // unknown preset value
959 errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE)
960 .message(tr("Presets do not contain property value"),
961 marktr("Value ''{0}'' for key ''{1}'' not in presets."), value, key)
962 .primitives(p)
963 .build());
964 withErrors.put(p, "UPV");
965 }
966 }
967
968 // see #19180
969 private void filterDeprecatedTags(OsmPrimitive p, String key, List<String> fixVals) {
970 if (fixVals.isEmpty() || deprecatedChecker == null)
971 return;
972
973 int unchangedDeprecated = countDeprecated(p);
974
975 // see #19895: create deep clone. This complex method works even with locked files
976 MergeSourceBuildingVisitor builder = new MergeSourceBuildingVisitor(p.getDataSet());
977 p.accept(builder);
978 DataSet clonedDs = builder.build();
979 OsmPrimitive clone = clonedDs.getPrimitiveById(p.getPrimitiveId());
980
981 Iterator<String> iter = fixVals.iterator();
982 while (iter.hasNext()) {
983 clone.put(key, iter.next());
984 if (countDeprecated(clone) > unchangedDeprecated)
985 iter.remove();
986 }
987 }
988
989
990 private int countDeprecated(OsmPrimitive p) {
991 if (deprecatedChecker == null)
992 return 0;
993 deprecatedChecker.getErrors().clear();
994 deprecatedChecker.runOnly("deprecated.mapcss", Collections.singleton(p));
995 return deprecatedChecker.getErrors().size();
996 }
997
998 private static boolean isNum(String harmonizedValue) {
999 try {
1000 Double.parseDouble(harmonizedValue);
1001 return true;
1002 } catch (NumberFormatException e) {
1003 return false;
1004 }
1005 }
1006
1007 private static boolean isFixme(String key, String value) {
1008 return key.toLowerCase(Locale.ENGLISH).contains("fixme") || key.contains("todo")
1009 || value.toLowerCase(Locale.ENGLISH).contains("fixme") || value.contains("check and delete");
1010 }
1011
1012 private static String harmonizeKey(String key) {
1013 return Utils.strip(key.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(':', '_').replace(' ', '_'), "-_;:,");
1014 }
1015
1016 private static String harmonizeValue(String value) {
1017 return Utils.strip(value.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(' ', '_'), "-_;:,");
1018 }
1019
1020 @Override
1021 public void startTest(ProgressMonitor monitor) {
1022 super.startTest(monitor);
1023 includeOtherSeverity = includeOtherSeverityChecks();
1024 checkKeys = Config.getPref().getBoolean(PREF_CHECK_KEYS, true);
1025 if (isBeforeUpload) {
1026 checkKeys = checkKeys && Config.getPref().getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true);
1027 }
1028
1029 checkValues = Config.getPref().getBoolean(PREF_CHECK_VALUES, true);
1030 if (isBeforeUpload) {
1031 checkValues = checkValues && Config.getPref().getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true);
1032 }
1033
1034 checkComplex = Config.getPref().getBoolean(PREF_CHECK_COMPLEX, true);
1035 if (isBeforeUpload) {
1036 checkComplex = checkComplex && Config.getPref().getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true);
1037 }
1038
1039 checkFixmes = includeOtherSeverity && Config.getPref().getBoolean(PREF_CHECK_FIXMES, true);
1040 if (isBeforeUpload) {
1041 checkFixmes = checkFixmes && Config.getPref().getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true);
1042 }
1043
1044 checkPresetsTypes = includeOtherSeverity && Config.getPref().getBoolean(PREF_CHECK_PRESETS_TYPES, true);
1045 if (isBeforeUpload) {
1046 checkPresetsTypes = checkPresetsTypes && Config.getPref().getBoolean(PREF_CHECK_PRESETS_TYPES_BEFORE_UPLOAD, true);
1047 }
1048 deprecatedChecker = OsmValidator.getTest(MapCSSTagChecker.class);
1049 }
1050
1051 @Override
1052 public void endTest() {
1053 deprecatedChecker = null;
1054 super.endTest();
1055 }
1056
1057 @Override
1058 public void visit(Collection<OsmPrimitive> selection) {
1059 if (checkKeys || checkValues || checkComplex || checkFixmes || checkPresetsTypes) {
1060 super.visit(selection);
1061 }
1062 }
1063
1064 @Override
1065 public void addGui(JPanel testPanel) {
1066 GBC a = GBC.eol();
1067 a.anchor = GridBagConstraints.EAST;
1068
1069 testPanel.add(new JLabel(name+" :"), GBC.eol().insets(3, 0, 0, 0));
1070
1071 prefCheckKeys = new JCheckBox(tr("Check property keys."), Config.getPref().getBoolean(PREF_CHECK_KEYS, true));
1072 prefCheckKeys.setToolTipText(tr("Validate that property keys are valid checking against list of words."));
1073 testPanel.add(prefCheckKeys, GBC.std().insets(20, 0, 0, 0));
1074
1075 prefCheckKeysBeforeUpload = new JCheckBox();
1076 prefCheckKeysBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true));
1077 testPanel.add(prefCheckKeysBeforeUpload, a);
1078
1079 prefCheckComplex = new JCheckBox(tr("Use complex property checker."), Config.getPref().getBoolean(PREF_CHECK_COMPLEX, true));
1080 prefCheckComplex.setToolTipText(tr("Validate property values and tags using complex rules."));
1081 testPanel.add(prefCheckComplex, GBC.std().insets(20, 0, 0, 0));
1082
1083 prefCheckComplexBeforeUpload = new JCheckBox();
1084 prefCheckComplexBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true));
1085 testPanel.add(prefCheckComplexBeforeUpload, a);
1086
1087 final Collection<String> sources = Config.getPref().getList(PREF_SOURCES, DEFAULT_SOURCES);
1088 sourcesList = new EditableList(tr("TagChecker source"));
1089 sourcesList.setItems(sources);
1090 testPanel.add(new JLabel(tr("Data sources ({0})", "*.cfg")), GBC.eol().insets(23, 0, 0, 0));
1091 testPanel.add(sourcesList, GBC.eol().fill(GridBagConstraints.HORIZONTAL).insets(23, 0, 0, 0));
1092
1093 ActionListener disableCheckActionListener = e -> handlePrefEnable();
1094 prefCheckKeys.addActionListener(disableCheckActionListener);
1095 prefCheckKeysBeforeUpload.addActionListener(disableCheckActionListener);
1096 prefCheckComplex.addActionListener(disableCheckActionListener);
1097 prefCheckComplexBeforeUpload.addActionListener(disableCheckActionListener);
1098
1099 handlePrefEnable();
1100
1101 prefCheckValues = new JCheckBox(tr("Check property values."), Config.getPref().getBoolean(PREF_CHECK_VALUES, true));
1102 prefCheckValues.setToolTipText(tr("Validate that property values are valid checking against presets."));
1103 testPanel.add(prefCheckValues, GBC.std().insets(20, 0, 0, 0));
1104
1105 prefCheckValuesBeforeUpload = new JCheckBox();
1106 prefCheckValuesBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true));
1107 testPanel.add(prefCheckValuesBeforeUpload, a);
1108
1109 prefCheckFixmes = new JCheckBox(tr("Check for FIXMES."), Config.getPref().getBoolean(PREF_CHECK_FIXMES, true));
1110 prefCheckFixmes.setToolTipText(tr("Looks for nodes or ways with FIXME in any property value."));
1111 testPanel.add(prefCheckFixmes, GBC.std().insets(20, 0, 0, 0));
1112
1113 prefCheckFixmesBeforeUpload = new JCheckBox();
1114 prefCheckFixmesBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true));
1115 testPanel.add(prefCheckFixmesBeforeUpload, a);
1116
1117 prefCheckPresetsTypes = new JCheckBox(tr("Check for presets types."), Config.getPref().getBoolean(PREF_CHECK_PRESETS_TYPES, true));
1118 prefCheckPresetsTypes.setToolTipText(tr("Validate that objects types are valid checking against presets."));
1119 testPanel.add(prefCheckPresetsTypes, GBC.std().insets(20, 0, 0, 0));
1120
1121 prefCheckPresetsTypesBeforeUpload = new JCheckBox();
1122 prefCheckPresetsTypesBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_PRESETS_TYPES_BEFORE_UPLOAD, true));
1123 testPanel.add(prefCheckPresetsTypesBeforeUpload, a);
1124 }
1125
1126 /**
1127 * Enables/disables the source list field
1128 */
1129 public void handlePrefEnable() {
1130 boolean selected = prefCheckKeys.isSelected() || prefCheckKeysBeforeUpload.isSelected()
1131 || prefCheckComplex.isSelected() || prefCheckComplexBeforeUpload.isSelected();
1132 sourcesList.setEnabled(selected);
1133 }
1134
1135 @Override
1136 public boolean ok() {
1137 enabled = prefCheckKeys.isSelected() || prefCheckValues.isSelected() || prefCheckComplex.isSelected() || prefCheckFixmes.isSelected();
1138 testBeforeUpload = prefCheckKeysBeforeUpload.isSelected() || prefCheckValuesBeforeUpload.isSelected()
1139 || prefCheckFixmesBeforeUpload.isSelected() || prefCheckComplexBeforeUpload.isSelected();
1140
1141 Config.getPref().putBoolean(PREF_CHECK_VALUES, prefCheckValues.isSelected());
1142 Config.getPref().putBoolean(PREF_CHECK_COMPLEX, prefCheckComplex.isSelected());
1143 Config.getPref().putBoolean(PREF_CHECK_KEYS, prefCheckKeys.isSelected());
1144 Config.getPref().putBoolean(PREF_CHECK_FIXMES, prefCheckFixmes.isSelected());
1145 Config.getPref().putBoolean(PREF_CHECK_PRESETS_TYPES, prefCheckPresetsTypes.isSelected());
1146 Config.getPref().putBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, prefCheckValuesBeforeUpload.isSelected());
1147 Config.getPref().putBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, prefCheckComplexBeforeUpload.isSelected());
1148 Config.getPref().putBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, prefCheckKeysBeforeUpload.isSelected());
1149 Config.getPref().putBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, prefCheckFixmesBeforeUpload.isSelected());
1150 Config.getPref().putBoolean(PREF_CHECK_PRESETS_TYPES_BEFORE_UPLOAD, prefCheckPresetsTypesBeforeUpload.isSelected());
1151 return Config.getPref().putList(PREF_SOURCES, sourcesList.getItems());
1152 }
1153
1154 @Override
1155 public Command fixError(TestError testError) {
1156 List<Command> commands = new ArrayList<>(50);
1157
1158 Collection<? extends OsmPrimitive> primitives = testError.getPrimitives();
1159 for (OsmPrimitive p : primitives) {
1160 Map<String, String> tags = p.getKeys();
1161 if (tags.isEmpty()) {
1162 continue;
1163 }
1164
1165 for (Entry<String, String> prop: tags.entrySet()) {
1166 String key = prop.getKey();
1167 String value = prop.getValue();
1168 if (value == null || value.trim().isEmpty()) {
1169 commands.add(new ChangePropertyCommand(p, key, null));
1170 } else if (value.startsWith(" ") || value.endsWith(" ") || value.contains(" ")) {
1171 commands.add(new ChangePropertyCommand(p, key, Utils.removeWhiteSpaces(value)));
1172 } else if (key.startsWith(" ") || key.endsWith(" ") || key.contains(" ")) {
1173 commands.add(new ChangePropertyKeyCommand(p, key, Utils.removeWhiteSpaces(key)));
1174 } else {
1175 String evalue = Entities.unescape(value);
1176 if (!evalue.equals(value)) {
1177 commands.add(new ChangePropertyCommand(p, key, evalue));
1178 }
1179 }
1180 }
1181 }
1182
1183 if (commands.isEmpty())
1184 return null;
1185 if (commands.size() == 1)
1186 return commands.get(0);
1187
1188 return new SequenceCommand(tr("Fix tags"), commands);
1189 }
1190
1191 @Override
1192 public boolean isFixable(TestError testError) {
1193 if (testError.getTester() instanceof TagChecker) {
1194 int code = testError.getCode();
1195 return code == EMPTY_VALUES || code == INVALID_SPACE ||
1196 code == INVALID_KEY_SPACE || code == INVALID_HTML ||
1197 code == MULTIPLE_SPACES;
1198 }
1199
1200 return false;
1201 }
1202
1203 @Override
1204 public void taggingPresetsModified() {
1205 try {
1206 initializeData();
1207 initializePresets();
1208 analysePresets();
1209 } catch (IOException e) {
1210 Logging.error(e);
1211 }
1212 }
1213}
Note: See TracBrowser for help on using the repository browser.