source: josm/trunk/src/org/openstreetmap/josm/data/validation/tests/TagChecker.java@ 17621

Last change on this file since 17621 was 17621, checked in by simon04, 3 years ago

see #19180 - Fix code duplication
  • Property svn:eol-style set to native
File size: 58.3 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.validation.tests;
3
4import static org.openstreetmap.josm.tools.I18n.marktr;
5import static org.openstreetmap.josm.tools.I18n.tr;
6import static org.openstreetmap.josm.tools.I18n.trn;
7
8import java.awt.GridBagConstraints;
9import java.awt.event.ActionListener;
10import java.io.BufferedReader;
11import java.io.IOException;
12import java.lang.Character.UnicodeBlock;
13import java.util.ArrayList;
14import java.util.Arrays;
15import java.util.Collection;
16import java.util.Collections;
17import java.util.EnumSet;
18import java.util.HashMap;
19import java.util.HashSet;
20import java.util.Iterator;
21import java.util.LinkedHashMap;
22import java.util.LinkedHashSet;
23import java.util.List;
24import java.util.Locale;
25import java.util.Map;
26import java.util.Map.Entry;
27import java.util.OptionalInt;
28import java.util.Set;
29import java.util.regex.Pattern;
30import java.util.stream.Collectors;
31
32import javax.swing.JCheckBox;
33import javax.swing.JLabel;
34import javax.swing.JPanel;
35
36import org.openstreetmap.josm.command.ChangePropertyCommand;
37import org.openstreetmap.josm.command.ChangePropertyKeyCommand;
38import org.openstreetmap.josm.command.Command;
39import org.openstreetmap.josm.command.SequenceCommand;
40import org.openstreetmap.josm.data.osm.AbstractPrimitive;
41import org.openstreetmap.josm.data.osm.DataSet;
42import org.openstreetmap.josm.data.osm.OsmPrimitive;
43import org.openstreetmap.josm.data.osm.OsmUtils;
44import org.openstreetmap.josm.data.osm.Relation;
45import org.openstreetmap.josm.data.osm.RelationMember;
46import org.openstreetmap.josm.data.osm.Tag;
47import org.openstreetmap.josm.data.osm.TagMap;
48import org.openstreetmap.josm.data.osm.Tagged;
49import org.openstreetmap.josm.data.osm.Way;
50import org.openstreetmap.josm.data.osm.visitor.MergeSourceBuildingVisitor;
51import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper;
52import org.openstreetmap.josm.data.validation.OsmValidator;
53import org.openstreetmap.josm.data.validation.Severity;
54import org.openstreetmap.josm.data.validation.Test.TagTest;
55import org.openstreetmap.josm.data.validation.TestError;
56import org.openstreetmap.josm.data.validation.util.Entities;
57import org.openstreetmap.josm.gui.progress.ProgressMonitor;
58import org.openstreetmap.josm.gui.tagging.presets.TaggingPreset;
59import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetItem;
60import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetListener;
61import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetType;
62import org.openstreetmap.josm.gui.tagging.presets.TaggingPresets;
63import org.openstreetmap.josm.gui.tagging.presets.items.Check;
64import org.openstreetmap.josm.gui.tagging.presets.items.CheckGroup;
65import org.openstreetmap.josm.gui.tagging.presets.items.KeyedItem;
66import org.openstreetmap.josm.gui.widgets.EditableList;
67import org.openstreetmap.josm.io.CachedFile;
68import org.openstreetmap.josm.spi.preferences.Config;
69import org.openstreetmap.josm.tools.GBC;
70import org.openstreetmap.josm.tools.Logging;
71import org.openstreetmap.josm.tools.MultiMap;
72import org.openstreetmap.josm.tools.Utils;
73
74/**
75 * Check for misspelled or wrong tags
76 *
77 * @author frsantos
78 * @since 3669
79 */
80public class TagChecker extends TagTest implements TaggingPresetListener {
81
82 /** The config file of ignored tags */
83 public static final String IGNORE_FILE = "resource://data/validator/ignoretags.cfg";
84 /** The config file of dictionary words */
85 public static final String SPELL_FILE = "resource://data/validator/words.cfg";
86
87 /** Normalized keys: the key should be substituted by the value if the key was not found in presets */
88 private static final Map<String, String> harmonizedKeys = new HashMap<>();
89 /** The spell check preset values which are not stored in TaggingPresets */
90 private static volatile HashSet<String> additionalPresetsValueData;
91 /** often used tags which are not in presets */
92 private static final MultiMap<String, String> oftenUsedTags = new MultiMap<>();
93 private static final Map<TaggingPreset, List<TaggingPresetItem>> presetIndex = new LinkedHashMap<>();
94
95 private static final Pattern UNWANTED_NON_PRINTING_CONTROL_CHARACTERS = Pattern.compile(
96 "[\\x00-\\x09\\x0B\\x0C\\x0E-\\x1F\\x7F\\u200e-\\u200f\\u202a-\\u202e]");
97
98 /** The TagChecker data */
99 private static final List<String> ignoreDataStartsWith = new ArrayList<>();
100 private static final Set<String> ignoreDataEquals = new HashSet<>();
101 private static final List<String> ignoreDataEndsWith = new ArrayList<>();
102 private static final List<Tag> ignoreDataTag = new ArrayList<>();
103 /** tag keys that have only numerical values in the presets */
104 private static final Set<String> ignoreForLevenshtein = new HashSet<>();
105
106 /** tag keys that are allowed to be the same on a multipolygon and an outer way */
107 private static final Set<String> ignoreForOuterMPSameTagCheck = new HashSet<>();
108
109 /** The preferences prefix */
110 protected static final String PREFIX = ValidatorPrefHelper.PREFIX + "." + TagChecker.class.getSimpleName();
111
112 MapCSSTagChecker deprecatedChecker;
113
114 /**
115 * The preference key to check values
116 */
117 public static final String PREF_CHECK_VALUES = PREFIX + ".checkValues";
118 /**
119 * The preference key to check keys
120 */
121 public static final String PREF_CHECK_KEYS = PREFIX + ".checkKeys";
122 /**
123 * The preference key to enable complex checks
124 */
125 public static final String PREF_CHECK_COMPLEX = PREFIX + ".checkComplex";
126 /**
127 * The preference key to search for fixme tags
128 */
129 public static final String PREF_CHECK_FIXMES = PREFIX + ".checkFixmes";
130 /**
131 * The preference key to check presets
132 */
133 public static final String PREF_CHECK_PRESETS_TYPES = PREFIX + ".checkPresetsTypes";
134
135 /**
136 * The preference key for source files
137 * @see #DEFAULT_SOURCES
138 */
139 public static final String PREF_SOURCES = PREFIX + ".source";
140
141 private static final String BEFORE_UPLOAD = "BeforeUpload";
142 /**
143 * The preference key to check keys - used before upload
144 */
145 public static final String PREF_CHECK_KEYS_BEFORE_UPLOAD = PREF_CHECK_KEYS + BEFORE_UPLOAD;
146 /**
147 * The preference key to check values - used before upload
148 */
149 public static final String PREF_CHECK_VALUES_BEFORE_UPLOAD = PREF_CHECK_VALUES + BEFORE_UPLOAD;
150 /**
151 * The preference key to run complex tests - used before upload
152 */
153 public static final String PREF_CHECK_COMPLEX_BEFORE_UPLOAD = PREF_CHECK_COMPLEX + BEFORE_UPLOAD;
154 /**
155 * The preference key to search for fixmes - used before upload
156 */
157 public static final String PREF_CHECK_FIXMES_BEFORE_UPLOAD = PREF_CHECK_FIXMES + BEFORE_UPLOAD;
158 /**
159 * The preference key to search for presets - used before upload
160 */
161 public static final String PREF_CHECK_PRESETS_TYPES_BEFORE_UPLOAD = PREF_CHECK_PRESETS_TYPES + BEFORE_UPLOAD;
162
163 /**
164 * The preference key for the list of tag keys that are allowed to be the same on a multipolygon and an outer way
165 */
166 public static final String PREF_KEYS_IGNORE_OUTER_MP_SAME_TAG = PREFIX + ".ignore-keys-outer-mp-same-tag";
167
168 private static final int MAX_LEVENSHTEIN_DISTANCE = 2;
169
170 protected boolean includeOtherSeverity;
171
172 protected boolean checkKeys;
173 protected boolean checkValues;
174 /** Was used for special configuration file, might be used to disable value spell checker. */
175 protected boolean checkComplex;
176 protected boolean checkFixmes;
177 protected boolean checkPresetsTypes;
178
179 protected JCheckBox prefCheckKeys;
180 protected JCheckBox prefCheckValues;
181 protected JCheckBox prefCheckComplex;
182 protected JCheckBox prefCheckFixmes;
183 protected JCheckBox prefCheckPresetsTypes;
184
185 protected JCheckBox prefCheckKeysBeforeUpload;
186 protected JCheckBox prefCheckValuesBeforeUpload;
187 protected JCheckBox prefCheckComplexBeforeUpload;
188 protected JCheckBox prefCheckFixmesBeforeUpload;
189 protected JCheckBox prefCheckPresetsTypesBeforeUpload;
190
191 // CHECKSTYLE.OFF: SingleSpaceSeparator
192 protected static final int EMPTY_VALUES = 1200;
193 protected static final int INVALID_KEY = 1201;
194 protected static final int INVALID_VALUE = 1202;
195 protected static final int FIXME = 1203;
196 protected static final int INVALID_SPACE = 1204;
197 protected static final int INVALID_KEY_SPACE = 1205;
198 protected static final int INVALID_HTML = 1206; /* 1207 was PAINT */
199 protected static final int LONG_VALUE = 1208;
200 protected static final int LONG_KEY = 1209;
201 protected static final int LOW_CHAR_VALUE = 1210;
202 protected static final int LOW_CHAR_KEY = 1211;
203 protected static final int MISSPELLED_VALUE = 1212;
204 protected static final int MISSPELLED_KEY = 1213;
205 protected static final int MULTIPLE_SPACES = 1214;
206 protected static final int MISSPELLED_VALUE_NO_FIX = 1215;
207 protected static final int UNUSUAL_UNICODE_CHAR_VALUE = 1216;
208 protected static final int INVALID_PRESETS_TYPE = 1217;
209 protected static final int MULTIPOLYGON_NO_AREA = 1218;
210 protected static final int MULTIPOLYGON_INCOMPLETE = 1219;
211 protected static final int MULTIPOLYGON_MAYBE_NO_AREA = 1220;
212 protected static final int MULTIPOLYGON_SAME_TAG_ON_OUTER = 1221;
213 // CHECKSTYLE.ON: SingleSpaceSeparator
214
215 protected EditableList sourcesList;
216
217 private static final List<String> DEFAULT_SOURCES = Arrays.asList(IGNORE_FILE, SPELL_FILE);
218
219 /**
220 * Constructor
221 */
222 public TagChecker() {
223 super(tr("Tag checker"), tr("This test checks for errors in tag keys and values."));
224 }
225
226 @Override
227 public void initialize() throws IOException {
228 TaggingPresets.addListener(this);
229 initializeData();
230 initializePresets();
231 analysePresets();
232 }
233
234 /**
235 * Add presets that contain only numerical values to the ignore list
236 */
237 private static void analysePresets() {
238 for (String key : TaggingPresets.getPresetKeys()) {
239 if (isKeyIgnored(key))
240 continue;
241 Set<String> values = TaggingPresets.getPresetValues(key);
242 boolean allNumerical = values != null && !values.isEmpty()
243 && values.stream().allMatch(TagChecker::isNum);
244 if (allNumerical) {
245 ignoreForLevenshtein.add(key);
246 }
247 }
248 }
249
250 /**
251 * Reads the spell-check file into a HashMap.
252 * The data file is a list of words, beginning with +/-. If it starts with +,
253 * the word is valid, but if it starts with -, the word should be replaced
254 * by the nearest + word before this.
255 *
256 * @throws IOException if any I/O error occurs
257 */
258 private static void initializeData() throws IOException {
259 ignoreDataStartsWith.clear();
260 ignoreDataEquals.clear();
261 ignoreDataEndsWith.clear();
262 ignoreDataTag.clear();
263 harmonizedKeys.clear();
264 ignoreForLevenshtein.clear();
265 oftenUsedTags.clear();
266 presetIndex.clear();
267 ignoreForOuterMPSameTagCheck.clear();
268
269 StringBuilder errorSources = new StringBuilder();
270 for (String source : Config.getPref().getList(PREF_SOURCES, DEFAULT_SOURCES)) {
271 try (
272 CachedFile cf = new CachedFile(source);
273 BufferedReader reader = cf.getContentReader()
274 ) {
275 String okValue = null;
276 boolean tagcheckerfile = false;
277 boolean ignorefile = false;
278 boolean isFirstLine = true;
279 String line;
280 while ((line = reader.readLine()) != null) {
281 if (line.isEmpty()) {
282 // ignore
283 } else if (line.startsWith("#")) {
284 if (line.startsWith("# JOSM TagChecker")) {
285 tagcheckerfile = true;
286 Logging.error(tr("Ignoring {0}. Support was dropped", source));
287 } else
288 if (line.startsWith("# JOSM IgnoreTags")) {
289 ignorefile = true;
290 if (!DEFAULT_SOURCES.contains(source)) {
291 Logging.info(tr("Adding {0} to ignore tags", source));
292 }
293 }
294 } else if (ignorefile) {
295 parseIgnoreFileLine(source, line);
296 } else if (tagcheckerfile) {
297 // ignore
298 } else if (line.charAt(0) == '+') {
299 okValue = line.substring(1);
300 } else if (line.charAt(0) == '-' && okValue != null) {
301 String hk = harmonizeKey(line.substring(1));
302 if (!okValue.equals(hk) && harmonizedKeys.put(hk, okValue) != null && Logging.isDebugEnabled()) {
303 Logging.debug("Line was ignored: " + line);
304 }
305 } else {
306 Logging.error(tr("Invalid spellcheck line: {0}", line));
307 }
308 if (isFirstLine) {
309 isFirstLine = false;
310 if (!(tagcheckerfile || ignorefile) && !DEFAULT_SOURCES.contains(source)) {
311 Logging.info(tr("Adding {0} to spellchecker", source));
312 }
313 }
314 }
315 } catch (IOException e) {
316 Logging.error(e);
317 errorSources.append(source).append('\n');
318 }
319 }
320
321 if (errorSources.length() > 0)
322 throw new IOException(trn(
323 "Could not access data file:\n{0}",
324 "Could not access data files:\n{0}", errorSources.length(), errorSources));
325 }
326
327 /**
328 * Parse a line found in a configuration file
329 * @param source name of configuration file
330 * @param line the line to parse
331 */
332 private static void parseIgnoreFileLine(String source, String line) {
333 line = line.trim();
334 if (line.length() < 4) {
335 return;
336 }
337 try {
338 String key = line.substring(0, 2);
339 line = line.substring(2);
340
341 switch (key) {
342 case "S:":
343 ignoreDataStartsWith.add(line);
344 break;
345 case "E:":
346 ignoreDataEquals.add(line);
347 addToKeyDictionary(line);
348 break;
349 case "F:":
350 ignoreDataEndsWith.add(line);
351 break;
352 case "K:":
353 Tag tag = Tag.ofString(line);
354 ignoreDataTag.add(tag);
355 oftenUsedTags.put(tag.getKey(), tag.getValue());
356 addToKeyDictionary(tag.getKey());
357 break;
358 default:
359 if (!key.startsWith(";")) {
360 Logging.warn("Unsupported TagChecker key: " + key);
361 }
362 }
363 } catch (IllegalArgumentException e) {
364 Logging.error("Invalid line in {0} : {1}", source, e.getMessage());
365 Logging.trace(e);
366 }
367 }
368
369 private static void addToKeyDictionary(String key) {
370 if (key != null) {
371 String hk = harmonizeKey(key);
372 if (!key.equals(hk)) {
373 harmonizedKeys.put(hk, key);
374 }
375 }
376 }
377
378 /**
379 * Reads the presets data.
380 *
381 */
382 public static void initializePresets() {
383
384 if (!Config.getPref().getBoolean(PREF_CHECK_VALUES, true))
385 return;
386
387 Collection<TaggingPreset> presets = TaggingPresets.getTaggingPresets();
388 if (!presets.isEmpty()) {
389 initAdditionalPresetsValueData();
390 for (TaggingPreset p : presets) {
391 List<TaggingPresetItem> minData = new ArrayList<>();
392 for (TaggingPresetItem i : p.data) {
393 if (i instanceof KeyedItem) {
394 if (!"none".equals(((KeyedItem) i).match))
395 minData.add(i);
396 addPresetValue((KeyedItem) i);
397 } else if (i instanceof CheckGroup) {
398 for (Check c : ((CheckGroup) i).checks) {
399 addPresetValue(c);
400 }
401 }
402 }
403 if (!minData.isEmpty()) {
404 presetIndex .put(p, minData);
405 }
406 }
407 }
408 }
409
410 private static void initAdditionalPresetsValueData() {
411 additionalPresetsValueData = new HashSet<>();
412 additionalPresetsValueData.addAll(AbstractPrimitive.getUninterestingKeys());
413 additionalPresetsValueData.addAll(Config.getPref().getList(
414 ValidatorPrefHelper.PREFIX + ".knownkeys",
415 Arrays.asList("is_in", "int_ref", "fixme", "population")));
416 }
417
418 private static void addPresetValue(KeyedItem ky) {
419 if (ky.key != null && ky.getValues() != null) {
420 addToKeyDictionary(ky.key);
421 }
422 }
423
424 /**
425 * Checks given string (key or value) if it contains unwanted non-printing control characters (either ASCII or Unicode bidi characters)
426 * @param s string to check
427 * @return {@code true} if {@code s} contains non-printing control characters
428 */
429 static boolean containsUnwantedNonPrintingControlCharacter(String s) {
430 return s != null && !s.isEmpty() && (
431 isJoiningChar(s.charAt(0)) ||
432 isJoiningChar(s.charAt(s.length() - 1)) ||
433 s.chars().anyMatch(c -> (isAsciiControlChar(c) && !isNewLineChar(c)) || isBidiControlChar(c))
434 );
435 }
436
437 private static boolean isAsciiControlChar(int c) {
438 return c < 0x20 || c == 0x7F;
439 }
440
441 private static boolean isNewLineChar(int c) {
442 return c == 0x0a || c == 0x0d;
443 }
444
445 private static boolean isJoiningChar(int c) {
446 return c == 0x200c || c == 0x200d; // ZWNJ, ZWJ
447 }
448
449 private static boolean isBidiControlChar(int c) {
450 /* check for range 0x200e to 0x200f (LRM, RLM) or
451 0x202a to 0x202e (LRE, RLE, PDF, LRO, RLO) */
452 return (c >= 0x200e && c <= 0x200f) || (c >= 0x202a && c <= 0x202e);
453 }
454
455 static String removeUnwantedNonPrintingControlCharacters(String s) {
456 // Remove all unwanted characters
457 String result = UNWANTED_NON_PRINTING_CONTROL_CHARACTERS.matcher(s).replaceAll("");
458 // Remove joining characters located at the beginning of the string
459 while (!result.isEmpty() && isJoiningChar(result.charAt(0))) {
460 result = result.substring(1);
461 }
462 // Remove joining characters located at the end of the string
463 while (!result.isEmpty() && isJoiningChar(result.charAt(result.length() - 1))) {
464 result = result.substring(0, result.length() - 1);
465 }
466 return result;
467 }
468
469 static boolean containsUnusualUnicodeCharacter(String key, String value) {
470 return getUnusualUnicodeCharacter(key, value).isPresent();
471 }
472
473 static OptionalInt getUnusualUnicodeCharacter(String key, String value) {
474 return value == null
475 ? OptionalInt.empty()
476 : value.chars().filter(c -> isUnusualUnicodeBlock(key, c)).findFirst();
477 }
478
479 /**
480 * Detects highly suspicious Unicode characters that have been seen in OSM database.
481 * @param key tag key
482 * @param c current character code point
483 * @return {@code true} if the current unicode block is very unusual for the given key
484 */
485 private static boolean isUnusualUnicodeBlock(String key, int c) {
486 UnicodeBlock b = UnicodeBlock.of(c);
487 return isUnusualPhoneticUse(key, b, c) || isUnusualBmpUse(b) || isUnusualSmpUse(b);
488 }
489
490 private static boolean isAllowedPhoneticCharacter(String key, int c) {
491 // CHECKSTYLE.OFF: BooleanExpressionComplexity
492 return c == 0x0259 || c == 0x018F // U+0259 is paired with the capital letter U+018F in Azeri, see #18740
493 || c == 0x0254 || c == 0x0186 // U+0254 is paired with the capital letter U+0186 in several African languages, see #18740
494 || c == 0x0257 || c == 0x018A // "ɗ/Ɗ" (U+0257/U+018A), see #19760
495 || c == 0x025B || c == 0x0190 // U+025B is paired with the capital letter U+0190 in several African languages, see #18740
496 || c == 0x0263 || c == 0x0194 // "ɣ/Ɣ" (U+0263/U+0194), see #18740
497 || c == 0x0268 || c == 0x0197 // "ɨ/Ɨ" (U+0268/U+0197), see #18740
498 || c == 0x0269 || c == 0x0196 // "ɩ/Ɩ" (U+0269/U+0196), see #20437
499 || c == 0x0272 || c == 0x019D // "ɲ/Ɲ" (U+0272/U+019D), see #18740
500 || c == 0x0273 || c == 0x019E // "ŋ/Ŋ" (U+0273/U+019E), see #18740
501 || (key.endsWith("ref") && 0x1D2C <= c && c <= 0x1D42); // allow uppercase superscript latin characters in *ref tags
502 }
503
504 private static boolean isUnusualPhoneticUse(String key, UnicodeBlock b, int c) {
505 return !isAllowedPhoneticCharacter(key, c)
506 && (b == UnicodeBlock.IPA_EXTENSIONS // U+0250..U+02AF
507 || b == UnicodeBlock.PHONETIC_EXTENSIONS // U+1D00..U+1D7F
508 || b == UnicodeBlock.PHONETIC_EXTENSIONS_SUPPLEMENT) // U+1D80..U+1DBF
509 && !key.endsWith(":pronunciation");
510 }
511
512 private static boolean isUnusualBmpUse(UnicodeBlock b) {
513 return b == UnicodeBlock.COMBINING_MARKS_FOR_SYMBOLS // U+20D0..U+20FF
514 || b == UnicodeBlock.MATHEMATICAL_OPERATORS // U+2200..U+22FF
515 || b == UnicodeBlock.ENCLOSED_ALPHANUMERICS // U+2460..U+24FF
516 || b == UnicodeBlock.BOX_DRAWING // U+2500..U+257F
517 || b == UnicodeBlock.GEOMETRIC_SHAPES // U+25A0..U+25FF
518 || b == UnicodeBlock.DINGBATS // U+2700..U+27BF
519 || b == UnicodeBlock.MISCELLANEOUS_SYMBOLS_AND_ARROWS // U+2B00..U+2BFF
520 || b == UnicodeBlock.GLAGOLITIC // U+2C00..U+2C5F
521 || b == UnicodeBlock.HANGUL_COMPATIBILITY_JAMO // U+3130..U+318F
522 || b == UnicodeBlock.ENCLOSED_CJK_LETTERS_AND_MONTHS // U+3200..U+32FF
523 || b == UnicodeBlock.LATIN_EXTENDED_D // U+A720..U+A7FF
524 || b == UnicodeBlock.CJK_COMPATIBILITY_IDEOGRAPHS // U+F900..U+FAFF
525 || b == UnicodeBlock.ALPHABETIC_PRESENTATION_FORMS // U+FB00..U+FB4F
526 || b == UnicodeBlock.VARIATION_SELECTORS // U+FE00..U+FE0F
527 || b == UnicodeBlock.SPECIALS; // U+FFF0..U+FFFF
528 // CHECKSTYLE.ON: BooleanExpressionComplexity
529 }
530
531 private static boolean isUnusualSmpUse(UnicodeBlock b) {
532 // UnicodeBlock.SUPPLEMENTAL_SYMBOLS_AND_PICTOGRAPHS is only defined in Java 9+
533 return b == UnicodeBlock.MUSICAL_SYMBOLS // U+1D100..U+1D1FF
534 || b == UnicodeBlock.ENCLOSED_ALPHANUMERIC_SUPPLEMENT // U+1F100..U+1F1FF
535 || b == UnicodeBlock.EMOTICONS // U+1F600..U+1F64F
536 || b == UnicodeBlock.TRANSPORT_AND_MAP_SYMBOLS; // U+1F680..U+1F6FF
537 }
538
539 /**
540 * Get set of preset values for the given key.
541 * @param key the key
542 * @return null if key is not in presets or in additionalPresetsValueData,
543 * else a set which might be empty.
544 */
545 private static Set<String> getPresetValues(String key) {
546 Set<String> res = TaggingPresets.getPresetValues(key);
547 if (res != null)
548 return res;
549 if (additionalPresetsValueData.contains(key))
550 return Collections.emptySet();
551 // null means key is not known
552 return null;
553 }
554
555 /**
556 * Determines if the given key is in internal presets.
557 * @param key key
558 * @return {@code true} if the given key is in internal presets
559 * @since 9023
560 */
561 public static boolean isKeyInPresets(String key) {
562 return TaggingPresets.getPresetValues(key) != null;
563 }
564
565 /**
566 * Determines if the given tag is in internal presets.
567 * @param key key
568 * @param value value
569 * @return {@code true} if the given tag is in internal presets
570 * @since 9023
571 */
572 public static boolean isTagInPresets(String key, String value) {
573 final Set<String> values = getPresetValues(key);
574 return values != null && values.contains(value);
575 }
576
577 /**
578 * Returns the list of ignored tags.
579 * @return the list of ignored tags
580 * @since 9023
581 */
582 public static List<Tag> getIgnoredTags() {
583 return new ArrayList<>(ignoreDataTag);
584 }
585
586 /**
587 * Determines if the given tag key is ignored for checks "key/tag not in presets".
588 * @param key key
589 * @return true if the given key is ignored
590 */
591 private static boolean isKeyIgnored(String key) {
592 return ignoreDataEquals.contains(key)
593 || ignoreDataStartsWith.stream().anyMatch(key::startsWith)
594 || ignoreDataEndsWith.stream().anyMatch(key::endsWith);
595 }
596
597 /**
598 * Determines if the given tag is ignored for checks "key/tag not in presets".
599 * @param key key
600 * @param value value
601 * @return {@code true} if the given tag is ignored
602 * @since 9023
603 */
604 public static boolean isTagIgnored(String key, String value) {
605 if (isKeyIgnored(key))
606 return true;
607 final Set<String> values = getPresetValues(key);
608 if (values != null && values.isEmpty())
609 return true;
610 if (!isTagInPresets(key, value)) {
611 return ignoreDataTag.stream()
612 .anyMatch(a -> key.equals(a.getKey()) && value.equals(a.getValue()));
613 }
614 return false;
615 }
616
617 /**
618 * Checks the primitive tags
619 * @param p The primitive to check
620 */
621 @Override
622 public void check(OsmPrimitive p) {
623 if (!p.isTagged())
624 return;
625
626 // Just a collection to know if a primitive has been already marked with error
627 MultiMap<OsmPrimitive, String> withErrors = new MultiMap<>();
628
629 for (Entry<String, String> prop : p.getKeys().entrySet()) {
630 String s = marktr("Tag ''{0}'' invalid.");
631 String key = prop.getKey();
632 String value = prop.getValue();
633
634 if (checkKeys) {
635 checkSingleTagKeySimple(withErrors, p, s, key);
636 }
637 if (checkValues) {
638 checkSingleTagValueSimple(withErrors, p, s, key, value);
639 checkSingleTagComplex(withErrors, p, key, value);
640 }
641 if (checkFixmes && key != null && value != null && !value.isEmpty() && isFixme(key, value) && !withErrors.contains(p, "FIXME")) {
642 errors.add(TestError.builder(this, Severity.OTHER, FIXME)
643 .message(tr("fixme"))
644 .primitives(p)
645 .build());
646 withErrors.put(p, "FIXME");
647 }
648 }
649
650 if (p instanceof Relation && p.hasTag("type", "multipolygon")) {
651 checkMultipolygonTags(p);
652 }
653
654 if (checkPresetsTypes) {
655 TagMap tags = p.getKeys();
656 TaggingPresetType presetType = TaggingPresetType.forPrimitive(p);
657 EnumSet<TaggingPresetType> presetTypes = EnumSet.of(presetType);
658
659 Collection<TaggingPreset> matchingPresets = presetIndex.entrySet().stream()
660 .filter(e -> TaggingPresetItem.matches(e.getValue(), tags))
661 .map(Entry::getKey)
662 .collect(Collectors.toCollection(LinkedHashSet::new));
663 Collection<TaggingPreset> matchingPresetsOK = matchingPresets.stream().filter(
664 tp -> tp.typeMatches(presetTypes)).collect(Collectors.toList());
665 Collection<TaggingPreset> matchingPresetsKO = matchingPresets.stream().filter(
666 tp -> !tp.typeMatches(presetTypes)).collect(Collectors.toList());
667
668 for (TaggingPreset tp : matchingPresetsKO) {
669 // Potential error, unless matching tags are all known by a supported preset
670 Map<String, String> matchingTags = tp.data.stream()
671 .filter(i -> Boolean.TRUE.equals(i.matches(tags)))
672 .filter(i -> i instanceof KeyedItem).map(i -> ((KeyedItem) i).key)
673 .collect(Collectors.toMap(k -> k, tags::get));
674 if (matchingPresetsOK.stream().noneMatch(
675 tp2 -> matchingTags.entrySet().stream().allMatch(
676 e -> tp2.data.stream().anyMatch(
677 i -> i instanceof KeyedItem && ((KeyedItem) i).key.equals(e.getKey()))))) {
678 errors.add(TestError.builder(this, Severity.OTHER, INVALID_PRESETS_TYPE)
679 .message(tr("Object type not in preset"),
680 marktr("Object type {0} is not supported by tagging preset: {1}"),
681 tr(presetType.getName()), tp.getLocaleName())
682 .primitives(p)
683 .build());
684 }
685 }
686 }
687 }
688
689 private static final Collection<String> NO_AREA_KEYS = Arrays.asList("name", "area", "ref", "access", "operator");
690
691 private void checkMultipolygonTags(OsmPrimitive p) {
692 if (p.isAnnotated() || p.keys()
693 .anyMatch(k -> k.matches("^(abandoned|construction|demolished|disused|planned|razed|removed|was).*")))
694 return;
695
696 checkOuterWaysOfRelation((Relation) p);
697
698 if (hasAcceptedPrimaryTagForMultipolygon(p))
699 return;
700 TestError.Builder builder = null;
701 if (p.hasKey("surface")) {
702 // accept often used tag surface=* as area tag
703 builder = TestError.builder(this, Severity.OTHER, MULTIPOLYGON_INCOMPLETE)
704 .message(tr("Multipolygon tags"), marktr("only {0} tag"), "surface");
705 } else {
706 Map<String, String> filteredTags = p.getInterestingTags();
707 filteredTags.remove("type");
708 NO_AREA_KEYS.forEach(filteredTags::remove);
709 filteredTags.keySet().removeIf(key -> !key.matches("[a-z0-9:_]+"));
710
711 if (filteredTags.isEmpty()) {
712 builder = TestError.builder(this, Severity.ERROR, MULTIPOLYGON_NO_AREA)
713 .message(tr("Multipolygon tags"), marktr("tag describing the area is missing"), new Object());
714
715 }
716 }
717 if (builder == null) {
718 // multipolygon has either no area tag or a rarely used one
719 builder = TestError.builder(this, Severity.WARNING, MULTIPOLYGON_MAYBE_NO_AREA)
720 .message(tr("Multipolygon tags"), marktr("tag describing the area might be missing"), new Object());
721 }
722 errors.add(builder.primitives(p).build());
723 }
724
725 /**
726 * Check if an outer way of the relation has the same tag as the relation.
727 * @param rel the relation
728 */
729 private void checkOuterWaysOfRelation(Relation rel) {
730 for (Entry<String, String> tag : rel.getInterestingTags().entrySet()) {
731 if (ignoreForOuterMPSameTagCheck.contains(tag.getKey()))
732 continue;
733
734 Set<Way> sameOuters = rel.getMembers().stream()
735 .filter(rm -> rm.isWay() && rm.getWay().isArea() && "outer".equals(rm.getRole())
736 && tag.getValue().equals(rm.getWay().get(tag.getKey())))
737 .map(RelationMember::getWay).collect(Collectors.toSet());
738 if (!sameOuters.isEmpty()) {
739 List<OsmPrimitive> primitives = new ArrayList<>(sameOuters.size() + 1);
740 primitives.add(rel);
741 primitives.addAll(sameOuters);
742 Way w = new Way();
743 w.put(tag.getKey(), tag.getValue());
744 if (hasAcceptedPrimaryTagForMultipolygon(w)) {
745 errors.add(TestError.builder(this, Severity.WARNING, MULTIPOLYGON_SAME_TAG_ON_OUTER)
746 .message(tr("Multipolygon outer way repeats major tag of relation"),
747 marktr("Same tag:''{0}''=''{1}''"), tag.getKey(), tag.getValue())
748 .primitives(primitives)
749 .build());
750 } else {
751 errors.add(TestError.builder(this, Severity.OTHER, MULTIPOLYGON_SAME_TAG_ON_OUTER)
752 .message(tr("Multipolygon outer way repeats tag of relation"),
753 marktr("Same tag:''{0}''=''{1}''"), tag.getKey(), tag.getValue())
754 .primitives(primitives)
755 .build());
756 }
757 }
758 }
759 }
760
761 /**
762 * Check if a multipolygon has a main tag that describes the type of area. Accepts also some deprecated tags and typos.
763 * @param p the multipolygon
764 * @return true if the multipolygon has a main tag that (likely) describes the type of area.
765 */
766 private static boolean hasAcceptedPrimaryTagForMultipolygon(OsmPrimitive p) {
767 if (p.hasKey("landuse", "amenity", "building", "building:part", "area:highway", "shop", "place", "boundary",
768 "landform", "piste:type", "sport", "golf", "landcover", "aeroway", "office", "healthcare", "craft", "room")
769 || p.hasTagDifferent("natural", "tree", "peek", "saddle", "tree_row")
770 || p.hasTagDifferent("man_made", "survey_point", "mast", "flagpole", "manhole", "watertap")
771 || p.hasTagDifferent("highway", "crossing", "bus_stop", "turning_circle", "street_lamp",
772 "traffic_signals", "stop", "milestone", "mini_roundabout", "motorway_junction", "passing_place",
773 "speed_camera", "traffic_mirror", "trailhead", "turning_circle", "turning_loop", "toll_gantry")
774 || p.hasTagDifferent("tourism", "attraction", "artwork")
775 || p.hasTagDifferent("leisure", "picnic_table", "slipway", "firepit")
776 || p.hasTagDifferent("historic", "wayside_cross", "milestone"))
777 return true;
778 if (p.hasTag("barrier", "hedge", "retaining_wall")
779 || p.hasTag("public_transport", "platform", "station")
780 || p.hasTag("railway", "platform")
781 || p.hasTag("waterway", "riverbank", "dam", "rapids", "dock", "boatyard", "fuel")
782 || p.hasTag("indoor", "corridor", "room", "area")
783 || p.hasTag("power", "substation", "generator", "plant", "switchgear", "converter", "sub_station")
784 || p.hasTag("seamark:type", "harbour", "fairway", "anchorage", "landmark", "berth", "harbour_basin",
785 "separation_zone")
786 || (p.get("seamark:type") != null && p.get("seamark:type").matches(".*\\_(area|zone)$")))
787 return true;
788 return p.hasTag("harbour", OsmUtils.TRUE_VALUE)
789 || p.hasTag("flood_prone", OsmUtils.TRUE_VALUE)
790 || p.hasTag("bridge", OsmUtils.TRUE_VALUE)
791 || p.hasTag("ruins", OsmUtils.TRUE_VALUE)
792 || p.hasTag("junction", OsmUtils.TRUE_VALUE);
793 }
794
795 private void checkSingleTagValueSimple(MultiMap<OsmPrimitive, String> withErrors, OsmPrimitive p, String s, String key, String value) {
796 if (!checkValues || value == null)
797 return;
798 if (containsUnwantedNonPrintingControlCharacter(value) && !withErrors.contains(p, "ICV")) {
799 errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_VALUE)
800 .message(tr("Tag value contains non-printing (usually invisible) character"), s, key)
801 .primitives(p)
802 .fix(() -> new ChangePropertyCommand(p, key, removeUnwantedNonPrintingControlCharacters(value)))
803 .build());
804 withErrors.put(p, "ICV");
805 }
806 final OptionalInt unusualUnicodeCharacter = getUnusualUnicodeCharacter(key, value);
807 if (unusualUnicodeCharacter.isPresent() && !withErrors.contains(p, "UUCV")) {
808 final String codepoint = String.format(Locale.ROOT, "U+%04X", unusualUnicodeCharacter.getAsInt());
809 errors.add(TestError.builder(this, Severity.WARNING, UNUSUAL_UNICODE_CHAR_VALUE)
810 .message(tr("Tag value contains unusual Unicode character {0}", codepoint), s, key)
811 .primitives(p)
812 .build());
813 withErrors.put(p, "UUCV");
814 }
815 if ((value.length() > Tagged.MAX_TAG_LENGTH) && !withErrors.contains(p, "LV")) {
816 errors.add(TestError.builder(this, Severity.ERROR, LONG_VALUE)
817 .message(tr("Tag value longer than {0} characters ({1} characters)", Tagged.MAX_TAG_LENGTH, value.length()), s, key)
818 .primitives(p)
819 .build());
820 withErrors.put(p, "LV");
821 }
822 if (value.trim().isEmpty() && !withErrors.contains(p, "EV")) {
823 errors.add(TestError.builder(this, Severity.WARNING, EMPTY_VALUES)
824 .message(tr("Tags with empty values"), s, key)
825 .primitives(p)
826 .build());
827 withErrors.put(p, "EV");
828 }
829 final String errTypeSpace = "SPACE";
830 if ((value.startsWith(" ") || value.endsWith(" ")) && !withErrors.contains(p, errTypeSpace)) {
831 errors.add(TestError.builder(this, Severity.WARNING, INVALID_SPACE)
832 .message(tr("Property values start or end with white space"), s, key)
833 .primitives(p)
834 .build());
835 withErrors.put(p, errTypeSpace);
836 }
837 if (value.contains(" ") && !withErrors.contains(p, errTypeSpace)) {
838 errors.add(TestError.builder(this, Severity.WARNING, MULTIPLE_SPACES)
839 .message(tr("Property values contain multiple white spaces"), s, key)
840 .primitives(p)
841 .build());
842 withErrors.put(p, errTypeSpace);
843 }
844 if (includeOtherSeverity && !value.equals(Entities.unescape(value)) && !withErrors.contains(p, "HTML")) {
845 errors.add(TestError.builder(this, Severity.OTHER, INVALID_HTML)
846 .message(tr("Property values contain HTML entity"), s, key)
847 .primitives(p)
848 .build());
849 withErrors.put(p, "HTML");
850 }
851 }
852
853 private void checkSingleTagKeySimple(MultiMap<OsmPrimitive, String> withErrors, OsmPrimitive p, String s, String key) {
854 if (!checkKeys || key == null)
855 return;
856 if (containsUnwantedNonPrintingControlCharacter(key) && !withErrors.contains(p, "ICK")) {
857 errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_KEY)
858 .message(tr("Tag key contains non-printing character"), s, key)
859 .primitives(p)
860 .fix(() -> new ChangePropertyCommand(p, key, removeUnwantedNonPrintingControlCharacters(key)))
861 .build());
862 withErrors.put(p, "ICK");
863 }
864 if (key.length() > Tagged.MAX_TAG_LENGTH && !withErrors.contains(p, "LK")) {
865 errors.add(TestError.builder(this, Severity.ERROR, LONG_KEY)
866 .message(tr("Tag key longer than {0} characters ({1} characters)", Tagged.MAX_TAG_LENGTH, key.length()), s, key)
867 .primitives(p)
868 .build());
869 withErrors.put(p, "LK");
870 }
871 if (key.indexOf(' ') >= 0 && !withErrors.contains(p, "IPK")) {
872 errors.add(TestError.builder(this, Severity.WARNING, INVALID_KEY_SPACE)
873 .message(tr("Invalid white space in property key"), s, key)
874 .primitives(p)
875 .build());
876 withErrors.put(p, "IPK");
877 }
878 }
879
880 private void checkSingleTagComplex(MultiMap<OsmPrimitive, String> withErrors, OsmPrimitive p, String key, String value) {
881 if (!checkValues || key == null || value == null || value.isEmpty())
882 return;
883 if (additionalPresetsValueData != null && !isTagIgnored(key, value)) {
884 if (!isKeyInPresets(key)) {
885 spellCheckKey(withErrors, p, key);
886 } else if (!isTagInPresets(key, value)) {
887 if (oftenUsedTags.contains(key, value)) {
888 // tag is quite often used but not in presets
889 errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE)
890 .message(tr("Presets do not contain property value"),
891 marktr("Value ''{0}'' for key ''{1}'' not in presets, but is known."), value, key)
892 .primitives(p)
893 .build());
894 withErrors.put(p, "UPV");
895 } else {
896 tryGuess(p, key, value, withErrors);
897 }
898 }
899 }
900 }
901
902 private void spellCheckKey(MultiMap<OsmPrimitive, String> withErrors, OsmPrimitive p, String key) {
903 String prettifiedKey = harmonizeKey(key);
904 String fixedKey;
905 if (ignoreDataEquals.contains(prettifiedKey)) {
906 fixedKey = prettifiedKey;
907 } else {
908 fixedKey = isKeyInPresets(prettifiedKey) ? prettifiedKey : harmonizedKeys.get(prettifiedKey);
909 }
910 if (fixedKey == null && ignoreDataTag.stream().anyMatch(a -> a.getKey().equals(prettifiedKey))) {
911 fixedKey = prettifiedKey;
912 }
913
914 if (fixedKey != null && !"".equals(fixedKey) && !fixedKey.equals(key)) {
915 final String proposedKey = fixedKey;
916 // misspelled preset key
917 final TestError.Builder error = TestError.builder(this, Severity.WARNING, MISSPELLED_KEY)
918 .message(tr("Misspelled property key"), marktr("Key ''{0}'' looks like ''{1}''."), key, proposedKey)
919 .primitives(p);
920 if (p.hasKey(fixedKey)) {
921 errors.add(error.build());
922 } else {
923 errors.add(error.fix(() -> new ChangePropertyKeyCommand(p, key, proposedKey)).build());
924 }
925 withErrors.put(p, "WPK");
926 } else if (includeOtherSeverity) {
927 errors.add(TestError.builder(this, Severity.OTHER, INVALID_KEY)
928 .message(tr("Presets do not contain property key"), marktr("Key ''{0}'' not in presets."), key)
929 .primitives(p)
930 .build());
931 withErrors.put(p, "UPK");
932 }
933 }
934
935 private void tryGuess(OsmPrimitive p, String key, String value, MultiMap<OsmPrimitive, String> withErrors) {
936 // try to fix common typos and check again if value is still unknown
937 final String harmonizedValue = harmonizeValue(value);
938 if (harmonizedValue == null || harmonizedValue.isEmpty())
939 return;
940 String fixedValue;
941 List<Set<String>> sets = new ArrayList<>();
942 Set<String> presetValues = getPresetValues(key);
943 if (presetValues != null)
944 sets.add(presetValues);
945 Set<String> usedValues = oftenUsedTags.get(key);
946 if (usedValues != null)
947 sets.add(usedValues);
948 fixedValue = sets.stream().anyMatch(possibleValues -> possibleValues.contains(harmonizedValue))
949 ? harmonizedValue : null;
950 if (fixedValue == null && !ignoreForLevenshtein.contains(key)) {
951 int maxPresetValueLen = 0;
952 List<String> fixVals = new ArrayList<>();
953 // use Levenshtein distance to find typical typos
954 int minDist = MAX_LEVENSHTEIN_DISTANCE + 1;
955 for (Set<String> possibleValues: sets) {
956 for (String possibleVal : possibleValues) {
957 if (possibleVal.isEmpty())
958 continue;
959 maxPresetValueLen = Math.max(maxPresetValueLen, possibleVal.length());
960 if (harmonizedValue.length() < 3 && possibleVal.length() >= harmonizedValue.length() + MAX_LEVENSHTEIN_DISTANCE) {
961 // don't suggest fix value when given value is short and lengths are too different
962 // for example surface=u would result in surface=mud
963 continue;
964 }
965 int dist = Utils.getLevenshteinDistance(possibleVal, harmonizedValue);
966 if (dist >= harmonizedValue.length()) {
967 // short value, all characters are different. Don't warn, might say Value '10' for key 'fee' looks like 'no'.
968 continue;
969 }
970 if (dist < minDist) {
971 minDist = dist;
972 fixVals.clear();
973 fixVals.add(possibleVal);
974 } else if (dist == minDist) {
975 fixVals.add(possibleVal);
976 }
977 }
978 }
979 if (minDist <= MAX_LEVENSHTEIN_DISTANCE && maxPresetValueLen > MAX_LEVENSHTEIN_DISTANCE
980 && !fixVals.isEmpty()
981 && (harmonizedValue.length() > 3 || minDist < MAX_LEVENSHTEIN_DISTANCE)) {
982 filterDeprecatedTags(p, key, fixVals);
983 if (!fixVals.isEmpty()) {
984 if (fixVals.size() < 2) {
985 fixedValue = fixVals.get(0);
986 } else {
987 Collections.sort(fixVals);
988 // misspelled preset value with multiple good alternatives
989 errors.add(TestError.builder(this, Severity.WARNING, MISSPELLED_VALUE_NO_FIX)
990 .message(tr("Unknown property value"),
991 marktr("Value ''{0}'' for key ''{1}'' is unknown, maybe one of {2} is meant?"),
992 value, key, fixVals)
993 .primitives(p).build());
994 withErrors.put(p, "WPV");
995 return;
996 }
997 }
998 }
999 }
1000 if (fixedValue != null && !fixedValue.equals(value)) {
1001 final String newValue = fixedValue;
1002 // misspelled preset value
1003 errors.add(TestError.builder(this, Severity.WARNING, MISSPELLED_VALUE)
1004 .message(tr("Unknown property value"),
1005 marktr("Value ''{0}'' for key ''{1}'' is unknown, maybe ''{2}'' is meant?"), value, key, newValue)
1006 .primitives(p)
1007 .build());
1008 withErrors.put(p, "WPV");
1009 } else if (includeOtherSeverity) {
1010 // unknown preset value
1011 errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE)
1012 .message(tr("Presets do not contain property value"),
1013 marktr("Value ''{0}'' for key ''{1}'' not in presets."), value, key)
1014 .primitives(p)
1015 .build());
1016 withErrors.put(p, "UPV");
1017 }
1018 }
1019
1020 // see #19180
1021 private void filterDeprecatedTags(OsmPrimitive p, String key, List<String> fixVals) {
1022 if (fixVals.isEmpty() || deprecatedChecker == null)
1023 return;
1024
1025 int unchangedDeprecated = countDeprecated(p);
1026
1027 // see #19895: create deep clone. This complex method works even with locked files
1028 MergeSourceBuildingVisitor builder = new MergeSourceBuildingVisitor(p.getDataSet());
1029 p.accept(builder);
1030 DataSet clonedDs = builder.build();
1031 OsmPrimitive clone = clonedDs.getPrimitiveById(p.getPrimitiveId());
1032
1033 Iterator<String> iter = fixVals.iterator();
1034 while (iter.hasNext()) {
1035 clone.put(key, iter.next());
1036 if (countDeprecated(clone) > unchangedDeprecated)
1037 iter.remove();
1038 }
1039 }
1040
1041 private int countDeprecated(OsmPrimitive p) {
1042 if (deprecatedChecker == null)
1043 return 0;
1044 deprecatedChecker.getErrors().clear();
1045 deprecatedChecker.visit(Collections.singleton(p), url -> url.endsWith("deprecated.mapcss"));
1046 return deprecatedChecker.getErrors().size();
1047 }
1048
1049 private static boolean isNum(String harmonizedValue) {
1050 try {
1051 Double.parseDouble(harmonizedValue);
1052 return true;
1053 } catch (NumberFormatException e) {
1054 return false;
1055 }
1056 }
1057
1058 private static boolean isFixme(String key, String value) {
1059 return key.toLowerCase(Locale.ENGLISH).contains("fixme") || key.contains("todo")
1060 || value.toLowerCase(Locale.ENGLISH).contains("fixme") || value.contains("check and delete");
1061 }
1062
1063 private static String harmonizeKey(String key) {
1064 return Utils.strip(key.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(':', '_').replace(' ', '_'), "-_;:,");
1065 }
1066
1067 private static String harmonizeValue(String value) {
1068 return Utils.strip(value.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(' ', '_'), "-_;:,");
1069 }
1070
1071 @Override
1072 public void startTest(ProgressMonitor monitor) {
1073 super.startTest(monitor);
1074 includeOtherSeverity = includeOtherSeverityChecks();
1075 checkKeys = Config.getPref().getBoolean(PREF_CHECK_KEYS, true);
1076 if (isBeforeUpload) {
1077 checkKeys = checkKeys && Config.getPref().getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true);
1078 }
1079
1080 checkValues = Config.getPref().getBoolean(PREF_CHECK_VALUES, true);
1081 if (isBeforeUpload) {
1082 checkValues = checkValues && Config.getPref().getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true);
1083 }
1084
1085 checkComplex = Config.getPref().getBoolean(PREF_CHECK_COMPLEX, true);
1086 if (isBeforeUpload) {
1087 checkComplex = checkComplex && Config.getPref().getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true);
1088 }
1089
1090 checkFixmes = includeOtherSeverity && Config.getPref().getBoolean(PREF_CHECK_FIXMES, true);
1091 if (isBeforeUpload) {
1092 checkFixmes = checkFixmes && Config.getPref().getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true);
1093 }
1094
1095 checkPresetsTypes = includeOtherSeverity && Config.getPref().getBoolean(PREF_CHECK_PRESETS_TYPES, true);
1096 if (isBeforeUpload) {
1097 checkPresetsTypes = checkPresetsTypes && Config.getPref().getBoolean(PREF_CHECK_PRESETS_TYPES_BEFORE_UPLOAD, true);
1098 }
1099 deprecatedChecker = OsmValidator.getTest(MapCSSTagChecker.class);
1100 ignoreForOuterMPSameTagCheck.addAll(Config.getPref().getList(PREF_KEYS_IGNORE_OUTER_MP_SAME_TAG, Collections.emptyList()));
1101 }
1102
1103 @Override
1104 public void endTest() {
1105 deprecatedChecker = null;
1106 super.endTest();
1107 }
1108
1109 @Override
1110 public void visit(Collection<OsmPrimitive> selection) {
1111 if (checkKeys || checkValues || checkComplex || checkFixmes || checkPresetsTypes) {
1112 super.visit(selection);
1113 }
1114 }
1115
1116 @Override
1117 public void addGui(JPanel testPanel) {
1118 GBC a = GBC.eol();
1119 a.anchor = GridBagConstraints.EAST;
1120
1121 testPanel.add(new JLabel(name+" :"), GBC.eol().insets(3, 0, 0, 0));
1122
1123 prefCheckKeys = new JCheckBox(tr("Check property keys."), Config.getPref().getBoolean(PREF_CHECK_KEYS, true));
1124 prefCheckKeys.setToolTipText(tr("Validate that property keys are valid checking against list of words."));
1125 testPanel.add(prefCheckKeys, GBC.std().insets(20, 0, 0, 0));
1126
1127 prefCheckKeysBeforeUpload = new JCheckBox();
1128 prefCheckKeysBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true));
1129 testPanel.add(prefCheckKeysBeforeUpload, a);
1130
1131 prefCheckComplex = new JCheckBox(tr("Use complex property checker."), Config.getPref().getBoolean(PREF_CHECK_COMPLEX, true));
1132 prefCheckComplex.setToolTipText(tr("Validate property values and tags using complex rules."));
1133 testPanel.add(prefCheckComplex, GBC.std().insets(20, 0, 0, 0));
1134
1135 prefCheckComplexBeforeUpload = new JCheckBox();
1136 prefCheckComplexBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true));
1137 testPanel.add(prefCheckComplexBeforeUpload, a);
1138
1139 final Collection<String> sources = Config.getPref().getList(PREF_SOURCES, DEFAULT_SOURCES);
1140 sourcesList = new EditableList(tr("TagChecker source"));
1141 sourcesList.setItems(sources);
1142 testPanel.add(new JLabel(tr("Data sources ({0})", "*.cfg")), GBC.eol().insets(23, 0, 0, 0));
1143 testPanel.add(sourcesList, GBC.eol().fill(GridBagConstraints.HORIZONTAL).insets(23, 0, 0, 0));
1144
1145 ActionListener disableCheckActionListener = e -> handlePrefEnable();
1146 prefCheckKeys.addActionListener(disableCheckActionListener);
1147 prefCheckKeysBeforeUpload.addActionListener(disableCheckActionListener);
1148 prefCheckComplex.addActionListener(disableCheckActionListener);
1149 prefCheckComplexBeforeUpload.addActionListener(disableCheckActionListener);
1150
1151 handlePrefEnable();
1152
1153 prefCheckValues = new JCheckBox(tr("Check property values."), Config.getPref().getBoolean(PREF_CHECK_VALUES, true));
1154 prefCheckValues.setToolTipText(tr("Validate that property values are valid checking against presets."));
1155 testPanel.add(prefCheckValues, GBC.std().insets(20, 0, 0, 0));
1156
1157 prefCheckValuesBeforeUpload = new JCheckBox();
1158 prefCheckValuesBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true));
1159 testPanel.add(prefCheckValuesBeforeUpload, a);
1160
1161 prefCheckFixmes = new JCheckBox(tr("Check for fixme."), Config.getPref().getBoolean(PREF_CHECK_FIXMES, true));
1162 prefCheckFixmes.setToolTipText(tr("Looks for nodes or ways with fixme in any property value."));
1163 testPanel.add(prefCheckFixmes, GBC.std().insets(20, 0, 0, 0));
1164
1165 prefCheckFixmesBeforeUpload = new JCheckBox();
1166 prefCheckFixmesBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true));
1167 testPanel.add(prefCheckFixmesBeforeUpload, a);
1168
1169 prefCheckPresetsTypes = new JCheckBox(tr("Check for presets types."), Config.getPref().getBoolean(PREF_CHECK_PRESETS_TYPES, true));
1170 prefCheckPresetsTypes.setToolTipText(tr("Validate that objects types are valid checking against presets."));
1171 testPanel.add(prefCheckPresetsTypes, GBC.std().insets(20, 0, 0, 0));
1172
1173 prefCheckPresetsTypesBeforeUpload = new JCheckBox();
1174 prefCheckPresetsTypesBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_PRESETS_TYPES_BEFORE_UPLOAD, true));
1175 testPanel.add(prefCheckPresetsTypesBeforeUpload, a);
1176 }
1177
1178 /**
1179 * Enables/disables the source list field
1180 */
1181 public void handlePrefEnable() {
1182 boolean selected = prefCheckKeys.isSelected() || prefCheckKeysBeforeUpload.isSelected()
1183 || prefCheckComplex.isSelected() || prefCheckComplexBeforeUpload.isSelected();
1184 sourcesList.setEnabled(selected);
1185 }
1186
1187 @Override
1188 public boolean ok() {
1189 enabled = prefCheckKeys.isSelected() || prefCheckValues.isSelected() || prefCheckComplex.isSelected() || prefCheckFixmes.isSelected();
1190 testBeforeUpload = prefCheckKeysBeforeUpload.isSelected() || prefCheckValuesBeforeUpload.isSelected()
1191 || prefCheckFixmesBeforeUpload.isSelected() || prefCheckComplexBeforeUpload.isSelected();
1192
1193 Config.getPref().putBoolean(PREF_CHECK_VALUES, prefCheckValues.isSelected());
1194 Config.getPref().putBoolean(PREF_CHECK_COMPLEX, prefCheckComplex.isSelected());
1195 Config.getPref().putBoolean(PREF_CHECK_KEYS, prefCheckKeys.isSelected());
1196 Config.getPref().putBoolean(PREF_CHECK_FIXMES, prefCheckFixmes.isSelected());
1197 Config.getPref().putBoolean(PREF_CHECK_PRESETS_TYPES, prefCheckPresetsTypes.isSelected());
1198 Config.getPref().putBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, prefCheckValuesBeforeUpload.isSelected());
1199 Config.getPref().putBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, prefCheckComplexBeforeUpload.isSelected());
1200 Config.getPref().putBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, prefCheckKeysBeforeUpload.isSelected());
1201 Config.getPref().putBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, prefCheckFixmesBeforeUpload.isSelected());
1202 Config.getPref().putBoolean(PREF_CHECK_PRESETS_TYPES_BEFORE_UPLOAD, prefCheckPresetsTypesBeforeUpload.isSelected());
1203 return Config.getPref().putList(PREF_SOURCES, sourcesList.getItems());
1204 }
1205
1206 @Override
1207 public Command fixError(TestError testError) {
1208 List<Command> commands = new ArrayList<>(50);
1209
1210 Collection<? extends OsmPrimitive> primitives = testError.getPrimitives();
1211 for (OsmPrimitive p : primitives) {
1212 Map<String, String> tags = p.getKeys();
1213 if (tags.isEmpty()) {
1214 continue;
1215 }
1216
1217 for (Entry<String, String> prop: tags.entrySet()) {
1218 String key = prop.getKey();
1219 String value = prop.getValue();
1220 if (value == null || value.trim().isEmpty()) {
1221 commands.add(new ChangePropertyCommand(p, key, null));
1222 } else if (value.startsWith(" ") || value.endsWith(" ") || value.contains(" ")) {
1223 commands.add(new ChangePropertyCommand(p, key, Utils.removeWhiteSpaces(value)));
1224 } else if (key.startsWith(" ") || key.endsWith(" ") || key.contains(" ")) {
1225 commands.add(new ChangePropertyKeyCommand(p, key, Utils.removeWhiteSpaces(key)));
1226 } else {
1227 String evalue = Entities.unescape(value);
1228 if (!evalue.equals(value)) {
1229 commands.add(new ChangePropertyCommand(p, key, evalue));
1230 }
1231 }
1232 }
1233 }
1234
1235 if (commands.isEmpty())
1236 return null;
1237 if (commands.size() == 1)
1238 return commands.get(0);
1239
1240 return new SequenceCommand(tr("Fix tags"), commands);
1241 }
1242
1243 @Override
1244 public boolean isFixable(TestError testError) {
1245 if (testError.getTester() instanceof TagChecker) {
1246 int code = testError.getCode();
1247 return code == EMPTY_VALUES || code == INVALID_SPACE ||
1248 code == INVALID_KEY_SPACE || code == INVALID_HTML ||
1249 code == MULTIPLE_SPACES;
1250 }
1251
1252 return false;
1253 }
1254
1255 @Override
1256 public void taggingPresetsModified() {
1257 try {
1258 initializeData();
1259 initializePresets();
1260 analysePresets();
1261 } catch (IOException e) {
1262 Logging.error(e);
1263 }
1264 }
1265}
Note: See TracBrowser for help on using the repository browser.