source: josm/trunk/src/org/openstreetmap/josm/data/validation/tests/TagChecker.java@ 18211

Last change on this file since 18211 was 18211, checked in by Don-vip, 3 years ago

global use of !Utils.isEmpty/isBlank
  • Property svn:eol-style set to native
File size: 58.2 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.validation.tests;
3
4import static org.openstreetmap.josm.tools.I18n.marktr;
5import static org.openstreetmap.josm.tools.I18n.tr;
6import static org.openstreetmap.josm.tools.I18n.trn;
7
8import java.awt.GridBagConstraints;
9import java.awt.event.ActionListener;
10import java.io.BufferedReader;
11import java.io.IOException;
12import java.lang.Character.UnicodeBlock;
13import java.util.ArrayList;
14import java.util.Arrays;
15import java.util.Collection;
16import java.util.Collections;
17import java.util.EnumSet;
18import java.util.HashMap;
19import java.util.HashSet;
20import java.util.Iterator;
21import java.util.LinkedHashMap;
22import java.util.LinkedHashSet;
23import java.util.List;
24import java.util.Locale;
25import java.util.Map;
26import java.util.Map.Entry;
27import java.util.OptionalInt;
28import java.util.Set;
29import java.util.regex.Pattern;
30import java.util.stream.Collectors;
31
32import javax.swing.JCheckBox;
33import javax.swing.JLabel;
34import javax.swing.JPanel;
35
36import org.openstreetmap.josm.command.ChangePropertyCommand;
37import org.openstreetmap.josm.command.ChangePropertyKeyCommand;
38import org.openstreetmap.josm.command.Command;
39import org.openstreetmap.josm.command.SequenceCommand;
40import org.openstreetmap.josm.data.osm.AbstractPrimitive;
41import org.openstreetmap.josm.data.osm.DataSet;
42import org.openstreetmap.josm.data.osm.OsmPrimitive;
43import org.openstreetmap.josm.data.osm.OsmUtils;
44import org.openstreetmap.josm.data.osm.Relation;
45import org.openstreetmap.josm.data.osm.RelationMember;
46import org.openstreetmap.josm.data.osm.Tag;
47import org.openstreetmap.josm.data.osm.TagMap;
48import org.openstreetmap.josm.data.osm.Tagged;
49import org.openstreetmap.josm.data.osm.Way;
50import org.openstreetmap.josm.data.osm.visitor.MergeSourceBuildingVisitor;
51import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper;
52import org.openstreetmap.josm.data.validation.OsmValidator;
53import org.openstreetmap.josm.data.validation.Severity;
54import org.openstreetmap.josm.data.validation.Test.TagTest;
55import org.openstreetmap.josm.data.validation.TestError;
56import org.openstreetmap.josm.data.validation.util.Entities;
57import org.openstreetmap.josm.gui.progress.ProgressMonitor;
58import org.openstreetmap.josm.gui.tagging.presets.TaggingPreset;
59import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetItem;
60import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetListener;
61import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetType;
62import org.openstreetmap.josm.gui.tagging.presets.TaggingPresets;
63import org.openstreetmap.josm.gui.tagging.presets.items.Check;
64import org.openstreetmap.josm.gui.tagging.presets.items.CheckGroup;
65import org.openstreetmap.josm.gui.tagging.presets.items.KeyedItem;
66import org.openstreetmap.josm.gui.widgets.EditableList;
67import org.openstreetmap.josm.io.CachedFile;
68import org.openstreetmap.josm.spi.preferences.Config;
69import org.openstreetmap.josm.tools.GBC;
70import org.openstreetmap.josm.tools.Logging;
71import org.openstreetmap.josm.tools.MultiMap;
72import org.openstreetmap.josm.tools.Utils;
73
74/**
75 * Check for misspelled or wrong tags
76 *
77 * @author frsantos
78 * @since 3669
79 */
80public class TagChecker extends TagTest implements TaggingPresetListener {
81
82 /** The config file of ignored tags */
83 public static final String IGNORE_FILE = "resource://data/validator/ignoretags.cfg";
84 /** The config file of dictionary words */
85 public static final String SPELL_FILE = "resource://data/validator/words.cfg";
86
87 /** Normalized keys: the key should be substituted by the value if the key was not found in presets */
88 private static final Map<String, String> harmonizedKeys = new HashMap<>();
89 /** The spell check preset values which are not stored in TaggingPresets */
90 private static volatile HashSet<String> additionalPresetsValueData;
91 /** often used tags which are not in presets */
92 private static final MultiMap<String, String> oftenUsedTags = new MultiMap<>();
93 private static final Map<TaggingPreset, List<TaggingPresetItem>> presetIndex = new LinkedHashMap<>();
94
95 private static final Pattern UNWANTED_NON_PRINTING_CONTROL_CHARACTERS = Pattern.compile(
96 "[\\x00-\\x09\\x0B\\x0C\\x0E-\\x1F\\x7F\\u200e-\\u200f\\u202a-\\u202e]");
97
98 /** The TagChecker data */
99 private static final List<String> ignoreDataStartsWith = new ArrayList<>();
100 private static final Set<String> ignoreDataEquals = new HashSet<>();
101 private static final List<String> ignoreDataEndsWith = new ArrayList<>();
102 private static final List<Tag> ignoreDataTag = new ArrayList<>();
103 /** tag keys that have only numerical values in the presets */
104 private static final Set<String> ignoreForLevenshtein = new HashSet<>();
105
106 /** tag keys that are allowed to be the same on a multipolygon and an outer way */
107 private static final Set<String> ignoreForOuterMPSameTagCheck = new HashSet<>();
108
109 /** The preferences prefix */
110 protected static final String PREFIX = ValidatorPrefHelper.PREFIX + "." + TagChecker.class.getSimpleName();
111
112 MapCSSTagChecker deprecatedChecker;
113
114 /**
115 * The preference key to check values
116 */
117 public static final String PREF_CHECK_VALUES = PREFIX + ".checkValues";
118 /**
119 * The preference key to check keys
120 */
121 public static final String PREF_CHECK_KEYS = PREFIX + ".checkKeys";
122 /**
123 * The preference key to enable complex checks
124 */
125 public static final String PREF_CHECK_COMPLEX = PREFIX + ".checkComplex";
126 /**
127 * The preference key to search for fixme tags
128 */
129 public static final String PREF_CHECK_FIXMES = PREFIX + ".checkFixmes";
130 /**
131 * The preference key to check presets
132 */
133 public static final String PREF_CHECK_PRESETS_TYPES = PREFIX + ".checkPresetsTypes";
134
135 /**
136 * The preference key for source files
137 * @see #DEFAULT_SOURCES
138 */
139 public static final String PREF_SOURCES = PREFIX + ".source";
140
141 private static final String BEFORE_UPLOAD = "BeforeUpload";
142 /**
143 * The preference key to check keys - used before upload
144 */
145 public static final String PREF_CHECK_KEYS_BEFORE_UPLOAD = PREF_CHECK_KEYS + BEFORE_UPLOAD;
146 /**
147 * The preference key to check values - used before upload
148 */
149 public static final String PREF_CHECK_VALUES_BEFORE_UPLOAD = PREF_CHECK_VALUES + BEFORE_UPLOAD;
150 /**
151 * The preference key to run complex tests - used before upload
152 */
153 public static final String PREF_CHECK_COMPLEX_BEFORE_UPLOAD = PREF_CHECK_COMPLEX + BEFORE_UPLOAD;
154 /**
155 * The preference key to search for fixmes - used before upload
156 */
157 public static final String PREF_CHECK_FIXMES_BEFORE_UPLOAD = PREF_CHECK_FIXMES + BEFORE_UPLOAD;
158 /**
159 * The preference key to search for presets - used before upload
160 */
161 public static final String PREF_CHECK_PRESETS_TYPES_BEFORE_UPLOAD = PREF_CHECK_PRESETS_TYPES + BEFORE_UPLOAD;
162
163 /**
164 * The preference key for the list of tag keys that are allowed to be the same on a multipolygon and an outer way
165 */
166 public static final String PREF_KEYS_IGNORE_OUTER_MP_SAME_TAG = PREFIX + ".ignore-keys-outer-mp-same-tag";
167
168 private static final int MAX_LEVENSHTEIN_DISTANCE = 2;
169
170 protected boolean includeOtherSeverity;
171
172 protected boolean checkKeys;
173 protected boolean checkValues;
174 /** Was used for special configuration file, might be used to disable value spell checker. */
175 protected boolean checkComplex;
176 protected boolean checkFixmes;
177 protected boolean checkPresetsTypes;
178
179 protected JCheckBox prefCheckKeys;
180 protected JCheckBox prefCheckValues;
181 protected JCheckBox prefCheckComplex;
182 protected JCheckBox prefCheckFixmes;
183 protected JCheckBox prefCheckPresetsTypes;
184
185 protected JCheckBox prefCheckKeysBeforeUpload;
186 protected JCheckBox prefCheckValuesBeforeUpload;
187 protected JCheckBox prefCheckComplexBeforeUpload;
188 protected JCheckBox prefCheckFixmesBeforeUpload;
189 protected JCheckBox prefCheckPresetsTypesBeforeUpload;
190
191 // CHECKSTYLE.OFF: SingleSpaceSeparator
192 protected static final int EMPTY_VALUES = 1200;
193 protected static final int INVALID_KEY = 1201;
194 protected static final int INVALID_VALUE = 1202;
195 protected static final int FIXME = 1203;
196 protected static final int INVALID_SPACE = 1204;
197 protected static final int INVALID_KEY_SPACE = 1205;
198 protected static final int INVALID_HTML = 1206; /* 1207 was PAINT */
199 protected static final int LONG_VALUE = 1208;
200 protected static final int LONG_KEY = 1209;
201 protected static final int LOW_CHAR_VALUE = 1210;
202 protected static final int LOW_CHAR_KEY = 1211;
203 protected static final int MISSPELLED_VALUE = 1212;
204 protected static final int MISSPELLED_KEY = 1213;
205 protected static final int MULTIPLE_SPACES = 1214;
206 protected static final int MISSPELLED_VALUE_NO_FIX = 1215;
207 protected static final int UNUSUAL_UNICODE_CHAR_VALUE = 1216;
208 protected static final int INVALID_PRESETS_TYPE = 1217;
209 protected static final int MULTIPOLYGON_NO_AREA = 1218;
210 protected static final int MULTIPOLYGON_INCOMPLETE = 1219;
211 protected static final int MULTIPOLYGON_MAYBE_NO_AREA = 1220;
212 protected static final int MULTIPOLYGON_SAME_TAG_ON_OUTER = 1221;
213 // CHECKSTYLE.ON: SingleSpaceSeparator
214
215 protected EditableList sourcesList;
216
217 private static final List<String> DEFAULT_SOURCES = Arrays.asList(IGNORE_FILE, SPELL_FILE);
218
219 /**
220 * Constructor
221 */
222 public TagChecker() {
223 super(tr("Tag checker"), tr("This test checks for errors in tag keys and values."));
224 }
225
226 @Override
227 public void initialize() throws IOException {
228 TaggingPresets.addListener(this);
229 initializeData();
230 initializePresets();
231 analysePresets();
232 }
233
234 /**
235 * Add presets that contain only numerical values to the ignore list
236 */
237 private static void analysePresets() {
238 for (String key : TaggingPresets.getPresetKeys()) {
239 if (isKeyIgnored(key))
240 continue;
241 Set<String> values = TaggingPresets.getPresetValues(key);
242 boolean allNumerical = !Utils.isEmpty(values)
243 && values.stream().allMatch(TagChecker::isNum);
244 if (allNumerical) {
245 ignoreForLevenshtein.add(key);
246 }
247 }
248 }
249
250 /**
251 * Reads the spell-check file into a HashMap.
252 * The data file is a list of words, beginning with +/-. If it starts with +,
253 * the word is valid, but if it starts with -, the word should be replaced
254 * by the nearest + word before this.
255 *
256 * @throws IOException if any I/O error occurs
257 */
258 private static void initializeData() throws IOException {
259 ignoreDataStartsWith.clear();
260 ignoreDataEquals.clear();
261 ignoreDataEndsWith.clear();
262 ignoreDataTag.clear();
263 harmonizedKeys.clear();
264 ignoreForLevenshtein.clear();
265 oftenUsedTags.clear();
266 presetIndex.clear();
267 ignoreForOuterMPSameTagCheck.clear();
268
269 StringBuilder errorSources = new StringBuilder();
270 for (String source : Config.getPref().getList(PREF_SOURCES, DEFAULT_SOURCES)) {
271 try (
272 CachedFile cf = new CachedFile(source);
273 BufferedReader reader = cf.getContentReader()
274 ) {
275 String okValue = null;
276 boolean tagcheckerfile = false;
277 boolean ignorefile = false;
278 boolean isFirstLine = true;
279 String line;
280 while ((line = reader.readLine()) != null) {
281 if (line.isEmpty()) {
282 // ignore
283 } else if (line.startsWith("#")) {
284 if (line.startsWith("# JOSM TagChecker")) {
285 tagcheckerfile = true;
286 Logging.error(tr("Ignoring {0}. Support was dropped", source));
287 } else
288 if (line.startsWith("# JOSM IgnoreTags")) {
289 ignorefile = true;
290 if (!DEFAULT_SOURCES.contains(source)) {
291 Logging.info(tr("Adding {0} to ignore tags", source));
292 }
293 }
294 } else if (ignorefile) {
295 parseIgnoreFileLine(source, line);
296 } else if (tagcheckerfile) {
297 // ignore
298 } else if (line.charAt(0) == '+') {
299 okValue = line.substring(1);
300 } else if (line.charAt(0) == '-' && okValue != null) {
301 String hk = harmonizeKey(line.substring(1));
302 if (!okValue.equals(hk) && harmonizedKeys.put(hk, okValue) != null && Logging.isDebugEnabled()) {
303 Logging.debug("Line was ignored: " + line);
304 }
305 } else {
306 Logging.error(tr("Invalid spellcheck line: {0}", line));
307 }
308 if (isFirstLine) {
309 isFirstLine = false;
310 if (!(tagcheckerfile || ignorefile) && !DEFAULT_SOURCES.contains(source)) {
311 Logging.info(tr("Adding {0} to spellchecker", source));
312 }
313 }
314 }
315 } catch (IOException e) {
316 Logging.error(e);
317 errorSources.append(source).append('\n');
318 }
319 }
320
321 if (errorSources.length() > 0)
322 throw new IOException(trn(
323 "Could not access data file:\n{0}",
324 "Could not access data files:\n{0}", errorSources.length(), errorSources));
325 }
326
327 /**
328 * Parse a line found in a configuration file
329 * @param source name of configuration file
330 * @param line the line to parse
331 */
332 private static void parseIgnoreFileLine(String source, String line) {
333 line = line.trim();
334 if (line.length() < 4) {
335 return;
336 }
337 try {
338 String key = line.substring(0, 2);
339 line = line.substring(2);
340
341 switch (key) {
342 case "S:":
343 ignoreDataStartsWith.add(line);
344 break;
345 case "E:":
346 ignoreDataEquals.add(line);
347 addToKeyDictionary(line);
348 break;
349 case "F:":
350 ignoreDataEndsWith.add(line);
351 break;
352 case "K:":
353 Tag tag = Tag.ofString(line);
354 ignoreDataTag.add(tag);
355 oftenUsedTags.put(tag.getKey(), tag.getValue());
356 addToKeyDictionary(tag.getKey());
357 break;
358 default:
359 if (!key.startsWith(";")) {
360 Logging.warn("Unsupported TagChecker key: " + key);
361 }
362 }
363 } catch (IllegalArgumentException e) {
364 Logging.error("Invalid line in {0} : {1}", source, e.getMessage());
365 Logging.trace(e);
366 }
367 }
368
369 private static void addToKeyDictionary(String key) {
370 if (key != null) {
371 String hk = harmonizeKey(key);
372 if (!key.equals(hk)) {
373 harmonizedKeys.put(hk, key);
374 }
375 }
376 }
377
378 /**
379 * Reads the presets data.
380 *
381 */
382 public static void initializePresets() {
383
384 if (!Config.getPref().getBoolean(PREF_CHECK_VALUES, true))
385 return;
386
387 Collection<TaggingPreset> presets = TaggingPresets.getTaggingPresets();
388 if (!presets.isEmpty()) {
389 initAdditionalPresetsValueData();
390 for (TaggingPreset p : presets) {
391 List<TaggingPresetItem> minData = new ArrayList<>();
392 for (TaggingPresetItem i : p.data) {
393 if (i instanceof KeyedItem) {
394 if (!"none".equals(((KeyedItem) i).match))
395 minData.add(i);
396 addPresetValue((KeyedItem) i);
397 } else if (i instanceof CheckGroup) {
398 for (Check c : ((CheckGroup) i).checks) {
399 addPresetValue(c);
400 }
401 }
402 }
403 if (!minData.isEmpty()) {
404 presetIndex .put(p, minData);
405 }
406 }
407 }
408 }
409
410 private static void initAdditionalPresetsValueData() {
411 additionalPresetsValueData = new HashSet<>();
412 additionalPresetsValueData.addAll(AbstractPrimitive.getUninterestingKeys());
413 additionalPresetsValueData.addAll(Config.getPref().getList(
414 ValidatorPrefHelper.PREFIX + ".knownkeys",
415 Arrays.asList("is_in", "int_ref", "fixme", "population")));
416 }
417
418 private static void addPresetValue(KeyedItem ky) {
419 if (ky.key != null && ky.getValues() != null) {
420 addToKeyDictionary(ky.key);
421 }
422 }
423
424 /**
425 * Checks given string (key or value) if it contains unwanted non-printing control characters (either ASCII or Unicode bidi characters)
426 * @param s string to check
427 * @return {@code true} if {@code s} contains non-printing control characters
428 */
429 static boolean containsUnwantedNonPrintingControlCharacter(String s) {
430 return !Utils.isEmpty(s) && (
431 isJoiningChar(s.charAt(0)) ||
432 isJoiningChar(s.charAt(s.length() - 1)) ||
433 s.chars().anyMatch(c -> (isAsciiControlChar(c) && !isNewLineChar(c)) || isBidiControlChar(c))
434 );
435 }
436
437 private static boolean isAsciiControlChar(int c) {
438 return c < 0x20 || c == 0x7F;
439 }
440
441 private static boolean isNewLineChar(int c) {
442 return c == 0x0a || c == 0x0d;
443 }
444
445 private static boolean isJoiningChar(int c) {
446 return c == 0x200c || c == 0x200d; // ZWNJ, ZWJ
447 }
448
449 private static boolean isBidiControlChar(int c) {
450 /* check for range 0x200e to 0x200f (LRM, RLM) or
451 0x202a to 0x202e (LRE, RLE, PDF, LRO, RLO) */
452 return (c >= 0x200e && c <= 0x200f) || (c >= 0x202a && c <= 0x202e);
453 }
454
455 static String removeUnwantedNonPrintingControlCharacters(String s) {
456 // Remove all unwanted characters
457 String result = UNWANTED_NON_PRINTING_CONTROL_CHARACTERS.matcher(s).replaceAll("");
458 // Remove joining characters located at the beginning of the string
459 while (!result.isEmpty() && isJoiningChar(result.charAt(0))) {
460 result = result.substring(1);
461 }
462 // Remove joining characters located at the end of the string
463 while (!result.isEmpty() && isJoiningChar(result.charAt(result.length() - 1))) {
464 result = result.substring(0, result.length() - 1);
465 }
466 return result;
467 }
468
469 static boolean containsUnusualUnicodeCharacter(String key, String value) {
470 return getUnusualUnicodeCharacter(key, value).isPresent();
471 }
472
473 static OptionalInt getUnusualUnicodeCharacter(String key, String value) {
474 return value == null
475 ? OptionalInt.empty()
476 : value.chars().filter(c -> isUnusualUnicodeBlock(key, c)).findFirst();
477 }
478
479 /**
480 * Detects highly suspicious Unicode characters that have been seen in OSM database.
481 * @param key tag key
482 * @param c current character code point
483 * @return {@code true} if the current unicode block is very unusual for the given key
484 */
485 private static boolean isUnusualUnicodeBlock(String key, int c) {
486 UnicodeBlock b = UnicodeBlock.of(c);
487 return isUnusualPhoneticUse(key, b, c) || isUnusualBmpUse(b) || isUnusualSmpUse(b);
488 }
489
490 private static boolean isAllowedPhoneticCharacter(String key, int c) {
491 // CHECKSTYLE.OFF: BooleanExpressionComplexity
492 return c == 0x0259 || c == 0x018F // U+0259 is paired with the capital letter U+018F in Azeri, see #18740
493 || c == 0x0254 || c == 0x0186 // U+0254 is paired with the capital letter U+0186 in several African languages, see #18740
494 || c == 0x0257 || c == 0x018A // "ɗ/Ɗ" (U+0257/U+018A), see #19760
495 || c == 0x025B || c == 0x0190 // U+025B is paired with the capital letter U+0190 in several African languages, see #18740
496 || c == 0x0263 || c == 0x0194 // "ɣ/Ɣ" (U+0263/U+0194), see #18740
497 || c == 0x0268 || c == 0x0197 // "ɨ/Ɨ" (U+0268/U+0197), see #18740
498 || c == 0x0269 || c == 0x0196 // "ɩ/Ɩ" (U+0269/U+0196), see #20437
499 || c == 0x0272 || c == 0x019D // "ɲ/Ɲ" (U+0272/U+019D), see #18740
500 || c == 0x0273 || c == 0x019E // "ŋ/Ŋ" (U+0273/U+019E), see #18740
501 || c == 0x0142 || c == 0x0294 // see #20754
502 || (key.endsWith("ref") && 0x1D2C <= c && c <= 0x1D42); // allow uppercase superscript latin characters in *ref tags
503 }
504
505 private static boolean isUnusualPhoneticUse(String key, UnicodeBlock b, int c) {
506 return !isAllowedPhoneticCharacter(key, c)
507 && (b == UnicodeBlock.IPA_EXTENSIONS // U+0250..U+02AF
508 || b == UnicodeBlock.PHONETIC_EXTENSIONS // U+1D00..U+1D7F
509 || b == UnicodeBlock.PHONETIC_EXTENSIONS_SUPPLEMENT) // U+1D80..U+1DBF
510 && !key.endsWith(":pronunciation");
511 }
512
513 private static boolean isUnusualBmpUse(UnicodeBlock b) {
514 return b == UnicodeBlock.COMBINING_MARKS_FOR_SYMBOLS // U+20D0..U+20FF
515 || b == UnicodeBlock.MATHEMATICAL_OPERATORS // U+2200..U+22FF
516 || b == UnicodeBlock.ENCLOSED_ALPHANUMERICS // U+2460..U+24FF
517 || b == UnicodeBlock.BOX_DRAWING // U+2500..U+257F
518 || b == UnicodeBlock.GEOMETRIC_SHAPES // U+25A0..U+25FF
519 || b == UnicodeBlock.DINGBATS // U+2700..U+27BF
520 || b == UnicodeBlock.MISCELLANEOUS_SYMBOLS_AND_ARROWS // U+2B00..U+2BFF
521 || b == UnicodeBlock.GLAGOLITIC // U+2C00..U+2C5F
522 || b == UnicodeBlock.HANGUL_COMPATIBILITY_JAMO // U+3130..U+318F
523 || b == UnicodeBlock.ENCLOSED_CJK_LETTERS_AND_MONTHS // U+3200..U+32FF
524 || b == UnicodeBlock.LATIN_EXTENDED_D // U+A720..U+A7FF
525 || b == UnicodeBlock.CJK_COMPATIBILITY_IDEOGRAPHS // U+F900..U+FAFF
526 || b == UnicodeBlock.ALPHABETIC_PRESENTATION_FORMS // U+FB00..U+FB4F
527 || b == UnicodeBlock.VARIATION_SELECTORS // U+FE00..U+FE0F
528 || b == UnicodeBlock.SPECIALS; // U+FFF0..U+FFFF
529 // CHECKSTYLE.ON: BooleanExpressionComplexity
530 }
531
532 private static boolean isUnusualSmpUse(UnicodeBlock b) {
533 // UnicodeBlock.SUPPLEMENTAL_SYMBOLS_AND_PICTOGRAPHS is only defined in Java 9+
534 return b == UnicodeBlock.MUSICAL_SYMBOLS // U+1D100..U+1D1FF
535 || b == UnicodeBlock.ENCLOSED_ALPHANUMERIC_SUPPLEMENT // U+1F100..U+1F1FF
536 || b == UnicodeBlock.EMOTICONS // U+1F600..U+1F64F
537 || b == UnicodeBlock.TRANSPORT_AND_MAP_SYMBOLS; // U+1F680..U+1F6FF
538 }
539
540 /**
541 * Get set of preset values for the given key.
542 * @param key the key
543 * @return null if key is not in presets or in additionalPresetsValueData,
544 * else a set which might be empty.
545 */
546 private static Set<String> getPresetValues(String key) {
547 Set<String> res = TaggingPresets.getPresetValues(key);
548 if (res != null)
549 return res;
550 if (additionalPresetsValueData.contains(key))
551 return Collections.emptySet();
552 // null means key is not known
553 return null;
554 }
555
556 /**
557 * Determines if the given key is in internal presets.
558 * @param key key
559 * @return {@code true} if the given key is in internal presets
560 * @since 9023
561 */
562 public static boolean isKeyInPresets(String key) {
563 return TaggingPresets.getPresetValues(key) != null;
564 }
565
566 /**
567 * Determines if the given tag is in internal presets.
568 * @param key key
569 * @param value value
570 * @return {@code true} if the given tag is in internal presets
571 * @since 9023
572 */
573 public static boolean isTagInPresets(String key, String value) {
574 final Set<String> values = getPresetValues(key);
575 return values != null && values.contains(value);
576 }
577
578 /**
579 * Returns the list of ignored tags.
580 * @return the list of ignored tags
581 * @since 9023
582 */
583 public static List<Tag> getIgnoredTags() {
584 return new ArrayList<>(ignoreDataTag);
585 }
586
587 /**
588 * Determines if the given tag key is ignored for checks "key/tag not in presets".
589 * @param key key
590 * @return true if the given key is ignored
591 */
592 private static boolean isKeyIgnored(String key) {
593 return ignoreDataEquals.contains(key)
594 || ignoreDataStartsWith.stream().anyMatch(key::startsWith)
595 || ignoreDataEndsWith.stream().anyMatch(key::endsWith);
596 }
597
598 /**
599 * Determines if the given tag is ignored for checks "key/tag not in presets".
600 * @param key key
601 * @param value value
602 * @return {@code true} if the given tag is ignored
603 * @since 9023
604 */
605 public static boolean isTagIgnored(String key, String value) {
606 if (isKeyIgnored(key))
607 return true;
608 final Set<String> values = getPresetValues(key);
609 if (values != null && values.isEmpty())
610 return true;
611 if (!isTagInPresets(key, value)) {
612 return ignoreDataTag.stream()
613 .anyMatch(a -> key.equals(a.getKey()) && value.equals(a.getValue()));
614 }
615 return false;
616 }
617
618 /**
619 * Checks the primitive tags
620 * @param p The primitive to check
621 */
622 @Override
623 public void check(OsmPrimitive p) {
624 if (!p.isTagged())
625 return;
626
627 // Just a collection to know if a primitive has been already marked with error
628 MultiMap<OsmPrimitive, String> withErrors = new MultiMap<>();
629
630 for (Entry<String, String> prop : p.getKeys().entrySet()) {
631 String s = marktr("Tag ''{0}'' invalid.");
632 String key = prop.getKey();
633 String value = prop.getValue();
634
635 if (checkKeys) {
636 checkSingleTagKeySimple(withErrors, p, s, key);
637 }
638 if (checkValues) {
639 checkSingleTagValueSimple(withErrors, p, s, key, value);
640 checkSingleTagComplex(withErrors, p, key, value);
641 }
642 if (checkFixmes && key != null && !Utils.isEmpty(value) && isFixme(key, value) && !withErrors.contains(p, "FIXME")) {
643 errors.add(TestError.builder(this, Severity.OTHER, FIXME)
644 .message(tr("fixme"))
645 .primitives(p)
646 .build());
647 withErrors.put(p, "FIXME");
648 }
649 }
650
651 if (p instanceof Relation && p.hasTag("type", "multipolygon")) {
652 checkMultipolygonTags(p);
653 }
654
655 if (checkPresetsTypes) {
656 TagMap tags = p.getKeys();
657 TaggingPresetType presetType = TaggingPresetType.forPrimitive(p);
658 EnumSet<TaggingPresetType> presetTypes = EnumSet.of(presetType);
659
660 Collection<TaggingPreset> matchingPresets = presetIndex.entrySet().stream()
661 .filter(e -> TaggingPresetItem.matches(e.getValue(), tags))
662 .map(Entry::getKey)
663 .collect(Collectors.toCollection(LinkedHashSet::new));
664 Collection<TaggingPreset> matchingPresetsOK = matchingPresets.stream().filter(
665 tp -> tp.typeMatches(presetTypes)).collect(Collectors.toList());
666 Collection<TaggingPreset> matchingPresetsKO = matchingPresets.stream().filter(
667 tp -> !tp.typeMatches(presetTypes)).collect(Collectors.toList());
668
669 for (TaggingPreset tp : matchingPresetsKO) {
670 // Potential error, unless matching tags are all known by a supported preset
671 Map<String, String> matchingTags = tp.data.stream()
672 .filter(i -> Boolean.TRUE.equals(i.matches(tags)))
673 .filter(i -> i instanceof KeyedItem).map(i -> ((KeyedItem) i).key)
674 .collect(Collectors.toMap(k -> k, tags::get));
675 if (matchingPresetsOK.stream().noneMatch(
676 tp2 -> matchingTags.entrySet().stream().allMatch(
677 e -> tp2.data.stream().anyMatch(
678 i -> i instanceof KeyedItem && ((KeyedItem) i).key.equals(e.getKey()))))) {
679 errors.add(TestError.builder(this, Severity.OTHER, INVALID_PRESETS_TYPE)
680 .message(tr("Object type not in preset"),
681 marktr("Object type {0} is not supported by tagging preset: {1}"),
682 tr(presetType.getName()), tp.getLocaleName())
683 .primitives(p)
684 .build());
685 }
686 }
687 }
688 }
689
690 private static final Collection<String> NO_AREA_KEYS = Arrays.asList("name", "area", "ref", "access", "operator");
691
692 private void checkMultipolygonTags(OsmPrimitive p) {
693 if (p.isAnnotated() || p.keys()
694 .anyMatch(k -> k.matches("^(abandoned|construction|demolished|disused|planned|razed|removed|was).*")))
695 return;
696
697 checkOuterWaysOfRelation((Relation) p);
698
699 if (hasAcceptedPrimaryTagForMultipolygon(p))
700 return;
701 TestError.Builder builder = null;
702 if (p.hasKey("surface")) {
703 // accept often used tag surface=* as area tag
704 builder = TestError.builder(this, Severity.OTHER, MULTIPOLYGON_INCOMPLETE)
705 .message(tr("Multipolygon tags"), marktr("only {0} tag"), "surface");
706 } else {
707 Map<String, String> filteredTags = p.getInterestingTags();
708 filteredTags.remove("type");
709 NO_AREA_KEYS.forEach(filteredTags::remove);
710 filteredTags.keySet().removeIf(key -> !key.matches("[a-z0-9:_]+"));
711
712 if (filteredTags.isEmpty()) {
713 builder = TestError.builder(this, Severity.ERROR, MULTIPOLYGON_NO_AREA)
714 .message(tr("Multipolygon tags"), marktr("tag describing the area is missing"), new Object());
715
716 }
717 }
718 if (builder == null) {
719 // multipolygon has either no area tag or a rarely used one
720 builder = TestError.builder(this, Severity.WARNING, MULTIPOLYGON_MAYBE_NO_AREA)
721 .message(tr("Multipolygon tags"), marktr("tag describing the area might be missing"), new Object());
722 }
723 errors.add(builder.primitives(p).build());
724 }
725
726 /**
727 * Check if an outer way of the relation has the same tag as the relation.
728 * @param rel the relation
729 */
730 private void checkOuterWaysOfRelation(Relation rel) {
731 for (Entry<String, String> tag : rel.getInterestingTags().entrySet()) {
732 if (ignoreForOuterMPSameTagCheck.contains(tag.getKey()))
733 continue;
734
735 Set<Way> sameOuters = rel.getMembers().stream()
736 .filter(rm -> rm.isWay() && rm.getWay().isArea() && "outer".equals(rm.getRole())
737 && tag.getValue().equals(rm.getWay().get(tag.getKey())))
738 .map(RelationMember::getWay).collect(Collectors.toSet());
739 if (!sameOuters.isEmpty()) {
740 List<OsmPrimitive> primitives = new ArrayList<>(sameOuters.size() + 1);
741 primitives.add(rel);
742 primitives.addAll(sameOuters);
743 Way w = new Way();
744 w.put(tag.getKey(), tag.getValue());
745 if (hasAcceptedPrimaryTagForMultipolygon(w)) {
746 errors.add(TestError.builder(this, Severity.WARNING, MULTIPOLYGON_SAME_TAG_ON_OUTER)
747 .message(tr("Multipolygon outer way repeats major tag of relation"),
748 marktr("Same tag:''{0}''=''{1}''"), tag.getKey(), tag.getValue())
749 .primitives(primitives)
750 .build());
751 } else {
752 errors.add(TestError.builder(this, Severity.OTHER, MULTIPOLYGON_SAME_TAG_ON_OUTER)
753 .message(tr("Multipolygon outer way repeats tag of relation"),
754 marktr("Same tag:''{0}''=''{1}''"), tag.getKey(), tag.getValue())
755 .primitives(primitives)
756 .build());
757 }
758 }
759 }
760 }
761
762 /**
763 * Check if a multipolygon has a main tag that describes the type of area. Accepts also some deprecated tags and typos.
764 * @param p the multipolygon
765 * @return true if the multipolygon has a main tag that (likely) describes the type of area.
766 */
767 private static boolean hasAcceptedPrimaryTagForMultipolygon(OsmPrimitive p) {
768 if (p.hasKey("landuse", "amenity", "building", "building:part", "area:highway", "shop", "place", "boundary",
769 "landform", "piste:type", "sport", "golf", "landcover", "aeroway", "office", "healthcare", "craft", "room")
770 || p.hasTagDifferent("natural", "tree", "peek", "saddle", "tree_row")
771 || p.hasTagDifferent("man_made", "survey_point", "mast", "flagpole", "manhole", "watertap")
772 || p.hasTagDifferent("highway", "crossing", "bus_stop", "turning_circle", "street_lamp",
773 "traffic_signals", "stop", "milestone", "mini_roundabout", "motorway_junction", "passing_place",
774 "speed_camera", "traffic_mirror", "trailhead", "turning_circle", "turning_loop", "toll_gantry")
775 || p.hasTagDifferent("tourism", "attraction", "artwork")
776 || p.hasTagDifferent("leisure", "picnic_table", "slipway", "firepit")
777 || p.hasTagDifferent("historic", "wayside_cross", "milestone"))
778 return true;
779 if (p.hasTag("barrier", "hedge", "retaining_wall")
780 || p.hasTag("public_transport", "platform", "station")
781 || p.hasTag("railway", "platform")
782 || p.hasTag("waterway", "riverbank", "dam", "rapids", "dock", "boatyard", "fuel")
783 || p.hasTag("indoor", "corridor", "room", "area")
784 || p.hasTag("power", "substation", "generator", "plant", "switchgear", "converter", "sub_station")
785 || p.hasTag("seamark:type", "harbour", "fairway", "anchorage", "landmark", "berth", "harbour_basin",
786 "separation_zone")
787 || (p.get("seamark:type") != null && p.get("seamark:type").matches(".*\\_(area|zone)$")))
788 return true;
789 return p.hasTag("harbour", OsmUtils.TRUE_VALUE)
790 || p.hasTag("flood_prone", OsmUtils.TRUE_VALUE)
791 || p.hasTag("bridge", OsmUtils.TRUE_VALUE)
792 || p.hasTag("ruins", OsmUtils.TRUE_VALUE)
793 || p.hasTag("junction", OsmUtils.TRUE_VALUE);
794 }
795
796 private void checkSingleTagValueSimple(MultiMap<OsmPrimitive, String> withErrors, OsmPrimitive p, String s, String key, String value) {
797 if (!checkValues || value == null)
798 return;
799 if (containsUnwantedNonPrintingControlCharacter(value) && !withErrors.contains(p, "ICV")) {
800 errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_VALUE)
801 .message(tr("Tag value contains non-printing (usually invisible) character"), s, key)
802 .primitives(p)
803 .fix(() -> new ChangePropertyCommand(p, key, removeUnwantedNonPrintingControlCharacters(value)))
804 .build());
805 withErrors.put(p, "ICV");
806 }
807 final OptionalInt unusualUnicodeCharacter = getUnusualUnicodeCharacter(key, value);
808 if (unusualUnicodeCharacter.isPresent() && !withErrors.contains(p, "UUCV")) {
809 final String codepoint = String.format(Locale.ROOT, "U+%04X", unusualUnicodeCharacter.getAsInt());
810 errors.add(TestError.builder(this, Severity.WARNING, UNUSUAL_UNICODE_CHAR_VALUE)
811 .message(tr("Tag value contains unusual Unicode character {0}", codepoint), s, key)
812 .primitives(p)
813 .build());
814 withErrors.put(p, "UUCV");
815 }
816 if ((value.length() > Tagged.MAX_TAG_LENGTH) && !withErrors.contains(p, "LV")) {
817 errors.add(TestError.builder(this, Severity.ERROR, LONG_VALUE)
818 .message(tr("Tag value longer than {0} characters ({1} characters)", Tagged.MAX_TAG_LENGTH, value.length()), s, key)
819 .primitives(p)
820 .build());
821 withErrors.put(p, "LV");
822 }
823 if (value.trim().isEmpty() && !withErrors.contains(p, "EV")) {
824 errors.add(TestError.builder(this, Severity.WARNING, EMPTY_VALUES)
825 .message(tr("Tags with empty values"), s, key)
826 .primitives(p)
827 .build());
828 withErrors.put(p, "EV");
829 }
830 final String errTypeSpace = "SPACE";
831 if ((value.startsWith(" ") || value.endsWith(" ")) && !withErrors.contains(p, errTypeSpace)) {
832 errors.add(TestError.builder(this, Severity.WARNING, INVALID_SPACE)
833 .message(tr("Property values start or end with white space"), s, key)
834 .primitives(p)
835 .build());
836 withErrors.put(p, errTypeSpace);
837 }
838 if (value.contains(" ") && !withErrors.contains(p, errTypeSpace)) {
839 errors.add(TestError.builder(this, Severity.WARNING, MULTIPLE_SPACES)
840 .message(tr("Property values contain multiple white spaces"), s, key)
841 .primitives(p)
842 .build());
843 withErrors.put(p, errTypeSpace);
844 }
845 if (includeOtherSeverity && !value.equals(Entities.unescape(value)) && !withErrors.contains(p, "HTML")) {
846 errors.add(TestError.builder(this, Severity.OTHER, INVALID_HTML)
847 .message(tr("Property values contain HTML entity"), s, key)
848 .primitives(p)
849 .build());
850 withErrors.put(p, "HTML");
851 }
852 }
853
854 private void checkSingleTagKeySimple(MultiMap<OsmPrimitive, String> withErrors, OsmPrimitive p, String s, String key) {
855 if (!checkKeys || key == null)
856 return;
857 if (containsUnwantedNonPrintingControlCharacter(key) && !withErrors.contains(p, "ICK")) {
858 errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_KEY)
859 .message(tr("Tag key contains non-printing character"), s, key)
860 .primitives(p)
861 .fix(() -> new ChangePropertyCommand(p, key, removeUnwantedNonPrintingControlCharacters(key)))
862 .build());
863 withErrors.put(p, "ICK");
864 }
865 if (key.length() > Tagged.MAX_TAG_LENGTH && !withErrors.contains(p, "LK")) {
866 errors.add(TestError.builder(this, Severity.ERROR, LONG_KEY)
867 .message(tr("Tag key longer than {0} characters ({1} characters)", Tagged.MAX_TAG_LENGTH, key.length()), s, key)
868 .primitives(p)
869 .build());
870 withErrors.put(p, "LK");
871 }
872 if (key.indexOf(' ') >= 0 && !withErrors.contains(p, "IPK")) {
873 errors.add(TestError.builder(this, Severity.WARNING, INVALID_KEY_SPACE)
874 .message(tr("Invalid white space in property key"), s, key)
875 .primitives(p)
876 .build());
877 withErrors.put(p, "IPK");
878 }
879 }
880
881 private void checkSingleTagComplex(MultiMap<OsmPrimitive, String> withErrors, OsmPrimitive p, String key, String value) {
882 if (!checkValues || key == null || Utils.isEmpty(value))
883 return;
884 if (additionalPresetsValueData != null && !isTagIgnored(key, value)) {
885 if (!isKeyInPresets(key)) {
886 spellCheckKey(withErrors, p, key);
887 } else if (!isTagInPresets(key, value)) {
888 if (oftenUsedTags.contains(key, value)) {
889 // tag is quite often used but not in presets
890 errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE)
891 .message(tr("Presets do not contain property value"),
892 marktr("Value ''{0}'' for key ''{1}'' not in presets, but is known."), value, key)
893 .primitives(p)
894 .build());
895 withErrors.put(p, "UPV");
896 } else {
897 tryGuess(p, key, value, withErrors);
898 }
899 }
900 }
901 }
902
903 private void spellCheckKey(MultiMap<OsmPrimitive, String> withErrors, OsmPrimitive p, String key) {
904 String prettifiedKey = harmonizeKey(key);
905 String fixedKey;
906 if (ignoreDataEquals.contains(prettifiedKey)) {
907 fixedKey = prettifiedKey;
908 } else {
909 fixedKey = isKeyInPresets(prettifiedKey) ? prettifiedKey : harmonizedKeys.get(prettifiedKey);
910 }
911 if (fixedKey == null && ignoreDataTag.stream().anyMatch(a -> a.getKey().equals(prettifiedKey))) {
912 fixedKey = prettifiedKey;
913 }
914
915 if (fixedKey != null && !"".equals(fixedKey) && !fixedKey.equals(key)) {
916 final String proposedKey = fixedKey;
917 // misspelled preset key
918 final TestError.Builder error = TestError.builder(this, Severity.WARNING, MISSPELLED_KEY)
919 .message(tr("Misspelled property key"), marktr("Key ''{0}'' looks like ''{1}''."), key, proposedKey)
920 .primitives(p);
921 if (p.hasKey(fixedKey)) {
922 errors.add(error.build());
923 } else {
924 errors.add(error.fix(() -> new ChangePropertyKeyCommand(p, key, proposedKey)).build());
925 }
926 withErrors.put(p, "WPK");
927 } else if (includeOtherSeverity) {
928 errors.add(TestError.builder(this, Severity.OTHER, INVALID_KEY)
929 .message(tr("Presets do not contain property key"), marktr("Key ''{0}'' not in presets."), key)
930 .primitives(p)
931 .build());
932 withErrors.put(p, "UPK");
933 }
934 }
935
936 private void tryGuess(OsmPrimitive p, String key, String value, MultiMap<OsmPrimitive, String> withErrors) {
937 // try to fix common typos and check again if value is still unknown
938 final String harmonizedValue = harmonizeValue(value);
939 if (Utils.isEmpty(harmonizedValue))
940 return;
941 String fixedValue;
942 List<Set<String>> sets = new ArrayList<>();
943 Set<String> presetValues = getPresetValues(key);
944 if (presetValues != null)
945 sets.add(presetValues);
946 Set<String> usedValues = oftenUsedTags.get(key);
947 if (usedValues != null)
948 sets.add(usedValues);
949 fixedValue = sets.stream().anyMatch(possibleValues -> possibleValues.contains(harmonizedValue))
950 ? harmonizedValue : null;
951 if (fixedValue == null && !ignoreForLevenshtein.contains(key)) {
952 int maxPresetValueLen = 0;
953 List<String> fixVals = new ArrayList<>();
954 // use Levenshtein distance to find typical typos
955 int minDist = MAX_LEVENSHTEIN_DISTANCE + 1;
956 for (Set<String> possibleValues: sets) {
957 for (String possibleVal : possibleValues) {
958 if (possibleVal.isEmpty())
959 continue;
960 maxPresetValueLen = Math.max(maxPresetValueLen, possibleVal.length());
961 if (harmonizedValue.length() < 3 && possibleVal.length() >= harmonizedValue.length() + MAX_LEVENSHTEIN_DISTANCE) {
962 // don't suggest fix value when given value is short and lengths are too different
963 // for example surface=u would result in surface=mud
964 continue;
965 }
966 int dist = Utils.getLevenshteinDistance(possibleVal, harmonizedValue);
967 if (dist >= harmonizedValue.length()) {
968 // short value, all characters are different. Don't warn, might say Value '10' for key 'fee' looks like 'no'.
969 continue;
970 }
971 if (dist < minDist) {
972 minDist = dist;
973 fixVals.clear();
974 fixVals.add(possibleVal);
975 } else if (dist == minDist) {
976 fixVals.add(possibleVal);
977 }
978 }
979 }
980 if (minDist <= MAX_LEVENSHTEIN_DISTANCE && maxPresetValueLen > MAX_LEVENSHTEIN_DISTANCE
981 && !fixVals.isEmpty()
982 && (harmonizedValue.length() > 3 || minDist < MAX_LEVENSHTEIN_DISTANCE)) {
983 filterDeprecatedTags(p, key, fixVals);
984 if (!fixVals.isEmpty()) {
985 if (fixVals.size() < 2) {
986 fixedValue = fixVals.get(0);
987 } else {
988 Collections.sort(fixVals);
989 // misspelled preset value with multiple good alternatives
990 errors.add(TestError.builder(this, Severity.WARNING, MISSPELLED_VALUE_NO_FIX)
991 .message(tr("Unknown property value"),
992 marktr("Value ''{0}'' for key ''{1}'' is unknown, maybe one of {2} is meant?"),
993 value, key, fixVals)
994 .primitives(p).build());
995 withErrors.put(p, "WPV");
996 return;
997 }
998 }
999 }
1000 }
1001 if (fixedValue != null && !fixedValue.equals(value)) {
1002 final String newValue = fixedValue;
1003 // misspelled preset value
1004 errors.add(TestError.builder(this, Severity.WARNING, MISSPELLED_VALUE)
1005 .message(tr("Unknown property value"),
1006 marktr("Value ''{0}'' for key ''{1}'' is unknown, maybe ''{2}'' is meant?"), value, key, newValue)
1007 .primitives(p)
1008 .build());
1009 withErrors.put(p, "WPV");
1010 } else if (includeOtherSeverity) {
1011 // unknown preset value
1012 errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE)
1013 .message(tr("Presets do not contain property value"),
1014 marktr("Value ''{0}'' for key ''{1}'' not in presets."), value, key)
1015 .primitives(p)
1016 .build());
1017 withErrors.put(p, "UPV");
1018 }
1019 }
1020
1021 // see #19180
1022 private void filterDeprecatedTags(OsmPrimitive p, String key, List<String> fixVals) {
1023 if (fixVals.isEmpty() || deprecatedChecker == null)
1024 return;
1025
1026 int unchangedDeprecated = countDeprecated(p);
1027
1028 // see #19895: create deep clone. This complex method works even with locked files
1029 MergeSourceBuildingVisitor builder = new MergeSourceBuildingVisitor(p.getDataSet());
1030 p.accept(builder);
1031 DataSet clonedDs = builder.build();
1032 OsmPrimitive clone = clonedDs.getPrimitiveById(p.getPrimitiveId());
1033
1034 Iterator<String> iter = fixVals.iterator();
1035 while (iter.hasNext()) {
1036 clone.put(key, iter.next());
1037 if (countDeprecated(clone) > unchangedDeprecated)
1038 iter.remove();
1039 }
1040 }
1041
1042 private int countDeprecated(OsmPrimitive p) {
1043 if (deprecatedChecker == null)
1044 return 0;
1045 deprecatedChecker.getErrors().clear();
1046 deprecatedChecker.visit(Collections.singleton(p), url -> url.endsWith("deprecated.mapcss"));
1047 return deprecatedChecker.getErrors().size();
1048 }
1049
1050 private static boolean isNum(String harmonizedValue) {
1051 try {
1052 Double.parseDouble(harmonizedValue);
1053 return true;
1054 } catch (NumberFormatException e) {
1055 return false;
1056 }
1057 }
1058
1059 private static boolean isFixme(String key, String value) {
1060 return key.toLowerCase(Locale.ENGLISH).contains("fixme") || key.contains("todo")
1061 || value.toLowerCase(Locale.ENGLISH).contains("fixme") || value.contains("check and delete");
1062 }
1063
1064 private static String harmonizeKey(String key) {
1065 return Utils.strip(key.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(':', '_').replace(' ', '_'), "-_;:,");
1066 }
1067
1068 private static String harmonizeValue(String value) {
1069 return Utils.strip(value.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(' ', '_'), "-_;:,");
1070 }
1071
1072 @Override
1073 public void startTest(ProgressMonitor monitor) {
1074 super.startTest(monitor);
1075 includeOtherSeverity = includeOtherSeverityChecks();
1076 checkKeys = Config.getPref().getBoolean(PREF_CHECK_KEYS, true);
1077 if (isBeforeUpload) {
1078 checkKeys = checkKeys && Config.getPref().getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true);
1079 }
1080
1081 checkValues = Config.getPref().getBoolean(PREF_CHECK_VALUES, true);
1082 if (isBeforeUpload) {
1083 checkValues = checkValues && Config.getPref().getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true);
1084 }
1085
1086 checkComplex = Config.getPref().getBoolean(PREF_CHECK_COMPLEX, true);
1087 if (isBeforeUpload) {
1088 checkComplex = checkComplex && Config.getPref().getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true);
1089 }
1090
1091 checkFixmes = includeOtherSeverity && Config.getPref().getBoolean(PREF_CHECK_FIXMES, true);
1092 if (isBeforeUpload) {
1093 checkFixmes = checkFixmes && Config.getPref().getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true);
1094 }
1095
1096 checkPresetsTypes = includeOtherSeverity && Config.getPref().getBoolean(PREF_CHECK_PRESETS_TYPES, true);
1097 if (isBeforeUpload) {
1098 checkPresetsTypes = checkPresetsTypes && Config.getPref().getBoolean(PREF_CHECK_PRESETS_TYPES_BEFORE_UPLOAD, true);
1099 }
1100 deprecatedChecker = OsmValidator.getTest(MapCSSTagChecker.class);
1101 ignoreForOuterMPSameTagCheck.addAll(Config.getPref().getList(PREF_KEYS_IGNORE_OUTER_MP_SAME_TAG, Collections.emptyList()));
1102 }
1103
1104 @Override
1105 public void endTest() {
1106 deprecatedChecker = null;
1107 super.endTest();
1108 }
1109
1110 @Override
1111 public void visit(Collection<OsmPrimitive> selection) {
1112 if (checkKeys || checkValues || checkComplex || checkFixmes || checkPresetsTypes) {
1113 super.visit(selection);
1114 }
1115 }
1116
1117 @Override
1118 public void addGui(JPanel testPanel) {
1119 GBC a = GBC.eol();
1120 a.anchor = GridBagConstraints.LINE_END;
1121
1122 testPanel.add(new JLabel(name+" :"), GBC.eol().insets(3, 0, 0, 0));
1123
1124 prefCheckKeys = new JCheckBox(tr("Check property keys."), Config.getPref().getBoolean(PREF_CHECK_KEYS, true));
1125 prefCheckKeys.setToolTipText(tr("Validate that property keys are valid checking against list of words."));
1126 testPanel.add(prefCheckKeys, GBC.std().insets(20, 0, 0, 0));
1127
1128 prefCheckKeysBeforeUpload = new JCheckBox();
1129 prefCheckKeysBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true));
1130 testPanel.add(prefCheckKeysBeforeUpload, a);
1131
1132 prefCheckComplex = new JCheckBox(tr("Use complex property checker."), Config.getPref().getBoolean(PREF_CHECK_COMPLEX, true));
1133 prefCheckComplex.setToolTipText(tr("Validate property values and tags using complex rules."));
1134 testPanel.add(prefCheckComplex, GBC.std().insets(20, 0, 0, 0));
1135
1136 prefCheckComplexBeforeUpload = new JCheckBox();
1137 prefCheckComplexBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true));
1138 testPanel.add(prefCheckComplexBeforeUpload, a);
1139
1140 final Collection<String> sources = Config.getPref().getList(PREF_SOURCES, DEFAULT_SOURCES);
1141 sourcesList = new EditableList(tr("TagChecker source"));
1142 sourcesList.setItems(sources);
1143 testPanel.add(new JLabel(tr("Data sources ({0})", "*.cfg")), GBC.eol().insets(23, 0, 0, 0));
1144 testPanel.add(sourcesList, GBC.eol().fill(GridBagConstraints.HORIZONTAL).insets(23, 0, 0, 0));
1145
1146 ActionListener disableCheckActionListener = e -> handlePrefEnable();
1147 prefCheckKeys.addActionListener(disableCheckActionListener);
1148 prefCheckKeysBeforeUpload.addActionListener(disableCheckActionListener);
1149 prefCheckComplex.addActionListener(disableCheckActionListener);
1150 prefCheckComplexBeforeUpload.addActionListener(disableCheckActionListener);
1151
1152 handlePrefEnable();
1153
1154 prefCheckValues = new JCheckBox(tr("Check property values."), Config.getPref().getBoolean(PREF_CHECK_VALUES, true));
1155 prefCheckValues.setToolTipText(tr("Validate that property values are valid checking against presets."));
1156 testPanel.add(prefCheckValues, GBC.std().insets(20, 0, 0, 0));
1157
1158 prefCheckValuesBeforeUpload = new JCheckBox();
1159 prefCheckValuesBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true));
1160 testPanel.add(prefCheckValuesBeforeUpload, a);
1161
1162 prefCheckFixmes = new JCheckBox(tr("Check for fixme."), Config.getPref().getBoolean(PREF_CHECK_FIXMES, true));
1163 prefCheckFixmes.setToolTipText(tr("Looks for nodes or ways with fixme in any property value."));
1164 testPanel.add(prefCheckFixmes, GBC.std().insets(20, 0, 0, 0));
1165
1166 prefCheckFixmesBeforeUpload = new JCheckBox();
1167 prefCheckFixmesBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true));
1168 testPanel.add(prefCheckFixmesBeforeUpload, a);
1169
1170 prefCheckPresetsTypes = new JCheckBox(tr("Check for presets types."), Config.getPref().getBoolean(PREF_CHECK_PRESETS_TYPES, true));
1171 prefCheckPresetsTypes.setToolTipText(tr("Validate that objects types are valid checking against presets."));
1172 testPanel.add(prefCheckPresetsTypes, GBC.std().insets(20, 0, 0, 0));
1173
1174 prefCheckPresetsTypesBeforeUpload = new JCheckBox();
1175 prefCheckPresetsTypesBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_PRESETS_TYPES_BEFORE_UPLOAD, true));
1176 testPanel.add(prefCheckPresetsTypesBeforeUpload, a);
1177 }
1178
1179 /**
1180 * Enables/disables the source list field
1181 */
1182 public void handlePrefEnable() {
1183 boolean selected = prefCheckKeys.isSelected() || prefCheckKeysBeforeUpload.isSelected()
1184 || prefCheckComplex.isSelected() || prefCheckComplexBeforeUpload.isSelected();
1185 sourcesList.setEnabled(selected);
1186 }
1187
1188 @Override
1189 public boolean ok() {
1190 enabled = prefCheckKeys.isSelected() || prefCheckValues.isSelected() || prefCheckComplex.isSelected() || prefCheckFixmes.isSelected();
1191 testBeforeUpload = prefCheckKeysBeforeUpload.isSelected() || prefCheckValuesBeforeUpload.isSelected()
1192 || prefCheckFixmesBeforeUpload.isSelected() || prefCheckComplexBeforeUpload.isSelected();
1193
1194 Config.getPref().putBoolean(PREF_CHECK_VALUES, prefCheckValues.isSelected());
1195 Config.getPref().putBoolean(PREF_CHECK_COMPLEX, prefCheckComplex.isSelected());
1196 Config.getPref().putBoolean(PREF_CHECK_KEYS, prefCheckKeys.isSelected());
1197 Config.getPref().putBoolean(PREF_CHECK_FIXMES, prefCheckFixmes.isSelected());
1198 Config.getPref().putBoolean(PREF_CHECK_PRESETS_TYPES, prefCheckPresetsTypes.isSelected());
1199 Config.getPref().putBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, prefCheckValuesBeforeUpload.isSelected());
1200 Config.getPref().putBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, prefCheckComplexBeforeUpload.isSelected());
1201 Config.getPref().putBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, prefCheckKeysBeforeUpload.isSelected());
1202 Config.getPref().putBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, prefCheckFixmesBeforeUpload.isSelected());
1203 Config.getPref().putBoolean(PREF_CHECK_PRESETS_TYPES_BEFORE_UPLOAD, prefCheckPresetsTypesBeforeUpload.isSelected());
1204 return Config.getPref().putList(PREF_SOURCES, sourcesList.getItems());
1205 }
1206
1207 @Override
1208 public Command fixError(TestError testError) {
1209 List<Command> commands = new ArrayList<>(50);
1210
1211 Collection<? extends OsmPrimitive> primitives = testError.getPrimitives();
1212 for (OsmPrimitive p : primitives) {
1213 Map<String, String> tags = p.getKeys();
1214 if (tags.isEmpty()) {
1215 continue;
1216 }
1217
1218 for (Entry<String, String> prop: tags.entrySet()) {
1219 String key = prop.getKey();
1220 String value = prop.getValue();
1221 if (Utils.isBlank(value)) {
1222 commands.add(new ChangePropertyCommand(p, key, null));
1223 } else if (value.startsWith(" ") || value.endsWith(" ") || value.contains(" ")) {
1224 commands.add(new ChangePropertyCommand(p, key, Utils.removeWhiteSpaces(value)));
1225 } else if (key.startsWith(" ") || key.endsWith(" ") || key.contains(" ")) {
1226 commands.add(new ChangePropertyKeyCommand(p, key, Utils.removeWhiteSpaces(key)));
1227 } else {
1228 String evalue = Entities.unescape(value);
1229 if (!evalue.equals(value)) {
1230 commands.add(new ChangePropertyCommand(p, key, evalue));
1231 }
1232 }
1233 }
1234 }
1235
1236 if (commands.isEmpty())
1237 return null;
1238 if (commands.size() == 1)
1239 return commands.get(0);
1240
1241 return new SequenceCommand(tr("Fix tags"), commands);
1242 }
1243
1244 @Override
1245 public boolean isFixable(TestError testError) {
1246 if (testError.getTester() instanceof TagChecker) {
1247 int code = testError.getCode();
1248 return code == EMPTY_VALUES || code == INVALID_SPACE ||
1249 code == INVALID_KEY_SPACE || code == INVALID_HTML ||
1250 code == MULTIPLE_SPACES;
1251 }
1252
1253 return false;
1254 }
1255
1256 @Override
1257 public void taggingPresetsModified() {
1258 try {
1259 initializeData();
1260 initializePresets();
1261 analysePresets();
1262 } catch (IOException e) {
1263 Logging.error(e);
1264 }
1265 }
1266}
Note: See TracBrowser for help on using the repository browser.