source: josm/trunk/src/org/openstreetmap/josm/data/validation/tests/TagChecker.java@ 8435

Last change on this file since 8435 was 8435, checked in by Don-vip, 9 years ago

fix #11498 - Warn about obvious misspelled tag values (modified patch by mdk) + javadoc
  • Property svn:eol-style set to native
File size: 34.7 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.validation.tests;
3
4import static org.openstreetmap.josm.tools.I18n.marktr;
5import static org.openstreetmap.josm.tools.I18n.tr;
6
7import java.awt.GridBagConstraints;
8import java.awt.event.ActionEvent;
9import java.awt.event.ActionListener;
10import java.io.BufferedReader;
11import java.io.FileNotFoundException;
12import java.io.IOException;
13import java.io.InputStream;
14import java.text.MessageFormat;
15import java.util.ArrayList;
16import java.util.Arrays;
17import java.util.Collection;
18import java.util.HashMap;
19import java.util.List;
20import java.util.Locale;
21import java.util.Map;
22import java.util.Map.Entry;
23import java.util.Set;
24import java.util.regex.Matcher;
25import java.util.regex.Pattern;
26import java.util.regex.PatternSyntaxException;
27
28import javax.swing.JCheckBox;
29import javax.swing.JLabel;
30import javax.swing.JPanel;
31
32import org.openstreetmap.josm.Main;
33import org.openstreetmap.josm.command.ChangePropertyCommand;
34import org.openstreetmap.josm.command.ChangePropertyKeyCommand;
35import org.openstreetmap.josm.command.Command;
36import org.openstreetmap.josm.command.SequenceCommand;
37import org.openstreetmap.josm.data.osm.OsmPrimitive;
38import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
39import org.openstreetmap.josm.data.osm.OsmUtils;
40import org.openstreetmap.josm.data.osm.Tag;
41import org.openstreetmap.josm.data.validation.FixableTestError;
42import org.openstreetmap.josm.data.validation.Severity;
43import org.openstreetmap.josm.data.validation.Test.TagTest;
44import org.openstreetmap.josm.data.validation.TestError;
45import org.openstreetmap.josm.data.validation.util.Entities;
46import org.openstreetmap.josm.gui.preferences.validator.ValidatorPreference;
47import org.openstreetmap.josm.gui.progress.ProgressMonitor;
48import org.openstreetmap.josm.gui.tagging.TaggingPreset;
49import org.openstreetmap.josm.gui.tagging.TaggingPresetItem;
50import org.openstreetmap.josm.gui.tagging.TaggingPresetItems.Check;
51import org.openstreetmap.josm.gui.tagging.TaggingPresetItems.CheckGroup;
52import org.openstreetmap.josm.gui.tagging.TaggingPresetItems.KeyedItem;
53import org.openstreetmap.josm.gui.tagging.TaggingPresets;
54import org.openstreetmap.josm.gui.widgets.EditableList;
55import org.openstreetmap.josm.io.CachedFile;
56import org.openstreetmap.josm.io.UTFInputStreamReader;
57import org.openstreetmap.josm.tools.GBC;
58import org.openstreetmap.josm.tools.MultiMap;
59import org.openstreetmap.josm.tools.Utils;
60
61/**
62 * Check for misspelled or wrong tags
63 *
64 * @author frsantos
65 * @since 3669
66 */
67public class TagChecker extends TagTest {
68
69 /** The config file of ignored tags */
70 public static final String IGNORE_FILE = "resource://data/validator/ignoretags.cfg";
71 /** The config file of dictionary words */
72 public static final String SPELL_FILE = "resource://data/validator/words.cfg";
73
74 /** The spell check key substitutions: the key should be substituted by the value */
75 private static volatile Map<String, String> spellCheckKeyData;
76 /** The spell check preset values */
77 private static volatile MultiMap<String, String> presetsValueData;
78 /** The TagChecker data */
79 private static final List<CheckerData> checkerData = new ArrayList<>();
80 private static final List<String> ignoreDataStartsWith = new ArrayList<>();
81 private static final List<String> ignoreDataEquals = new ArrayList<>();
82 private static final List<String> ignoreDataEndsWith = new ArrayList<>();
83 private static final List<IgnoreKeyPair> ignoreDataKeyPair = new ArrayList<>();
84
85 /** The preferences prefix */
86 protected static final String PREFIX = ValidatorPreference.PREFIX + "." + TagChecker.class.getSimpleName();
87
88 public static final String PREF_CHECK_VALUES = PREFIX + ".checkValues";
89 public static final String PREF_CHECK_KEYS = PREFIX + ".checkKeys";
90 public static final String PREF_CHECK_COMPLEX = PREFIX + ".checkComplex";
91 public static final String PREF_CHECK_FIXMES = PREFIX + ".checkFixmes";
92
93 public static final String PREF_SOURCES = PREFIX + ".source";
94
95 public static final String PREF_CHECK_KEYS_BEFORE_UPLOAD = PREF_CHECK_KEYS + "BeforeUpload";
96 public static final String PREF_CHECK_VALUES_BEFORE_UPLOAD = PREF_CHECK_VALUES + "BeforeUpload";
97 public static final String PREF_CHECK_COMPLEX_BEFORE_UPLOAD = PREF_CHECK_COMPLEX + "BeforeUpload";
98 public static final String PREF_CHECK_FIXMES_BEFORE_UPLOAD = PREF_CHECK_FIXMES + "BeforeUpload";
99
100 protected boolean checkKeys = false;
101 protected boolean checkValues = false;
102 protected boolean checkComplex = false;
103 protected boolean checkFixmes = false;
104
105 protected JCheckBox prefCheckKeys;
106 protected JCheckBox prefCheckValues;
107 protected JCheckBox prefCheckComplex;
108 protected JCheckBox prefCheckFixmes;
109 protected JCheckBox prefCheckPaint;
110
111 protected JCheckBox prefCheckKeysBeforeUpload;
112 protected JCheckBox prefCheckValuesBeforeUpload;
113 protected JCheckBox prefCheckComplexBeforeUpload;
114 protected JCheckBox prefCheckFixmesBeforeUpload;
115 protected JCheckBox prefCheckPaintBeforeUpload;
116
117 protected static final int EMPTY_VALUES = 1200;
118 protected static final int INVALID_KEY = 1201;
119 protected static final int INVALID_VALUE = 1202;
120 protected static final int FIXME = 1203;
121 protected static final int INVALID_SPACE = 1204;
122 protected static final int INVALID_KEY_SPACE = 1205;
123 protected static final int INVALID_HTML = 1206; /* 1207 was PAINT */
124 protected static final int LONG_VALUE = 1208;
125 protected static final int LONG_KEY = 1209;
126 protected static final int LOW_CHAR_VALUE = 1210;
127 protected static final int LOW_CHAR_KEY = 1211;
128 protected static final int MISSPELLED_VALUE = 1212;
129 /** 1250 and up is used by tagcheck */
130
131 protected EditableList sourcesList;
132
133 protected static final Entities entities = new Entities();
134
135 private static final List<String> DEFAULT_SOURCES = Arrays.asList(/*DATA_FILE, */IGNORE_FILE, SPELL_FILE);
136
137 /**
138 * Constructor
139 */
140 public TagChecker() {
141 super(tr("Tag checker"), tr("This test checks for errors in tag keys and values."));
142 }
143
144 @Override
145 public void initialize() throws IOException {
146 initializeData();
147 initializePresets();
148 }
149
150 /**
151 * Reads the spellcheck file into a HashMap.
152 * The data file is a list of words, beginning with +/-. If it starts with +,
153 * the word is valid, but if it starts with -, the word should be replaced
154 * by the nearest + word before this.
155 *
156 * @throws FileNotFoundException
157 * @throws IOException
158 */
159 private static void initializeData() throws IOException {
160 checkerData.clear();
161 ignoreDataStartsWith.clear();
162 ignoreDataEquals.clear();
163 ignoreDataEndsWith.clear();
164 ignoreDataKeyPair.clear();
165
166 spellCheckKeyData = new HashMap<>();
167
168 String errorSources = "";
169 for (String source : Main.pref.getCollection(PREF_SOURCES, DEFAULT_SOURCES)) {
170 try (
171 InputStream s = new CachedFile(source).getInputStream();
172 BufferedReader reader = new BufferedReader(UTFInputStreamReader.create(s));
173 ) {
174 String okValue = null;
175 boolean tagcheckerfile = false;
176 boolean ignorefile = false;
177 boolean isFirstLine = true;
178 String line;
179 while ((line = reader.readLine()) != null && (tagcheckerfile || line.length() != 0)) {
180 if (line.startsWith("#")) {
181 if (line.startsWith("# JOSM TagChecker")) {
182 tagcheckerfile = true;
183 if (!DEFAULT_SOURCES.contains(source)) {
184 Main.info(tr("Adding {0} to tag checker", source));
185 }
186 } else
187 if (line.startsWith("# JOSM IgnoreTags")) {
188 ignorefile = true;
189 if (!DEFAULT_SOURCES.contains(source)) {
190 Main.info(tr("Adding {0} to ignore tags", source));
191 }
192 }
193 } else if (ignorefile) {
194 line = line.trim();
195 if (line.length() < 4) {
196 continue;
197 }
198
199 String key = line.substring(0, 2);
200 line = line.substring(2);
201
202 switch (key) {
203 case "S:":
204 ignoreDataStartsWith.add(line);
205 break;
206 case "E:":
207 ignoreDataEquals.add(line);
208 break;
209 case "F:":
210 ignoreDataEndsWith.add(line);
211 break;
212 case "K:":
213 IgnoreKeyPair tmp = new IgnoreKeyPair();
214 int mid = line.indexOf('=');
215 tmp.key = line.substring(0, mid);
216 tmp.value = line.substring(mid+1);
217 ignoreDataKeyPair.add(tmp);
218 }
219 } else if (tagcheckerfile) {
220 if (line.length() > 0) {
221 CheckerData d = new CheckerData();
222 String err = d.getData(line);
223
224 if (err == null) {
225 checkerData.add(d);
226 } else {
227 Main.error(tr("Invalid tagchecker line - {0}: {1}", err, line));
228 }
229 }
230 } else if (line.charAt(0) == '+') {
231 okValue = line.substring(1);
232 } else if (line.charAt(0) == '-' && okValue != null) {
233 spellCheckKeyData.put(line.substring(1), okValue);
234 } else {
235 Main.error(tr("Invalid spellcheck line: {0}", line));
236 }
237 if (isFirstLine) {
238 isFirstLine = false;
239 if (!(tagcheckerfile || ignorefile) && !DEFAULT_SOURCES.contains(source)) {
240 Main.info(tr("Adding {0} to spellchecker", source));
241 }
242 }
243 }
244 } catch (IOException e) {
245 errorSources += source + "\n";
246 }
247 }
248
249 if (errorSources.length() > 0)
250 throw new IOException( tr("Could not access data file(s):\n{0}", errorSources) );
251 }
252
253 /**
254 * Reads the presets data.
255 *
256 */
257 public static void initializePresets() {
258
259 if (!Main.pref.getBoolean(PREF_CHECK_VALUES, true))
260 return;
261
262 Collection<TaggingPreset> presets = TaggingPresets.getTaggingPresets();
263 if (!presets.isEmpty()) {
264 presetsValueData = new MultiMap<>();
265 for (String a : OsmPrimitive.getUninterestingKeys()) {
266 presetsValueData.putVoid(a);
267 }
268 // TODO directionKeys are no longer in OsmPrimitive (search pattern is used instead)
269 /* for(String a : OsmPrimitive.getDirectionKeys())
270 presetsValueData.add(a);
271 */
272 for (String a : Main.pref.getCollection(ValidatorPreference.PREFIX + ".knownkeys",
273 Arrays.asList(new String[]{"is_in", "int_ref", "fixme", "population"}))) {
274 presetsValueData.putVoid(a);
275 }
276 for (TaggingPreset p : presets) {
277 for (TaggingPresetItem i : p.data) {
278 if (i instanceof KeyedItem) {
279 addPresetValue(p, (KeyedItem) i);
280 } else if (i instanceof CheckGroup) {
281 for (Check c : ((CheckGroup) i).checks) {
282 addPresetValue(p, c);
283 }
284 }
285 }
286 }
287 }
288 }
289
290 private static void addPresetValue(TaggingPreset p, KeyedItem ky) {
291 Collection<String> values = ky.getValues();
292 if (ky.key != null && values != null) {
293 try {
294 presetsValueData.putAll(ky.key, values);
295 } catch (NullPointerException e) {
296 Main.error(p+": Unable to initialize "+ky);
297 }
298 }
299 }
300
301 /**
302 * Checks given string (key or value) if it contains characters with code below 0x20 (either newline or some other special characters)
303 * @param s string to check
304 */
305 private boolean containsLow(String s) {
306 if (s == null)
307 return false;
308 for (int i = 0; i < s.length(); i++) {
309 if (s.charAt(i) < 0x20)
310 return true;
311 }
312 return false;
313 }
314
315 /**
316 * Checks the primitive tags
317 * @param p The primitive to check
318 */
319 @Override
320 public void check(OsmPrimitive p) {
321 // Just a collection to know if a primitive has been already marked with error
322 MultiMap<OsmPrimitive, String> withErrors = new MultiMap<>();
323
324 if (checkComplex) {
325 Map<String, String> keys = p.getKeys();
326 for (CheckerData d : checkerData) {
327 if (d.match(p, keys)) {
328 errors.add(new TestError(this, d.getSeverity(), tr("Suspicious tag/value combinations"),
329 d.getDescription(), d.getDescriptionOrig(), d.getCode(), p));
330 withErrors.put(p, "TC");
331 }
332 }
333 }
334
335 for (Entry<String, String> prop : p.getKeys().entrySet()) {
336 String s = marktr("Key ''{0}'' invalid.");
337 String key = prop.getKey();
338 String value = prop.getValue();
339 if (checkValues && (containsLow(value)) && !withErrors.contains(p, "ICV")) {
340 errors.add(new TestError(this, Severity.WARNING, tr("Tag value contains character with code less than 0x20"),
341 tr(s, key), MessageFormat.format(s, key), LOW_CHAR_VALUE, p));
342 withErrors.put(p, "ICV");
343 }
344 if (checkKeys && (containsLow(key)) && !withErrors.contains(p, "ICK")) {
345 errors.add(new TestError(this, Severity.WARNING, tr("Tag key contains character with code less than 0x20"),
346 tr(s, key), MessageFormat.format(s, key), LOW_CHAR_KEY, p));
347 withErrors.put(p, "ICK");
348 }
349 if (checkValues && (value!=null && value.length() > 255) && !withErrors.contains(p, "LV")) {
350 errors.add(new TestError(this, Severity.ERROR, tr("Tag value longer than allowed"),
351 tr(s, key), MessageFormat.format(s, key), LONG_VALUE, p));
352 withErrors.put(p, "LV");
353 }
354 if (checkKeys && (key!=null && key.length() > 255) && !withErrors.contains(p, "LK")) {
355 errors.add(new TestError(this, Severity.ERROR, tr("Tag key longer than allowed"),
356 tr(s, key), MessageFormat.format(s, key), LONG_KEY, p));
357 withErrors.put(p, "LK");
358 }
359 if (checkValues && (value==null || value.trim().isEmpty()) && !withErrors.contains(p, "EV")) {
360 errors.add(new TestError(this, Severity.WARNING, tr("Tags with empty values"),
361 tr(s, key), MessageFormat.format(s, key), EMPTY_VALUES, p));
362 withErrors.put(p, "EV");
363 }
364 if (checkKeys && spellCheckKeyData.containsKey(key) && !withErrors.contains(p, "IPK")) {
365 errors.add(new TestError(this, Severity.WARNING, tr("Invalid property key"),
366 tr(s, key), MessageFormat.format(s, key), INVALID_KEY, p));
367 withErrors.put(p, "IPK");
368 }
369 if (checkKeys && key != null && key.indexOf(' ') >= 0 && !withErrors.contains(p, "IPK")) {
370 errors.add(new TestError(this, Severity.WARNING, tr("Invalid white space in property key"),
371 tr(s, key), MessageFormat.format(s, key), INVALID_KEY_SPACE, p));
372 withErrors.put(p, "IPK");
373 }
374 if (checkValues && value != null && (value.startsWith(" ") || value.endsWith(" ")) && !withErrors.contains(p, "SPACE")) {
375 errors.add(new TestError(this, Severity.WARNING, tr("Property values start or end with white space"),
376 tr(s, key), MessageFormat.format(s, key), INVALID_SPACE, p));
377 withErrors.put(p, "SPACE");
378 }
379 if (checkValues && value != null && !value.equals(entities.unescape(value)) && !withErrors.contains(p, "HTML")) {
380 errors.add(new TestError(this, Severity.OTHER, tr("Property values contain HTML entity"),
381 tr(s, key), MessageFormat.format(s, key), INVALID_HTML, p));
382 withErrors.put(p, "HTML");
383 }
384 if (checkValues && key != null && value != null && value.length() > 0 && presetsValueData != null) {
385 final Set<String> values = presetsValueData.get(key);
386 final boolean keyInPresets = values != null;
387 final boolean tagInPresets = values != null && (values.isEmpty() || values.contains(prop.getValue()));
388
389 boolean ignore = false;
390 for (String a : ignoreDataStartsWith) {
391 if (key.startsWith(a)) {
392 ignore = true;
393 }
394 }
395 for (String a : ignoreDataEquals) {
396 if(key.equals(a)) {
397 ignore = true;
398 }
399 }
400 for (String a : ignoreDataEndsWith) {
401 if(key.endsWith(a)) {
402 ignore = true;
403 }
404 }
405
406 if (!tagInPresets) {
407 for (IgnoreKeyPair a : ignoreDataKeyPair) {
408 if (key.equals(a.key) && value.equals(a.value)) {
409 ignore = true;
410 }
411 }
412 }
413
414 if (!ignore) {
415 if (!keyInPresets) {
416 String i = marktr("Key ''{0}'' not in presets.");
417 errors.add(new TestError(this, Severity.OTHER, tr("Presets do not contain property key"),
418 tr(i, key), MessageFormat.format(i, key), INVALID_VALUE, p));
419 withErrors.put(p, "UPK");
420 } else if (!tagInPresets) {
421 // try to fix common typos and check again if value is still unknown
422 String fixedValue = prettifyValue(prop.getValue());
423 if (values != null && values.contains(fixedValue)) {
424 // misspelled preset value
425 String i = marktr("Value ''{0}'' for key ''{1}'' looks like ''{2}}.");
426 errors.add(new FixableTestError(this, Severity.WARNING, tr("Misspelled property value"),
427 tr(i, prop.getValue(), key, fixedValue), MessageFormat.format(i, prop.getValue(), fixedValue),
428 MISSPELLED_VALUE, p, new ChangePropertyCommand(p, key, fixedValue)));
429 withErrors.put(p, "WPV");
430 } else {
431 // unknown preset value
432 String i = marktr("Value ''{0}'' for key ''{1}'' not in presets.");
433 errors.add(new TestError(this, Severity.OTHER, tr("Presets do not contain property value"),
434 tr(i, prop.getValue(), key), MessageFormat.format(i, prop.getValue(), key), INVALID_VALUE, p));
435 withErrors.put(p, "UPV");
436 }
437 }
438 }
439 }
440 if (checkFixmes && key != null && value != null && value.length() > 0) {
441 if ((value.toLowerCase(Locale.ENGLISH).contains("fixme")
442 || value.contains("check and delete")
443 || key.contains("todo") || key.toLowerCase(Locale.ENGLISH).contains("fixme"))
444 && !withErrors.contains(p, "FIXME")) {
445 errors.add(new TestError(this, Severity.OTHER,
446 tr("FIXMES"), FIXME, p));
447 withErrors.put(p, "FIXME");
448 }
449 }
450 }
451 }
452
453 private static String prettifyValue(String value) {
454 // convert to lower case, replace ' ' or '-' with '_'
455 value = value.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(' ', '_');
456 // remove trailing or leading special chars
457 return Utils.strip(value, "-_;:,");
458 }
459
460 @Override
461 public void startTest(ProgressMonitor monitor) {
462 super.startTest(monitor);
463 checkKeys = Main.pref.getBoolean(PREF_CHECK_KEYS, true);
464 if (isBeforeUpload) {
465 checkKeys = checkKeys && Main.pref.getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true);
466 }
467
468 checkValues = Main.pref.getBoolean(PREF_CHECK_VALUES, true);
469 if (isBeforeUpload) {
470 checkValues = checkValues && Main.pref.getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true);
471 }
472
473 checkComplex = Main.pref.getBoolean(PREF_CHECK_COMPLEX, true);
474 if (isBeforeUpload) {
475 checkComplex = checkValues && Main.pref.getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true);
476 }
477
478 checkFixmes = Main.pref.getBoolean(PREF_CHECK_FIXMES, true);
479 if (isBeforeUpload) {
480 checkFixmes = checkFixmes && Main.pref.getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true);
481 }
482 }
483
484 @Override
485 public void visit(Collection<OsmPrimitive> selection) {
486 if (checkKeys || checkValues || checkComplex || checkFixmes) {
487 super.visit(selection);
488 }
489 }
490
491 @Override
492 public void addGui(JPanel testPanel) {
493 GBC a = GBC.eol();
494 a.anchor = GridBagConstraints.EAST;
495
496 testPanel.add(new JLabel(name+" :"), GBC.eol().insets(3,0,0,0));
497
498 prefCheckKeys = new JCheckBox(tr("Check property keys."), Main.pref.getBoolean(PREF_CHECK_KEYS, true));
499 prefCheckKeys.setToolTipText(tr("Validate that property keys are valid checking against list of words."));
500 testPanel.add(prefCheckKeys, GBC.std().insets(20,0,0,0));
501
502 prefCheckKeysBeforeUpload = new JCheckBox();
503 prefCheckKeysBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true));
504 testPanel.add(prefCheckKeysBeforeUpload, a);
505
506 prefCheckComplex = new JCheckBox(tr("Use complex property checker."), Main.pref.getBoolean(PREF_CHECK_COMPLEX, true));
507 prefCheckComplex.setToolTipText(tr("Validate property values and tags using complex rules."));
508 testPanel.add(prefCheckComplex, GBC.std().insets(20,0,0,0));
509
510 prefCheckComplexBeforeUpload = new JCheckBox();
511 prefCheckComplexBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true));
512 testPanel.add(prefCheckComplexBeforeUpload, a);
513
514 final Collection<String> sources = Main.pref.getCollection(PREF_SOURCES, DEFAULT_SOURCES);
515 sourcesList = new EditableList(tr("TagChecker source"));
516 sourcesList.setItems(sources);
517 testPanel.add(new JLabel(tr("Data sources ({0})", "*.cfg")), GBC.eol().insets(23, 0, 0, 0));
518 testPanel.add(sourcesList, GBC.eol().fill(GridBagConstraints.HORIZONTAL).insets(23, 0, 0, 0));
519
520 ActionListener disableCheckActionListener = new ActionListener() {
521 @Override
522 public void actionPerformed(ActionEvent e) {
523 handlePrefEnable();
524 }
525 };
526 prefCheckKeys.addActionListener(disableCheckActionListener);
527 prefCheckKeysBeforeUpload.addActionListener(disableCheckActionListener);
528 prefCheckComplex.addActionListener(disableCheckActionListener);
529 prefCheckComplexBeforeUpload.addActionListener(disableCheckActionListener);
530
531 handlePrefEnable();
532
533 prefCheckValues = new JCheckBox(tr("Check property values."), Main.pref.getBoolean(PREF_CHECK_VALUES, true));
534 prefCheckValues.setToolTipText(tr("Validate that property values are valid checking against presets."));
535 testPanel.add(prefCheckValues, GBC.std().insets(20,0,0,0));
536
537 prefCheckValuesBeforeUpload = new JCheckBox();
538 prefCheckValuesBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true));
539 testPanel.add(prefCheckValuesBeforeUpload, a);
540
541 prefCheckFixmes = new JCheckBox(tr("Check for FIXMES."), Main.pref.getBoolean(PREF_CHECK_FIXMES, true));
542 prefCheckFixmes.setToolTipText(tr("Looks for nodes or ways with FIXME in any property value."));
543 testPanel.add(prefCheckFixmes, GBC.std().insets(20,0,0,0));
544
545 prefCheckFixmesBeforeUpload = new JCheckBox();
546 prefCheckFixmesBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true));
547 testPanel.add(prefCheckFixmesBeforeUpload, a);
548 }
549
550 public void handlePrefEnable() {
551 boolean selected = prefCheckKeys.isSelected() || prefCheckKeysBeforeUpload.isSelected()
552 || prefCheckComplex.isSelected() || prefCheckComplexBeforeUpload.isSelected();
553 sourcesList.setEnabled(selected);
554 }
555
556 @Override
557 public boolean ok() {
558 enabled = prefCheckKeys.isSelected() || prefCheckValues.isSelected() || prefCheckComplex.isSelected() || prefCheckFixmes.isSelected();
559 testBeforeUpload = prefCheckKeysBeforeUpload.isSelected() || prefCheckValuesBeforeUpload.isSelected()
560 || prefCheckFixmesBeforeUpload.isSelected() || prefCheckComplexBeforeUpload.isSelected();
561
562 Main.pref.put(PREF_CHECK_VALUES, prefCheckValues.isSelected());
563 Main.pref.put(PREF_CHECK_COMPLEX, prefCheckComplex.isSelected());
564 Main.pref.put(PREF_CHECK_KEYS, prefCheckKeys.isSelected());
565 Main.pref.put(PREF_CHECK_FIXMES, prefCheckFixmes.isSelected());
566 Main.pref.put(PREF_CHECK_VALUES_BEFORE_UPLOAD, prefCheckValuesBeforeUpload.isSelected());
567 Main.pref.put(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, prefCheckComplexBeforeUpload.isSelected());
568 Main.pref.put(PREF_CHECK_KEYS_BEFORE_UPLOAD, prefCheckKeysBeforeUpload.isSelected());
569 Main.pref.put(PREF_CHECK_FIXMES_BEFORE_UPLOAD, prefCheckFixmesBeforeUpload.isSelected());
570 return Main.pref.putCollection(PREF_SOURCES, sourcesList.getItems());
571 }
572
573 @Override
574 public Command fixError(TestError testError) {
575 List<Command> commands = new ArrayList<>(50);
576
577 if (testError instanceof FixableTestError) {
578 commands.add(testError.getFix());
579 } else {
580 Collection<? extends OsmPrimitive> primitives = testError.getPrimitives();
581 for (OsmPrimitive p : primitives) {
582 Map<String, String> tags = p.getKeys();
583 if (tags == null || tags.isEmpty()) {
584 continue;
585 }
586
587 for (Entry<String, String> prop: tags.entrySet()) {
588 String key = prop.getKey();
589 String value = prop.getValue();
590 if (value == null || value.trim().isEmpty()) {
591 commands.add(new ChangePropertyCommand(p, key, null));
592 } else if (value.startsWith(" ") || value.endsWith(" ")) {
593 commands.add(new ChangePropertyCommand(p, key, Tag.removeWhiteSpaces(value)));
594 } else if (key.startsWith(" ") || key.endsWith(" ")) {
595 commands.add(new ChangePropertyKeyCommand(p, key, Tag.removeWhiteSpaces(key)));
596 } else {
597 String evalue = entities.unescape(value);
598 if (!evalue.equals(value)) {
599 commands.add(new ChangePropertyCommand(p, key, evalue));
600 } else {
601 String replacementKey = spellCheckKeyData.get(key);
602 if (replacementKey != null) {
603 commands.add(new ChangePropertyKeyCommand(p, key, replacementKey));
604 }
605 }
606 }
607 }
608 }
609 }
610
611 if (commands.isEmpty())
612 return null;
613 if (commands.size() == 1)
614 return commands.get(0);
615
616 return new SequenceCommand(tr("Fix tags"), commands);
617 }
618
619 @Override
620 public boolean isFixable(TestError testError) {
621 if (testError.getTester() instanceof TagChecker) {
622 int code = testError.getCode();
623 return code == INVALID_KEY || code == EMPTY_VALUES || code == INVALID_SPACE ||
624 code == INVALID_KEY_SPACE || code == INVALID_HTML || code == MISSPELLED_VALUE;
625 }
626
627 return false;
628 }
629
630 protected static class IgnoreKeyPair {
631 public String key;
632 public String value;
633 }
634
635 protected static class CheckerData {
636 private String description;
637 protected List<CheckerElement> data = new ArrayList<>();
638 private OsmPrimitiveType type;
639 private int code;
640 protected Severity severity;
641 protected static final int TAG_CHECK_ERROR = 1250;
642 protected static final int TAG_CHECK_WARN = 1260;
643 protected static final int TAG_CHECK_INFO = 1270;
644
645 protected static class CheckerElement {
646 public Object tag;
647 public Object value;
648 public boolean noMatch;
649 public boolean tagAll = false;
650 public boolean valueAll = false;
651 public boolean valueBool = false;
652
653 private Pattern getPattern(String str) throws PatternSyntaxException {
654 if (str.endsWith("/i"))
655 return Pattern.compile(str.substring(1,str.length()-2), Pattern.CASE_INSENSITIVE);
656 if (str.endsWith("/"))
657 return Pattern.compile(str.substring(1,str.length()-1));
658
659 throw new IllegalStateException();
660 }
661 public CheckerElement(String exp) throws PatternSyntaxException {
662 Matcher m = Pattern.compile("(.+)([!=]=)(.+)").matcher(exp);
663 m.matches();
664
665 String n = m.group(1).trim();
666
667 if ("*".equals(n)) {
668 tagAll = true;
669 } else {
670 tag = n.startsWith("/") ? getPattern(n) : n;
671 noMatch = "!=".equals(m.group(2));
672 n = m.group(3).trim();
673 if ("*".equals(n)) {
674 valueAll = true;
675 } else if ("BOOLEAN_TRUE".equals(n)) {
676 valueBool = true;
677 value = OsmUtils.trueval;
678 } else if ("BOOLEAN_FALSE".equals(n)) {
679 valueBool = true;
680 value = OsmUtils.falseval;
681 } else {
682 value = n.startsWith("/") ? getPattern(n) : n;
683 }
684 }
685 }
686
687 public boolean match(OsmPrimitive osm, Map<String, String> keys) {
688 for (Entry<String, String> prop: keys.entrySet()) {
689 String key = prop.getKey();
690 String val = valueBool ? OsmUtils.getNamedOsmBoolean(prop.getValue()) : prop.getValue();
691 if ((tagAll || (tag instanceof Pattern ? ((Pattern) tag).matcher(key).matches() : key.equals(tag)))
692 && (valueAll || (value instanceof Pattern ? ((Pattern) value).matcher(val).matches() : val.equals(value))))
693 return !noMatch;
694 }
695 return noMatch;
696 }
697 }
698
699 private static final Pattern CLEAN_STR_PATTERN = Pattern.compile(" *# *([^#]+) *$");
700 private static final Pattern SPLIT_TRIMMED_PATTERN = Pattern.compile(" *: *");
701 private static final Pattern SPLIT_ELEMENTS_PATTERN = Pattern.compile(" *&& *");
702
703 public String getData(final String str) {
704 Matcher m = CLEAN_STR_PATTERN.matcher(str);
705 String trimmed = m.replaceFirst("").trim();
706 try {
707 description = m.group(1);
708 if (description != null && description.isEmpty()) {
709 description = null;
710 }
711 } catch (IllegalStateException e) {
712 description = null;
713 }
714 String[] n = SPLIT_TRIMMED_PATTERN.split(trimmed, 3);
715 switch (n[0]) {
716 case "way":
717 type = OsmPrimitiveType.WAY;
718 break;
719 case "node":
720 type = OsmPrimitiveType.NODE;
721 break;
722 case "relation":
723 type = OsmPrimitiveType.RELATION;
724 break;
725 case "*":
726 type = null;
727 break;
728 default:
729 return tr("Could not find element type");
730 }
731 if (n.length != 3)
732 return tr("Incorrect number of parameters");
733
734 switch (n[1]) {
735 case "W":
736 severity = Severity.WARNING;
737 code = TAG_CHECK_WARN;
738 break;
739 case "E":
740 severity = Severity.ERROR;
741 code = TAG_CHECK_ERROR;
742 break;
743 case "I":
744 severity = Severity.OTHER;
745 code = TAG_CHECK_INFO;
746 break;
747 default:
748 return tr("Could not find warning level");
749 }
750 for (String exp: SPLIT_ELEMENTS_PATTERN.split(n[2])) {
751 try {
752 data.add(new CheckerElement(exp));
753 } catch (IllegalStateException e) {
754 return tr("Illegal expression ''{0}''", exp);
755 } catch (PatternSyntaxException e) {
756 return tr("Illegal regular expression ''{0}''", exp);
757 }
758 }
759 return null;
760 }
761
762 public boolean match(OsmPrimitive osm, Map<String, String> keys) {
763 if (type != null && OsmPrimitiveType.from(osm) != type)
764 return false;
765
766 for (CheckerElement ce : data) {
767 if (!ce.match(osm, keys))
768 return false;
769 }
770 return true;
771 }
772
773 public String getDescription() {
774 return tr(description);
775 }
776
777 public String getDescriptionOrig() {
778 return description;
779 }
780
781 public Severity getSeverity() {
782 return severity;
783 }
784
785 public int getCode() {
786 if (type == null)
787 return code;
788
789 return code + type.ordinal() + 1;
790 }
791 }
792}
Note: See TracBrowser for help on using the repository browser.