Changeset 14933 in josm for trunk

Timestamp:

2019-03-26T01:58:41+01:00 (5 years ago)

Author:

Don-vip

Message:

fix #15645, fix #17521 - Detect and fix non-printing control characters

Location:

trunk

Files:

• src/org/openstreetmap/josm/data/validation/tests/TagChecker.java (modified) (5 diffs)
• test/unit/org/openstreetmap/josm/data/validation/tests/TagCheckerTest.java (modified) (1 diff)

trunk/src/org/openstreetmap/josm/data/validation/tests/TagChecker.java

@@ -20 +20 @@
 import java.util.Map.Entry;
 import java.util.Set;
+import java.util.regex.Pattern;
 
 import javax.swing.JCheckBox;
@@ -72 +73 @@
     /** often used tags which are not in presets */
     private static volatile MultiMap<String, String> oftenUsedTags = new MultiMap<>();
+
+    private static final Pattern NON_PRINTING_CONTROL_CHARACTERS = Pattern.compile(
+            "[\\x00-\\x09\\x0B\\x0C\\x0E-\\x1F\\x7F\\u200c-\\u200f\\u202a-\\u202e]");
 
     /** The TagChecker data */
@@ -372 +376 @@
 
     /**
-     * Checks given string (key or value) if it contains characters with code below 0x20 (either newline or some other special characters)
+     * Checks given string (key or value) if it contains non-printing control characters (either ASCII or Unicode bidi characters)
      * @param s string to check
-     * @return {@code true} if {@code s} contains characters with code below 0x20
-     */
-    private static boolean containsLow(String s) {
+     * @return {@code true} if {@code s} contains non-printing control characters
+     */
+    private static boolean containsNonPrintingControlCharacter(String s) {
         if (s == null)
             return false;
         for (int i = 0; i < s.length(); i++) {
-            if (s.charAt(i) < 0x20)
+            char c = s.charAt(i);
+            if ((IsAsciiControlChar(c) && !isNewLineChar(c)) || IsBidiControlChar(c))
                 return true;
         }
         return false;
+    }
+
+    private static boolean IsAsciiControlChar(char c) {
+        return c < 0x20 || c == 0x7F;
+    }
+
+    private static boolean isNewLineChar(char c) {
+        return c == 0x0a || c == 0x0d;
+    }
+
+    private static boolean IsBidiControlChar(char c) {
+        /* check for range 0x200c to 0x200f (ZWNJ, ZWJ, LRM, RLM) or
+                           0x202a to 0x202e (LRE, RLE, PDF, LRO, RLO) */
+        return (((c & 0xfffffffc) == 0x200c) || ((c >= 0x202a) && (c <= 0x202e)));
+    }
+
+    static String removeNonPrintingControlCharacters(String s) {
+        return NON_PRINTING_CONTROL_CHARACTERS.matcher(s).replaceAll("");
     }
 
@@ -515 +538 @@
         if (!checkValues || value == null)
             return;
-        if ((containsLow(value)) && !withErrors.contains(p, "ICV")) {
+        if ((containsNonPrintingControlCharacter(value)) && !withErrors.contains(p, "ICV")) {
             errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_VALUE)
-                    .message(tr("Tag value contains character with code less than 0x20"), s, key)
-                    .primitives(p)
+                    .message(tr("Tag value contains non-printing character"), s, key)
+                    .primitives(p)
+                    .fix(() -> new ChangePropertyCommand(p, key, removeNonPrintingControlCharacters(value)))
                     .build());
             withErrors.put(p, "ICV");
@@ -563 +587 @@
         if (!checkKeys || key == null)
             return;
-        if ((containsLow(key)) && !withErrors.contains(p, "ICK")) {
+        if ((containsNonPrintingControlCharacter(key)) && !withErrors.contains(p, "ICK")) {
             errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_KEY)
-                    .message(tr("Tag key contains character with code less than 0x20"), s, key)
-                    .primitives(p)
+                    .message(tr("Tag key contains non-printing character"), s, key)
+                    .primitives(p)
+                    .fix(() -> new ChangePropertyCommand(p, key, removeNonPrintingControlCharacters(key)))
                     .build());
             withErrors.put(p, "ICK");

trunk/test/unit/org/openstreetmap/josm/data/validation/tests/TagCheckerTest.java

@@ -248 +248 @@
     }
 
+    /**
+     * Unit test of {@link TagChecker#removeNonPrintingControlCharacters}
+     */
+    @Test
+    public void testRemoveUnprintableControlCharacters() {
+        // Check 65 ASCII control characters are removed, except new lines
+        for (char c = 0x0; c < 0x20; c++) {
+            if (c != '\r' && c != '\n') {
+                assertTrue(TagChecker.removeNonPrintingControlCharacters(Character.toString(c)).isEmpty());
+            } else {
+                assertFalse(TagChecker.removeNonPrintingControlCharacters(Character.toString(c)).isEmpty());
+            }
+        }
+        assertTrue(TagChecker.removeNonPrintingControlCharacters(Character.toString((char) 0x7F)).isEmpty());
+        // Check 9 Unicode bidi control characters are removed
+        for (char c = 0x200c; c <= 0x200f; c++) {
+            assertTrue(TagChecker.removeNonPrintingControlCharacters(Character.toString(c)).isEmpty());
+        }
+        for (char c = 0x202a; c <= 0x202e; c++) {
+            assertTrue(TagChecker.removeNonPrintingControlCharacters(Character.toString(c)).isEmpty());
+        }
+    }
 }