Changeset 7824 in josm for trunk/src/org

Timestamp:

2014-12-18T23:47:21+01:00 (9 years ago)

Author:

Don-vip

Message:

fix #10862 - proper validation of IDN (Internationalized Domain Name) URLs, both in their Unicode and ASCII form => patch of Apache DomainValidator routine

Location:

trunk/src/org/openstreetmap/josm/data/validation

Files:

• routines/DomainValidator.java (modified) (6 diffs)
• tests/InternetTags.java (modified) (5 diffs)

trunk/src/org/openstreetmap/josm/data/validation/routines/DomainValidator.java

@@ -46 +46 @@
  *     <li>{@link #isValidGenericTld} - validates generic TLDs
  *         (<code>.com, .org</code>, etc.)</li>
+ *     <li>{@link #isValidIdnTld} - validates IDN TLDs
+ *         (<code>.xn--*</code>, etc.)</li>
  *     <li>{@link #isValidCountryCodeTld} - validates country code TLDs
  *         (<code>.us, .uk, .cn</code>, etc.)</li>
@@ -64 +66 @@
     private static final String DOMAIN_LABEL_REGEX = "\\p{Alnum}(?>[\\p{Alnum}-]*\\p{Alnum})*";
     private static final String TOP_LABEL_REGEX = "\\p{Alpha}{2,}";
+    // JOSM PATCH BEGIN
+    // See #10862 - IDN TLDs in ASCII form
+    private static final String TOP_LABEL_IDN_REGEX = "(?:xn|XN)--\\p{Alnum}{2,}(?:-\\p{Alpha}{2,})?";
     private static final String DOMAIN_NAME_REGEX =
-            "^(?:" + DOMAIN_LABEL_REGEX + "\\.)+" + "(" + TOP_LABEL_REGEX + ")$";
+            "^(?:" + DOMAIN_LABEL_REGEX + "\\.)+" + "(" + TOP_LABEL_REGEX + "|" + TOP_LABEL_IDN_REGEX + ")$";
+    // JOSM PATCH END
 
     private final boolean allowLocal;
@@ -152 +158 @@
         return isValidInfrastructureTld(tld)
                 || isValidGenericTld(tld)
+                || isValidIdnTld(tld)
                 || isValidCountryCodeTld(tld);
     }
@@ -175 +182 @@
     public boolean isValidGenericTld(String gTld) {
         return Arrays.binarySearch(GENERIC_TLDS, chompLeadingDot(gTld.toLowerCase())) >= 0;
+    }
+
+    /**
+     * Returns true if the specified <code>String</code> matches any
+     * IANA-defined IDN top-level domain. Leading dots are ignored
+     * if present. The search is case-sensitive.
+     * @param iTld the parameter to check for IDN TLD status
+     * @return true if the parameter is an IDN TLD
+     */
+    public boolean isValidIdnTld(String iTld) {
+        return Arrays.binarySearch(IDN_TLDS, chompLeadingDot(iTld.toUpperCase())) >= 0;
     }
 
@@ -634 +652 @@
     };
 
+    // JOSM PATCH BEGIN
+    // see #10862 - list of IDN TLDs taken from IANA on 2014-12-18
+    private static final String[] IDN_TLDS = new String[] {
+        "XN--1QQW23A",
+        "XN--3BST00M",
+        "XN--3DS443G",
+        "XN--3E0B707E",
+        "XN--45BRJ9C",
+        "XN--45Q11C",
+        "XN--4GBRIM",
+        "XN--55QW42G",
+        "XN--55QX5D",
+        "XN--6FRZ82G",
+        "XN--6QQ986B3XL",
+        "XN--80ADXHKS",
+        "XN--80AO21A",
+        "XN--80ASEHDB",
+        "XN--80ASWG",
+        "XN--90A3AC",
+        "XN--C1AVG",
+        "XN--CG4BKI",
+        "XN--CLCHC0EA0B2G2A9GCD",
+        "XN--CZR694B",
+        "XN--CZRS0T",
+        "XN--CZRU2D",
+        "XN--D1ACJ3B",
+        "XN--D1ALF",
+        "XN--FIQ228C5HS",
+        "XN--FIQ64B",
+        "XN--FIQS8S",
+        "XN--FIQZ9S",
+        "XN--FLW351E",
+        "XN--FPCRJ9C3D",
+        "XN--FZC2C9E2C",
+        "XN--GECRJ9C",
+        "XN--H2BRJ9C",
+        "XN--HXT814E",
+        "XN--I1B6B1A6A2E",
+        "XN--IO0A7I",
+        "XN--J1AMH",
+        "XN--J6W193G",
+        "XN--KPRW13D",
+        "XN--KPRY57D",
+        "XN--KPUT3I",
+        "XN--L1ACC",
+        "XN--LGBBAT1AD8J",
+        "XN--MGB9AWBF",
+        "XN--MGBA3A4F16A",
+        "XN--MGBAAM7A8H",
+        "XN--MGBAB2BD",
+        "XN--MGBAYH7GPA",
+        "XN--MGBBH1A71E",
+        "XN--MGBC0A9AZCG",
+        "XN--MGBERP4A5D4AR",
+        "XN--MGBX4CD0AB",
+        "XN--NGBC5AZD",
+        "XN--NODE",
+        "XN--NQV7F",
+        "XN--NQV7FS00EMA",
+        "XN--O3CW4H",
+        "XN--OGBPF8FL",
+        "XN--P1ACF",
+        "XN--P1AI",
+        "XN--PGBS0DH",
+        "XN--Q9JYB4C",
+        "XN--QCKA1PMC",
+        "XN--RHQV96G",
+        "XN--S9BRJ9C",
+        "XN--SES554G",
+        "XN--UNUP4Y",
+        "XN--VERMGENSBERATER-CTB",
+        "XN--VERMGENSBERATUNG-PWB",
+        "XN--VHQUV",
+        "XN--WGBH1C",
+        "XN--WGBL6A",
+        "XN--XHQ521B",
+        "XN--XKC2AL3HYE2A",
+        "XN--XKC2DL3A5EE0H",
+        "XN--YFRO4I67O",
+        "XN--YGBI2AMMX",
+        "XN--ZFR164B",
+    };
+    // END JOSM PATCH
+
     private static final String[] COUNTRY_CODE_TLDS = new String[] {
         "ac",                 // Ascension Island
@@ -897 +999 @@
         Arrays.sort(COUNTRY_CODE_TLDS);
         Arrays.sort(GENERIC_TLDS);
+        Arrays.sort(IDN_TLDS);
         Arrays.sort(LOCAL_TLDS);
     }

trunk/src/org/openstreetmap/josm/data/validation/tests/InternetTags.java

@@ -3 +3 @@
 
 import static org.openstreetmap.josm.tools.I18n.tr;
+
+import java.net.IDN;
+import java.util.regex.Pattern;
 
 import org.openstreetmap.josm.command.ChangePropertyCommand;
@@ -23 +26 @@
 public class InternetTags extends Test {
 
-    protected static final int INVALID_URL = 3301;
-    protected static final int INVALID_EMAIL = 3302;
+    /** Error code for an invalid URL */
+    public static final int INVALID_URL = 3301;
+    /** Error code for an invalid e-mail */
+    public static final int INVALID_EMAIL = 3302;
+
+    private static final Pattern ASCII_PATTERN = Pattern.compile("^\\p{ASCII}+$");
 
     /**
      * List of keys subject to URL validation.
      */
-    public static String[] URL_KEYS = new String[] {
+    private static String[] URL_KEYS = new String[] {
         "url", "source:url",
         "website", "contact:website", "heritage:website", "source:website"
@@ -37 +44 @@
      * List of keys subject to email validation.
      */
-    public static String[] EMAIL_KEYS = new String[] {
+    private static String[] EMAIL_KEYS = new String[] {
         "email", "contact:email"
     };
@@ -48 +55 @@
     }
 
+    /**
+     * Potentially validates a given primitive key against a given validator.
+     * @param p The OSM primitive to test
+     * @param k The key to validate
+     * @param keys The list of keys to check. If {@code k} is not inside this collection, do nothing
+     * @param validator The validator to run if {@code k} is inside {@code keys}
+     * @param code The error code to set if the validation fails
+     * @return {@code true} if the validation fails. In this case, a new error has been created.
+     */
     private boolean doTest(OsmPrimitive p, String k, String[] keys, AbstractValidator validator, int code) {
         for (String i : keys) {
             if (i.equals(k)) {
-                if (!validator.isValid(p.get(k))) {
-                    TestError error;
-                    String msg = tr("''{0}'': {1}", k, validator.getErrorMessage());
-                    String fix = validator.getFix();
-                    if (fix != null) {
-                        error = new FixableTestError(this, Severity.WARNING, msg, code, p,
-                                new ChangePropertyCommand(p, k, fix));
-                    } else {
-                        error = new TestError(this, Severity.WARNING, msg, code, p);
-                    }
-                    return errors.add(error);
+                TestError error = validateTag(p, k, validator, code);
+                if (error != null) {
+                    errors.add(error);
                 }
                 break;
@@ -69 +77 @@
     }
 
+    /**
+     * Validates a given primitive tag against a given validator.
+     * @param p The OSM primitive to test
+     * @param k The key to validate
+     * @param validator The validator to run
+     * @param code The error code to set if the validation fails
+     * @return The error if the validation fails, {@code null} otherwise
+     * @since 7824
+     */
+    public TestError validateTag(OsmPrimitive p, String k, AbstractValidator validator, int code) {
+        TestError error = doValidateTag(p, k, null, validator, code);
+        if (error != null) {
+            // Workaround to https://issues.apache.org/jira/browse/VALIDATOR-290
+            // Apache Commons Validator 1.4.1-SNAPSHOT does not support yet IDN URLs
+            // To remove if it gets fixed on Apache side
+            String v = p.get(k);
+            if (!ASCII_PATTERN.matcher(v).matches()) {
+                try {
+                    String protocol = "";
+                    if (v.contains("://")) {
+                        protocol = v.substring(0, v.indexOf("://")+3);
+                    }
+                    String domain = !protocol.isEmpty() ? v.substring(protocol.length(), v.length()) : v;
+                    String ending = "";
+                    if (domain.contains("/")) {
+                        int idx = domain.indexOf("/");
+                        ending = domain.substring(idx, domain.length());
+                        domain = domain.substring(0, idx);
+                    }
+                    // Try to apply ToASCII algorithm
+                    error = doValidateTag(p, k, protocol+IDN.toASCII(domain)+ending, validator, code);
+                } catch (IllegalArgumentException e) {
+                    error.setMessage(error.getMessage() +
+                            tr(" URL cannot be converted to ASCII: {0}", e.getMessage()));
+                }
+            }
+        }
+        return error;
+    }
+
+    /**
+     * Validates a given primitive tag against a given validator.
+     * @param p The OSM primitive to test
+     * @param k The key to validate
+     * @param v The value to validate. May be {@code null} to use {@code p.get(k)}
+     * @param validator The validator to run
+     * @param code The error code to set if the validation fails
+     * @return The error if the validation fails, {@code null} otherwise
+     */
+    private TestError doValidateTag(OsmPrimitive p, String k, String v, AbstractValidator validator, int code) {
+        TestError error = null;
+        if (!validator.isValid(v != null ? v : p.get(k))) {
+            String msg = tr("''{0}'': {1}", k, validator.getErrorMessage());
+            String fix = validator.getFix();
+            if (fix != null) {
+                error = new FixableTestError(this, Severity.WARNING, msg, code, p,
+                        new ChangePropertyCommand(p, k, fix));
+            } else {
+                error = new TestError(this, Severity.WARNING, msg, code, p);
+            }
+        }
+        return error;
+    }
+
     private void test(OsmPrimitive p) {
         for (String k : p.keySet()) {
+            // Test key against URL validator
             if (!doTest(p, k, URL_KEYS, UrlValidator.getInstance(), INVALID_URL)) {
+                // Test key against e-mail validator only if the URL validator did not fail
                 doTest(p, k, EMAIL_KEYS, EmailValidator.getInstance(), INVALID_EMAIL);
             }