Changeset 16262 in josm

Timestamp:

2020-04-11T18:00:31+02:00 (5 years ago)

Author:

simon04

Message:

fix #18164 - Migrate OverpassTurboQueryWizard to Java

The new OverpassTurboQueryWizard first invokes SearchCompiler, and then turns the AST into an Overpass QL.

Location:

trunk

Files:

• README (modified) (1 diff)
• resources/data/overpass-wizard.js (deleted)
• src/org/openstreetmap/josm/data/osm/search/SearchCompiler.java (modified) (10 diffs)
• src/org/openstreetmap/josm/gui/download/OverpassQueryWizardDialog.java (modified) (1 diff)
• src/org/openstreetmap/josm/tools/OverpassTurboQueryWizard.java (modified) (3 diffs)
• test/unit/org/openstreetmap/josm/tools/OverpassTurboQueryWizardTest.java (modified) (7 diffs)

trunk/README

@@ -123 +123 @@
     - help-browser.css      CSS file for the help sites (HTML content is downloaded from the website
                             on demand, but displayed inside the programm in a Java web browser component.)
-    - overpass-wizard.js    Javascript code to provide a wizard-GUI for creating Overpass requests
-                            (external library, see https://github.com/tyrasd/overpass-wizard)
     - *.lang                translation data
     - *.xsd                 xml schema files for validation of configuration files

trunk/src/org/openstreetmap/josm/data/osm/search/SearchCompiler.java

@@ -505 +505 @@
      * Matches if the value of the corresponding key is ''yes'', ''true'', ''1'' or ''on''.
      */
-    private static class BooleanMatch extends TaggedMatch {
+    public static class BooleanMatch extends TaggedMatch {
         private final String key;
         private final boolean defaultValue;
@@ -512 +512 @@
             this.key = key;
             this.defaultValue = defaultValue;
+        }
+
+        public String getKey() {
+            return key;
         }
 
@@ -710 +714 @@
      * Matches objects with the given key-value pair.
      */
-    private static class KeyValue extends TaggedMatch {
+    public static class KeyValue extends TaggedMatch {
         private final String key;
         private final Pattern keyPattern;
@@ -775 +779 @@
         }
 
+        public String getKey() {
+            return key;
+        }
+
+        public String getValue() {
+            return value;
+        }
+
         @Override
         public String toString() {
@@ -891 +903 @@
     public static class ExactKeyValue extends TaggedMatch {
 
-        enum Mode {
+        public enum Mode {
             ANY, ANY_KEY, ANY_VALUE, EXACT, NONE, MISSING_KEY,
             ANY_KEY_REGEXP, ANY_VALUE_REGEXP, EXACT_REGEXP, MISSING_KEY_REGEXP;
@@ -1003 +1015 @@
         }
 
+        public String getKey() {
+            return key;
+        }
+
+        public String getValue() {
+            return value;
+        }
+
+        public Mode getMode() {
+            return mode;
+        }
+
         @Override
         public String toString() {
@@ -1148 +1172 @@
     }
 
-    private static class ExactType extends Match {
+    public static class ExactType extends Match {
         private final OsmPrimitiveType type;
 
@@ -1157 +1181 @@
         }
 
+        public OsmPrimitiveType getType() {
+            return type;
+        }
+
         @Override
         public boolean match(OsmPrimitive osm) {
@@ -1186 +1214 @@
      * Matches objects last changed by the given username.
      */
-    private static class UserMatch extends Match {
+    public static class UserMatch extends Match {
         private String user;
 
@@ -1195 +1223 @@
                 this.user = user;
             }
+        }
+
+        public String getUser() {
+            return user;
         }
 

trunk/src/org/openstreetmap/josm/gui/download/OverpassQueryWizardDialog.java

@@ -221 +221 @@
                 .append(TR_START).append(TD_START)
                 .append(Utils.joinAsHtmlUnorderedList(Arrays.asList("<i>key=value</i>", "<i>key=*</i>", "<i>key~regex</i>",
-                        "<i>key!=value</i>", "<i>key!~regex</i>", "<i>key=\"combined value\"</i>")))
+                        "<i>-key=value</i>", "<i>-key~regex</i>", "<i>key=\"combined value\"</i>")))
                 .append(TD_END).append(TD_START)
                 .append(tr("<span>Download objects that have some concrete key/value pair, only the key with any contents for the value, " +

trunk/src/org/openstreetmap/josm/tools/OverpassTurboQueryWizard.java

@@ -2 +2 @@
 package org.openstreetmap.josm.tools;
 
-import java.io.IOException;
-import java.io.Reader;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Optional;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
-import javax.script.Invocable;
-import javax.script.ScriptEngine;
-import javax.script.ScriptException;
-
-import org.openstreetmap.josm.io.CachedFile;
-import org.openstreetmap.josm.spi.preferences.Config;
+import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
+import org.openstreetmap.josm.data.osm.search.SearchCompiler;
+import org.openstreetmap.josm.data.osm.search.SearchCompiler.Match;
+import org.openstreetmap.josm.data.osm.search.SearchParseError;
 
 /**
- * Uses <a href="https://github.com/tyrasd/overpass-wizard/">Overpass Turbo query wizard</a> code (MIT Licensed)
- * to build an Overpass QL from a {@link org.openstreetmap.josm.actions.search.SearchAction} like query.
+ * Builds an Overpass QL from a {@link org.openstreetmap.josm.actions.search.SearchAction} query.
  *
- * Requires a JavaScript {@link ScriptEngine}.
- * @since 8744
+ * @since 8744 (using tyrasd/overpass-wizard), 16262 (standalone)
  */
 public final class OverpassTurboQueryWizard {
 
-    private static OverpassTurboQueryWizard instance;
-    private final ScriptEngine engine = Utils.getJavaScriptEngine();
+    private static final OverpassTurboQueryWizard instance = new OverpassTurboQueryWizard();
 
     /**
@@ -29 +32 @@
      * @return the unique instance of this class
      */
-    public static synchronized OverpassTurboQueryWizard getInstance() {
-        if (instance == null) {
-            instance = new OverpassTurboQueryWizard();
-        }
+    public static OverpassTurboQueryWizard getInstance() {
         return instance;
     }
 
     private OverpassTurboQueryWizard() {
-        try (CachedFile file = new CachedFile("resource://data/overpass-wizard.js");
-             Reader reader = file.getContentReader()) {
-            if (engine != null) {
-                engine.eval("var console = {error: " + Logging.class.getCanonicalName() + ".warn};");
-                engine.eval("var global = {};");
-                engine.eval(reader);
-                engine.eval("var overpassWizardJOSM = function(query) {" +
-                        "  return overpassWizard(query, {" +
-                        "    comment: false," +
-                        "    timeout: " + Config.getPref().getInt("overpass.wizard.timeout", 90) + "," +
-                        "    outputFormat: 'xml'," +
-                        "    outputMode: 'recursive_meta'" +
-                        "  });" +
-                        "}");
-            }
-        } catch (ScriptException | IOException ex) {
-            throw new IllegalStateException("Failed to initialize OverpassTurboQueryWizard", ex);
-        }
+        // private constructor for utility class
     }
 
@@ -63 +46 @@
      * @throws UncheckedParseException when the parsing fails
      */
-    public String constructQuery(String search) {
-        if (engine == null) {
-            throw new IllegalStateException("Failed to retrieve JavaScript engine");
-        }
+    public String constructQuery(final String search) {
         try {
-            final Object result = ((Invocable) engine).invokeFunction("overpassWizardJOSM", search);
-            if (Boolean.FALSE.equals(result)) {
-                throw new UncheckedParseException();
+            Matcher matcher = Pattern.compile("\\s+GLOBAL\\s*$", Pattern.CASE_INSENSITIVE).matcher(search);
+            if (matcher.find()) {
+                final Match match = SearchCompiler.compile(matcher.replaceFirst(""));
+                return constructQuery(match, ";", "");
             }
-            return (String) result;
-        } catch (NoSuchMethodException e) {
-            throw new IllegalStateException(e);
-        } catch (ScriptException e) {
-            throw new UncheckedParseException("Failed to execute OverpassTurboQueryWizard", e);
+
+            matcher = Pattern.compile("\\s+IN BBOX\\s*$", Pattern.CASE_INSENSITIVE).matcher(search);
+            if (matcher.find()) {
+                final Match match = SearchCompiler.compile(matcher.replaceFirst(""));
+                return constructQuery(match, "[bbox:{{bbox}}];", "");
+            }
+
+            matcher = Pattern.compile("\\s+(?<mode>IN|AROUND)\\s+(?<area>[^\" ]+|\"[^\"]+\")\\s*$", Pattern.CASE_INSENSITIVE).matcher(search);
+            if (matcher.find()) {
+                final Match match = SearchCompiler.compile(matcher.replaceFirst(""));
+                final String mode = matcher.group("mode").toUpperCase(Locale.ENGLISH);
+                final String area = Utils.strip(matcher.group("area"), "\"");
+                if ("IN".equals(mode)) {
+                    return constructQuery(match, ";\n{{geocodeArea:" + area + "}}->.searchArea;", "(area.searchArea)");
+                } else if ("AROUND".equals(mode)) {
+                    return constructQuery(match, ";\n{{radius=1000}}", "(around:{{radius}},{{geocodeCoords:" + area + "}})");
+                } else {
+                    throw new IllegalStateException(mode);
+                }
+            }
+            
+            final Match match = SearchCompiler.compile(search);
+            return constructQuery(match, "[bbox:{{bbox}}];", "");
+        } catch (SearchParseError | UnsupportedOperationException e) {
+            throw new UncheckedParseException(e);
         }
     }
+
+    private String constructQuery(final Match match, final String bounds, final String queryLineSuffix) {
+        final List<Match> normalized = normalizeToDNF(match);
+        final List<String> queryLines = new ArrayList<>();
+        queryLines.add("[out:xml][timeout:90]" + bounds);
+        queryLines.add("(");
+        for (Match conjunction : normalized) {
+            final EnumSet<OsmPrimitiveType> types = EnumSet.noneOf(OsmPrimitiveType.class);
+            final String query = constructQuery(conjunction, types);
+            for (Object type : types.isEmpty() || types.size() == 3 ? Collections.singleton("nwr") : types) {
+                queryLines.add("  " + type + query + queryLineSuffix + ";");
+            }
+        }
+        queryLines.add(");");
+        queryLines.add("(._;>;);");
+        queryLines.add("out meta;");
+        return String.join("\n", queryLines);
+    }
+
+    private static String constructQuery(Match match, final Set<OsmPrimitiveType> types) {
+        final boolean negated;
+        if (match instanceof SearchCompiler.Not) {
+            negated = true;
+            match = ((SearchCompiler.Not) match).getMatch();
+        } else {
+            negated = false;
+        }
+        if (match instanceof SearchCompiler.And) {
+            return ((SearchCompiler.And) match).map(m -> constructQuery(m, types), (s1, s2) -> s1 + s2);
+        } else if (match instanceof SearchCompiler.KeyValue) {
+            final String key = ((SearchCompiler.KeyValue) match).getKey();
+            final String value = ((SearchCompiler.KeyValue) match).getValue();
+            return "[~" + quote(key) + "~" + quote(value) + "]";
+        } else if (match instanceof SearchCompiler.ExactKeyValue) {
+            // https://wiki.openstreetmap.org/wiki/Overpass_API/Language_Guide
+            // ["key"]             -- filter objects tagged with this key and any value
+            // [!"key"]            -- filter objects not tagged with this key and any value
+            // ["key"="value"]     -- filter objects tagged with this key and this value
+            // ["key"!="value"]    -- filter objects tagged with this key but not this value, or not tagged with this key
+            // ["key"~"value"]     -- filter objects tagged with this key and a value matching a regular expression
+            // ["key"!~"value"]    -- filter objects tagged with this key but a value not matching a regular expression
+            // [~"key"~"value"]    -- filter objects tagged with a key and a value matching regular expressions
+            // [~"key"~"value", i] -- filter objects tagged with a key and a case-insensitive value matching regular expressions
+            final String key = ((SearchCompiler.ExactKeyValue) match).getKey();
+            final String value = ((SearchCompiler.ExactKeyValue) match).getValue();
+            final SearchCompiler.ExactKeyValue.Mode mode = ((SearchCompiler.ExactKeyValue) match).getMode();
+            switch (mode) {
+                case ANY_VALUE:
+                    return "[" + (negated ? "!" : "") + quote(key) + "]";
+                case EXACT:
+                    return "[" + quote(key) + (negated ? "!=" : "=") + quote(value) + "]";
+                case EXACT_REGEXP:
+                    final Matcher matcher = Pattern.compile("/(?<regex>.*)/(?<flags>i)?").matcher(value);
+                    final String valueQuery = matcher.matches()
+                            ? quote(matcher.group("regex")) + Optional.ofNullable(matcher.group("flags")).map(f -> "," + f).orElse("")
+                            : quote(value);
+                    return "[" + quote(key) + (negated ? "!~" : "~") + valueQuery + "]";
+                case MISSING_KEY:
+                    // special case for empty values, see https://github.com/drolbr/Overpass-API/issues/53
+                    return "[" + quote(key) + (negated ? "!~" : "~") + quote("^$") + "]";
+                default:
+                    return "";
+            }
+        } else if (match instanceof SearchCompiler.BooleanMatch) {
+            final String key = ((SearchCompiler.BooleanMatch) match).getKey();
+            return negated
+                    ? "[" + quote(key) + "~\"false|no|0|off\"]"
+                    : "[" + quote(key) + "~\"true|yes|1|on\"]";
+        } else if (match instanceof SearchCompiler.UserMatch) {
+            final String user = ((SearchCompiler.UserMatch) match).getUser();
+            return user.matches("\\d+")
+                    ? "(uid:" + user + ")"
+                    : "(user:" + quote(user) + ")";
+        } else if (match instanceof SearchCompiler.ExactType) {
+            types.add(((SearchCompiler.ExactType) match).getType());
+            return "";
+        }
+        Logging.warn("Unsupported match type {0}: {1}", match.getClass(), match);
+        return "/*" + match + "*/";
+    }
+
+    /**
+     * Quotes the given string for its use in Overpass QL
+     */
+    private static String quote(final String s) {
+        return "\"" + s.replace("\\", "\\\\").replace("\"", "\\\"") + "\"";
+    }
+
+    /**
+     * Normalizes the match to disjunctive normal form: A∧(B∨C) ⇔ (A∧B)∨(A∧C)
+     */
+    private static List<Match> normalizeToDNF(final Match match) {
+        if (match instanceof SearchCompiler.And) {
+            return ((SearchCompiler.And) match).map(OverpassTurboQueryWizard::normalizeToDNF, (lhs, rhs) -> lhs.stream()
+                    .flatMap(l -> rhs.stream().map(r -> new SearchCompiler.And(l, r)))
+                    .collect(Collectors.toList()));
+        } else if (match instanceof SearchCompiler.Or) {
+            return ((SearchCompiler.Or) match).map(OverpassTurboQueryWizard::normalizeToDNF, CompositeList::new);
+        } else if (match instanceof SearchCompiler.Xor) {
+            throw new UnsupportedOperationException(match.toString());
+        } else if (match instanceof SearchCompiler.Not) {
+            // only support negated KeyValue or ExactKeyValue matches
+            final Match innerMatch = ((SearchCompiler.Not) match).getMatch();
+            if (innerMatch instanceof SearchCompiler.BooleanMatch
+                    || innerMatch instanceof SearchCompiler.KeyValue
+                    || innerMatch instanceof SearchCompiler.ExactKeyValue) {
+                return Collections.singletonList(match);
+            }
+            throw new UnsupportedOperationException(match.toString());
+        } else {
+            return Collections.singletonList(match);
+        }
+    }
+
 }

trunk/test/unit/org/openstreetmap/josm/tools/OverpassTurboQueryWizardTest.java

@@ -39 +39 @@
     public void testKeyValue() {
         assertQueryEquals("  nwr[\"amenity\"=\"drinking_water\"];\n", "amenity=drinking_water");
+        assertQueryEquals("  nwr[\"amenity\"];\n", "amenity=*");
     }
 
@@ -46 +47 @@
     @Test
     public void testKeyNotValue() {
-        assertQueryEquals("  nwr[\"amenity\"!=\"drinking_water\"];\n", "amenity!=drinking_water");
-        assertQueryEquals("  nwr[\"amenity\"!=\"drinking_water\"];\n", "amenity<>drinking_water");
+        assertQueryEquals("  nwr[\"amenity\"!=\"drinking_water\"];\n", "-amenity=drinking_water");
+        assertQueryEquals("  nwr[!\"amenity\"];\n", "-amenity=*");
     }
 
@@ -57 +58 @@
         assertQueryEquals("  nwr[\"foo\"~\"bar\"];\n", "foo~bar");
         assertQueryEquals("  nwr[\"foo\"~\"bar\"];\n", "foo~/bar/");
-        assertQueryEquals("  nwr[\"foo\"~\"bar\"];\n", "foo~=bar");
-        assertQueryEquals("  nwr[\"foo\"~\"bar\"];\n", "foo~=/bar/");
-        assertQueryEquals("  nwr[\"foo\"~\"bar\"];\n", "foo like bar");
-        assertQueryEquals("  nwr[\"foo\"~\"bar\"];\n", "foo like /bar/");
         // case insensitive
         assertQueryEquals("  nwr[\"foo\"~\"bar\",i];\n", "foo~/bar/i");
         // negated
-        assertQueryEquals("  nwr[\"foo\"!~\"bar\"];\n", "foo!~bar");
-        assertQueryEquals("  nwr[\"foo\"!~\"bar\"];\n", "foo not like bar");
+        assertQueryEquals("  nwr[\"foo\"!~\"bar\"];\n", "-foo~bar");
+        assertQueryEquals("  nwr[\"foo\"!~\"bar\",i];\n", "-foo~/bar/i");
+    }
+
+    /**
+     * Test OSM boolean true/false.
+     */
+    @Test
+    public void testOsmBoolean() {
+        assertQueryEquals("  nwr[\"highway\"][\"oneway\"~\"true|yes|1|on\"];\n", "highway=* AND oneway?");
+        assertQueryEquals("  nwr[\"highway\"][\"oneway\"~\"false|no|0|off\"];\n", "highway=* AND -oneway?");
     }
 
@@ -84 +90 @@
     public void testBooleanOr() {
         assertQueryEquals("  nwr[\"foo\"=\"bar\"];\n  nwr[\"baz\"=\"42\"];\n", "foo=bar or baz=42");
-        assertQueryEquals("  nwr[\"foo\"=\"bar\"];\n  nwr[\"baz\"=\"42\"];\n", "foo=bar || baz=42");
         assertQueryEquals("  nwr[\"foo\"=\"bar\"];\n  nwr[\"baz\"=\"42\"];\n", "foo=bar | baz=42");
     }
@@ -118 +123 @@
     @Test
     public void testUser() {
-        assertQueryEquals("  nwr(user:\"foo\");\n  nwr(uid:42);\n", "user:foo or uid:42");
+        assertQueryEquals("  nwr(user:\"foo\");\n  nwr(uid:42);\n", "user:foo or user:42");
     }
 
@@ -126 +131 @@
     @Test
     public void testEmpty() {
-        assertQueryEquals("  way[\"foo\"~\"^$\"];\n", "foo='' and type:way");
+        assertQueryEquals("  way[\"foo\"~\"^$\"];\n", "foo=\"\" and type:way");
     }
 
@@ -220 +225 @@
     @Test(expected = UncheckedParseException.class)
     public void testErroneous() {
-        OverpassTurboQueryWizard.getInstance().constructQuery("foo");
+        OverpassTurboQueryWizard.getInstance().constructQuery("-(foo or bar)");
     }
 }