Changeset 16262 in josm


Ignore:
Timestamp:
2020-04-11T18:00:31+02:00 (18 months ago)
Author:
simon04
Message:

fix #18164 - Migrate OverpassTurboQueryWizard to Java

The new OverpassTurboQueryWizard first invokes SearchCompiler, and then turns the AST into an Overpass QL.

Location:
trunk
Files:
1 deleted
5 edited

Legend:

Unmodified
Added
Removed
  • trunk/README

    r16169 r16262  
    123123    - help-browser.css      CSS file for the help sites (HTML content is downloaded from the website
    124124                            on demand, but displayed inside the programm in a Java web browser component.)
    125     - overpass-wizard.js    Javascript code to provide a wizard-GUI for creating Overpass requests
    126                             (external library, see https://github.com/tyrasd/overpass-wizard)
    127125    - *.lang                translation data
    128126    - *.xsd                 xml schema files for validation of configuration files
  • trunk/src/org/openstreetmap/josm/data/osm/search/SearchCompiler.java

    r16260 r16262  
    505505     * Matches if the value of the corresponding key is ''yes'', ''true'', ''1'' or ''on''.
    506506     */
    507     private static class BooleanMatch extends TaggedMatch {
     507    public static class BooleanMatch extends TaggedMatch {
    508508        private final String key;
    509509        private final boolean defaultValue;
     
    512512            this.key = key;
    513513            this.defaultValue = defaultValue;
     514        }
     515
     516        public String getKey() {
     517            return key;
    514518        }
    515519
     
    710714     * Matches objects with the given key-value pair.
    711715     */
    712     private static class KeyValue extends TaggedMatch {
     716    public static class KeyValue extends TaggedMatch {
    713717        private final String key;
    714718        private final Pattern keyPattern;
     
    775779        }
    776780
     781        public String getKey() {
     782            return key;
     783        }
     784
     785        public String getValue() {
     786            return value;
     787        }
     788
    777789        @Override
    778790        public String toString() {
     
    891903    public static class ExactKeyValue extends TaggedMatch {
    892904
    893         enum Mode {
     905        public enum Mode {
    894906            ANY, ANY_KEY, ANY_VALUE, EXACT, NONE, MISSING_KEY,
    895907            ANY_KEY_REGEXP, ANY_VALUE_REGEXP, EXACT_REGEXP, MISSING_KEY_REGEXP;
     
    10031015        }
    10041016
     1017        public String getKey() {
     1018            return key;
     1019        }
     1020
     1021        public String getValue() {
     1022            return value;
     1023        }
     1024
     1025        public Mode getMode() {
     1026            return mode;
     1027        }
     1028
    10051029        @Override
    10061030        public String toString() {
     
    11481172    }
    11491173
    1150     private static class ExactType extends Match {
     1174    public static class ExactType extends Match {
    11511175        private final OsmPrimitiveType type;
    11521176
     
    11571181        }
    11581182
     1183        public OsmPrimitiveType getType() {
     1184            return type;
     1185        }
     1186
    11591187        @Override
    11601188        public boolean match(OsmPrimitive osm) {
     
    11861214     * Matches objects last changed by the given username.
    11871215     */
    1188     private static class UserMatch extends Match {
     1216    public static class UserMatch extends Match {
    11891217        private String user;
    11901218
     
    11951223                this.user = user;
    11961224            }
     1225        }
     1226
     1227        public String getUser() {
     1228            return user;
    11971229        }
    11981230
  • trunk/src/org/openstreetmap/josm/gui/download/OverpassQueryWizardDialog.java

    r15551 r16262  
    221221                .append(TR_START).append(TD_START)
    222222                .append(Utils.joinAsHtmlUnorderedList(Arrays.asList("<i>key=value</i>", "<i>key=*</i>", "<i>key~regex</i>",
    223                         "<i>key!=value</i>", "<i>key!~regex</i>", "<i>key=\"combined value\"</i>")))
     223                        "<i>-key=value</i>", "<i>-key~regex</i>", "<i>key=\"combined value\"</i>")))
    224224                .append(TD_END).append(TD_START)
    225225                .append(tr("<span>Download objects that have some concrete key/value pair, only the key with any contents for the value, " +
  • trunk/src/org/openstreetmap/josm/tools/OverpassTurboQueryWizard.java

    r15814 r16262  
    22package org.openstreetmap.josm.tools;
    33
    4 import java.io.IOException;
    5 import java.io.Reader;
     4import java.util.ArrayList;
     5import java.util.Collections;
     6import java.util.EnumSet;
     7import java.util.List;
     8import java.util.Locale;
     9import java.util.Optional;
     10import java.util.Set;
     11import java.util.regex.Matcher;
     12import java.util.regex.Pattern;
     13import java.util.stream.Collectors;
    614
    7 import javax.script.Invocable;
    8 import javax.script.ScriptEngine;
    9 import javax.script.ScriptException;
    10 
    11 import org.openstreetmap.josm.io.CachedFile;
    12 import org.openstreetmap.josm.spi.preferences.Config;
     15import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
     16import org.openstreetmap.josm.data.osm.search.SearchCompiler;
     17import org.openstreetmap.josm.data.osm.search.SearchCompiler.Match;
     18import org.openstreetmap.josm.data.osm.search.SearchParseError;
    1319
    1420/**
    15  * Uses <a href="https://github.com/tyrasd/overpass-wizard/">Overpass Turbo query wizard</a> code (MIT Licensed)
    16  * to build an Overpass QL from a {@link org.openstreetmap.josm.actions.search.SearchAction} like query.
     21 * Builds an Overpass QL from a {@link org.openstreetmap.josm.actions.search.SearchAction} query.
    1722 *
    18  * Requires a JavaScript {@link ScriptEngine}.
    19  * @since 8744
     23 * @since 8744 (using tyrasd/overpass-wizard), 16262 (standalone)
    2024 */
    2125public final class OverpassTurboQueryWizard {
    2226
    23     private static OverpassTurboQueryWizard instance;
    24     private final ScriptEngine engine = Utils.getJavaScriptEngine();
     27    private static final OverpassTurboQueryWizard instance = new OverpassTurboQueryWizard();
    2528
    2629    /**
     
    2932     * @return the unique instance of this class
    3033     */
    31     public static synchronized OverpassTurboQueryWizard getInstance() {
    32         if (instance == null) {
    33             instance = new OverpassTurboQueryWizard();
    34         }
     34    public static OverpassTurboQueryWizard getInstance() {
    3535        return instance;
    3636    }
    3737
    3838    private OverpassTurboQueryWizard() {
    39         try (CachedFile file = new CachedFile("resource://data/overpass-wizard.js");
    40              Reader reader = file.getContentReader()) {
    41             if (engine != null) {
    42                 engine.eval("var console = {error: " + Logging.class.getCanonicalName() + ".warn};");
    43                 engine.eval("var global = {};");
    44                 engine.eval(reader);
    45                 engine.eval("var overpassWizardJOSM = function(query) {" +
    46                         "  return overpassWizard(query, {" +
    47                         "    comment: false," +
    48                         "    timeout: " + Config.getPref().getInt("overpass.wizard.timeout", 90) + "," +
    49                         "    outputFormat: 'xml'," +
    50                         "    outputMode: 'recursive_meta'" +
    51                         "  });" +
    52                         "}");
    53             }
    54         } catch (ScriptException | IOException ex) {
    55             throw new IllegalStateException("Failed to initialize OverpassTurboQueryWizard", ex);
    56         }
     39        // private constructor for utility class
    5740    }
    5841
     
    6346     * @throws UncheckedParseException when the parsing fails
    6447     */
    65     public String constructQuery(String search) {
    66         if (engine == null) {
    67             throw new IllegalStateException("Failed to retrieve JavaScript engine");
    68         }
     48    public String constructQuery(final String search) {
    6949        try {
    70             final Object result = ((Invocable) engine).invokeFunction("overpassWizardJOSM", search);
    71             if (Boolean.FALSE.equals(result)) {
    72                 throw new UncheckedParseException();
     50            Matcher matcher = Pattern.compile("\\s+GLOBAL\\s*$", Pattern.CASE_INSENSITIVE).matcher(search);
     51            if (matcher.find()) {
     52                final Match match = SearchCompiler.compile(matcher.replaceFirst(""));
     53                return constructQuery(match, ";", "");
    7354            }
    74             return (String) result;
    75         } catch (NoSuchMethodException e) {
    76             throw new IllegalStateException(e);
    77         } catch (ScriptException e) {
    78             throw new UncheckedParseException("Failed to execute OverpassTurboQueryWizard", e);
     55
     56            matcher = Pattern.compile("\\s+IN BBOX\\s*$", Pattern.CASE_INSENSITIVE).matcher(search);
     57            if (matcher.find()) {
     58                final Match match = SearchCompiler.compile(matcher.replaceFirst(""));
     59                return constructQuery(match, "[bbox:{{bbox}}];", "");
     60            }
     61
     62            matcher = Pattern.compile("\\s+(?<mode>IN|AROUND)\\s+(?<area>[^\" ]+|\"[^\"]+\")\\s*$", Pattern.CASE_INSENSITIVE).matcher(search);
     63            if (matcher.find()) {
     64                final Match match = SearchCompiler.compile(matcher.replaceFirst(""));
     65                final String mode = matcher.group("mode").toUpperCase(Locale.ENGLISH);
     66                final String area = Utils.strip(matcher.group("area"), "\"");
     67                if ("IN".equals(mode)) {
     68                    return constructQuery(match, ";\n{{geocodeArea:" + area + "}}->.searchArea;", "(area.searchArea)");
     69                } else if ("AROUND".equals(mode)) {
     70                    return constructQuery(match, ";\n{{radius=1000}}", "(around:{{radius}},{{geocodeCoords:" + area + "}})");
     71                } else {
     72                    throw new IllegalStateException(mode);
     73                }
     74            }
     75           
     76            final Match match = SearchCompiler.compile(search);
     77            return constructQuery(match, "[bbox:{{bbox}}];", "");
     78        } catch (SearchParseError | UnsupportedOperationException e) {
     79            throw new UncheckedParseException(e);
    7980        }
    8081    }
     82
     83    private String constructQuery(final Match match, final String bounds, final String queryLineSuffix) {
     84        final List<Match> normalized = normalizeToDNF(match);
     85        final List<String> queryLines = new ArrayList<>();
     86        queryLines.add("[out:xml][timeout:90]" + bounds);
     87        queryLines.add("(");
     88        for (Match conjunction : normalized) {
     89            final EnumSet<OsmPrimitiveType> types = EnumSet.noneOf(OsmPrimitiveType.class);
     90            final String query = constructQuery(conjunction, types);
     91            for (Object type : types.isEmpty() || types.size() == 3 ? Collections.singleton("nwr") : types) {
     92                queryLines.add("  " + type + query + queryLineSuffix + ";");
     93            }
     94        }
     95        queryLines.add(");");
     96        queryLines.add("(._;>;);");
     97        queryLines.add("out meta;");
     98        return String.join("\n", queryLines);
     99    }
     100
     101    private static String constructQuery(Match match, final Set<OsmPrimitiveType> types) {
     102        final boolean negated;
     103        if (match instanceof SearchCompiler.Not) {
     104            negated = true;
     105            match = ((SearchCompiler.Not) match).getMatch();
     106        } else {
     107            negated = false;
     108        }
     109        if (match instanceof SearchCompiler.And) {
     110            return ((SearchCompiler.And) match).map(m -> constructQuery(m, types), (s1, s2) -> s1 + s2);
     111        } else if (match instanceof SearchCompiler.KeyValue) {
     112            final String key = ((SearchCompiler.KeyValue) match).getKey();
     113            final String value = ((SearchCompiler.KeyValue) match).getValue();
     114            return "[~" + quote(key) + "~" + quote(value) + "]";
     115        } else if (match instanceof SearchCompiler.ExactKeyValue) {
     116            // https://wiki.openstreetmap.org/wiki/Overpass_API/Language_Guide
     117            // ["key"]             -- filter objects tagged with this key and any value
     118            // [!"key"]            -- filter objects not tagged with this key and any value
     119            // ["key"="value"]     -- filter objects tagged with this key and this value
     120            // ["key"!="value"]    -- filter objects tagged with this key but not this value, or not tagged with this key
     121            // ["key"~"value"]     -- filter objects tagged with this key and a value matching a regular expression
     122            // ["key"!~"value"]    -- filter objects tagged with this key but a value not matching a regular expression
     123            // [~"key"~"value"]    -- filter objects tagged with a key and a value matching regular expressions
     124            // [~"key"~"value", i] -- filter objects tagged with a key and a case-insensitive value matching regular expressions
     125            final String key = ((SearchCompiler.ExactKeyValue) match).getKey();
     126            final String value = ((SearchCompiler.ExactKeyValue) match).getValue();
     127            final SearchCompiler.ExactKeyValue.Mode mode = ((SearchCompiler.ExactKeyValue) match).getMode();
     128            switch (mode) {
     129                case ANY_VALUE:
     130                    return "[" + (negated ? "!" : "") + quote(key) + "]";
     131                case EXACT:
     132                    return "[" + quote(key) + (negated ? "!=" : "=") + quote(value) + "]";
     133                case EXACT_REGEXP:
     134                    final Matcher matcher = Pattern.compile("/(?<regex>.*)/(?<flags>i)?").matcher(value);
     135                    final String valueQuery = matcher.matches()
     136                            ? quote(matcher.group("regex")) + Optional.ofNullable(matcher.group("flags")).map(f -> "," + f).orElse("")
     137                            : quote(value);
     138                    return "[" + quote(key) + (negated ? "!~" : "~") + valueQuery + "]";
     139                case MISSING_KEY:
     140                    // special case for empty values, see https://github.com/drolbr/Overpass-API/issues/53
     141                    return "[" + quote(key) + (negated ? "!~" : "~") + quote("^$") + "]";
     142                default:
     143                    return "";
     144            }
     145        } else if (match instanceof SearchCompiler.BooleanMatch) {
     146            final String key = ((SearchCompiler.BooleanMatch) match).getKey();
     147            return negated
     148                    ? "[" + quote(key) + "~\"false|no|0|off\"]"
     149                    : "[" + quote(key) + "~\"true|yes|1|on\"]";
     150        } else if (match instanceof SearchCompiler.UserMatch) {
     151            final String user = ((SearchCompiler.UserMatch) match).getUser();
     152            return user.matches("\\d+")
     153                    ? "(uid:" + user + ")"
     154                    : "(user:" + quote(user) + ")";
     155        } else if (match instanceof SearchCompiler.ExactType) {
     156            types.add(((SearchCompiler.ExactType) match).getType());
     157            return "";
     158        }
     159        Logging.warn("Unsupported match type {0}: {1}", match.getClass(), match);
     160        return "/*" + match + "*/";
     161    }
     162
     163    /**
     164     * Quotes the given string for its use in Overpass QL
     165     */
     166    private static String quote(final String s) {
     167        return "\"" + s.replace("\\", "\\\\").replace("\"", "\\\"") + "\"";
     168    }
     169
     170    /**
     171     * Normalizes the match to disjunctive normal form: A∧(B∨C) ⇔ (A∧B)∨(A∧C)
     172     */
     173    private static List<Match> normalizeToDNF(final Match match) {
     174        if (match instanceof SearchCompiler.And) {
     175            return ((SearchCompiler.And) match).map(OverpassTurboQueryWizard::normalizeToDNF, (lhs, rhs) -> lhs.stream()
     176                    .flatMap(l -> rhs.stream().map(r -> new SearchCompiler.And(l, r)))
     177                    .collect(Collectors.toList()));
     178        } else if (match instanceof SearchCompiler.Or) {
     179            return ((SearchCompiler.Or) match).map(OverpassTurboQueryWizard::normalizeToDNF, CompositeList::new);
     180        } else if (match instanceof SearchCompiler.Xor) {
     181            throw new UnsupportedOperationException(match.toString());
     182        } else if (match instanceof SearchCompiler.Not) {
     183            // only support negated KeyValue or ExactKeyValue matches
     184            final Match innerMatch = ((SearchCompiler.Not) match).getMatch();
     185            if (innerMatch instanceof SearchCompiler.BooleanMatch
     186                    || innerMatch instanceof SearchCompiler.KeyValue
     187                    || innerMatch instanceof SearchCompiler.ExactKeyValue) {
     188                return Collections.singletonList(match);
     189            }
     190            throw new UnsupportedOperationException(match.toString());
     191        } else {
     192            return Collections.singletonList(match);
     193        }
     194    }
     195
    81196}
  • trunk/test/unit/org/openstreetmap/josm/tools/OverpassTurboQueryWizardTest.java

    r16261 r16262  
    3939    public void testKeyValue() {
    4040        assertQueryEquals("  nwr[\"amenity\"=\"drinking_water\"];\n", "amenity=drinking_water");
     41        assertQueryEquals("  nwr[\"amenity\"];\n", "amenity=*");
    4142    }
    4243
     
    4647    @Test
    4748    public void testKeyNotValue() {
    48         assertQueryEquals("  nwr[\"amenity\"!=\"drinking_water\"];\n", "amenity!=drinking_water");
    49         assertQueryEquals("  nwr[\"amenity\"!=\"drinking_water\"];\n", "amenity<>drinking_water");
     49        assertQueryEquals("  nwr[\"amenity\"!=\"drinking_water\"];\n", "-amenity=drinking_water");
     50        assertQueryEquals("  nwr[!\"amenity\"];\n", "-amenity=*");
    5051    }
    5152
     
    5758        assertQueryEquals("  nwr[\"foo\"~\"bar\"];\n", "foo~bar");
    5859        assertQueryEquals("  nwr[\"foo\"~\"bar\"];\n", "foo~/bar/");
    59         assertQueryEquals("  nwr[\"foo\"~\"bar\"];\n", "foo~=bar");
    60         assertQueryEquals("  nwr[\"foo\"~\"bar\"];\n", "foo~=/bar/");
    61         assertQueryEquals("  nwr[\"foo\"~\"bar\"];\n", "foo like bar");
    62         assertQueryEquals("  nwr[\"foo\"~\"bar\"];\n", "foo like /bar/");
    6360        // case insensitive
    6461        assertQueryEquals("  nwr[\"foo\"~\"bar\",i];\n", "foo~/bar/i");
    6562        // negated
    66         assertQueryEquals("  nwr[\"foo\"!~\"bar\"];\n", "foo!~bar");
    67         assertQueryEquals("  nwr[\"foo\"!~\"bar\"];\n", "foo not like bar");
     63        assertQueryEquals("  nwr[\"foo\"!~\"bar\"];\n", "-foo~bar");
     64        assertQueryEquals("  nwr[\"foo\"!~\"bar\",i];\n", "-foo~/bar/i");
     65    }
     66
     67    /**
     68     * Test OSM boolean true/false.
     69     */
     70    @Test
     71    public void testOsmBoolean() {
     72        assertQueryEquals("  nwr[\"highway\"][\"oneway\"~\"true|yes|1|on\"];\n", "highway=* AND oneway?");
     73        assertQueryEquals("  nwr[\"highway\"][\"oneway\"~\"false|no|0|off\"];\n", "highway=* AND -oneway?");
    6874    }
    6975
     
    8490    public void testBooleanOr() {
    8591        assertQueryEquals("  nwr[\"foo\"=\"bar\"];\n  nwr[\"baz\"=\"42\"];\n", "foo=bar or baz=42");
    86         assertQueryEquals("  nwr[\"foo\"=\"bar\"];\n  nwr[\"baz\"=\"42\"];\n", "foo=bar || baz=42");
    8792        assertQueryEquals("  nwr[\"foo\"=\"bar\"];\n  nwr[\"baz\"=\"42\"];\n", "foo=bar | baz=42");
    8893    }
     
    118123    @Test
    119124    public void testUser() {
    120         assertQueryEquals("  nwr(user:\"foo\");\n  nwr(uid:42);\n", "user:foo or uid:42");
     125        assertQueryEquals("  nwr(user:\"foo\");\n  nwr(uid:42);\n", "user:foo or user:42");
    121126    }
    122127
     
    126131    @Test
    127132    public void testEmpty() {
    128         assertQueryEquals("  way[\"foo\"~\"^$\"];\n", "foo='' and type:way");
     133        assertQueryEquals("  way[\"foo\"~\"^$\"];\n", "foo=\"\" and type:way");
    129134    }
    130135
     
    220225    @Test(expected = UncheckedParseException.class)
    221226    public void testErroneous() {
    222         OverpassTurboQueryWizard.getInstance().constructQuery("foo");
     227        OverpassTurboQueryWizard.getInstance().constructQuery("-(foo or bar)");
    223228    }
    224229}
Note: See TracChangeset for help on using the changeset viewer.