Index: src/org/openstreetmap/josm/actions/downloadtasks/DownloadReferrersTask.java =================================================================== --- src/org/openstreetmap/josm/actions/downloadtasks/DownloadReferrersTask.java (revision 15849) +++ src/org/openstreetmap/josm/actions/downloadtasks/DownloadReferrersTask.java (working copy) @@ -14,6 +14,7 @@ import javax.swing.JOptionPane; import javax.swing.SwingUtilities; +import org.openstreetmap.josm.data.Bounds; import org.openstreetmap.josm.data.osm.DataSet; import org.openstreetmap.josm.data.osm.DataSetMerger; import org.openstreetmap.josm.data.osm.Node; @@ -26,10 +27,12 @@ import org.openstreetmap.josm.gui.PleaseWaitRunnable; import org.openstreetmap.josm.gui.layer.OsmDataLayer; import org.openstreetmap.josm.gui.progress.ProgressMonitor; +import org.openstreetmap.josm.io.MultiFetchOverpassObjectReader; import org.openstreetmap.josm.io.MultiFetchServerObjectReader; import org.openstreetmap.josm.io.OsmServerBackreferenceReader; import org.openstreetmap.josm.io.OsmServerReader; import org.openstreetmap.josm.io.OsmTransferException; +import org.openstreetmap.josm.io.OverpassDownloadReader; import org.openstreetmap.josm.tools.CheckParameterUtil; import org.openstreetmap.josm.tools.ExceptionUtil; import org.xml.sax.SAXException; @@ -150,37 +153,45 @@ @Override protected void realRun() throws SAXException, IOException, OsmTransferException { try { - progressMonitor.setTicksCount(children.size()); - int i = 1; - for (PrimitiveId p : children) { - if (canceled) - return; - String msg; - String id = Long.toString(p.getUniqueId()); - switch(p.getType()) { - case NODE: msg = tr("({0}/{1}) Loading parents of node {2}", i, children.size(), id); break; - case WAY: msg = tr("({0}/{1}) Loading parents of way {2}", i, children.size(), id); break; - case RELATION: msg = tr("({0}/{1}) Loading parents of relation {2}", i, children.size(), id); break; - default: throw new AssertionError(); + if (Boolean.TRUE.equals(OverpassDownloadReader.FOR_MULTI_FETCH.get())) { + String request = MultiFetchOverpassObjectReader.genOverpassQuery(children, false, true, false); + reader = new OverpassDownloadReader(new Bounds(0, 0, 0, 0), + OverpassDownloadReader.OVERPASS_SERVER.get(), request); + DataSet ds = reader.parseOsm(progressMonitor.createSubTaskMonitor(1, false)); + new DataSetMerger(parents, ds).merge(); + } else { + progressMonitor.setTicksCount(children.size()); + int i = 1; + for (PrimitiveId p : children) { + if (canceled) + return; + String msg; + String id = Long.toString(p.getUniqueId()); + switch(p.getType()) { + case NODE: msg = tr("({0}/{1}) Loading parents of node {2}", i, children.size(), id); break; + case WAY: msg = tr("({0}/{1}) Loading parents of way {2}", i, children.size(), id); break; + case RELATION: msg = tr("({0}/{1}) Loading parents of relation {2}", i, children.size(), id); break; + default: throw new AssertionError(); + } + progressMonitor.subTask(msg); + downloadParents(p.getUniqueId(), p.getType(), progressMonitor); + i++; } - progressMonitor.subTask(msg); - downloadParents(p.getUniqueId(), p.getType(), progressMonitor); - i++; - } - Collection ways = parents.getWays(); + Collection ways = parents.getWays(); - if (!ways.isEmpty()) { - // Collect incomplete nodes of parent ways - Set nodes = ways.stream().flatMap(w -> w.getNodes().stream().filter(OsmPrimitive::isIncomplete)) - .collect(Collectors.toSet()); - if (!nodes.isEmpty()) { - reader = MultiFetchServerObjectReader.create(); - ((MultiFetchServerObjectReader) reader).append(nodes); - DataSet wayNodes = reader.parseOsm(progressMonitor.createSubTaskMonitor(1, false)); - synchronized (this) { // avoid race condition in cancel() - reader = null; + if (!ways.isEmpty()) { + // Collect incomplete nodes of parent ways + Set nodes = ways.stream().flatMap(w -> w.getNodes().stream().filter(OsmPrimitive::isIncomplete)) + .collect(Collectors.toSet()); + if (!nodes.isEmpty()) { + reader = MultiFetchServerObjectReader.create(); + ((MultiFetchServerObjectReader) reader).append(nodes); + DataSet wayNodes = reader.parseOsm(progressMonitor.createSubTaskMonitor(1, false)); + synchronized (this) { // avoid race condition in cancel() + reader = null; + } + new DataSetMerger(parents, wayNodes).merge(); } - new DataSetMerger(parents, wayNodes).merge(); } } } catch (OsmTransferException e) { @@ -189,4 +200,5 @@ lastException = e; } } + } Index: src/org/openstreetmap/josm/gui/io/DownloadFromOverpassTask.java =================================================================== --- src/org/openstreetmap/josm/gui/io/DownloadFromOverpassTask.java (nonexistent) +++ src/org/openstreetmap/josm/gui/io/DownloadFromOverpassTask.java (working copy) @@ -0,0 +1,70 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.gui.io; + +import static org.openstreetmap.josm.tools.I18n.tr; + +import java.io.IOException; + +import org.openstreetmap.josm.data.Bounds; +import org.openstreetmap.josm.data.osm.DataSet; +import org.openstreetmap.josm.data.osm.DataSetMerger; +import org.openstreetmap.josm.gui.ExceptionDialogUtil; +import org.openstreetmap.josm.gui.PleaseWaitRunnable; +import org.openstreetmap.josm.gui.progress.ProgressMonitor; +import org.openstreetmap.josm.io.OsmTransferException; +import org.openstreetmap.josm.io.OverpassDownloadReader; +import org.xml.sax.SAXException; + +/** + * Download OSM data from Overpass API + * + */ +public class DownloadFromOverpassTask extends PleaseWaitRunnable { + private boolean canceled; + private final String request; + private final DataSet ds; + private Exception lastException; + + /** + * Constructor + * @param request the overpass query + * @param ds the {@code DataSet} instance that should contain the downloaded data + * @param monitor ProgressMonitor to use or null to create a new one. + */ + public DownloadFromOverpassTask(String request, DataSet ds, ProgressMonitor monitor) { + super(tr("Download objects via Overpass API"), monitor, false); + this.request = request; + this.ds = ds; + } + + @Override + protected void cancel() { + canceled = true; + } + + @Override + protected void realRun() throws SAXException, IOException, OsmTransferException { + try { + OverpassDownloadReader reader = new OverpassDownloadReader(new Bounds(0, 0, 0, 0), + OverpassDownloadReader.OVERPASS_SERVER.get(), request); + DataSet tmpDs = reader.parseOsm(progressMonitor.createSubTaskMonitor(1, false)); + if (!canceled) { + new DataSetMerger(ds, tmpDs).merge(); + } + } catch (OsmTransferException e) { + if (canceled) + return; + lastException = e; + } + + } + + @Override + protected void finish() { + if (canceled) + return; + if (lastException != null) { + ExceptionDialogUtil.explainException(lastException); + } + } +} Index: src/org/openstreetmap/josm/gui/io/DownloadPrimitivesWithReferrersTask.java =================================================================== --- src/org/openstreetmap/josm/gui/io/DownloadPrimitivesWithReferrersTask.java (revision 15849) +++ src/org/openstreetmap/josm/gui/io/DownloadPrimitivesWithReferrersTask.java (working copy) @@ -31,7 +31,9 @@ import org.openstreetmap.josm.gui.util.GuiHelper; import org.openstreetmap.josm.gui.widgets.HtmlPanel; import org.openstreetmap.josm.gui.widgets.JosmTextArea; +import org.openstreetmap.josm.io.MultiFetchOverpassObjectReader; import org.openstreetmap.josm.io.OsmTransferException; +import org.openstreetmap.josm.io.OverpassDownloadReader; import org.openstreetmap.josm.tools.GBC; import org.xml.sax.SAXException; @@ -50,12 +52,11 @@ /** Temporary layer where downloaded primitives are put */ private final OsmDataLayer tmpLayer; - /** Reference to the task that download requested primitives */ - private DownloadPrimitivesTask mainTask; /** Flag indicated that user ask for cancel this task */ private boolean canceled; /** Reference to the task currently running */ private PleaseWaitRunnable currentTask; + private Set missingPrimitives; /** * Constructor @@ -101,9 +102,29 @@ @Override protected void realRun() throws SAXException, IOException, OsmTransferException { + if (Boolean.TRUE.equals(OverpassDownloadReader.FOR_MULTI_FETCH.get())) { + useOverpassApi(); + } else { + useOSMApi(); + } + } + + private void useOverpassApi() { + String request = MultiFetchOverpassObjectReader.genOverpassQuery(ids, true, downloadReferrers, full); + currentTask = new DownloadFromOverpassTask(request, tmpLayer.data, getProgressMonitor().createSubTaskMonitor(1, false)); + currentTask.run(); + missingPrimitives = new HashSet<>(); + for (PrimitiveId id : ids) { + if (tmpLayer.data.getPrimitiveById(id) == null) + missingPrimitives.add(id); + } + } + + private void useOSMApi() { getProgressMonitor().setTicksCount(ids.size()+1); // First, download primitives - mainTask = new DownloadPrimitivesTask(tmpLayer, ids, full, getProgressMonitor().createSubTaskMonitor(1, false)); + DownloadPrimitivesTask mainTask = new DownloadPrimitivesTask(tmpLayer, ids, full, + getProgressMonitor().createSubTaskMonitor(1, false)); synchronized (this) { currentTask = mainTask; if (canceled) { @@ -111,7 +132,11 @@ return; } } + currentTask.run(); + + missingPrimitives = mainTask.getMissingPrimitives(); + // Then, download referrers for each primitive if (downloadReferrers) { currentTask = new DownloadReferrersTask(tmpLayer, ids); @@ -139,7 +164,7 @@ layer.mergeFrom(tmpLayer); // Warm about missing primitives - final Set errs = mainTask.getMissingPrimitives(); + final Set errs = missingPrimitives; if (errs != null && !errs.isEmpty()) GuiHelper.runInEDTAndWait(() -> reportProblemDialog(errs, trn("Object could not be downloaded", "Some objects could not be downloaded", errs.size()), @@ -185,7 +210,7 @@ return null; } List downloaded = new ArrayList<>(ids); - downloaded.removeAll(mainTask.getMissingPrimitives()); + downloaded.removeAll(missingPrimitives); return downloaded; } Index: src/org/openstreetmap/josm/io/MultiFetchOverpassObjectReader.java =================================================================== --- src/org/openstreetmap/josm/io/MultiFetchOverpassObjectReader.java (revision 15849) +++ src/org/openstreetmap/josm/io/MultiFetchOverpassObjectReader.java (working copy) @@ -1,11 +1,16 @@ // License: GPL. For details, see LICENSE file. package org.openstreetmap.josm.io; +import java.util.Collection; +import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; import java.util.stream.Collectors; import org.openstreetmap.josm.data.osm.OsmPrimitiveType; +import org.openstreetmap.josm.data.osm.PrimitiveId; import org.openstreetmap.josm.tools.Logging; /** @@ -17,7 +22,7 @@ private static String getPackageString(final OsmPrimitiveType type, Set idPackage) { return idPackage.stream().map(String::valueOf) - .collect(Collectors.joining(",", type.getAPIName() + (idPackage.size() == 1 ? "(" : "(id:"), ");")); + .collect(Collectors.joining(",", type.getAPIName() + (idPackage.size() == 1 ? "(" : "(id:"), ")")); } /** @@ -30,7 +35,7 @@ for (Entry> e : primitivesMap.entrySet()) { if (!e.getValue().isEmpty()) { countTypes++; - String list = getPackageString(e.getKey(), e.getValue()); + String list = getPackageString(e.getKey(), e.getValue()) + ";"; switch (e.getKey()) { case MULTIPOLYGON: case RELATION: @@ -56,6 +61,87 @@ return query; } + /** + * Generate single overpass query to retrieve multiple primitives. Can be used to download parents, + * children, the objects, or any combination of them. + * @param ids the collection of ids + * @param includeObjects if false, don't retrieve the primitives (e.g. only the referrers) + * @param recurseUp if true, referrers (parents) of the objects are downloaded and all nodes of parent ways + * @param recurseDownRelations true: yes, recurse down to retrieve complete relations + * @return the overpass query + */ + public static String genOverpassQuery(Collection ids, boolean includeObjects, boolean recurseUp, + boolean recurseDownRelations) { + Map> primitivesMap = new TreeMap<>(); + for (PrimitiveId p : ids) { + primitivesMap.computeIfAbsent(p.getType(), k -> new TreeSet<>()).add(p.getUniqueId()); + } + return genOverpassQuery(primitivesMap, includeObjects, recurseUp, recurseDownRelations); + } + + /** + * Generate single overpass query to retrieve multiple primitives. Can be used to download parents, + * children, the objects, or any combination of them. + * @param primitivesMap map containing the primitives + * @param includeObjects if false, don't retrieve the primitives (e.g. only the referrers) + * @param recurseUp if true, referrers (parents) of the objects are downloaded and all nodes of parent ways + * @param recurseDownRelations true: yes, recurse down to retrieve complete relations + * @return the overpass query + */ + public static String genOverpassQuery(Map> primitivesMap, boolean includeObjects, + boolean recurseUp, boolean recurseDownRelations) { + StringBuilder sb = new StringBuilder(128); + StringBuilder setsToInclude = new StringBuilder("("); + for (Entry> e : primitivesMap.entrySet()) { + if (!e.getValue().isEmpty()) { + sb.append(getPackageString(e.getKey(), e.getValue())); + switch (e.getKey()) { + case NODE: + sb.append("->.n;"); + if (includeObjects) { + setsToInclude.append(".n;"); + } + if (recurseUp) { + sb.append(".n;way(bn)->.wn;.n;rel(bn)->.rn;"); + setsToInclude.append(".wn;node(w);.rn;"); + } + break; + case CLOSEDWAY: + case WAY: + sb.append("->.w;"); + if (includeObjects) { + setsToInclude.append(".w;>;"); + } + if (recurseUp) { + sb.append(".w;rel(bw)->.pw;"); + setsToInclude.append(".pw;"); + } + break; + case MULTIPOLYGON: + case RELATION: + sb.append("->.r;"); + if (includeObjects) { + setsToInclude.append(".r;"); + } + if (recurseUp) { + sb.append(".r;rel(br)->.pr;"); + setsToInclude.append(".pr;"); + } + if (recurseDownRelations) { + sb.append(".r;>>->.rm;"); + setsToInclude.append(".rm;"); + } + break; + } + } + } + setsToInclude.append(");"); + sb.append(setsToInclude).append("out meta;"); + String query = sb.toString(); + Logging.debug("{0} {1}", "Possible Overpass query:", query); + return query; + } + @Override protected String getBaseUrl() { return OverpassDownloadReader.OVERPASS_SERVER.get(); Index: src/org/openstreetmap/josm/io/MultiFetchServerObjectReader.java =================================================================== --- src/org/openstreetmap/josm/io/MultiFetchServerObjectReader.java (revision 15849) +++ src/org/openstreetmap/josm/io/MultiFetchServerObjectReader.java (working copy) @@ -12,12 +12,12 @@ import java.util.Collections; import java.util.HashSet; import java.util.Iterator; -import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import java.util.TreeMap; import java.util.concurrent.Callable; import java.util.concurrent.CompletionService; import java.util.concurrent.ExecutionException; @@ -89,7 +89,7 @@ relations = new LinkedHashSet<>(); this.outputDataSet = new DataSet(); this.missingPrimitives = new LinkedHashSet<>(); - primitivesMap = new LinkedHashMap<>(); + primitivesMap = new TreeMap<>(); primitivesMap.put(OsmPrimitiveType.RELATION, relations); primitivesMap.put(OsmPrimitiveType.WAY, ways); primitivesMap.put(OsmPrimitiveType.NODE, nodes); @@ -382,11 +382,11 @@ try { if (this instanceof MultiFetchOverpassObjectReader) { // calculate a single request for all the objects - String request = ((MultiFetchOverpassObjectReader) this).buildComplexRequestString(); + String request = MultiFetchOverpassObjectReader.genOverpassQuery(primitivesMap, true, false, recurseDownRelations); if (isCanceled()) return null; OverpassDownloadReader reader = new OverpassDownloadReader(new Bounds(0, 0, 0, 0), getBaseUrl(), request); - DataSet ds = reader.parseOsm(progressMonitor.createSubTaskMonitor(n, false)); + DataSet ds = reader.parseOsm(progressMonitor.createSubTaskMonitor(1, false)); new DataSetMerger(outputDataSet, ds).merge(); checkMissing(outputDataSet, progressMonitor); } else {