// License: GPL. For details, see LICENSE file. package org.openstreetmap.josm.actions.downloadtasks; import static java.util.function.Predicate.not; import static org.openstreetmap.josm.tools.I18n.tr; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.Future; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Stream; import org.openstreetmap.josm.data.Bounds; import org.openstreetmap.josm.data.DataSource; import org.openstreetmap.josm.data.ProjectionBounds; import org.openstreetmap.josm.data.ViewportData; import org.openstreetmap.josm.data.coor.LatLon; import org.openstreetmap.josm.data.osm.DataSet; import org.openstreetmap.josm.data.osm.OsmPrimitive; import org.openstreetmap.josm.data.osm.Relation; import org.openstreetmap.josm.data.osm.Way; import org.openstreetmap.josm.data.osm.visitor.BoundingXYVisitor; import org.openstreetmap.josm.gui.MainApplication; import org.openstreetmap.josm.gui.MapFrame; import org.openstreetmap.josm.gui.PleaseWaitRunnable; import org.openstreetmap.josm.gui.io.UpdatePrimitivesTask; import org.openstreetmap.josm.gui.layer.OsmDataLayer; import org.openstreetmap.josm.gui.progress.NullProgressMonitor; import org.openstreetmap.josm.gui.progress.ProgressMonitor; import org.openstreetmap.josm.io.BoundingBoxDownloader; import org.openstreetmap.josm.io.Compression; import org.openstreetmap.josm.io.OsmServerLocationReader; import org.openstreetmap.josm.io.OsmServerReader; import org.openstreetmap.josm.io.OsmTransferCanceledException; import org.openstreetmap.josm.io.OsmTransferException; import org.openstreetmap.josm.io.OverpassDownloadReader; import org.openstreetmap.josm.io.UrlPatterns.OsmUrlPattern; import org.openstreetmap.josm.spi.preferences.Config; import org.openstreetmap.josm.tools.Logging; import org.openstreetmap.josm.tools.Utils; import org.xml.sax.SAXException; /** * Open the download dialog and download the data. * Run in the worker thread. */ public class DownloadOsmTask extends AbstractDownloadTask { protected Bounds currentBounds; protected DownloadTask downloadTask; protected String newLayerName; /** This allows subclasses to ignore this warning */ protected boolean warnAboutEmptyArea = true; protected static final String OVERPASS_INTERPRETER_DATA = "interpreter?data="; private static final String NO_DATA_FOUND = tr("No data found in this area."); static { PostDownloadHandler.addNoDataErrorMessage(NO_DATA_FOUND); } @Override public String[] getPatterns() { if (this.getClass() == DownloadOsmTask.class) { return patterns(OsmUrlPattern.class); } else { return super.getPatterns(); } } @Override public String getTitle() { if (this.getClass() == DownloadOsmTask.class) { return tr("Download OSM"); } else { return super.getTitle(); } } @Override public Future download(DownloadParams settings, Bounds downloadArea, ProgressMonitor progressMonitor) { return download(new BoundingBoxDownloader(downloadArea), settings, downloadArea, progressMonitor); } /** * Asynchronously launches the download task for a given bounding box. * * Set progressMonitor to null, if the task should create, open, and close a progress monitor. * Set progressMonitor to {@link NullProgressMonitor#INSTANCE} if progress information is to * be discarded. * * You can wait for the asynchronous download task to finish by synchronizing on the returned * {@link Future}, but make sure not to freeze up JOSM. Example: * 
     *    Future<?> future = task.download(...);
     *    // DON'T run this on the Swing EDT or JOSM will freeze
     *    future.get(); // waits for the download task to complete
     *
* * The following example uses a pattern which is better suited if a task is launched from * the Swing EDT: * 
     *    final Future<?> future = task.download(...);
     *    Runnable runAfterTask = new Runnable() {
     *       public void run() {
     *           // this is not strictly necessary because of the type of executor service
     *           // Main.worker is initialized with, but it doesn't harm either
     *           //
     *           future.get(); // wait for the download task to complete
     *           doSomethingAfterTheTaskCompleted();
     *       }
     *    }
     *    MainApplication.worker.submit(runAfterTask);
     *
* @param reader the reader used to parse OSM data (see {@link OsmServerReader#parseOsm}) * @param settings download settings * @param downloadArea the area to download * @param progressMonitor the progressMonitor * @return the future representing the asynchronous task * @since 13927 */ public Future download(OsmServerReader reader, DownloadParams settings, Bounds downloadArea, ProgressMonitor progressMonitor) { return download(new DownloadTask(settings, reader, progressMonitor, zoomAfterDownload), downloadArea); } protected Future download(DownloadTask downloadTask, Bounds downloadArea) { this.downloadTask = downloadTask; this.currentBounds = new Bounds(downloadArea); // We need submit instead of execute so we can wait for it to finish and get the error // message if necessary. If no one calls getErrorMessage() it just behaves like execute. return MainApplication.worker.submit(downloadTask); } /** * This allows subclasses to perform operations on the URL before {@link #loadUrl} is performed. * @param url the original URL * @return the modified URL */ protected String modifyUrlBeforeLoad(String url) { return url; } /** * Loads a given URL from the OSM Server * @param settings download settings * @param url The URL as String */ @Override public Future loadUrl(DownloadParams settings, String url, ProgressMonitor progressMonitor) { String newUrl = modifyUrlBeforeLoad(url); Optional urlPattern = Arrays.stream(OsmUrlPattern.values()).filter(p -> p.matches(newUrl)).findFirst(); downloadTask = new DownloadTask(settings, getOsmServerReader(newUrl), progressMonitor, true, Compression.byExtension(newUrl)); currentBounds = null; // Extract .osm filename from URL to set the new layer name extractOsmFilename(settings, urlPattern.orElse(OsmUrlPattern.EXTERNAL_OSM_FILE).pattern(), newUrl); return MainApplication.worker.submit(downloadTask); } protected OsmServerReader getOsmServerReader(String url) { try { String host = new URL(url).getHost(); for (String knownOverpassServer : OverpassDownloadReader.OVERPASS_SERVER_HISTORY.get()) { if (host.equals(new URL(knownOverpassServer).getHost())) { int index = url.indexOf(OVERPASS_INTERPRETER_DATA); if (index > 0) { return new OverpassDownloadReader(new Bounds(LatLon.ZERO), knownOverpassServer, Utils.decodeUrl(url.substring(index + OVERPASS_INTERPRETER_DATA.length()))); } } } } catch (MalformedURLException e) { Logging.error(e); } return new OsmServerLocationReader(url); } protected final void extractOsmFilename(DownloadParams settings, String pattern, String url) { newLayerName = settings.getLayerName(); if (Utils.isEmpty(newLayerName)) { Matcher matcher = Pattern.compile(pattern).matcher(url); newLayerName = matcher.matches() && matcher.groupCount() > 0 ? Utils.decodeUrl(matcher.group(1)) : null; } } @Override public void cancel() { if (downloadTask != null) { downloadTask.cancel(); } } @Override public boolean isSafeForRemotecontrolRequests() { return true; } @Override public ProjectionBounds getDownloadProjectionBounds() { return downloadTask != null ? downloadTask.computeBbox(currentBounds).orElse(null) : null; } protected Collection searchPotentiallyDeletedPrimitives(DataSet ds) { return downloadTask.searchPrimitivesToUpdate(currentBounds, ds); } protected final void rememberDownloadedBounds(Bounds bounds) { if (bounds != null) { Config.getPref().put("osm-download.bounds", bounds.encodeAsString(";")); } } /** * Superclass of internal download task. * @since 7636 */ public abstract static class AbstractInternalTask extends PleaseWaitRunnable { protected final DownloadParams settings; protected final boolean zoomAfterDownload; protected DataSet dataSet; /** * Constructs a new {@code AbstractInternalTask}. * @param settings download settings * @param title message for the user * @param ignoreException If true, exception will be propagated to calling code. If false then * exception will be thrown directly in EDT. When this runnable is executed using executor framework * then use false unless you read result of task (because exception will get lost if you don't) * @param zoomAfterDownload If true, the map view will zoom to download area after download */ protected AbstractInternalTask(DownloadParams settings, String title, boolean ignoreException, boolean zoomAfterDownload) { super(title, ignoreException); this.settings = Objects.requireNonNull(settings); this.zoomAfterDownload = zoomAfterDownload; } /** * Constructs a new {@code AbstractInternalTask}. * @param settings download settings * @param title message for the user * @param progressMonitor progress monitor * @param ignoreException If true, exception will be propagated to calling code. If false then * exception will be thrown directly in EDT. When this runnable is executed using executor framework * then use false unless you read result of task (because exception will get lost if you don't) * @param zoomAfterDownload If true, the map view will zoom to download area after download */ protected AbstractInternalTask(DownloadParams settings, String title, ProgressMonitor progressMonitor, boolean ignoreException, boolean zoomAfterDownload) { super(title, progressMonitor, ignoreException); this.settings = Objects.requireNonNull(settings); this.zoomAfterDownload = zoomAfterDownload; } protected OsmDataLayer getEditLayer() { return MainApplication.getLayerManager().getEditLayer(); } private static Stream getModifiableDataLayers() { return MainApplication.getLayerManager().getLayersOfType(OsmDataLayer.class) .stream().filter(OsmDataLayer::isDownloadable); } /** * Returns the number of modifiable data layers * @return number of modifiable data layers * @since 13434 */ protected long getNumModifiableDataLayers() { return getModifiableDataLayers().count(); } /** * Returns the first modifiable data layer * @return the first modifiable data layer * @since 13434 */ protected OsmDataLayer getFirstModifiableDataLayer() { return getModifiableDataLayers().findFirst().orElse(null); } /** * Creates a name for a new layer by utilizing the settings ({@link DownloadParams#getLayerName()}) or * {@link OsmDataLayer#createNewName()} if the former option is {@code null}. * * @return a name for a new layer * @since 14347 */ protected String generateLayerName() { return Optional.ofNullable(settings.getLayerName()) .filter(not(Utils::isStripEmpty)) .orElse(OsmDataLayer.createNewName()); } /** * Can be overridden (e.g. by plugins) if a subclass of {@link OsmDataLayer} is needed. * If you want to change how the name is determined, consider overriding * {@link #generateLayerName()} instead. * * @param ds the dataset on which the layer is based, must be non-null * @param layerName the name of the new layer, must be either non-blank or non-present * @return a new instance of {@link OsmDataLayer} constructed with the given arguments * @since 14347 */ protected OsmDataLayer createNewLayer(final DataSet ds, final Optional layerName) { if (layerName.filter(Utils::isStripEmpty).isPresent()) { throw new IllegalArgumentException("Blank layer name!"); } return new OsmDataLayer( Objects.requireNonNull(ds, "dataset parameter"), layerName.orElseGet(this::generateLayerName), null ); } /** * Convenience method for {@link #createNewLayer(DataSet, Optional)}, uses the dataset * from field {@link #dataSet} and applies the settings from field {@link #settings}. * * @param layerName an optional layer name, must be non-blank if the [Optional] is present * @return a newly constructed layer * @since 14347 */ protected final OsmDataLayer createNewLayer(final Optional layerName) { Optional.ofNullable(settings.getDownloadPolicy()) .ifPresent(dataSet::setDownloadPolicy); Optional.ofNullable(settings.getUploadPolicy()) .ifPresent(dataSet::setUploadPolicy); if (dataSet.isLocked() && !settings.isLocked()) { dataSet.unlock(); } else if (!dataSet.isLocked() && settings.isLocked()) { dataSet.lock(); } return createNewLayer(dataSet, layerName); } protected Optional computeBbox(Bounds bounds) { BoundingXYVisitor v = new BoundingXYVisitor(); if (bounds != null) { v.visit(bounds); } else { v.computeBoundingBox(dataSet.getNodes()); } return Optional.ofNullable(v.getBounds()); } protected OsmDataLayer addNewLayerIfRequired(String newLayerName) { long numDataLayers = getNumModifiableDataLayers(); if (settings.isNewLayer() || numDataLayers == 0 || (numDataLayers > 1 && getEditLayer() == null)) { // the user explicitly wants a new layer, we don't have any layer at all // or it is not clear which layer to merge to final OsmDataLayer layer = createNewLayer(Optional.ofNullable(newLayerName).filter(not(Utils::isStripEmpty))); MainApplication.getLayerManager().addLayer(layer, zoomAfterDownload); return layer; } return null; } protected void loadData(String newLayerName, Bounds bounds) { OsmDataLayer layer = addNewLayerIfRequired(newLayerName); if (layer == null) { layer = getEditLayer(); if (layer == null || !layer.isDownloadable()) { layer = getFirstModifiableDataLayer(); } Collection primitivesToUpdate = searchPrimitivesToUpdate(bounds, layer.getDataSet()); layer.mergeFrom(dataSet); MapFrame map = MainApplication.getMap(); if (map != null && zoomAfterDownload) { computeBbox(bounds).map(ViewportData::new).ifPresent(map.mapView::zoomTo); } if (!primitivesToUpdate.isEmpty()) { MainApplication.worker.submit(new UpdatePrimitivesTask(layer, primitivesToUpdate)); } } layer.onPostDownloadFromServer(); // for existing and newly added layer, see #19816 } /** * Look for primitives deleted on server (thus absent from downloaded data) * but still present in existing data layer * @param bounds download bounds * @param ds existing data set * @return the primitives to update */ protected Collection searchPrimitivesToUpdate(Bounds bounds, DataSet ds) { if (bounds == null) return Collections.emptySet(); Collection col = new ArrayList<>(); ds.searchNodes(bounds.toBBox()).stream().filter(n -> !n.isNew() && !dataSet.containsNode(n)).forEachOrdered(col::add); if (!col.isEmpty()) { Set ways = new HashSet<>(); Set rels = new HashSet<>(); for (OsmPrimitive n : col) { for (OsmPrimitive ref : n.getReferrers()) { if (ref.isNew()) { continue; } else if (ref instanceof Way) { ways.add((Way) ref); } else if (ref instanceof Relation) { rels.add((Relation) ref); } } } ways.stream().filter(w -> !dataSet.containsWay(w)).forEachOrdered(col::add); rels.stream().filter(r -> !dataSet.containsRelation(r)).forEachOrdered(col::add); } return Collections.unmodifiableCollection(col); } } protected class DownloadTask extends AbstractInternalTask { protected final OsmServerReader reader; protected final Compression compression; /** * Constructs a new {@code DownloadTask}. * @param settings download settings * @param reader OSM data reader * @param progressMonitor progress monitor * @since 13927 */ public DownloadTask(DownloadParams settings, OsmServerReader reader, ProgressMonitor progressMonitor) { this(settings, reader, progressMonitor, true); } /** * Constructs a new {@code DownloadTask}. * @param settings download settings * @param reader OSM data reader * @param progressMonitor progress monitor * @param zoomAfterDownload If true, the map view will zoom to download area after download * @since 13927 */ public DownloadTask(DownloadParams settings, OsmServerReader reader, ProgressMonitor progressMonitor, boolean zoomAfterDownload) { this(settings, reader, progressMonitor, zoomAfterDownload, Compression.NONE); } /** * Constructs a new {@code DownloadTask}. * @param settings download settings * @param reader OSM data reader * @param progressMonitor progress monitor * @param zoomAfterDownload If true, the map view will zoom to download area after download * @param compression compression to use * @since 15784 */ public DownloadTask(DownloadParams settings, OsmServerReader reader, ProgressMonitor progressMonitor, boolean zoomAfterDownload, Compression compression) { super(settings, tr("Downloading data"), progressMonitor, false, zoomAfterDownload); this.reader = Objects.requireNonNull(reader); this.compression = compression; } protected DataSet parseDataSet() throws OsmTransferException { ProgressMonitor subTaskMonitor = progressMonitor.createSubTaskMonitor(ProgressMonitor.ALL_TICKS, false); // Don't call parseOsm signature with compression if not needed, too many implementations to update before to avoid side effects return compression != null && compression != Compression.NONE ? reader.parseOsm(subTaskMonitor, compression) : reader.parseOsm(subTaskMonitor); } @Override public void realRun() throws IOException, SAXException, OsmTransferException { try { if (isCanceled()) return; dataSet = parseDataSet(); } catch (OsmTransferException e) { if (isCanceled()) { Logging.info(tr("Ignoring exception because download has been canceled. Exception was: {0}", e.toString())); return; } if (e instanceof OsmTransferCanceledException) { setCanceled(true); return; } else { rememberException(e); } DownloadOsmTask.this.setFailed(true); } } @Override protected void finish() { if (isFailed() || isCanceled()) return; if (dataSet == null) return; // user canceled download or error occurred if (dataSet.allPrimitives().isEmpty()) { if (warnAboutEmptyArea) { rememberErrorMessage(NO_DATA_FOUND); } String remark = dataSet.getRemark(); if (!Utils.isEmpty(remark)) { rememberErrorMessage(remark); } if (!(reader instanceof BoundingBoxDownloader) || ((BoundingBoxDownloader) reader).considerAsFullDownload()) { // need to synthesize a download bounds lest the visual indication of downloaded area doesn't work dataSet.addDataSource(new DataSource( currentBounds != null ? currentBounds : new Bounds(LatLon.ZERO), "OpenStreetMap server")); } } rememberDownloadedBounds(currentBounds); rememberDownloadedData(dataSet); loadData(newLayerName, currentBounds); } @Override protected void cancel() { setCanceled(true); if (reader != null) { reader.cancel(); } } } @Override public String getConfirmationMessage(URL url) { if (OsmUrlPattern.OSM_API_URL.matches(url)) { Collection items = new ArrayList<>(); items.add(tr("OSM Server URL:") + ' ' + url.getHost()); items.add(tr("Command")+": "+url.getPath()); if (url.getQuery() != null) { items.add(tr("Request details: {0}", url.getQuery().replaceAll(",\\s*", ", "))); } return Utils.joinAsHtmlUnorderedList(items); } return null; } }