source: josm/trunk/src/org/openstreetmap/josm/actions/downloadtasks/DownloadOsmTask.java@ 15784

Last change on this file since 15784 was 15784, checked in by Don-vip, 4 years ago

see #18613 - rework download tasks / URL patterns
  • Property svn:eol-style set to native
File size: 22.5 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.actions.downloadtasks;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.IOException;
7import java.net.MalformedURLException;
8import java.net.URL;
9import java.util.ArrayList;
10import java.util.Arrays;
11import java.util.Collection;
12import java.util.Collections;
13import java.util.HashSet;
14import java.util.Objects;
15import java.util.Optional;
16import java.util.Set;
17import java.util.concurrent.Future;
18import java.util.regex.Matcher;
19import java.util.regex.Pattern;
20import java.util.stream.Stream;
21
22import org.openstreetmap.josm.data.Bounds;
23import org.openstreetmap.josm.data.DataSource;
24import org.openstreetmap.josm.data.ProjectionBounds;
25import org.openstreetmap.josm.data.ViewportData;
26import org.openstreetmap.josm.data.coor.LatLon;
27import org.openstreetmap.josm.data.osm.DataSet;
28import org.openstreetmap.josm.data.osm.OsmPrimitive;
29import org.openstreetmap.josm.data.osm.Relation;
30import org.openstreetmap.josm.data.osm.Way;
31import org.openstreetmap.josm.data.osm.visitor.BoundingXYVisitor;
32import org.openstreetmap.josm.gui.MainApplication;
33import org.openstreetmap.josm.gui.MapFrame;
34import org.openstreetmap.josm.gui.PleaseWaitRunnable;
35import org.openstreetmap.josm.gui.io.UpdatePrimitivesTask;
36import org.openstreetmap.josm.gui.layer.OsmDataLayer;
37import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
38import org.openstreetmap.josm.gui.progress.ProgressMonitor;
39import org.openstreetmap.josm.io.BoundingBoxDownloader;
40import org.openstreetmap.josm.io.Compression;
41import org.openstreetmap.josm.io.OsmServerLocationReader;
42import org.openstreetmap.josm.io.OsmServerReader;
43import org.openstreetmap.josm.io.OsmTransferCanceledException;
44import org.openstreetmap.josm.io.OsmTransferException;
45import org.openstreetmap.josm.io.OverpassDownloadReader;
46import org.openstreetmap.josm.io.UrlPatterns.OsmUrlPattern;
47import org.openstreetmap.josm.tools.Logging;
48import org.openstreetmap.josm.tools.Utils;
49import org.xml.sax.SAXException;
50
51/**
52 * Open the download dialog and download the data.
53 * Run in the worker thread.
54 */
55public class DownloadOsmTask extends AbstractDownloadTask<DataSet> {
56
57 protected Bounds currentBounds;
58 protected DownloadTask downloadTask;
59
60 protected String newLayerName;
61
62 /** This allows subclasses to ignore this warning */
63 protected boolean warnAboutEmptyArea = true;
64
65 protected static final String OVERPASS_INTERPRETER_DATA = "interpreter?data=";
66
67 private static final String NO_DATA_FOUND = tr("No data found in this area.");
68 static {
69 PostDownloadHandler.addNoDataErrorMessage(NO_DATA_FOUND);
70 }
71
72 @Override
73 public String[] getPatterns() {
74 if (this.getClass() == DownloadOsmTask.class) {
75 return patterns(OsmUrlPattern.class);
76 } else {
77 return super.getPatterns();
78 }
79 }
80
81 @Override
82 public String getTitle() {
83 if (this.getClass() == DownloadOsmTask.class) {
84 return tr("Download OSM");
85 } else {
86 return super.getTitle();
87 }
88 }
89
90 @Override
91 public Future<?> download(DownloadParams settings, Bounds downloadArea, ProgressMonitor progressMonitor) {
92 return download(new BoundingBoxDownloader(downloadArea), settings, downloadArea, progressMonitor);
93 }
94
95 /**
96 * Asynchronously launches the download task for a given bounding box.
97 *
98 * Set <code>progressMonitor</code> to null, if the task should create, open, and close a progress monitor.
99 * Set progressMonitor to {@link NullProgressMonitor#INSTANCE} if progress information is to
100 * be discarded.
101 *
102 * You can wait for the asynchronous download task to finish by synchronizing on the returned
103 * {@link Future}, but make sure not to freeze up JOSM. Example:
104 * <pre>
105 * Future&lt;?&gt; future = task.download(...);
106 * // DON'T run this on the Swing EDT or JOSM will freeze
107 * future.get(); // waits for the dowload task to complete
108 * </pre>
109 *
110 * The following example uses a pattern which is better suited if a task is launched from
111 * the Swing EDT:
112 * <pre>
113 * final Future&lt;?&gt; future = task.download(...);
114 * Runnable runAfterTask = new Runnable() {
115 * public void run() {
116 * // this is not strictly necessary because of the type of executor service
117 * // Main.worker is initialized with, but it doesn't harm either
118 * //
119 * future.get(); // wait for the download task to complete
120 * doSomethingAfterTheTaskCompleted();
121 * }
122 * }
123 * MainApplication.worker.submit(runAfterTask);
124 * </pre>
125 * @param reader the reader used to parse OSM data (see {@link OsmServerReader#parseOsm})
126 * @param settings download settings
127 * @param downloadArea the area to download
128 * @param progressMonitor the progressMonitor
129 * @return the future representing the asynchronous task
130 * @since 13927
131 */
132 public Future<?> download(OsmServerReader reader, DownloadParams settings, Bounds downloadArea, ProgressMonitor progressMonitor) {
133 return download(new DownloadTask(settings, reader, progressMonitor, zoomAfterDownload), downloadArea);
134 }
135
136 protected Future<?> download(DownloadTask downloadTask, Bounds downloadArea) {
137 this.downloadTask = downloadTask;
138 this.currentBounds = new Bounds(downloadArea);
139 // We need submit instead of execute so we can wait for it to finish and get the error
140 // message if necessary. If no one calls getErrorMessage() it just behaves like execute.
141 return MainApplication.worker.submit(downloadTask);
142 }
143
144 /**
145 * This allows subclasses to perform operations on the URL before {@link #loadUrl} is performed.
146 * @param url the original URL
147 * @return the modified URL
148 */
149 protected String modifyUrlBeforeLoad(String url) {
150 return url;
151 }
152
153 /**
154 * Loads a given URL from the OSM Server
155 * @param settings download settings
156 * @param url The URL as String
157 */
158 @Override
159 public Future<?> loadUrl(DownloadParams settings, String url, ProgressMonitor progressMonitor) {
160 String newUrl = modifyUrlBeforeLoad(url);
161 OsmUrlPattern urlPattern = Arrays.stream(OsmUrlPattern.values()).filter(p -> p.matches(newUrl)).findFirst()
162 .orElseThrow(() -> new IllegalArgumentException("URL does not match any OSM URL pattern: " + newUrl));
163 downloadTask = new DownloadTask(settings, getOsmServerReader(newUrl), progressMonitor, true, Compression.byExtension(newUrl));
164 currentBounds = null;
165 // Extract .osm filename from URL to set the new layer name
166 extractOsmFilename(settings, urlPattern.pattern(), newUrl);
167 return MainApplication.worker.submit(downloadTask);
168 }
169
170 protected OsmServerReader getOsmServerReader(String url) {
171 try {
172 String host = new URL(url).getHost();
173 for (String knownOverpassServer : OverpassDownloadReader.OVERPASS_SERVER_HISTORY.get()) {
174 if (host.equals(new URL(knownOverpassServer).getHost())) {
175 int index = url.indexOf(OVERPASS_INTERPRETER_DATA);
176 if (index > 0) {
177 return new OverpassDownloadReader(new Bounds(LatLon.ZERO), knownOverpassServer,
178 Utils.decodeUrl(url.substring(index + OVERPASS_INTERPRETER_DATA.length())));
179 }
180 }
181 }
182 } catch (MalformedURLException e) {
183 Logging.error(e);
184 }
185 return new OsmServerLocationReader(url);
186 }
187
188 protected final void extractOsmFilename(DownloadParams settings, String pattern, String url) {
189 newLayerName = settings.getLayerName();
190 if (newLayerName == null || newLayerName.isEmpty()) {
191 Matcher matcher = Pattern.compile(pattern).matcher(url);
192 newLayerName = matcher.matches() ? matcher.group(1) : null;
193 }
194 }
195
196 @Override
197 public void cancel() {
198 if (downloadTask != null) {
199 downloadTask.cancel();
200 }
201 }
202
203 @Override
204 public boolean isSafeForRemotecontrolRequests() {
205 return true;
206 }
207
208 @Override
209 public ProjectionBounds getDownloadProjectionBounds() {
210 return downloadTask != null ? downloadTask.computeBbox(currentBounds).orElse(null) : null;
211 }
212
213 protected Collection<OsmPrimitive> searchPotentiallyDeletedPrimitives(DataSet ds) {
214 return downloadTask.searchPrimitivesToUpdate(currentBounds, ds);
215 }
216
217 /**
218 * Superclass of internal download task.
219 * @since 7636
220 */
221 public abstract static class AbstractInternalTask extends PleaseWaitRunnable {
222
223 protected final DownloadParams settings;
224 protected final boolean zoomAfterDownload;
225 protected DataSet dataSet;
226
227 /**
228 * Constructs a new {@code AbstractInternalTask}.
229 * @param settings download settings
230 * @param title message for the user
231 * @param ignoreException If true, exception will be propagated to calling code. If false then
232 * exception will be thrown directly in EDT. When this runnable is executed using executor framework
233 * then use false unless you read result of task (because exception will get lost if you don't)
234 * @param zoomAfterDownload If true, the map view will zoom to download area after download
235 */
236 public AbstractInternalTask(DownloadParams settings, String title, boolean ignoreException, boolean zoomAfterDownload) {
237 super(title, ignoreException);
238 this.settings = Objects.requireNonNull(settings);
239 this.zoomAfterDownload = zoomAfterDownload;
240 }
241
242 /**
243 * Constructs a new {@code AbstractInternalTask}.
244 * @param settings download settings
245 * @param title message for the user
246 * @param progressMonitor progress monitor
247 * @param ignoreException If true, exception will be propagated to calling code. If false then
248 * exception will be thrown directly in EDT. When this runnable is executed using executor framework
249 * then use false unless you read result of task (because exception will get lost if you don't)
250 * @param zoomAfterDownload If true, the map view will zoom to download area after download
251 */
252 public AbstractInternalTask(DownloadParams settings, String title, ProgressMonitor progressMonitor, boolean ignoreException,
253 boolean zoomAfterDownload) {
254 super(title, progressMonitor, ignoreException);
255 this.settings = Objects.requireNonNull(settings);
256 this.zoomAfterDownload = zoomAfterDownload;
257 }
258
259 protected OsmDataLayer getEditLayer() {
260 return MainApplication.getLayerManager().getEditLayer();
261 }
262
263 private static Stream<OsmDataLayer> getModifiableDataLayers() {
264 return MainApplication.getLayerManager().getLayersOfType(OsmDataLayer.class)
265 .stream().filter(OsmDataLayer::isDownloadable);
266 }
267
268 /**
269 * Returns the number of modifiable data layers
270 * @return number of modifiable data layers
271 * @since 13434
272 */
273 protected long getNumModifiableDataLayers() {
274 return getModifiableDataLayers().count();
275 }
276
277 /**
278 * Returns the first modifiable data layer
279 * @return the first modifiable data layer
280 * @since 13434
281 */
282 protected OsmDataLayer getFirstModifiableDataLayer() {
283 return getModifiableDataLayers().findFirst().orElse(null);
284 }
285
286 /**
287 * Creates a name for a new layer by utilizing the settings ({@link DownloadParams#getLayerName()}) or
288 * {@link OsmDataLayer#createNewName()} if the former option is {@code null}.
289 *
290 * @return a name for a new layer
291 * @since 14347
292 */
293 protected String generateLayerName() {
294 return Optional.ofNullable(settings.getLayerName())
295 .filter(layerName -> !Utils.isStripEmpty(layerName))
296 .orElse(OsmDataLayer.createNewName());
297 }
298
299 /**
300 * Can be overridden (e.g. by plugins) if a subclass of {@link OsmDataLayer} is needed.
301 * If you want to change how the name is determined, consider overriding
302 * {@link #generateLayerName()} instead.
303 *
304 * @param ds the dataset on which the layer is based, must be non-null
305 * @param layerName the name of the new layer, must be either non-blank or non-present
306 * @return a new instance of {@link OsmDataLayer} constructed with the given arguments
307 * @since 14347
308 */
309 protected OsmDataLayer createNewLayer(final DataSet ds, final Optional<String> layerName) {
310 if (layerName.filter(Utils::isStripEmpty).isPresent()) {
311 throw new IllegalArgumentException("Blank layer name!");
312 }
313 return new OsmDataLayer(
314 Objects.requireNonNull(ds, "dataset parameter"),
315 layerName.orElseGet(this::generateLayerName),
316 null
317 );
318 }
319
320 /**
321 * Convenience method for {@link #createNewLayer(DataSet, Optional)}, uses the dataset
322 * from field {@link #dataSet} and applies the settings from field {@link #settings}.
323 *
324 * @param layerName an optional layer name, must be non-blank if the [Optional] is present
325 * @return a newly constructed layer
326 * @since 14347
327 */
328 protected final OsmDataLayer createNewLayer(final Optional<String> layerName) {
329 Optional.ofNullable(settings.getDownloadPolicy())
330 .ifPresent(dataSet::setDownloadPolicy);
331 Optional.ofNullable(settings.getUploadPolicy())
332 .ifPresent(dataSet::setUploadPolicy);
333 if (dataSet.isLocked() && !settings.isLocked()) {
334 dataSet.unlock();
335 } else if (!dataSet.isLocked() && settings.isLocked()) {
336 dataSet.lock();
337 }
338 return createNewLayer(dataSet, layerName);
339 }
340
341 protected Optional<ProjectionBounds> computeBbox(Bounds bounds) {
342 BoundingXYVisitor v = new BoundingXYVisitor();
343 if (bounds != null) {
344 v.visit(bounds);
345 } else {
346 v.computeBoundingBox(dataSet.getNodes());
347 }
348 return Optional.ofNullable(v.getBounds());
349 }
350
351 protected OsmDataLayer addNewLayerIfRequired(String newLayerName) {
352 long numDataLayers = getNumModifiableDataLayers();
353 if (settings.isNewLayer() || numDataLayers == 0 || (numDataLayers > 1 && getEditLayer() == null)) {
354 // the user explicitly wants a new layer, we don't have any layer at all
355 // or it is not clear which layer to merge to
356 final OsmDataLayer layer = createNewLayer(Optional.ofNullable(newLayerName).filter(it -> !Utils.isStripEmpty(it)));
357 MainApplication.getLayerManager().addLayer(layer, zoomAfterDownload);
358 return layer;
359 }
360 return null;
361 }
362
363 protected void loadData(String newLayerName, Bounds bounds) {
364 OsmDataLayer layer = addNewLayerIfRequired(newLayerName);
365 if (layer == null) {
366 layer = getEditLayer();
367 if (layer == null || !layer.isDownloadable()) {
368 layer = getFirstModifiableDataLayer();
369 }
370 Collection<OsmPrimitive> primitivesToUpdate = searchPrimitivesToUpdate(bounds, layer.getDataSet());
371 layer.mergeFrom(dataSet);
372 MapFrame map = MainApplication.getMap();
373 if (map != null && zoomAfterDownload) {
374 computeBbox(bounds).map(ViewportData::new).ifPresent(map.mapView::zoomTo);
375 }
376 if (!primitivesToUpdate.isEmpty()) {
377 MainApplication.worker.submit(new UpdatePrimitivesTask(layer, primitivesToUpdate));
378 }
379 layer.onPostDownloadFromServer();
380 }
381 }
382
383 /**
384 * Look for primitives deleted on server (thus absent from downloaded data)
385 * but still present in existing data layer
386 * @param bounds download bounds
387 * @param ds existing data set
388 * @return the primitives to update
389 */
390 protected Collection<OsmPrimitive> searchPrimitivesToUpdate(Bounds bounds, DataSet ds) {
391 if (bounds == null)
392 return Collections.emptySet();
393 Collection<OsmPrimitive> col = new ArrayList<>();
394 ds.searchNodes(bounds.toBBox()).stream().filter(n -> !n.isNew() && !dataSet.containsNode(n)).forEachOrdered(col::add);
395 if (!col.isEmpty()) {
396 Set<Way> ways = new HashSet<>();
397 Set<Relation> rels = new HashSet<>();
398 for (OsmPrimitive n : col) {
399 for (OsmPrimitive ref : n.getReferrers()) {
400 if (ref.isNew()) {
401 continue;
402 } else if (ref instanceof Way) {
403 ways.add((Way) ref);
404 } else if (ref instanceof Relation) {
405 rels.add((Relation) ref);
406 }
407 }
408 }
409 ways.stream().filter(w -> !dataSet.containsWay(w)).forEachOrdered(col::add);
410 rels.stream().filter(r -> !dataSet.containsRelation(r)).forEachOrdered(col::add);
411 }
412 return col;
413 }
414 }
415
416 protected class DownloadTask extends AbstractInternalTask {
417 protected final OsmServerReader reader;
418 protected final Compression compression;
419
420 /**
421 * Constructs a new {@code DownloadTask}.
422 * @param settings download settings
423 * @param reader OSM data reader
424 * @param progressMonitor progress monitor
425 * @since 13927
426 */
427 public DownloadTask(DownloadParams settings, OsmServerReader reader, ProgressMonitor progressMonitor) {
428 this(settings, reader, progressMonitor, true);
429 }
430
431 /**
432 * Constructs a new {@code DownloadTask}.
433 * @param settings download settings
434 * @param reader OSM data reader
435 * @param progressMonitor progress monitor
436 * @param zoomAfterDownload If true, the map view will zoom to download area after download
437 * @since 13927
438 */
439 public DownloadTask(DownloadParams settings, OsmServerReader reader, ProgressMonitor progressMonitor, boolean zoomAfterDownload) {
440 this(settings, reader, progressMonitor, zoomAfterDownload, Compression.NONE);
441 }
442
443 /**
444 * Constructs a new {@code DownloadTask}.
445 * @param settings download settings
446 * @param reader OSM data reader
447 * @param progressMonitor progress monitor
448 * @param zoomAfterDownload If true, the map view will zoom to download area after download
449 * @param compression compression to use
450 * @since 15784
451 */
452 public DownloadTask(DownloadParams settings, OsmServerReader reader, ProgressMonitor progressMonitor, boolean zoomAfterDownload,
453 Compression compression) {
454 super(settings, tr("Downloading data"), progressMonitor, false, zoomAfterDownload);
455 this.reader = Objects.requireNonNull(reader);
456 this.compression = compression;
457 }
458
459 protected DataSet parseDataSet() throws OsmTransferException {
460 ProgressMonitor subTaskMonitor = progressMonitor.createSubTaskMonitor(ProgressMonitor.ALL_TICKS, false);
461 // Don't call parseOsm signature with compression if not needed, too many implementations to update before to avoid side effects
462 return compression != null && compression != Compression.NONE ?
463 reader.parseOsm(subTaskMonitor, compression) : reader.parseOsm(subTaskMonitor);
464 }
465
466 @Override
467 public void realRun() throws IOException, SAXException, OsmTransferException {
468 try {
469 if (isCanceled())
470 return;
471 dataSet = parseDataSet();
472 } catch (OsmTransferException e) {
473 if (isCanceled()) {
474 Logging.info(tr("Ignoring exception because download has been canceled. Exception was: {0}", e.toString()));
475 return;
476 }
477 if (e instanceof OsmTransferCanceledException) {
478 setCanceled(true);
479 return;
480 } else {
481 rememberException(e);
482 }
483 DownloadOsmTask.this.setFailed(true);
484 }
485 }
486
487 @Override
488 protected void finish() {
489 if (isFailed() || isCanceled())
490 return;
491 if (dataSet == null)
492 return; // user canceled download or error occurred
493 if (dataSet.allPrimitives().isEmpty()) {
494 if (warnAboutEmptyArea) {
495 rememberErrorMessage(NO_DATA_FOUND);
496 }
497 String remark = dataSet.getRemark();
498 if (remark != null && !remark.isEmpty()) {
499 rememberErrorMessage(remark);
500 }
501 if (!(reader instanceof BoundingBoxDownloader)
502 || ((BoundingBoxDownloader) reader).considerAsFullDownload()) {
503 // need to synthesize a download bounds lest the visual indication of downloaded area doesn't work
504 dataSet.addDataSource(new DataSource(
505 currentBounds != null ? currentBounds : new Bounds(LatLon.ZERO), "OpenStreetMap server"));
506 }
507 }
508
509 rememberDownloadedData(dataSet);
510 loadData(newLayerName, currentBounds);
511 }
512
513 @Override
514 protected void cancel() {
515 setCanceled(true);
516 if (reader != null) {
517 reader.cancel();
518 }
519 }
520 }
521
522 @Override
523 public String getConfirmationMessage(URL url) {
524 if (OsmUrlPattern.OSM_API_URL.matches(url)) {
525 Collection<String> items = new ArrayList<>();
526 items.add(tr("OSM Server URL:") + ' ' + url.getHost());
527 items.add(tr("Command")+": "+url.getPath());
528 if (url.getQuery() != null) {
529 items.add(tr("Request details: {0}", url.getQuery().replaceAll(",\\s*", ", ")));
530 }
531 return Utils.joinAsHtmlUnorderedList(items);
532 }
533 return null;
534 }
535}
Note: See TracBrowser for help on using the repository browser.