source: josm/trunk/src/org/openstreetmap/josm/gui/io/DownloadPrimitivesWithReferrersTask.java@ 17318

Last change on this file since 17318 was 16611, checked in by GerdP, 4 years ago

fix #12303: When downloading objects with Overpass API, use recurse up to fetch referrers

  • new static method genOverpassQuery() to generate a single overpass query for all wanted objects
  • use POST instead of PUT to send the query
  • add handling for missing primitives (Overpass doesn't return invisible objects and doesn't a rc 404)
  • Property svn:eol-style set to native
File size: 11.0 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.gui.io;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5import static org.openstreetmap.josm.tools.I18n.trn;
6
7import java.awt.Font;
8import java.awt.GridBagLayout;
9import java.io.IOException;
10import java.text.MessageFormat;
11import java.util.ArrayList;
12import java.util.Collections;
13import java.util.HashSet;
14import java.util.LinkedHashSet;
15import java.util.List;
16import java.util.Set;
17import java.util.stream.Collectors;
18
19import javax.swing.JLabel;
20import javax.swing.JOptionPane;
21import javax.swing.JPanel;
22import javax.swing.JScrollPane;
23
24import org.openstreetmap.josm.actions.downloadtasks.DownloadReferrersTask;
25import org.openstreetmap.josm.data.osm.DataSet;
26import org.openstreetmap.josm.data.osm.OsmPrimitive;
27import org.openstreetmap.josm.data.osm.PrimitiveId;
28import org.openstreetmap.josm.gui.ExtendedDialog;
29import org.openstreetmap.josm.gui.MainApplication;
30import org.openstreetmap.josm.gui.PleaseWaitRunnable;
31import org.openstreetmap.josm.gui.layer.OsmDataLayer;
32import org.openstreetmap.josm.gui.progress.ProgressMonitor;
33import org.openstreetmap.josm.gui.util.GuiHelper;
34import org.openstreetmap.josm.gui.widgets.HtmlPanel;
35import org.openstreetmap.josm.gui.widgets.JosmTextArea;
36import org.openstreetmap.josm.io.MultiFetchOverpassObjectReader;
37import org.openstreetmap.josm.io.OsmTransferException;
38import org.openstreetmap.josm.io.OverpassDownloadReader;
39import org.openstreetmap.josm.tools.GBC;
40import org.xml.sax.SAXException;
41
42/**
43 * Task for downloading a set of primitives with all referrers.
44 */
45public class DownloadPrimitivesWithReferrersTask extends PleaseWaitRunnable {
46 /** If true download into a new layer */
47 private final boolean newLayer;
48 /** List of primitives id to download */
49 private final List<PrimitiveId> ids;
50 /** If true, download members for relation */
51 private final boolean full;
52 /** If true, download also referrers */
53 private final boolean downloadReferrers;
54
55 /** Temporary layer where downloaded primitives are put */
56 private final OsmDataLayer tmpLayer;
57 /** Flag indicated that user ask for cancel this task */
58 private boolean canceled;
59 /** Reference to the task currently running */
60 private PleaseWaitRunnable currentTask;
61
62 /** set of missing ids, with overpass API these are also deleted objects */
63 private Set<PrimitiveId> missingPrimitives;
64
65 /**
66 * Constructor
67 *
68 * @param newLayer if the data should be downloaded into a new layer
69 * @param ids List of primitive id to download
70 * @param downloadReferrers if the referrers of the object should be downloaded as well,
71 * i.e., parent relations, and for nodes, additionally, parent ways
72 * @param full if the members of a relation should be downloaded as well
73 * @param newLayerName the name to use for the new layer, can be null.
74 * @param monitor ProgressMonitor to use, or null to create a new one
75 */
76 public DownloadPrimitivesWithReferrersTask(boolean newLayer, List<PrimitiveId> ids, boolean downloadReferrers,
77 boolean full, String newLayerName, ProgressMonitor monitor) {
78 super(tr("Download objects"), monitor, false);
79 this.ids = ids;
80 this.downloadReferrers = downloadReferrers;
81 this.full = full;
82 this.newLayer = newLayer;
83 // Check we don't try to download new primitives
84 for (PrimitiveId primitiveId : ids) {
85 if (primitiveId.isNew()) {
86 throw new IllegalArgumentException(MessageFormat.format(
87 "Cannot download new primitives (ID {0})", primitiveId.getUniqueId()));
88 }
89 }
90 // All downloaded primitives are put in a tmpLayer
91 tmpLayer = new OsmDataLayer(new DataSet(), newLayerName != null ? newLayerName : OsmDataLayer.createNewName(), null);
92 }
93
94 /**
95 * Cancel recursively the task. Do not call directly
96 * @see DownloadPrimitivesWithReferrersTask#operationCanceled()
97 */
98 @Override
99 protected void cancel() {
100 synchronized (this) {
101 canceled = true;
102 if (currentTask != null)
103 currentTask.operationCanceled();
104 }
105 }
106
107 @Override
108 protected void realRun() throws SAXException, IOException, OsmTransferException {
109 if (Boolean.TRUE.equals(OverpassDownloadReader.FOR_MULTI_FETCH.get())) {
110 useOverpassApi();
111 } else {
112 useOSMApi();
113 }
114 }
115
116 private void useOverpassApi() {
117 String request = MultiFetchOverpassObjectReader.genOverpassQuery(ids, true, downloadReferrers, full);
118 currentTask = new DownloadFromOverpassTask(request, tmpLayer.data, getProgressMonitor().createSubTaskMonitor(1, false));
119 currentTask.run();
120 missingPrimitives = ids.stream()
121 .filter(id -> tmpLayer.data.getPrimitiveById(id) == null)
122 .collect(Collectors.toSet());
123 }
124
125 private void useOSMApi() {
126 getProgressMonitor().setTicksCount(ids.size()+1);
127 // First, download primitives
128 DownloadPrimitivesTask mainTask = new DownloadPrimitivesTask(tmpLayer, ids, full,
129 getProgressMonitor().createSubTaskMonitor(1, false));
130 synchronized (this) {
131 currentTask = mainTask;
132 if (canceled) {
133 currentTask = null;
134 return;
135 }
136 }
137 currentTask.run();
138
139 missingPrimitives = mainTask.getMissingPrimitives();
140
141 // Then, download referrers for each primitive
142 if (downloadReferrers && tmpLayer.data != null) {
143 // see #18895: don't try to download parents for invisible objects
144 List<PrimitiveId> visible = ids.stream().map(tmpLayer.data::getPrimitiveById)
145 .filter(p -> p != null && p.isVisible()).collect(Collectors.toList());
146 if (!visible.isEmpty()) {
147 currentTask = new DownloadReferrersTask(tmpLayer, visible);
148 currentTask.run();
149 synchronized (this) {
150 if (currentTask.getProgressMonitor().isCanceled())
151 cancel();
152 }
153 }
154 }
155 currentTask = null;
156 }
157
158 @Override
159 protected void finish() {
160 synchronized (this) {
161 if (canceled)
162 return;
163 }
164
165 // Append downloaded data to JOSM
166 OsmDataLayer layer = MainApplication.getLayerManager().getEditLayer();
167 if (layer == null || this.newLayer || !layer.isDownloadable())
168 MainApplication.getLayerManager().addLayer(tmpLayer);
169 else
170 layer.mergeFrom(tmpLayer);
171
172 // Collect known deleted primitives
173 final Set<PrimitiveId> del = new HashSet<>();
174 DataSet ds = MainApplication.getLayerManager().getEditDataSet();
175 for (PrimitiveId id : ids) {
176 OsmPrimitive osm = ds.getPrimitiveById(id);
177 if (osm != null && osm.isDeleted()) {
178 del.add(id);
179 }
180 }
181 final Set<PrimitiveId> errs;
182 if (missingPrimitives != null) {
183 errs = missingPrimitives.stream().filter(id -> !del.contains(id)).collect(Collectors.toCollection(LinkedHashSet::new));
184 } else {
185 errs = Collections.emptySet();
186 }
187
188 // Warm about missing primitives
189 if (!errs.isEmpty()) {
190 final String assumedApiRC;
191 if (Boolean.TRUE.equals(OverpassDownloadReader.FOR_MULTI_FETCH.get())) {
192 assumedApiRC = trn("The server did not return data for the requested object, it was either deleted or does not exist.",
193 "The server did not return data for the requested objects, they were either deleted or do not exist.",
194 errs.size());
195
196 } else {
197 assumedApiRC = tr("The server replied with response code 404.<br>"
198 + "This usually means, the server does not know an object with the requested id.");
199 }
200 GuiHelper.runInEDTAndWait(() -> reportProblemDialog(errs,
201 trn("Object could not be downloaded", "Some objects could not be downloaded", errs.size()),
202 trn("One object could not be downloaded.<br>",
203 "{0} objects could not be downloaded.<br>",
204 errs.size(),
205 errs.size())
206 + assumedApiRC,
207 tr("missing objects:"),
208 JOptionPane.ERROR_MESSAGE
209 ).showDialog());
210 }
211
212 // Warm about deleted primitives
213 if (!del.isEmpty())
214 GuiHelper.runInEDTAndWait(() -> reportProblemDialog(del,
215 trn("Object deleted", "Objects deleted", del.size()),
216 trn(
217 "One downloaded object is deleted.",
218 "{0} downloaded objects are deleted.",
219 del.size(),
220 del.size()),
221 null,
222 JOptionPane.WARNING_MESSAGE
223 ).showDialog());
224 }
225
226 /**
227 * Return ids of really downloaded primitives.
228 * @return List of primitives id or null if no primitives were downloaded
229 */
230 public List<PrimitiveId> getDownloadedId() {
231 synchronized (this) {
232 if (canceled)
233 return null;
234 }
235 List<PrimitiveId> downloaded = new ArrayList<>(ids);
236 downloaded.removeAll(missingPrimitives);
237 return downloaded;
238 }
239
240 /**
241 * Dialog for report a problem during download.
242 * @param errs Primitives involved
243 * @param title Title of dialog
244 * @param text Detail message
245 * @param listLabel List of primitives description
246 * @param msgType Type of message, see {@link JOptionPane}
247 * @return The Dialog object
248 */
249 public static ExtendedDialog reportProblemDialog(Set<PrimitiveId> errs,
250 String title, String text, String listLabel, int msgType) {
251 JPanel p = new JPanel(new GridBagLayout());
252 p.add(new HtmlPanel(text), GBC.eop());
253 JosmTextArea txt = new JosmTextArea();
254 if (listLabel != null) {
255 JLabel missing = new JLabel(listLabel);
256 missing.setFont(missing.getFont().deriveFont(Font.PLAIN));
257 missing.setLabelFor(txt);
258 p.add(missing, GBC.eol());
259 }
260 txt.setFont(GuiHelper.getMonospacedFont(txt));
261 txt.setEditable(false);
262 txt.setBackground(p.getBackground());
263 txt.setColumns(40);
264 txt.setRows(1);
265 txt.setText(errs.stream().map(pid -> pid.getType().getAPIName().substring(0, 1) + pid.getUniqueId())
266 .collect(Collectors.joining(", ")));
267 JScrollPane scroll = new JScrollPane(txt);
268 p.add(scroll, GBC.eop().weight(1.0, 0.0).fill(GBC.HORIZONTAL));
269
270 return new ExtendedDialog(
271 MainApplication.getMainFrame(),
272 title,
273 tr("Ok"))
274 .setButtonIcons("ok")
275 .setIcon(msgType)
276 .setContent(p, false);
277 }
278}
Note: See TracBrowser for help on using the repository browser.