source: josm/trunk/src/org/openstreetmap/josm/io/MultiFetchServerObjectReader.java

Last change on this file was 19050, checked in by taylor.smock, 34 hours ago

Revert most var changes from r19048, fix most new compile warnings and checkstyle issues

Also, document why various ErrorProne checks were originally disabled and fix
generic SonarLint issues.
  • Property svn:eol-style set to native
File size: 31.0 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.io;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5import static org.openstreetmap.josm.tools.I18n.trn;
6
7import java.io.IOException;
8import java.io.InputStream;
9import java.net.HttpURLConnection;
10import java.util.ArrayList;
11import java.util.Collection;
12import java.util.Collections;
13import java.util.HashSet;
14import java.util.Iterator;
15import java.util.LinkedHashMap;
16import java.util.LinkedHashSet;
17import java.util.List;
18import java.util.Map;
19import java.util.Map.Entry;
20import java.util.Set;
21import java.util.concurrent.Callable;
22import java.util.concurrent.CompletionService;
23import java.util.concurrent.ExecutionException;
24import java.util.concurrent.ExecutorCompletionService;
25import java.util.concurrent.ExecutorService;
26import java.util.concurrent.Executors;
27import java.util.concurrent.Future;
28import java.util.stream.Collectors;
29
30import org.openstreetmap.josm.data.Bounds;
31import org.openstreetmap.josm.data.osm.DataSet;
32import org.openstreetmap.josm.data.osm.DataSetMerger;
33import org.openstreetmap.josm.data.osm.Node;
34import org.openstreetmap.josm.data.osm.OsmPrimitive;
35import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
36import org.openstreetmap.josm.data.osm.PrimitiveId;
37import org.openstreetmap.josm.data.osm.Relation;
38import org.openstreetmap.josm.data.osm.RelationMember;
39import org.openstreetmap.josm.data.osm.SimplePrimitiveId;
40import org.openstreetmap.josm.data.osm.Way;
41import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
42import org.openstreetmap.josm.gui.progress.ProgressMonitor;
43import org.openstreetmap.josm.spi.preferences.Config;
44import org.openstreetmap.josm.tools.Logging;
45import org.openstreetmap.josm.tools.Utils;
46
47/**
48 * Retrieves a set of {@link OsmPrimitive}s from an OSM server using the so called
49 * Multi Fetch API.
50 * <p>
51 * Usage:
52 * <pre>
53 * MultiFetchServerObjectReader reader = MultiFetchServerObjectReader()
54 * .append(new Node(72343));
55 * reader.parseOsm();
56 * if (!reader.getMissingPrimitives().isEmpty()) {
57 * Logging.info("There are missing primitives: " + reader.getMissingPrimitives());
58 * }
59 * if (!reader.getSkippedWays().isEmpty()) {
60 * Logging.info("There are skipped ways: " + reader.getMissingPrimitives());
61 * }
62 * </pre>
63 */
64public class MultiFetchServerObjectReader extends OsmServerReader {
65 /**
66 * the max. number of primitives retrieved in one step. Assuming IDs with 10 digits,
67 * this leads to a max. request URL of ~ 1900 Bytes ((10 digits + 1 Separator) * 170),
68 * which should be safe according to the
69 * <a href="https://web.archive.org/web/20190902193246/https://boutell.com/newfaq/misc/urllength.html">WWW FAQ</a>.
70 */
71 private static final int MAX_IDS_PER_REQUEST = 170;
72
73 private final Set<Long> nodes;
74 private final Set<Long> ways;
75 private final Set<Long> relations;
76 private final Set<PrimitiveId> missingPrimitives;
77 private final DataSet outputDataSet;
78 protected final Map<OsmPrimitiveType, Set<Long>> primitivesMap;
79
80 protected boolean recurseDownRelations;
81 private boolean recurseDownAppended = true;
82
83 private ExecutorService exec;
84
85 /**
86 * Constructs a {@code MultiFetchServerObjectReader}.
87 */
88 protected MultiFetchServerObjectReader() {
89 nodes = new LinkedHashSet<>();
90 ways = new LinkedHashSet<>();
91 relations = new LinkedHashSet<>();
92 this.outputDataSet = new DataSet();
93 this.missingPrimitives = new LinkedHashSet<>();
94 primitivesMap = new LinkedHashMap<>();
95 primitivesMap.put(OsmPrimitiveType.RELATION, relations);
96 primitivesMap.put(OsmPrimitiveType.WAY, ways);
97 primitivesMap.put(OsmPrimitiveType.NODE, nodes);
98 }
99
100 /**
101 * Creates a new instance of {@link MultiFetchServerObjectReader} or {@link MultiFetchOverpassObjectReader}
102 * depending on the {@link OverpassDownloadReader#FOR_MULTI_FETCH preference}.
103 *
104 * @return a new instance
105 * @since 9241
106 */
107 public static MultiFetchServerObjectReader create() {
108 return create(OverpassDownloadReader.FOR_MULTI_FETCH.get());
109 }
110
111 /**
112 * Creates a new instance of {@link MultiFetchServerObjectReader} or {@link MultiFetchOverpassObjectReader}
113 * depending on the {@code fromMirror} parameter.
114 *
115 * @param fromMirror {@code false} for {@link MultiFetchServerObjectReader}, {@code true} for {@link MultiFetchOverpassObjectReader}
116 * @return a new instance
117 * @since 15520 (changed visibility)
118 */
119 public static MultiFetchServerObjectReader create(final boolean fromMirror) {
120 if (fromMirror) {
121 return new MultiFetchOverpassObjectReader();
122 } else {
123 return new MultiFetchServerObjectReader();
124 }
125 }
126
127 /**
128 * Remembers an {@link OsmPrimitive}'s id. The id will
129 * later be fetched as part of a Multi Get request.
130 * <p>
131 * Ignore the id if it represents a new primitives.
132 *
133 * @param id the id
134 */
135 public void append(PrimitiveId id) {
136 if (id.isNew()) return;
137 switch (id.getType()) {
138 case NODE: nodes.add(id.getUniqueId()); break;
139 case WAY: ways.add(id.getUniqueId()); break;
140 case RELATION: relations.add(id.getUniqueId()); break;
141 default: throw new AssertionError();
142 }
143 }
144
145 /**
146 * appends a {@link OsmPrimitive} id to the list of ids which will be fetched from the server.
147 *
148 * @param ds the {@link DataSet} to which the primitive belongs
149 * @param id the primitive id
150 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY},
151 * {@link OsmPrimitiveType#RELATION RELATION}
152 * @return this
153 */
154 public MultiFetchServerObjectReader append(DataSet ds, long id, OsmPrimitiveType type) {
155 OsmPrimitive p = ds.getPrimitiveById(id, type);
156 return append(p);
157 }
158
159 /**
160 * appends a {@link Node} id to the list of ids which will be fetched from the server.
161 *
162 * @param node the node (ignored, if null)
163 * @return this
164 */
165 public MultiFetchServerObjectReader appendNode(Node node) {
166 if (node == null || node.isNew()) return this;
167 append(node.getPrimitiveId());
168 return this;
169 }
170
171 /**
172 * appends a {@link Way} id and the list of ids of nodes the way refers to the list of ids which will be fetched from the server.
173 *
174 * @param way the way (ignored, if null)
175 * @return this
176 */
177 public MultiFetchServerObjectReader appendWay(Way way) {
178 if (way == null || way.isNew()) return this;
179 if (recurseDownAppended) {
180 append(way.getNodes());
181 }
182 append(way.getPrimitiveId());
183 return this;
184 }
185
186 /**
187 * appends a {@link Relation} id to the list of ids which will be fetched from the server.
188 *
189 * @param relation the relation (ignored, if null)
190 * @return this
191 */
192 protected MultiFetchServerObjectReader appendRelation(Relation relation) {
193 if (relation == null || relation.isNew()) return this;
194 append(relation.getPrimitiveId());
195 if (recurseDownAppended) {
196 for (RelationMember member : relation.getMembers()) {
197 // avoid infinite recursion in case of cyclic dependencies in relations
198 if (OsmPrimitiveType.from(member.getMember()) == OsmPrimitiveType.RELATION
199 && relations.contains(member.getMember().getId())) {
200 continue;
201 }
202 if (!member.getMember().isIncomplete()) {
203 append(member.getMember());
204 }
205 }
206 }
207 return this;
208 }
209
210 /**
211 * appends an {@link OsmPrimitive} to the list of ids which will be fetched from the server.
212 * @param primitive the primitive
213 * @return this
214 */
215 public MultiFetchServerObjectReader append(OsmPrimitive primitive) {
216 if (primitive instanceof Node) {
217 return appendNode((Node) primitive);
218 } else if (primitive instanceof Way) {
219 return appendWay((Way) primitive);
220 } else if (primitive instanceof Relation) {
221 return appendRelation((Relation) primitive);
222 }
223 return this;
224 }
225
226 /**
227 * appends a list of {@link OsmPrimitive} to the list of ids which will be fetched from the server.
228 *
229 * @param primitives the list of primitives (ignored, if null)
230 * @return this
231 *
232 * @see #append(OsmPrimitive)
233 */
234 public MultiFetchServerObjectReader append(Collection<? extends OsmPrimitive> primitives) {
235 if (primitives == null) return this;
236 primitives.forEach(this::append);
237 return this;
238 }
239
240 /**
241 * extracts a subset of max {@link #MAX_IDS_PER_REQUEST} ids from <code>ids</code> and
242 * replies the subset. The extracted subset is removed from <code>ids</code>.
243 *
244 * @param ids a set of ids
245 * @return the subset of ids
246 */
247 protected Set<Long> extractIdPackage(Set<Long> ids) {
248 Set<Long> pkg = new HashSet<>();
249 if (ids.isEmpty())
250 return pkg;
251 if (ids.size() > MAX_IDS_PER_REQUEST) {
252 Iterator<Long> it = ids.iterator();
253 for (int i = 0; i < MAX_IDS_PER_REQUEST; i++) {
254 pkg.add(it.next());
255 }
256 ids.removeAll(pkg);
257 } else {
258 pkg.addAll(ids);
259 ids.clear();
260 }
261 return pkg;
262 }
263
264 /**
265 * builds the Multi Get request string for a set of ids and a given {@link OsmPrimitiveType}.
266 *
267 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY},
268 * {@link OsmPrimitiveType#RELATION RELATION}
269 * @param idPackage the package of ids
270 * @return the request string
271 */
272 protected String buildRequestString(final OsmPrimitiveType type, Set<Long> idPackage) {
273 return type.getAPIName() + "s?" + type.getAPIName() + "s=" + idPackage.stream().map(String::valueOf).collect(Collectors.joining(","));
274 }
275
276 protected void rememberNodesOfIncompleteWaysToLoad(DataSet from) {
277 for (Way w: from.getWays()) {
278 for (Node n: w.getNodes()) {
279 if (n.isIncomplete()) {
280 nodes.add(n.getId());
281 }
282 }
283 }
284 }
285
286 /**
287 * merges the dataset <code>from</code> to {@link #outputDataSet}.
288 *
289 * @param from the other dataset
290 */
291 protected void merge(DataSet from) {
292 final DataSetMerger visitor = new DataSetMerger(outputDataSet, from);
293 visitor.merge();
294 }
295
296 /**
297 * fetches a set of ids of a given {@link OsmPrimitiveType} from the server
298 *
299 * @param ids the set of ids
300 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY},
301 * {@link OsmPrimitiveType#RELATION RELATION}
302 * @param progressMonitor progress monitor
303 * @throws OsmTransferException if an error occurs while communicating with the API server
304 */
305 protected void fetchPrimitives(Set<Long> ids, OsmPrimitiveType type, ProgressMonitor progressMonitor) throws OsmTransferException {
306 String msg;
307 final String baseUrl = getBaseUrl();
308 switch (type) {
309 // CHECKSTYLE.OFF: SingleSpaceSeparator
310 case NODE: msg = tr("Fetching a package of nodes from ''{0}''", baseUrl); break;
311 case WAY: msg = tr("Fetching a package of ways from ''{0}''", baseUrl); break;
312 case RELATION: msg = tr("Fetching a package of relations from ''{0}''", baseUrl); break;
313 // CHECKSTYLE.ON: SingleSpaceSeparator
314 default: throw new AssertionError();
315 }
316 progressMonitor.setTicksCount(ids.size());
317 progressMonitor.setTicks(0);
318 // The complete set containing all primitives to fetch
319 Set<Long> toFetch = new HashSet<>(ids);
320 // Build a list of fetchers that will download smaller sets containing only MAX_IDS_PER_REQUEST (200) primitives each.
321 // we will run up to MAX_DOWNLOAD_THREADS concurrent fetchers.
322 int threadsNumber = Config.getPref().getInt("osm.download.threads", OsmApi.MAX_DOWNLOAD_THREADS);
323 threadsNumber = Utils.clamp(threadsNumber, 1, OsmApi.MAX_DOWNLOAD_THREADS);
324 exec = Executors.newFixedThreadPool(
325 threadsNumber, Utils.newThreadFactory(getClass() + "-%d", Thread.NORM_PRIORITY));
326 CompletionService<FetchResult> ecs = new ExecutorCompletionService<>(exec);
327 List<Future<FetchResult>> jobs = new ArrayList<>();
328 // There exists a race condition where this is cancelled after isCanceled is called, such that
329 // the exec ThreadPool has been shut down. This can cause a RejectedExecutionException.
330 synchronized (this) {
331 while (!toFetch.isEmpty() && !isCanceled()) {
332 jobs.add(ecs.submit(new Fetcher(type, extractIdPackage(toFetch), progressMonitor)));
333 }
334 }
335 // Run the fetchers
336 for (int i = 0; i < jobs.size() && !isCanceled(); i++) {
337 progressMonitor.subTask(msg + "... " + progressMonitor.getTicks() + '/' + progressMonitor.getTicksCount());
338 try {
339 FetchResult result = ecs.take().get();
340 if (result.rc404 != null) {
341 List<Long> toSplit = new ArrayList<>(result.rc404);
342 int n = toSplit.size() / 2;
343 jobs.add(ecs.submit(new Fetcher(type, new HashSet<>(toSplit.subList(0, n)), progressMonitor)));
344 jobs.add(ecs.submit(new Fetcher(type, new HashSet<>(toSplit.subList(n, toSplit.size())), progressMonitor)));
345 }
346 if (result.missingPrimitives != null) {
347 missingPrimitives.addAll(result.missingPrimitives);
348 }
349 if (result.dataSet != null && !isCanceled()) {
350 rememberNodesOfIncompleteWaysToLoad(result.dataSet);
351 merge(result.dataSet);
352 }
353 } catch (InterruptedException | ExecutionException e) {
354 if (e instanceof InterruptedException) {
355 Thread.currentThread().interrupt();
356 }
357 Logging.error(e);
358 if (e.getCause() instanceof OsmTransferException)
359 throw (OsmTransferException) e.getCause();
360 }
361 }
362 exec.shutdown();
363 // Cancel requests if the user chose to
364 if (isCanceled()) {
365 for (Future<FetchResult> job : jobs) {
366 job.cancel(true);
367 }
368 }
369 exec = null;
370 }
371
372 /**
373 * invokes one or more Multi Gets to fetch the {@link OsmPrimitive}s and replies
374 * the dataset of retrieved primitives. Note that the dataset includes non visible primitives too!
375 * In contrast to a simple Get for a node, a way, or a relation, a Multi Get always replies
376 * the latest version of the primitive (if any), even if the primitive is not visible (i.e. if
377 * visible==false).
378 * <p>
379 * Invoke {@link #getMissingPrimitives()} to get a list of primitives which have not been
380 * found on the server (the server response code was 404)
381 *
382 * @param progressMonitor progress monitor
383 * @return the parsed data
384 * @throws OsmTransferException if an error occurs while communicating with the API server
385 * @see #getMissingPrimitives()
386 *
387 */
388 @Override
389 public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException {
390 missingPrimitives.clear();
391 int n = nodes.size() + ways.size() + relations.size();
392 progressMonitor.beginTask(trn("Downloading {0} object from ''{1}''",
393 "Downloading {0} objects from ''{1}''", n, n, getBaseUrl()));
394 try {
395 if (this instanceof MultiFetchOverpassObjectReader) {
396 // calculate a single request for all the objects
397 String request = MultiFetchOverpassObjectReader.genOverpassQuery(primitivesMap, true, false, recurseDownRelations);
398 if (isCanceled())
399 return null;
400 OverpassDownloadReader reader = new OverpassDownloadReader(new Bounds(0, 0, 0, 0), getBaseUrl(), request);
401 DataSet ds = reader.parseOsm(progressMonitor.createSubTaskMonitor(1, false));
402 new DataSetMerger(outputDataSet, ds).merge();
403 checkMissing(outputDataSet, progressMonitor);
404 } else {
405 downloadRelations(progressMonitor);
406 if (isCanceled())
407 return null;
408 fetchPrimitives(ways, OsmPrimitiveType.WAY, progressMonitor);
409 if (isCanceled())
410 return null;
411 fetchPrimitives(nodes, OsmPrimitiveType.NODE, progressMonitor);
412 }
413 outputDataSet.deleteInvisible();
414 return outputDataSet;
415 } finally {
416 progressMonitor.finishTask();
417 }
418 }
419
420 /**
421 * Workaround for difference in Overpass API.
422 * As of now (version 7.55) Overpass api doesn't return invisible objects.
423 * Check if we have objects which do not appear in the dataset and fetch them from OSM instead.
424 * @param ds the dataset
425 * @param progressMonitor progress monitor
426 * @throws OsmTransferException if an error occurs while communicating with the API server
427 */
428 private void checkMissing(DataSet ds, ProgressMonitor progressMonitor) throws OsmTransferException {
429 Set<OsmPrimitive> missing = new LinkedHashSet<>();
430 for (Entry<OsmPrimitiveType, Set<Long>> e : primitivesMap.entrySet()) {
431 for (long id : e.getValue()) {
432 if (ds.getPrimitiveById(id, e.getKey()) == null)
433 missing.add(e.getKey().newInstance(id, true));
434 }
435 }
436 if (isCanceled() || missing.isEmpty())
437 return;
438
439 MultiFetchServerObjectReader missingReader = MultiFetchServerObjectReader.create(false);
440 missingReader.setRecurseDownAppended(false);
441 missingReader.setRecurseDownRelations(false);
442 missingReader.append(missing);
443 DataSet mds = missingReader.parseOsm(progressMonitor.createSubTaskMonitor(missing.size(), false));
444 new DataSetMerger(ds, mds).merge();
445 missingPrimitives.addAll(missingReader.getMissingPrimitives());
446 }
447
448 /**
449 * Finds best way to download a set of relations.
450 * @param progressMonitor progress monitor
451 * @throws OsmTransferException if an error occurs while communicating with the API server
452 * @see #getMissingPrimitives()
453 */
454 private void downloadRelations(ProgressMonitor progressMonitor) throws OsmTransferException {
455 boolean removeIncomplete = outputDataSet.isEmpty();
456 Set<Long> toDownload = new LinkedHashSet<>(relations);
457 fetchPrimitives(toDownload, OsmPrimitiveType.RELATION, progressMonitor);
458 if (!recurseDownRelations) {
459 return;
460 }
461 // OSM multi-fetch api may return invisible objects, we don't try to get details for them
462 for (Relation r : outputDataSet.getRelations()) {
463 if (!r.isVisible()) {
464 toDownload.remove(r.getUniqueId());
465 } else if (removeIncomplete) {
466 outputDataSet.removePrimitive(r);
467 }
468 }
469
470 // fetch full info for all visible relations
471 for (long id : toDownload) {
472 if (isCanceled())
473 return;
474 OsmServerObjectReader reader = new OsmServerObjectReader(id, OsmPrimitiveType.RELATION, true/* full*/);
475 DataSet ds = reader.parseOsm(progressMonitor.createSubTaskMonitor(1, false));
476 merge(ds);
477 }
478 }
479
480 /**
481 * replies the set of ids of all primitives for which a fetch request to the
482 * server was submitted but which are not available from the server (the server
483 * replied a return code of 404)
484 *
485 * @return the set of ids of missing primitives
486 */
487 public Set<PrimitiveId> getMissingPrimitives() {
488 return missingPrimitives;
489 }
490
491 /**
492 * Should downloaded relations be complete?
493 * @param recurseDownRelations true: yes, recurse down to retrieve the members of the relation
494 * This will download sub relations, complete way members and nodes. Members of sub relations are not
495 * retrieved unless they are also members of the relations. See #18835.
496 * @return this
497 * @since 15811
498 */
499 public MultiFetchServerObjectReader setRecurseDownRelations(boolean recurseDownRelations) {
500 this.recurseDownRelations = recurseDownRelations;
501 return this;
502 }
503
504 /**
505 * Determine how appended objects are treated. By default, all children of an appended object are also appended.
506 * @param recurseAppended false: do not append known children of appended objects, i.e. all nodes of way and all members of a relation
507 * @return this
508 * @since 15811
509 */
510 public MultiFetchServerObjectReader setRecurseDownAppended(boolean recurseAppended) {
511 this.recurseDownAppended = recurseAppended;
512 return this;
513 }
514
515 /**
516 * The class holding the results given by {@link Fetcher}.
517 * It is only a wrapper of the resulting {@link DataSet} and the collection of {@link PrimitiveId} that could not have been loaded.
518 */
519 protected static class FetchResult {
520
521 /**
522 * The resulting data set
523 */
524 public final DataSet dataSet;
525
526 /**
527 * The collection of primitive ids that could not have been loaded
528 */
529 public final Set<PrimitiveId> missingPrimitives;
530
531 private Set<Long> rc404;
532
533 /**
534 * Constructs a {@code FetchResult}
535 * @param dataSet The resulting data set
536 * @param missingPrimitives The collection of primitive ids that could not have been loaded
537 */
538 public FetchResult(DataSet dataSet, Set<PrimitiveId> missingPrimitives) {
539 this.dataSet = dataSet;
540 this.missingPrimitives = missingPrimitives;
541 }
542 }
543
544 /**
545 * The class that actually download data from OSM API.
546 * Several instances of this class are used by {@link MultiFetchServerObjectReader} (one per set of primitives to fetch).
547 * The inheritance of {@link OsmServerReader} is only explained by the need to have a distinct OSM connection by {@code Fetcher} instance.
548 * @see FetchResult
549 */
550 protected class Fetcher extends OsmServerReader implements Callable<FetchResult> {
551
552 private final Set<Long> pkg;
553 private final OsmPrimitiveType type;
554 private final ProgressMonitor progressMonitor;
555
556 /**
557 * Constructs a {@code Fetcher}
558 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY},
559 * {@link OsmPrimitiveType#RELATION RELATION}
560 * @param idsPackage The set of primitives ids to fetch
561 * @param progressMonitor The progress monitor
562 */
563 public Fetcher(OsmPrimitiveType type, Set<Long> idsPackage, ProgressMonitor progressMonitor) {
564 this.pkg = idsPackage;
565 this.type = type;
566 this.progressMonitor = progressMonitor;
567 }
568
569 @Override
570 public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException {
571 // This method is implemented because of the OsmServerReader inheritance, but not used,
572 // as the main target of this class is the call() method.
573 return fetch(progressMonitor).dataSet;
574 }
575
576 @Override
577 public FetchResult call() throws Exception {
578 return fetch(progressMonitor);
579 }
580
581 /**
582 * fetches the requested primitives and updates the specified progress monitor.
583 * @param progressMonitor the progress monitor
584 * @return the {@link FetchResult} of this operation
585 * @throws OsmTransferException if an error occurs while communicating with the API server
586 */
587 protected FetchResult fetch(ProgressMonitor progressMonitor) throws OsmTransferException {
588 try {
589 return multiGetIdPackage(type, pkg, progressMonitor);
590 } catch (OsmApiException e) {
591 if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) {
592 if (pkg.size() > 4) {
593 FetchResult res = new FetchResult(null, null);
594 res.rc404 = pkg;
595 return res;
596 }
597 if (pkg.size() == 1) {
598 FetchResult res = new FetchResult(new DataSet(), new HashSet<>());
599 res.missingPrimitives.add(new SimplePrimitiveId(pkg.iterator().next(), type));
600 return res;
601 } else {
602 Logging.info(tr("Server replied with response code 404, retrying with an individual request for each object."));
603 return singleGetIdPackage(type, pkg, progressMonitor);
604 }
605 } else {
606 throw e;
607 }
608 }
609 }
610
611 @Override
612 protected String getBaseUrl() {
613 return MultiFetchServerObjectReader.this.getBaseUrl();
614 }
615
616 /**
617 * invokes a Multi Get for a set of ids and a given {@link OsmPrimitiveType}.
618 * The retrieved primitives are merged to {@link #outputDataSet}.
619 *
620 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY},
621 * {@link OsmPrimitiveType#RELATION RELATION}
622 * @param pkg the package of ids
623 * @param progressMonitor progress monitor
624 * @return the {@link FetchResult} of this operation
625 * @throws OsmTransferException if an error occurs while communicating with the API server
626 */
627 protected FetchResult multiGetIdPackage(OsmPrimitiveType type, Set<Long> pkg, ProgressMonitor progressMonitor)
628 throws OsmTransferException {
629 String request = buildRequestString(type, pkg);
630 FetchResult result = null;
631 try (InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE)) {
632 if (in == null) return null;
633 progressMonitor.subTask(tr("Downloading OSM data..."));
634 try {
635 result = new FetchResult(OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(pkg.size(), false)), null);
636 } catch (IllegalDataException e) {
637 throw new OsmTransferException(e);
638 }
639 } catch (IOException ex) {
640 Logging.warn(ex);
641 throw new OsmTransferException(ex);
642 }
643 return result;
644 }
645
646 /**
647 * invokes a Multi Get for a single id and a given {@link OsmPrimitiveType}.
648 * The retrieved primitive is merged to {@link #outputDataSet}.
649 *
650 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY},
651 * {@link OsmPrimitiveType#RELATION RELATION}
652 * @param id the id
653 * @param progressMonitor progress monitor
654 * @return the {@link DataSet} resulting of this operation
655 * @throws OsmTransferException if an error occurs while communicating with the API server
656 */
657 protected DataSet singleGetId(OsmPrimitiveType type, long id, ProgressMonitor progressMonitor) throws OsmTransferException {
658 String request = buildRequestString(type, Collections.singleton(id));
659 DataSet result = null;
660 try (InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE)) {
661 if (in == null) return null;
662 progressMonitor.subTask(tr("Downloading OSM data..."));
663 try {
664 result = OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(1, false));
665 } catch (IllegalDataException e) {
666 throw new OsmTransferException(e);
667 }
668 } catch (IOException ex) {
669 Logging.warn(ex);
670 }
671 return result;
672 }
673
674 /**
675 * invokes a sequence of Multi Gets for individual ids in a set of ids and a given {@link OsmPrimitiveType}.
676 * The retrieved primitives are merged to {@link #outputDataSet}.
677 * <p>
678 * This method is used if one of the ids in pkg doesn't exist (the server replies with return code 404).
679 * If the set is fetched with this method it is possible to find out which of the ids doesn't exist.
680 * Unfortunately, the server does not provide an error header or an error body for a 404 reply.
681 *
682 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY},
683 * {@link OsmPrimitiveType#RELATION RELATION}
684 * @param pkg the set of ids
685 * @param progressMonitor progress monitor
686 * @return the {@link FetchResult} of this operation
687 * @throws OsmTransferException if an error occurs while communicating with the API server
688 */
689 protected FetchResult singleGetIdPackage(OsmPrimitiveType type, Set<Long> pkg, ProgressMonitor progressMonitor)
690 throws OsmTransferException {
691 FetchResult result = new FetchResult(new DataSet(), new HashSet<>());
692 String baseUrl = OsmApi.getOsmApi().getBaseUrl();
693 for (long id : pkg) {
694 try {
695 String msg;
696 switch (type) {
697 // CHECKSTYLE.OFF: SingleSpaceSeparator
698 case NODE: msg = tr("Fetching node with id {0} from ''{1}''", id, baseUrl); break;
699 case WAY: msg = tr("Fetching way with id {0} from ''{1}''", id, baseUrl); break;
700 case RELATION: msg = tr("Fetching relation with id {0} from ''{1}''", id, baseUrl); break;
701 // CHECKSTYLE.ON: SingleSpaceSeparator
702 default: throw new AssertionError();
703 }
704 progressMonitor.setCustomText(msg);
705 result.dataSet.mergeFrom(singleGetId(type, id, progressMonitor));
706 } catch (OsmApiException e) {
707 if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) {
708 Logging.info(tr("Server replied with response code 404 for id {0}. Skipping.", Long.toString(id)));
709 result.missingPrimitives.add(new SimplePrimitiveId(id, type));
710 } else {
711 throw e;
712 }
713 }
714 }
715 return result;
716 }
717 }
718
719 @Override
720 public void cancel() {
721 super.cancel();
722 // Synchronized to avoid a RejectedExecutionException in fetchPrimitives
723 // We don't want to synchronize on the super.cancel() call.
724 synchronized (this) {
725 if (exec != null) {
726 exec.shutdownNow();
727 }
728 }
729 }
730}
Note: See TracBrowser for help on using the repository browser.