| 1 | // License: GPL. For details, see LICENSE file.
|
|---|
| 2 | package org.openstreetmap.josm.io;
|
|---|
| 3 |
|
|---|
| 4 | import static org.openstreetmap.josm.tools.I18n.tr;
|
|---|
| 5 | import static org.openstreetmap.josm.tools.I18n.trn;
|
|---|
| 6 |
|
|---|
| 7 | import java.io.IOException;
|
|---|
| 8 | import java.io.InputStream;
|
|---|
| 9 | import java.net.HttpURLConnection;
|
|---|
| 10 | import java.util.ArrayList;
|
|---|
| 11 | import java.util.Collection;
|
|---|
| 12 | import java.util.Collections;
|
|---|
| 13 | import java.util.HashSet;
|
|---|
| 14 | import java.util.Iterator;
|
|---|
| 15 | import java.util.LinkedHashSet;
|
|---|
| 16 | import java.util.List;
|
|---|
| 17 | import java.util.Set;
|
|---|
| 18 | import java.util.concurrent.Callable;
|
|---|
| 19 | import java.util.concurrent.CompletionService;
|
|---|
| 20 | import java.util.concurrent.ExecutionException;
|
|---|
| 21 | import java.util.concurrent.ExecutorCompletionService;
|
|---|
| 22 | import java.util.concurrent.ExecutorService;
|
|---|
| 23 | import java.util.concurrent.Executors;
|
|---|
| 24 | import java.util.concurrent.Future;
|
|---|
| 25 | import java.util.stream.Collectors;
|
|---|
| 26 |
|
|---|
| 27 | import org.openstreetmap.josm.data.osm.DataSet;
|
|---|
| 28 | import org.openstreetmap.josm.data.osm.DataSetMerger;
|
|---|
| 29 | import org.openstreetmap.josm.data.osm.Node;
|
|---|
| 30 | import org.openstreetmap.josm.data.osm.OsmPrimitive;
|
|---|
| 31 | import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
|
|---|
| 32 | import org.openstreetmap.josm.data.osm.PrimitiveId;
|
|---|
| 33 | import org.openstreetmap.josm.data.osm.Relation;
|
|---|
| 34 | import org.openstreetmap.josm.data.osm.RelationMember;
|
|---|
| 35 | import org.openstreetmap.josm.data.osm.SimplePrimitiveId;
|
|---|
| 36 | import org.openstreetmap.josm.data.osm.Way;
|
|---|
| 37 | import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
|
|---|
| 38 | import org.openstreetmap.josm.gui.progress.ProgressMonitor;
|
|---|
| 39 | import org.openstreetmap.josm.spi.preferences.Config;
|
|---|
| 40 | import org.openstreetmap.josm.tools.Logging;
|
|---|
| 41 | import org.openstreetmap.josm.tools.Utils;
|
|---|
| 42 |
|
|---|
| 43 | /**
|
|---|
| 44 | * Retrieves a set of {@link OsmPrimitive}s from an OSM server using the so called
|
|---|
| 45 | * Multi Fetch API.
|
|---|
| 46 | *
|
|---|
| 47 | * Usage:
|
|---|
| 48 | * <pre>
|
|---|
| 49 | * MultiFetchServerObjectReader reader = MultiFetchServerObjectReader()
|
|---|
| 50 | * .append(2345,2334,4444)
|
|---|
| 51 | * .append(new Node(72343));
|
|---|
| 52 | * reader.parseOsm();
|
|---|
| 53 | * if (!reader.getMissingPrimitives().isEmpty()) {
|
|---|
| 54 | * Logging.info("There are missing primitives: " + reader.getMissingPrimitives());
|
|---|
| 55 | * }
|
|---|
| 56 | * if (!reader.getSkippedWays().isEmpty()) {
|
|---|
| 57 | * Logging.info("There are skipped ways: " + reader.getMissingPrimitives());
|
|---|
| 58 | * }
|
|---|
| 59 | * </pre>
|
|---|
| 60 | */
|
|---|
| 61 | public class MultiFetchServerObjectReader extends OsmServerReader {
|
|---|
| 62 | /**
|
|---|
| 63 | * the max. number of primitives retrieved in one step. Assuming IDs with 10 digits,
|
|---|
| 64 | * this leads to a max. request URL of ~ 1900 Bytes ((10 digits + 1 Separator) * 170),
|
|---|
| 65 | * which should be safe according to the
|
|---|
| 66 | * <a href="http://www.boutell.com/newfaq/misc/urllength.html">WWW FAQ</a>.
|
|---|
| 67 | */
|
|---|
| 68 | private static final int MAX_IDS_PER_REQUEST = 170;
|
|---|
| 69 |
|
|---|
| 70 | private final Set<Long> nodes;
|
|---|
| 71 | private final Set<Long> ways;
|
|---|
| 72 | private final Set<Long> relations;
|
|---|
| 73 | private Set<PrimitiveId> missingPrimitives;
|
|---|
| 74 | private final DataSet outputDataSet;
|
|---|
| 75 |
|
|---|
| 76 | /**
|
|---|
| 77 | * Constructs a {@code MultiFetchServerObjectReader}.
|
|---|
| 78 | */
|
|---|
| 79 | protected MultiFetchServerObjectReader() {
|
|---|
| 80 | nodes = new LinkedHashSet<>();
|
|---|
| 81 | ways = new LinkedHashSet<>();
|
|---|
| 82 | relations = new LinkedHashSet<>();
|
|---|
| 83 | this.outputDataSet = new DataSet();
|
|---|
| 84 | this.missingPrimitives = new LinkedHashSet<>();
|
|---|
| 85 | }
|
|---|
| 86 |
|
|---|
| 87 | /**
|
|---|
| 88 | * Creates a new instance of {@link MultiFetchServerObjectReader} or {@link MultiFetchOverpassObjectReader}
|
|---|
| 89 | * depending on the {@link OverpassDownloadReader#FOR_MULTI_FETCH preference}.
|
|---|
| 90 | *
|
|---|
| 91 | * @return a new instance
|
|---|
| 92 | * @since 9241
|
|---|
| 93 | */
|
|---|
| 94 | public static MultiFetchServerObjectReader create() {
|
|---|
| 95 | return create(OverpassDownloadReader.FOR_MULTI_FETCH.get());
|
|---|
| 96 | }
|
|---|
| 97 |
|
|---|
| 98 | /**
|
|---|
| 99 | * Creates a new instance of {@link MultiFetchServerObjectReader} or {@link MultiFetchOverpassObjectReader}
|
|---|
| 100 | * depending on the {@code fromMirror} parameter.
|
|---|
| 101 | *
|
|---|
| 102 | * @param fromMirror {@code false} for {@link MultiFetchServerObjectReader}, {@code true} for {@link MultiFetchOverpassObjectReader}
|
|---|
| 103 | * @return a new instance
|
|---|
| 104 | * @since 15520 (changed visibility)
|
|---|
| 105 | */
|
|---|
| 106 | public static MultiFetchServerObjectReader create(final boolean fromMirror) {
|
|---|
| 107 | if (fromMirror) {
|
|---|
| 108 | return new MultiFetchOverpassObjectReader();
|
|---|
| 109 | } else {
|
|---|
| 110 | return new MultiFetchServerObjectReader();
|
|---|
| 111 | }
|
|---|
| 112 | }
|
|---|
| 113 |
|
|---|
| 114 | /**
|
|---|
| 115 | * Remembers an {@link OsmPrimitive}'s id. The id will
|
|---|
| 116 | * later be fetched as part of a Multi Get request.
|
|---|
| 117 | *
|
|---|
| 118 | * Ignore the id if it represents a new primitives.
|
|---|
| 119 | *
|
|---|
| 120 | * @param id the id
|
|---|
| 121 | */
|
|---|
| 122 | protected void remember(PrimitiveId id) {
|
|---|
| 123 | if (id.isNew()) return;
|
|---|
| 124 | switch(id.getType()) {
|
|---|
| 125 | case NODE: nodes.add(id.getUniqueId()); break;
|
|---|
| 126 | case WAY: ways.add(id.getUniqueId()); break;
|
|---|
| 127 | case RELATION: relations.add(id.getUniqueId()); break;
|
|---|
| 128 | default: throw new AssertionError();
|
|---|
| 129 | }
|
|---|
| 130 | }
|
|---|
| 131 |
|
|---|
| 132 | /**
|
|---|
| 133 | * appends a {@link OsmPrimitive} id to the list of ids which will be fetched from the server.
|
|---|
| 134 | *
|
|---|
| 135 | * @param ds the {@link DataSet} to which the primitive belongs
|
|---|
| 136 | * @param id the primitive id
|
|---|
| 137 | * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY},
|
|---|
| 138 | * {@link OsmPrimitiveType#RELATION RELATION}
|
|---|
| 139 | * @return this
|
|---|
| 140 | */
|
|---|
| 141 | public MultiFetchServerObjectReader append(DataSet ds, long id, OsmPrimitiveType type) {
|
|---|
| 142 | OsmPrimitive p = ds.getPrimitiveById(id, type);
|
|---|
| 143 | switch(type) {
|
|---|
| 144 | case NODE:
|
|---|
| 145 | return appendNode((Node) p);
|
|---|
| 146 | case WAY:
|
|---|
| 147 | return appendWay((Way) p);
|
|---|
| 148 | case RELATION:
|
|---|
| 149 | return appendRelation((Relation) p);
|
|---|
| 150 | default:
|
|---|
| 151 | return this;
|
|---|
| 152 | }
|
|---|
| 153 | }
|
|---|
| 154 |
|
|---|
| 155 | /**
|
|---|
| 156 | * appends a {@link Node} id to the list of ids which will be fetched from the server.
|
|---|
| 157 | *
|
|---|
| 158 | * @param node the node (ignored, if null)
|
|---|
| 159 | * @return this
|
|---|
| 160 | */
|
|---|
| 161 | public MultiFetchServerObjectReader appendNode(Node node) {
|
|---|
| 162 | if (node == null) return this;
|
|---|
| 163 | remember(node.getPrimitiveId());
|
|---|
| 164 | return this;
|
|---|
| 165 | }
|
|---|
| 166 |
|
|---|
| 167 | /**
|
|---|
| 168 | * appends a {@link Way} id and the list of ids of nodes the way refers to the list of ids which will be fetched from the server.
|
|---|
| 169 | *
|
|---|
| 170 | * @param way the way (ignored, if null)
|
|---|
| 171 | * @return this
|
|---|
| 172 | */
|
|---|
| 173 | public MultiFetchServerObjectReader appendWay(Way way) {
|
|---|
| 174 | if (way == null) return this;
|
|---|
| 175 | if (way.isNew()) return this;
|
|---|
| 176 | for (Node node: !recursesDown() ? way.getNodes() : Collections.<Node>emptyList()) {
|
|---|
| 177 | if (!node.isNew()) {
|
|---|
| 178 | remember(node.getPrimitiveId());
|
|---|
| 179 | }
|
|---|
| 180 | }
|
|---|
| 181 | remember(way.getPrimitiveId());
|
|---|
| 182 | return this;
|
|---|
| 183 | }
|
|---|
| 184 |
|
|---|
| 185 | /**
|
|---|
| 186 | * appends a {@link Relation} id to the list of ids which will be fetched from the server.
|
|---|
| 187 | *
|
|---|
| 188 | * @param relation the relation (ignored, if null)
|
|---|
| 189 | * @return this
|
|---|
| 190 | */
|
|---|
| 191 | protected MultiFetchServerObjectReader appendRelation(Relation relation) {
|
|---|
| 192 | if (relation == null) return this;
|
|---|
| 193 | if (relation.isNew()) return this;
|
|---|
| 194 | remember(relation.getPrimitiveId());
|
|---|
| 195 | for (RelationMember member : !recursesDown() ? relation.getMembers() : Collections.<RelationMember>emptyList()) {
|
|---|
| 196 | // avoid infinite recursion in case of cyclic dependencies in relations
|
|---|
| 197 | if (OsmPrimitiveType.from(member.getMember()) == OsmPrimitiveType.RELATION
|
|---|
| 198 | && relations.contains(member.getMember().getId())) {
|
|---|
| 199 | continue;
|
|---|
| 200 | }
|
|---|
| 201 | if (!member.getMember().isIncomplete()) {
|
|---|
| 202 | append(member.getMember());
|
|---|
| 203 | }
|
|---|
| 204 | }
|
|---|
| 205 | return this;
|
|---|
| 206 | }
|
|---|
| 207 |
|
|---|
| 208 | /**
|
|---|
| 209 | * appends an {@link OsmPrimitive} to the list of ids which will be fetched from the server.
|
|---|
| 210 | * @param primitive the primitive
|
|---|
| 211 | * @return this
|
|---|
| 212 | */
|
|---|
| 213 | public MultiFetchServerObjectReader append(OsmPrimitive primitive) {
|
|---|
| 214 | if (primitive instanceof Node) {
|
|---|
| 215 | return appendNode((Node) primitive);
|
|---|
| 216 | } else if (primitive instanceof Way) {
|
|---|
| 217 | return appendWay((Way) primitive);
|
|---|
| 218 | } else if (primitive instanceof Relation) {
|
|---|
| 219 | return appendRelation((Relation) primitive);
|
|---|
| 220 | }
|
|---|
| 221 | return this;
|
|---|
| 222 | }
|
|---|
| 223 |
|
|---|
| 224 | /**
|
|---|
| 225 | * appends a list of {@link OsmPrimitive} to the list of ids which will be fetched from the server.
|
|---|
| 226 | *
|
|---|
| 227 | * @param primitives the list of primitives (ignored, if null)
|
|---|
| 228 | * @return this
|
|---|
| 229 | *
|
|---|
| 230 | * @see #append(OsmPrimitive)
|
|---|
| 231 | */
|
|---|
| 232 | public MultiFetchServerObjectReader append(Collection<? extends OsmPrimitive> primitives) {
|
|---|
| 233 | if (primitives == null) return this;
|
|---|
| 234 | for (OsmPrimitive primitive : primitives) {
|
|---|
| 235 | append(primitive);
|
|---|
| 236 | }
|
|---|
| 237 | return this;
|
|---|
| 238 | }
|
|---|
| 239 |
|
|---|
| 240 | /**
|
|---|
| 241 | * extracts a subset of max {@link #MAX_IDS_PER_REQUEST} ids from <code>ids</code> and
|
|---|
| 242 | * replies the subset. The extracted subset is removed from <code>ids</code>.
|
|---|
| 243 | *
|
|---|
| 244 | * @param ids a set of ids
|
|---|
| 245 | * @return the subset of ids
|
|---|
| 246 | */
|
|---|
| 247 | protected Set<Long> extractIdPackage(Set<Long> ids) {
|
|---|
| 248 | Set<Long> pkg = new HashSet<>();
|
|---|
| 249 | if (ids.isEmpty())
|
|---|
| 250 | return pkg;
|
|---|
| 251 | if (ids.size() > MAX_IDS_PER_REQUEST) {
|
|---|
| 252 | Iterator<Long> it = ids.iterator();
|
|---|
| 253 | for (int i = 0; i < MAX_IDS_PER_REQUEST; i++) {
|
|---|
| 254 | pkg.add(it.next());
|
|---|
| 255 | }
|
|---|
| 256 | ids.removeAll(pkg);
|
|---|
| 257 | } else {
|
|---|
| 258 | pkg.addAll(ids);
|
|---|
| 259 | ids.clear();
|
|---|
| 260 | }
|
|---|
| 261 | return pkg;
|
|---|
| 262 | }
|
|---|
| 263 |
|
|---|
| 264 | /**
|
|---|
| 265 | * builds the Multi Get request string for a set of ids and a given {@link OsmPrimitiveType}.
|
|---|
| 266 | *
|
|---|
| 267 | * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY},
|
|---|
| 268 | * {@link OsmPrimitiveType#RELATION RELATION}
|
|---|
| 269 | * @param idPackage the package of ids
|
|---|
| 270 | * @return the request string
|
|---|
| 271 | */
|
|---|
| 272 | protected String buildRequestString(final OsmPrimitiveType type, Set<Long> idPackage) {
|
|---|
| 273 | return type.getAPIName() + "s?" + type.getAPIName() + "s=" + idPackage.stream().map(String::valueOf).collect(Collectors.joining(","));
|
|---|
| 274 | }
|
|---|
| 275 |
|
|---|
| 276 | protected void rememberNodesOfIncompleteWaysToLoad(DataSet from) {
|
|---|
| 277 | for (Way w: from.getWays()) {
|
|---|
| 278 | if (w.hasIncompleteNodes()) {
|
|---|
| 279 | for (Node n: w.getNodes()) {
|
|---|
| 280 | if (n.isIncomplete()) {
|
|---|
| 281 | nodes.add(n.getId());
|
|---|
| 282 | }
|
|---|
| 283 | }
|
|---|
| 284 | }
|
|---|
| 285 | }
|
|---|
| 286 | }
|
|---|
| 287 |
|
|---|
| 288 | /**
|
|---|
| 289 | * merges the dataset <code>from</code> to {@link #outputDataSet}.
|
|---|
| 290 | *
|
|---|
| 291 | * @param from the other dataset
|
|---|
| 292 | */
|
|---|
| 293 | protected void merge(DataSet from) {
|
|---|
| 294 | final DataSetMerger visitor = new DataSetMerger(outputDataSet, from);
|
|---|
| 295 | visitor.merge();
|
|---|
| 296 | }
|
|---|
| 297 |
|
|---|
| 298 | /**
|
|---|
| 299 | * fetches a set of ids of a given {@link OsmPrimitiveType} from the server
|
|---|
| 300 | *
|
|---|
| 301 | * @param ids the set of ids
|
|---|
| 302 | * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY},
|
|---|
| 303 | * {@link OsmPrimitiveType#RELATION RELATION}
|
|---|
| 304 | * @param progressMonitor progress monitor
|
|---|
| 305 | * @throws OsmTransferException if an error occurs while communicating with the API server
|
|---|
| 306 | */
|
|---|
| 307 | protected void fetchPrimitives(Set<Long> ids, OsmPrimitiveType type, ProgressMonitor progressMonitor) throws OsmTransferException {
|
|---|
| 308 | String msg;
|
|---|
| 309 | final String baseUrl = getBaseUrl();
|
|---|
| 310 | switch (type) {
|
|---|
| 311 | // CHECKSTYLE.OFF: SingleSpaceSeparator
|
|---|
| 312 | case NODE: msg = tr("Fetching a package of nodes from ''{0}''", baseUrl); break;
|
|---|
| 313 | case WAY: msg = tr("Fetching a package of ways from ''{0}''", baseUrl); break;
|
|---|
| 314 | case RELATION: msg = tr("Fetching a package of relations from ''{0}''", baseUrl); break;
|
|---|
| 315 | // CHECKSTYLE.ON: SingleSpaceSeparator
|
|---|
| 316 | default: throw new AssertionError();
|
|---|
| 317 | }
|
|---|
| 318 | progressMonitor.setTicksCount(ids.size());
|
|---|
| 319 | progressMonitor.setTicks(0);
|
|---|
| 320 | // The complete set containing all primitives to fetch
|
|---|
| 321 | Set<Long> toFetch = new HashSet<>(ids);
|
|---|
| 322 | // Build a list of fetchers that will download smaller sets containing only MAX_IDS_PER_REQUEST (200) primitives each.
|
|---|
| 323 | // we will run up to MAX_DOWNLOAD_THREADS concurrent fetchers.
|
|---|
| 324 | int threadsNumber = Config.getPref().getInt("osm.download.threads", OsmApi.MAX_DOWNLOAD_THREADS);
|
|---|
| 325 | threadsNumber = Utils.clamp(threadsNumber, 1, OsmApi.MAX_DOWNLOAD_THREADS);
|
|---|
| 326 | final ExecutorService exec = Executors.newFixedThreadPool(
|
|---|
| 327 | threadsNumber, Utils.newThreadFactory(getClass() + "-%d", Thread.NORM_PRIORITY));
|
|---|
| 328 | CompletionService<FetchResult> ecs = new ExecutorCompletionService<>(exec);
|
|---|
| 329 | List<Future<FetchResult>> jobs = new ArrayList<>();
|
|---|
| 330 | while (!toFetch.isEmpty()) {
|
|---|
| 331 | jobs.add(ecs.submit(new Fetcher(type, extractIdPackage(toFetch), progressMonitor)));
|
|---|
| 332 | }
|
|---|
| 333 | // Run the fetchers
|
|---|
| 334 | for (int i = 0; i < jobs.size() && !isCanceled(); i++) {
|
|---|
| 335 | progressMonitor.subTask(msg + "... " + progressMonitor.getTicks() + '/' + progressMonitor.getTicksCount());
|
|---|
| 336 | try {
|
|---|
| 337 | FetchResult result = ecs.take().get();
|
|---|
| 338 | if (result.rc404 != null) {
|
|---|
| 339 | List<Long> toSplit = new ArrayList<>(result.rc404);
|
|---|
| 340 | int n = toSplit.size() / 2;
|
|---|
| 341 | jobs.add(ecs.submit(new Fetcher(type, new HashSet<>(toSplit.subList(0, n)), progressMonitor)));
|
|---|
| 342 | jobs.add(ecs.submit(new Fetcher(type, new HashSet<>(toSplit.subList(n, toSplit.size())), progressMonitor)));
|
|---|
| 343 | }
|
|---|
| 344 | if (result.missingPrimitives != null) {
|
|---|
| 345 | missingPrimitives.addAll(result.missingPrimitives);
|
|---|
| 346 | }
|
|---|
| 347 | if (result.dataSet != null && !isCanceled()) {
|
|---|
| 348 | rememberNodesOfIncompleteWaysToLoad(result.dataSet);
|
|---|
| 349 | merge(result.dataSet);
|
|---|
| 350 | }
|
|---|
| 351 | } catch (InterruptedException | ExecutionException e) {
|
|---|
| 352 | Logging.error(e);
|
|---|
| 353 | }
|
|---|
| 354 | }
|
|---|
| 355 | exec.shutdown();
|
|---|
| 356 | // Cancel requests if the user chose to
|
|---|
| 357 | if (isCanceled()) {
|
|---|
| 358 | for (Future<FetchResult> job : jobs) {
|
|---|
| 359 | job.cancel(true);
|
|---|
| 360 | }
|
|---|
| 361 | }
|
|---|
| 362 | }
|
|---|
| 363 |
|
|---|
| 364 | /**
|
|---|
| 365 | * invokes one or more Multi Gets to fetch the {@link OsmPrimitive}s and replies
|
|---|
| 366 | * the dataset of retrieved primitives. Note that the dataset includes non visible primitives too!
|
|---|
| 367 | * In contrast to a simple Get for a node, a way, or a relation, a Multi Get always replies
|
|---|
| 368 | * the latest version of the primitive (if any), even if the primitive is not visible (i.e. if
|
|---|
| 369 | * visible==false).
|
|---|
| 370 | *
|
|---|
| 371 | * Invoke {@link #getMissingPrimitives()} to get a list of primitives which have not been
|
|---|
| 372 | * found on the server (the server response code was 404)
|
|---|
| 373 | *
|
|---|
| 374 | * @return the parsed data
|
|---|
| 375 | * @throws OsmTransferException if an error occurs while communicating with the API server
|
|---|
| 376 | * @see #getMissingPrimitives()
|
|---|
| 377 | *
|
|---|
| 378 | */
|
|---|
| 379 | @Override
|
|---|
| 380 | public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException {
|
|---|
| 381 | int n = nodes.size() + ways.size() + relations.size();
|
|---|
| 382 | progressMonitor.beginTask(trn("Downloading {0} object from ''{1}''",
|
|---|
| 383 | "Downloading {0} objects from ''{1}''", n, n, OsmApi.getOsmApi().getBaseUrl()));
|
|---|
| 384 | try {
|
|---|
| 385 | missingPrimitives = new HashSet<>();
|
|---|
| 386 | if (isCanceled()) return null;
|
|---|
| 387 | fetchPrimitives(ways, OsmPrimitiveType.WAY, progressMonitor);
|
|---|
| 388 | if (isCanceled()) return null;
|
|---|
| 389 | fetchPrimitives(nodes, OsmPrimitiveType.NODE, progressMonitor);
|
|---|
| 390 | if (isCanceled()) return null;
|
|---|
| 391 | fetchPrimitives(relations, OsmPrimitiveType.RELATION, progressMonitor);
|
|---|
| 392 | if (outputDataSet != null) {
|
|---|
| 393 | outputDataSet.deleteInvisible();
|
|---|
| 394 | }
|
|---|
| 395 | return outputDataSet;
|
|---|
| 396 | } finally {
|
|---|
| 397 | progressMonitor.finishTask();
|
|---|
| 398 | }
|
|---|
| 399 | }
|
|---|
| 400 |
|
|---|
| 401 | /**
|
|---|
| 402 | * replies the set of ids of all primitives for which a fetch request to the
|
|---|
| 403 | * server was submitted but which are not available from the server (the server
|
|---|
| 404 | * replied a return code of 404)
|
|---|
| 405 | *
|
|---|
| 406 | * @return the set of ids of missing primitives
|
|---|
| 407 | */
|
|---|
| 408 | public Set<PrimitiveId> getMissingPrimitives() {
|
|---|
| 409 | return missingPrimitives;
|
|---|
| 410 | }
|
|---|
| 411 |
|
|---|
| 412 | /**
|
|---|
| 413 | * Whether this reader fetches nodes when loading ways, or members when loading relations.
|
|---|
| 414 | *
|
|---|
| 415 | * @return {@code true} if the reader recurses down
|
|---|
| 416 | */
|
|---|
| 417 | protected boolean recursesDown() {
|
|---|
| 418 | return false;
|
|---|
| 419 | }
|
|---|
| 420 |
|
|---|
| 421 | /**
|
|---|
| 422 | * The class holding the results given by {@link Fetcher}.
|
|---|
| 423 | * It is only a wrapper of the resulting {@link DataSet} and the collection of {@link PrimitiveId} that could not have been loaded.
|
|---|
| 424 | */
|
|---|
| 425 | protected static class FetchResult {
|
|---|
| 426 |
|
|---|
| 427 | /**
|
|---|
| 428 | * The resulting data set
|
|---|
| 429 | */
|
|---|
| 430 | public final DataSet dataSet;
|
|---|
| 431 |
|
|---|
| 432 | /**
|
|---|
| 433 | * The collection of primitive ids that could not have been loaded
|
|---|
| 434 | */
|
|---|
| 435 | public final Set<PrimitiveId> missingPrimitives;
|
|---|
| 436 |
|
|---|
| 437 | private Set<Long> rc404;
|
|---|
| 438 |
|
|---|
| 439 | /**
|
|---|
| 440 | * Constructs a {@code FetchResult}
|
|---|
| 441 | * @param dataSet The resulting data set
|
|---|
| 442 | * @param missingPrimitives The collection of primitive ids that could not have been loaded
|
|---|
| 443 | */
|
|---|
| 444 | public FetchResult(DataSet dataSet, Set<PrimitiveId> missingPrimitives) {
|
|---|
| 445 | this.dataSet = dataSet;
|
|---|
| 446 | this.missingPrimitives = missingPrimitives;
|
|---|
| 447 | }
|
|---|
| 448 | }
|
|---|
| 449 |
|
|---|
| 450 | /**
|
|---|
| 451 | * The class that actually download data from OSM API.
|
|---|
| 452 | * Several instances of this class are used by {@link MultiFetchServerObjectReader} (one per set of primitives to fetch).
|
|---|
| 453 | * The inheritance of {@link OsmServerReader} is only explained by the need to have a distinct OSM connection by {@code Fetcher} instance.
|
|---|
| 454 | * @see FetchResult
|
|---|
| 455 | */
|
|---|
| 456 | protected class Fetcher extends OsmServerReader implements Callable<FetchResult> {
|
|---|
| 457 |
|
|---|
| 458 | private final Set<Long> pkg;
|
|---|
| 459 | private final OsmPrimitiveType type;
|
|---|
| 460 | private final ProgressMonitor progressMonitor;
|
|---|
| 461 |
|
|---|
| 462 | /**
|
|---|
| 463 | * Constructs a {@code Fetcher}
|
|---|
| 464 | * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY},
|
|---|
| 465 | * {@link OsmPrimitiveType#RELATION RELATION}
|
|---|
| 466 | * @param idsPackage The set of primitives ids to fetch
|
|---|
| 467 | * @param progressMonitor The progress monitor
|
|---|
| 468 | */
|
|---|
| 469 | public Fetcher(OsmPrimitiveType type, Set<Long> idsPackage, ProgressMonitor progressMonitor) {
|
|---|
| 470 | this.pkg = idsPackage;
|
|---|
| 471 | this.type = type;
|
|---|
| 472 | this.progressMonitor = progressMonitor;
|
|---|
| 473 | }
|
|---|
| 474 |
|
|---|
| 475 | @Override
|
|---|
| 476 | public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException {
|
|---|
| 477 | // This method is implemented because of the OsmServerReader inheritance, but not used,
|
|---|
| 478 | // as the main target of this class is the call() method.
|
|---|
| 479 | return fetch(progressMonitor).dataSet;
|
|---|
| 480 | }
|
|---|
| 481 |
|
|---|
| 482 | @Override
|
|---|
| 483 | public FetchResult call() throws Exception {
|
|---|
| 484 | return fetch(progressMonitor);
|
|---|
| 485 | }
|
|---|
| 486 |
|
|---|
| 487 | /**
|
|---|
| 488 | * fetches the requested primitives and updates the specified progress monitor.
|
|---|
| 489 | * @param progressMonitor the progress monitor
|
|---|
| 490 | * @return the {@link FetchResult} of this operation
|
|---|
| 491 | * @throws OsmTransferException if an error occurs while communicating with the API server
|
|---|
| 492 | */
|
|---|
| 493 | protected FetchResult fetch(ProgressMonitor progressMonitor) throws OsmTransferException {
|
|---|
| 494 | try {
|
|---|
| 495 | return multiGetIdPackage(type, pkg, progressMonitor);
|
|---|
| 496 | } catch (OsmApiException e) {
|
|---|
| 497 | if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) {
|
|---|
| 498 | if (pkg.size() > 4) {
|
|---|
| 499 | FetchResult res = new FetchResult(null, null);
|
|---|
| 500 | res.rc404 = pkg;
|
|---|
| 501 | return res;
|
|---|
| 502 | }
|
|---|
| 503 | Logging.info(tr("Server replied with response code 404, retrying with an individual request for each object."));
|
|---|
| 504 | return singleGetIdPackage(type, pkg, progressMonitor);
|
|---|
| 505 | } else {
|
|---|
| 506 | throw e;
|
|---|
| 507 | }
|
|---|
| 508 | }
|
|---|
| 509 | }
|
|---|
| 510 |
|
|---|
| 511 | @Override
|
|---|
| 512 | protected String getBaseUrl() {
|
|---|
| 513 | return MultiFetchServerObjectReader.this.getBaseUrl();
|
|---|
| 514 | }
|
|---|
| 515 |
|
|---|
| 516 | /**
|
|---|
| 517 | * invokes a Multi Get for a set of ids and a given {@link OsmPrimitiveType}.
|
|---|
| 518 | * The retrieved primitives are merged to {@link #outputDataSet}.
|
|---|
| 519 | *
|
|---|
| 520 | * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY},
|
|---|
| 521 | * {@link OsmPrimitiveType#RELATION RELATION}
|
|---|
| 522 | * @param pkg the package of ids
|
|---|
| 523 | * @param progressMonitor progress monitor
|
|---|
| 524 | * @return the {@link FetchResult} of this operation
|
|---|
| 525 | * @throws OsmTransferException if an error occurs while communicating with the API server
|
|---|
| 526 | */
|
|---|
| 527 | protected FetchResult multiGetIdPackage(OsmPrimitiveType type, Set<Long> pkg, ProgressMonitor progressMonitor)
|
|---|
| 528 | throws OsmTransferException {
|
|---|
| 529 | String request = buildRequestString(type, pkg);
|
|---|
| 530 | FetchResult result = null;
|
|---|
| 531 | try (InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE)) {
|
|---|
| 532 | if (in == null) return null;
|
|---|
| 533 | progressMonitor.subTask(tr("Downloading OSM data..."));
|
|---|
| 534 | try {
|
|---|
| 535 | result = new FetchResult(OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(pkg.size(), false)), null);
|
|---|
| 536 | } catch (IllegalDataException e) {
|
|---|
| 537 | throw new OsmTransferException(e);
|
|---|
| 538 | }
|
|---|
| 539 | } catch (IOException ex) {
|
|---|
| 540 | Logging.warn(ex);
|
|---|
| 541 | }
|
|---|
| 542 | return result;
|
|---|
| 543 | }
|
|---|
| 544 |
|
|---|
| 545 | /**
|
|---|
| 546 | * invokes a Multi Get for a single id and a given {@link OsmPrimitiveType}.
|
|---|
| 547 | * The retrieved primitive is merged to {@link #outputDataSet}.
|
|---|
| 548 | *
|
|---|
| 549 | * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY},
|
|---|
| 550 | * {@link OsmPrimitiveType#RELATION RELATION}
|
|---|
| 551 | * @param id the id
|
|---|
| 552 | * @param progressMonitor progress monitor
|
|---|
| 553 | * @return the {@link DataSet} resulting of this operation
|
|---|
| 554 | * @throws OsmTransferException if an error occurs while communicating with the API server
|
|---|
| 555 | */
|
|---|
| 556 | protected DataSet singleGetId(OsmPrimitiveType type, long id, ProgressMonitor progressMonitor) throws OsmTransferException {
|
|---|
| 557 | String request = buildRequestString(type, Collections.singleton(id));
|
|---|
| 558 | DataSet result = null;
|
|---|
| 559 | try (InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE)) {
|
|---|
| 560 | if (in == null) return null;
|
|---|
| 561 | progressMonitor.subTask(tr("Downloading OSM data..."));
|
|---|
| 562 | try {
|
|---|
| 563 | result = OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(1, false));
|
|---|
| 564 | } catch (IllegalDataException e) {
|
|---|
| 565 | throw new OsmTransferException(e);
|
|---|
| 566 | }
|
|---|
| 567 | } catch (IOException ex) {
|
|---|
| 568 | Logging.warn(ex);
|
|---|
| 569 | }
|
|---|
| 570 | return result;
|
|---|
| 571 | }
|
|---|
| 572 |
|
|---|
| 573 | /**
|
|---|
| 574 | * invokes a sequence of Multi Gets for individual ids in a set of ids and a given {@link OsmPrimitiveType}.
|
|---|
| 575 | * The retrieved primitives are merged to {@link #outputDataSet}.
|
|---|
| 576 | *
|
|---|
| 577 | * This method is used if one of the ids in pkg doesn't exist (the server replies with return code 404).
|
|---|
| 578 | * If the set is fetched with this method it is possible to find out which of the ids doesn't exist.
|
|---|
| 579 | * Unfortunately, the server does not provide an error header or an error body for a 404 reply.
|
|---|
| 580 | *
|
|---|
| 581 | * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY},
|
|---|
| 582 | * {@link OsmPrimitiveType#RELATION RELATION}
|
|---|
| 583 | * @param pkg the set of ids
|
|---|
| 584 | * @param progressMonitor progress monitor
|
|---|
| 585 | * @return the {@link FetchResult} of this operation
|
|---|
| 586 | * @throws OsmTransferException if an error occurs while communicating with the API server
|
|---|
| 587 | */
|
|---|
| 588 | protected FetchResult singleGetIdPackage(OsmPrimitiveType type, Set<Long> pkg, ProgressMonitor progressMonitor)
|
|---|
| 589 | throws OsmTransferException {
|
|---|
| 590 | FetchResult result = new FetchResult(new DataSet(), new HashSet<PrimitiveId>());
|
|---|
| 591 | String baseUrl = OsmApi.getOsmApi().getBaseUrl();
|
|---|
| 592 | for (long id : pkg) {
|
|---|
| 593 | try {
|
|---|
| 594 | String msg;
|
|---|
| 595 | switch (type) {
|
|---|
| 596 | // CHECKSTYLE.OFF: SingleSpaceSeparator
|
|---|
| 597 | case NODE: msg = tr("Fetching node with id {0} from ''{1}''", id, baseUrl); break;
|
|---|
| 598 | case WAY: msg = tr("Fetching way with id {0} from ''{1}''", id, baseUrl); break;
|
|---|
| 599 | case RELATION: msg = tr("Fetching relation with id {0} from ''{1}''", id, baseUrl); break;
|
|---|
| 600 | // CHECKSTYLE.ON: SingleSpaceSeparator
|
|---|
| 601 | default: throw new AssertionError();
|
|---|
| 602 | }
|
|---|
| 603 | progressMonitor.setCustomText(msg);
|
|---|
| 604 | result.dataSet.mergeFrom(singleGetId(type, id, progressMonitor));
|
|---|
| 605 | } catch (OsmApiException e) {
|
|---|
| 606 | if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) {
|
|---|
| 607 | Logging.info(tr("Server replied with response code 404 for id {0}. Skipping.", Long.toString(id)));
|
|---|
| 608 | result.missingPrimitives.add(new SimplePrimitiveId(id, type));
|
|---|
| 609 | } else {
|
|---|
| 610 | throw e;
|
|---|
| 611 | }
|
|---|
| 612 | }
|
|---|
| 613 | }
|
|---|
| 614 | return result;
|
|---|
| 615 | }
|
|---|
| 616 | }
|
|---|
| 617 | }
|
|---|