Index: core/src/org/openstreetmap/josm/io/MultiFetchServerObjectReader.java =================================================================== --- core/src/org/openstreetmap/josm/io/MultiFetchServerObjectReader.java (revision 5384) +++ core/src/org/openstreetmap/josm/io/MultiFetchServerObjectReader.java (working copy) @@ -12,6 +12,12 @@ import java.util.LinkedHashSet; import java.util.NoSuchElementException; import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletionService; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.Executors; import org.openstreetmap.josm.data.osm.DataSet; import org.openstreetmap.josm.data.osm.DataSetMerger; @@ -253,7 +259,7 @@ * @param idPackage the package of ids * @return the request string */ - protected String buildRequestString(OsmPrimitiveType type, Set idPackage) { + protected static String buildRequestString(OsmPrimitiveType type, Set idPackage) { StringBuilder sb = new StringBuilder(); sb.append(type.getAPIName()).append("s?") .append(type.getAPIName()).append("s="); @@ -276,7 +282,7 @@ * @param id the id * @return the request string */ - protected String buildRequestString(OsmPrimitiveType type, long id) { + protected static String buildRequestString(OsmPrimitiveType type, long id) { StringBuilder sb = new StringBuilder(); sb.append(type.getAPIName()).append("s?") .append(type.getAPIName()).append("s=") @@ -284,88 +290,6 @@ return sb.toString(); } - /** - * invokes a Multi Get for a set of ids and a given {@link OsmPrimitiveType}. - * The retrieved primitives are merged to {@link #outputDataSet}. - * - * @param type the type - * @param pkg the package of ids - * @exception OsmTransferException thrown if an error occurs while communicating with the API server - * - */ - protected void multiGetIdPackage(OsmPrimitiveType type, Set pkg, ProgressMonitor progressMonitor) throws OsmTransferException { - String request = buildRequestString(type, pkg); - final InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE); - if (in == null) return; - progressMonitor.subTask(tr("Downloading OSM data...")); - try { - DataSet loaded = OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(pkg.size(), false)); - rememberNodesOfIncompleteWaysToLoad(loaded); - merge(loaded); - } catch(Exception e) { - throw new OsmTransferException(e); - } - } - - /** - * invokes a Multi Get for a single id and a given {@link OsmPrimitiveType}. - * The retrieved primitive is merged to {@link #outputDataSet}. - * - * @param type the type - * @param id the id - * @exception OsmTransferException thrown if an error occurs while communicating with the API server - * - */ - protected void singleGetId(OsmPrimitiveType type, long id, ProgressMonitor progressMonitor) throws OsmTransferException { - String request = buildRequestString(type, id); - final InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE); - if (in == null) - return; - progressMonitor.subTask(tr("Downloading OSM data...")); - try { - DataSet loaded = OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(1, false)); - rememberNodesOfIncompleteWaysToLoad(loaded); - merge(loaded); - } catch(Exception e) { - throw new OsmTransferException(e); - } - } - - /** - * invokes a sequence of Multi Gets for individual ids in a set of ids and a given {@link OsmPrimitiveType}. - * The retrieved primitives are merged to {@link #outputDataSet}. - * - * This method is used if one of the ids in pkg doesn't exist (the server replies with return code 404). - * If the set is fetched with this method it is possible to find out which of the ids doesn't exist. - * Unfortunatelly, the server does not provide an error header or an error body for a 404 reply. - * - * @param type the type - * @param pkg the set of ids - * @exception OsmTransferException thrown if an error occurs while communicating with the API server - * - */ - protected void singleGetIdPackage(OsmPrimitiveType type, Set pkg, ProgressMonitor progressMonitor) throws OsmTransferException { - for (long id : pkg) { - try { - String msg = ""; - switch(type) { - case NODE: msg = tr("Fetching node with id {0} from ''{1}''", id, OsmApi.getOsmApi().getBaseUrl()); break; - case WAY: msg = tr("Fetching way with id {0} from ''{1}''", id, OsmApi.getOsmApi().getBaseUrl()); break; - case RELATION: msg = tr("Fetching relation with id {0} from ''{1}''", id, OsmApi.getOsmApi().getBaseUrl()); break; - } - progressMonitor.setCustomText(msg); - singleGetId(type, id, progressMonitor); - } catch(OsmApiException e) { - if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) { - System.out.println(tr("Server replied with response code 404 for id {0}. Skipping.", Long.toString(id))); - missingPrimitives.add(new SimplePrimitiveId(id, type)); - continue; - } - throw e; - } - } - } - protected void rememberNodesOfIncompleteWaysToLoad(DataSet from) { for (Way w: from.getWays()) { if (w.hasIncompleteNodes()) { @@ -396,7 +320,7 @@ * @param type the type * @exception OsmTransferException thrown if an error occurs while communicating with the API server */ - protected void fetchPrimitives(Set ids, OsmPrimitiveType type, ProgressMonitor progressMonitor) throws OsmTransferException{ + protected void fetchPrimitives(Set ids, OsmPrimitiveType type, ProgressMonitor progressMonitor) throws OsmTransferException { String msg = ""; switch(type) { case NODE: msg = tr("Fetching a package of nodes from ''{0}''", OsmApi.getOsmApi().getBaseUrl()); break; @@ -405,18 +329,34 @@ } progressMonitor.setTicksCount(ids.size()); progressMonitor.setTicks(0); + // The complete set containg all primitives to fetch Set toFetch = new HashSet(ids); - while(! toFetch.isEmpty() && !isCanceled()) { - Set pkg = extractIdPackage(toFetch); + // Build a list of fetchers that will download smaller sets containing only MAX_IDS_PER_REQUEST (200) primitives each + // On a multicore architecture, we will run up to MAX_DOWNLOAD_THREADS concurrent fetchers. + Executor exec = Executors.newFixedThreadPool( + Math.min(OsmApi.MAX_DOWNLOAD_THREADS, Runtime.getRuntime().availableProcessors())); + CompletionService ecs = new ExecutorCompletionService(exec); + int n = 0; + while (!toFetch.isEmpty()) { + ecs.submit(new Fetcher(type, extractIdPackage(toFetch), progressMonitor)); + n++; + } + // Run the fetchers + for (int i = 0; i < n && !isCanceled(); i++) { progressMonitor.subTask(msg + "... " + progressMonitor.getTicks() + "/" + progressMonitor.getTicksCount()); try { - multiGetIdPackage(type, pkg, progressMonitor); - } catch(OsmApiException e) { - if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) { - System.out.println(tr("Server replied with response code 404, retrying with an individual request for each object.")); - singleGetIdPackage(type, pkg, progressMonitor); - } else - throw e; + FetchResult result = ecs.take().get(); + if (result.missingPrimitives != null) { + missingPrimitives.addAll(result.missingPrimitives); + } + if (result.dataSet != null) { + rememberNodesOfIncompleteWaysToLoad(result.dataSet); + merge(result.dataSet); + } + } catch (InterruptedException e) { + e.printStackTrace(); + } catch (ExecutionException e) { + e.printStackTrace(); } } } @@ -467,4 +407,130 @@ public Set getMissingPrimitives() { return missingPrimitives; } + + protected static class FetchResult { + public final DataSet dataSet; + public final Set missingPrimitives; + public FetchResult(DataSet dataSet, Set missingPrimitives) { + this.dataSet = dataSet; + this.missingPrimitives = missingPrimitives; + } + } + + protected static class Fetcher extends OsmServerReader implements Callable { + + private final Set pkg; + private final OsmPrimitiveType type; + private final ProgressMonitor progressMonitor; + + public Fetcher(OsmPrimitiveType type, Set extractIdPackage, ProgressMonitor progressMonitor) { + this.pkg = extractIdPackage; + this.type = type; + this.progressMonitor = progressMonitor; + } + + @Override + public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException { + return fetch(progressMonitor).dataSet; + } + + @Override + public FetchResult call() throws Exception { + return fetch(progressMonitor); + } + + private FetchResult fetch(ProgressMonitor progressMonitor) throws OsmTransferException { + try { + return multiGetIdPackage(type, pkg, progressMonitor); + } catch (OsmApiException e) { + if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) { + System.out.println(tr("Server replied with response code 404, retrying with an individual request for each object.")); + return singleGetIdPackage(type, pkg, progressMonitor); + } else { + throw e; + } + } + } + + /** + * invokes a Multi Get for a set of ids and a given {@link OsmPrimitiveType}. + * The retrieved primitives are merged to {@link #outputDataSet}. + * + * @param type the type + * @param pkg the package of ids + * @exception OsmTransferException thrown if an error occurs while communicating with the API server + * + */ + protected FetchResult multiGetIdPackage(OsmPrimitiveType type, Set pkg, ProgressMonitor progressMonitor) throws OsmTransferException { + String request = buildRequestString(type, pkg); + final InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE); + if (in == null) return null; + progressMonitor.subTask(tr("Downloading OSM data...")); + try { + return new FetchResult(OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(pkg.size(), false)), null); + } catch (Exception e) { + throw new OsmTransferException(e); + } + } + + /** + * invokes a Multi Get for a single id and a given {@link OsmPrimitiveType}. + * The retrieved primitive is merged to {@link #outputDataSet}. + * + * @param type the type + * @param id the id + * @exception OsmTransferException thrown if an error occurs while communicating with the API server + * + */ + protected DataSet singleGetId(OsmPrimitiveType type, long id, ProgressMonitor progressMonitor) throws OsmTransferException { + String request = buildRequestString(type, id); + final InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE); + if (in == null) return null; + progressMonitor.subTask(tr("Downloading OSM data...")); + try { + return OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(1, false)); + } catch (Exception e) { + throw new OsmTransferException(e); + } + } + + /** + * invokes a sequence of Multi Gets for individual ids in a set of ids and a given {@link OsmPrimitiveType}. + * The retrieved primitives are merged to {@link #outputDataSet}. + * + * This method is used if one of the ids in pkg doesn't exist (the server replies with return code 404). + * If the set is fetched with this method it is possible to find out which of the ids doesn't exist. + * Unfortunatelly, the server does not provide an error header or an error body for a 404 reply. + * + * @param type the type + * @param pkg the set of ids + * @return + * @exception OsmTransferException thrown if an error occurs while communicating with the API server + * + */ + protected FetchResult singleGetIdPackage(OsmPrimitiveType type, Set pkg, ProgressMonitor progressMonitor) throws OsmTransferException { + FetchResult result = new FetchResult(new DataSet(), new HashSet()); + String baseUrl = OsmApi.getOsmApi().getBaseUrl(); + for (long id : pkg) { + try { + String msg = ""; + switch (type) { + case NODE: msg = tr("Fetching node with id {0} from ''{1}''", id, baseUrl); break; + case WAY: msg = tr("Fetching way with id {0} from ''{1}''", id, baseUrl); break; + case RELATION: msg = tr("Fetching relation with id {0} from ''{1}''", id, baseUrl); break; + } + progressMonitor.setCustomText(msg); + result.dataSet.mergeFrom(singleGetId(type, id, progressMonitor)); + } catch (OsmApiException e) { + if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) { + System.out.println(tr("Server replied with response code 404 for id {0}. Skipping.", Long.toString(id))); + result.missingPrimitives.add(new SimplePrimitiveId(id, type)); + } else { + throw e; + } + } + } + return result; + } + } } Index: core/src/org/openstreetmap/josm/io/OsmApi.java =================================================================== --- core/src/org/openstreetmap/josm/io/OsmApi.java (revision 5384) +++ core/src/org/openstreetmap/josm/io/OsmApi.java (working copy) @@ -54,6 +54,13 @@ /** max number of retries to send a request in case of HTTP 500 errors or timeouts */ static public final int DEFAULT_MAX_NUM_RETRIES = 5; + /** + * max number of concurrent download threads, imposed by + * + * OSM API usage policy. + */ + static public final int MAX_DOWNLOAD_THREADS = 2; + /** the collection of instantiated OSM APIs */ private static HashMap instances = new HashMap();