Index: trunk/src/org/openstreetmap/josm/io/MultiFetchServerObjectReader.java
===================================================================
--- trunk/src/org/openstreetmap/josm/io/MultiFetchServerObjectReader.java (revision 5386)
+++ trunk/src/org/openstreetmap/josm/io/MultiFetchServerObjectReader.java (revision 5387)
@@ -7,11 +7,21 @@
import java.io.InputStream;
import java.net.HttpURLConnection;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
+import java.util.List;
import java.util.NoSuchElementException;
import java.util.Set;
-
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import org.openstreetmap.josm.Main;
import org.openstreetmap.josm.data.osm.DataSet;
import org.openstreetmap.josm.data.osm.DataSetMerger;
@@ -45,6 +55,4 @@
* }
*
- *
- *
*/
public class MultiFetchServerObjectReader extends OsmServerReader{
@@ -54,5 +62,4 @@
* which should be safe according to the
* WWW FAQ.
- *
*/
static private int MAX_IDS_PER_REQUEST = 200;
@@ -65,6 +72,5 @@
/**
- * constructor
- *
+ * Constructs a {@code MultiFetchServerObjectReader}.
*/
public MultiFetchServerObjectReader() {
@@ -101,8 +107,8 @@
*
* @param ds the dataset (must not be null)
- * @param id the id
- * @exception IllegalArgumentException thrown, if ds is null
- * @exception NoSuchElementException thrown, if ds doesn't include an {@link OsmPrimitive} with
- * id=id
+ * @param id the primitive id
+ * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION}
+ * @throws IllegalArgumentException if ds is null
+ * @throws NoSuchElementException if ds does not include an {@link OsmPrimitive} with id=id
*/
protected void remember(DataSet ds, long id, OsmPrimitiveType type) throws IllegalArgumentException, NoSuchElementException{
@@ -116,18 +122,21 @@
}
+ /**
+ * appends a {@link OsmPrimitive} id to the list of ids which will be fetched from the server.
+ *
+ * @param ds the {@link DataSet} to which the primitive belongs
+ * @param id the primitive id
+ * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION}
+ * @return this
+ */
public MultiFetchServerObjectReader append(DataSet ds, long id, OsmPrimitiveType type) {
+ OsmPrimitive p = ds.getPrimitiveById(id,type);
switch(type) {
case NODE:
- Node n = (Node)ds.getPrimitiveById(id,type);
- appendNode(n);
- break;
+ return appendNode((Node)p);
case WAY:
- Way w= (Way)ds.getPrimitiveById(id,type);
- appendWay(w);
- break;
+ return appendWay((Way)p);
case RELATION:
- Relation r = (Relation)ds.getPrimitiveById(id,type);
- appendRelation(r);
- break;
+ return appendRelation((Relation)p);
}
return this;
@@ -135,9 +144,8 @@
/**
- * appends a {@link Node}s id to the list of ids which will be fetched from the server.
+ * appends a {@link Node} id to the list of ids which will be fetched from the server.
*
* @param node the node (ignored, if null)
* @return this
- *
*/
public MultiFetchServerObjectReader appendNode(Node node) {
@@ -148,9 +156,8 @@
/**
- * appends a {@link Way}s id and the list of ids of nodes the way refers to the list of ids which will be fetched from the server.
+ * appends a {@link Way} id and the list of ids of nodes the way refers to the list of ids which will be fetched from the server.
*
* @param way the way (ignored, if null)
* @return this
- *
*/
public MultiFetchServerObjectReader appendWay(Way way) {
@@ -167,9 +174,8 @@
/**
- * appends a {@link Relation}s id to the list of ids which will be fetched from the server.
+ * appends a {@link Relation} id to the list of ids which will be fetched from the server.
*
* @param relation the relation (ignored, if null)
* @return this
- *
*/
protected MultiFetchServerObjectReader appendRelation(Relation relation) {
@@ -192,12 +198,17 @@
}
+ /**
+ * appends an {@link OsmPrimitive} to the list of ids which will be fetched from the server.
+ * @param primitive the primitive
+ * @return this
+ */
public MultiFetchServerObjectReader append(OsmPrimitive primitive) {
- if (OsmPrimitiveType.from(primitive).equals(OsmPrimitiveType.NODE))
- return appendNode((Node)primitive);
- else if (OsmPrimitiveType.from(primitive).equals(OsmPrimitiveType.WAY))
- return appendWay((Way)primitive);
- else if (OsmPrimitiveType.from(primitive).equals(OsmPrimitiveType.RELATION))
- return appendRelation((Relation)primitive);
-
+ if (primitive != null) {
+ switch (OsmPrimitiveType.from(primitive)) {
+ case NODE: return appendNode((Node)primitive);
+ case WAY: return appendWay((Way)primitive);
+ case RELATION: return appendRelation((Relation)primitive);
+ }
+ }
return this;
}
@@ -209,8 +220,5 @@
* @return this
*
- * @see #append(Node)
- * @see #append(Way)
- * @see #append(Relation)
- *
+ * @see #append(OsmPrimitive)
*/
public MultiFetchServerObjectReader append(Collection primitives) {
@@ -235,5 +243,5 @@
if (ids.size() > MAX_IDS_PER_REQUEST) {
Iterator it = ids.iterator();
- for (int i =0;i idPackage) {
+ protected static String buildRequestString(OsmPrimitiveType type, Set idPackage) {
StringBuilder sb = new StringBuilder();
sb.append(type.getAPIName()).append("s?")
@@ -260,5 +268,5 @@
Iterator it = idPackage.iterator();
- for (int i=0; i< idPackage.size();i++) {
+ for (int i=0; i pkg, ProgressMonitor progressMonitor) throws OsmTransferException {
- String request = buildRequestString(type, pkg);
- final InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE);
- if (in == null) return;
- progressMonitor.subTask(tr("Downloading OSM data..."));
- try {
- DataSet loaded = OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(pkg.size(), false));
- rememberNodesOfIncompleteWaysToLoad(loaded);
- merge(loaded);
- } catch(Exception e) {
- throw new OsmTransferException(e);
- }
- }
-
- /**
- * invokes a Multi Get for a single id and a given {@link OsmPrimitiveType}.
- * The retrieved primitive is merged to {@link #outputDataSet}.
- *
- * @param type the type
- * @param id the id
- * @exception OsmTransferException thrown if an error occurs while communicating with the API server
- *
- */
- protected void singleGetId(OsmPrimitiveType type, long id, ProgressMonitor progressMonitor) throws OsmTransferException {
- String request = buildRequestString(type, id);
- final InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE);
- if (in == null)
- return;
- progressMonitor.subTask(tr("Downloading OSM data..."));
- try {
- DataSet loaded = OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(1, false));
- rememberNodesOfIncompleteWaysToLoad(loaded);
- merge(loaded);
- } catch(Exception e) {
- throw new OsmTransferException(e);
- }
- }
-
- /**
- * invokes a sequence of Multi Gets for individual ids in a set of ids and a given {@link OsmPrimitiveType}.
- * The retrieved primitives are merged to {@link #outputDataSet}.
- *
- * This method is used if one of the ids in pkg doesn't exist (the server replies with return code 404).
- * If the set is fetched with this method it is possible to find out which of the ids doesn't exist.
- * Unfortunatelly, the server does not provide an error header or an error body for a 404 reply.
- *
- * @param type the type
- * @param pkg the set of ids
- * @exception OsmTransferException thrown if an error occurs while communicating with the API server
- *
- */
- protected void singleGetIdPackage(OsmPrimitiveType type, Set pkg, ProgressMonitor progressMonitor) throws OsmTransferException {
- for (long id : pkg) {
- try {
- String msg = "";
- switch(type) {
- case NODE: msg = tr("Fetching node with id {0} from ''{1}''", id, OsmApi.getOsmApi().getBaseUrl()); break;
- case WAY: msg = tr("Fetching way with id {0} from ''{1}''", id, OsmApi.getOsmApi().getBaseUrl()); break;
- case RELATION: msg = tr("Fetching relation with id {0} from ''{1}''", id, OsmApi.getOsmApi().getBaseUrl()); break;
- }
- progressMonitor.setCustomText(msg);
- singleGetId(type, id, progressMonitor);
- } catch(OsmApiException e) {
- if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) {
- System.out.println(tr("Server replied with response code 404 for id {0}. Skipping.", Long.toString(id)));
- missingPrimitives.add(new SimplePrimitiveId(id, type));
- continue;
- }
- throw e;
- }
- }
}
@@ -383,5 +309,4 @@
*
* @param from the other dataset
- *
*/
protected void merge(DataSet from) {
@@ -394,28 +319,51 @@
*
* @param ids the set of ids
- * @param type the type
- * @exception OsmTransferException thrown if an error occurs while communicating with the API server
- */
- protected void fetchPrimitives(Set ids, OsmPrimitiveType type, ProgressMonitor progressMonitor) throws OsmTransferException{
+ * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION}
+ * @throws OsmTransferException if an error occurs while communicating with the API server
+ */
+ protected void fetchPrimitives(Set ids, OsmPrimitiveType type, ProgressMonitor progressMonitor) throws OsmTransferException {
String msg = "";
- switch(type) {
- case NODE: msg = tr("Fetching a package of nodes from ''{0}''", OsmApi.getOsmApi().getBaseUrl()); break;
- case WAY: msg = tr("Fetching a package of ways from ''{0}''", OsmApi.getOsmApi().getBaseUrl()); break;
- case RELATION: msg = tr("Fetching a package of relations from ''{0}''", OsmApi.getOsmApi().getBaseUrl()); break;
+ String baseUrl = OsmApi.getOsmApi().getBaseUrl();
+ switch (type) {
+ case NODE: msg = tr("Fetching a package of nodes from ''{0}''", baseUrl); break;
+ case WAY: msg = tr("Fetching a package of ways from ''{0}''", baseUrl); break;
+ case RELATION: msg = tr("Fetching a package of relations from ''{0}''", baseUrl); break;
}
progressMonitor.setTicksCount(ids.size());
progressMonitor.setTicks(0);
+ // The complete set containg all primitives to fetch
Set toFetch = new HashSet(ids);
- while(! toFetch.isEmpty() && !isCanceled()) {
- Set pkg = extractIdPackage(toFetch);
+ // Build a list of fetchers that will download smaller sets containing only MAX_IDS_PER_REQUEST (200) primitives each.
+ // we will run up to MAX_DOWNLOAD_THREADS concurrent fetchers.
+ int threadsNumber = Main.pref.getInteger("osm.download.threads", OsmApi.MAX_DOWNLOAD_THREADS);
+ threadsNumber = Math.min(Math.max(threadsNumber, 1), OsmApi.MAX_DOWNLOAD_THREADS);
+ Executor exec = Executors.newFixedThreadPool(threadsNumber);
+ CompletionService ecs = new ExecutorCompletionService(exec);
+ List> jobs = new ArrayList>();
+ while (!toFetch.isEmpty()) {
+ jobs.add(ecs.submit(new Fetcher(type, extractIdPackage(toFetch), progressMonitor)));
+ }
+ // Run the fetchers
+ for (int i = 0; i < jobs.size() && !isCanceled(); i++) {
progressMonitor.subTask(msg + "... " + progressMonitor.getTicks() + "/" + progressMonitor.getTicksCount());
try {
- multiGetIdPackage(type, pkg, progressMonitor);
- } catch(OsmApiException e) {
- if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) {
- System.out.println(tr("Server replied with response code 404, retrying with an individual request for each object."));
- singleGetIdPackage(type, pkg, progressMonitor);
- } else
- throw e;
+ FetchResult result = ecs.take().get();
+ if (result.missingPrimitives != null) {
+ missingPrimitives.addAll(result.missingPrimitives);
+ }
+ if (result.dataSet != null && !isCanceled()) {
+ rememberNodesOfIncompleteWaysToLoad(result.dataSet);
+ merge(result.dataSet);
+ }
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ } catch (ExecutionException e) {
+ e.printStackTrace();
+ }
+ }
+ // Cancel requests if the user choosed to
+ if (isCanceled()) {
+ for (Future job : jobs) {
+ job.cancel(true);
}
}
@@ -433,5 +381,5 @@
*
* @return the parsed data
- * @exception OsmTransferException thrown if an error occurs while communicating with the API server
+ * @throws OsmTransferException if an error occurs while communicating with the API server
* @see #getMissingPrimitives()
*
@@ -443,9 +391,9 @@
try {
missingPrimitives = new HashSet();
- if (isCanceled())return null;
+ if (isCanceled()) return null;
fetchPrimitives(ways,OsmPrimitiveType.WAY, progressMonitor);
- if (isCanceled())return null;
+ if (isCanceled()) return null;
fetchPrimitives(nodes,OsmPrimitiveType.NODE, progressMonitor);
- if (isCanceled())return null;
+ if (isCanceled()) return null;
fetchPrimitives(relations,OsmPrimitiveType.RELATION, progressMonitor);
if (outputDataSet != null) {
@@ -468,3 +416,164 @@
return missingPrimitives;
}
+
+ /**
+ * The class holding the results given by {@link Fetcher}.
+ * It is only a wrapper of the resulting {@link DataSet} and the collection of {@link PrimitiveId} that could not have been loaded.
+ */
+ protected static class FetchResult {
+
+ /**
+ * The resulting data set
+ */
+ public final DataSet dataSet;
+
+ /**
+ * The collection of primitive ids that could not have been loaded
+ */
+ public final Set missingPrimitives;
+
+ /**
+ * Constructs a {@code FetchResult}
+ * @param dataSet The resulting data set
+ * @param missingPrimitives The collection of primitive ids that could not have been loaded
+ */
+ public FetchResult(DataSet dataSet, Set missingPrimitives) {
+ this.dataSet = dataSet;
+ this.missingPrimitives = missingPrimitives;
+ }
+ }
+
+ /**
+ * The class that actually download data from OSM API. Several instances of this class are used by {@link MultiFetchServerObjectReader} (one per set of primitives to fetch).
+ * The inheritance of {@link OsmServerReader} is only explained by the need to have a distinct OSM connection by {@code Fetcher} instance.
+ * @see FetchResult
+ */
+ protected static class Fetcher extends OsmServerReader implements Callable {
+
+ private final Set pkg;
+ private final OsmPrimitiveType type;
+ private final ProgressMonitor progressMonitor;
+
+ /**
+ * Constructs a {@code Fetcher}
+ * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION}
+ * @param idsPackage The set of primitives ids to fetch
+ * @param progressMonitor The progress monitor
+ */
+ public Fetcher(OsmPrimitiveType type, Set idsPackage, ProgressMonitor progressMonitor) {
+ this.pkg = idsPackage;
+ this.type = type;
+ this.progressMonitor = progressMonitor;
+ }
+
+ @Override
+ public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException {
+ // This method is implemented because of the OsmServerReader inheritance, but not used, as the main target of this class is the call() method.
+ return fetch(progressMonitor).dataSet;
+ }
+
+ @Override
+ public FetchResult call() throws Exception {
+ return fetch(progressMonitor);
+ }
+
+ /**
+ * fetches the requested primitives and updates the specified progress monitor.
+ * @param progressMonitor the progress monitor
+ * @return the {@link FetchResult} of this operation
+ * @throws OsmTransferException if an error occurs while communicating with the API server
+ */
+ protected FetchResult fetch(ProgressMonitor progressMonitor) throws OsmTransferException {
+ try {
+ return multiGetIdPackage(type, pkg, progressMonitor);
+ } catch (OsmApiException e) {
+ if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) {
+ System.out.println(tr("Server replied with response code 404, retrying with an individual request for each object."));
+ return singleGetIdPackage(type, pkg, progressMonitor);
+ } else {
+ throw e;
+ }
+ }
+ }
+
+ /**
+ * invokes a Multi Get for a set of ids and a given {@link OsmPrimitiveType}.
+ * The retrieved primitives are merged to {@link #outputDataSet}.
+ *
+ * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION}
+ * @param pkg the package of ids
+ * @return the {@link FetchResult} of this operation
+ * @throws OsmTransferException if an error occurs while communicating with the API server
+ */
+ protected FetchResult multiGetIdPackage(OsmPrimitiveType type, Set pkg, ProgressMonitor progressMonitor) throws OsmTransferException {
+ String request = buildRequestString(type, pkg);
+ final InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE);
+ if (in == null) return null;
+ progressMonitor.subTask(tr("Downloading OSM data..."));
+ try {
+ return new FetchResult(OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(pkg.size(), false)), null);
+ } catch (Exception e) {
+ throw new OsmTransferException(e);
+ }
+ }
+
+ /**
+ * invokes a Multi Get for a single id and a given {@link OsmPrimitiveType}.
+ * The retrieved primitive is merged to {@link #outputDataSet}.
+ *
+ * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION}
+ * @param id the id
+ * @return the {@link DataSet} resulting of this operation
+ * @throws OsmTransferException if an error occurs while communicating with the API server
+ */
+ protected DataSet singleGetId(OsmPrimitiveType type, long id, ProgressMonitor progressMonitor) throws OsmTransferException {
+ String request = buildRequestString(type, id);
+ final InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE);
+ if (in == null) return null;
+ progressMonitor.subTask(tr("Downloading OSM data..."));
+ try {
+ return OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(1, false));
+ } catch (Exception e) {
+ throw new OsmTransferException(e);
+ }
+ }
+
+ /**
+ * invokes a sequence of Multi Gets for individual ids in a set of ids and a given {@link OsmPrimitiveType}.
+ * The retrieved primitives are merged to {@link #outputDataSet}.
+ *
+ * This method is used if one of the ids in pkg doesn't exist (the server replies with return code 404).
+ * If the set is fetched with this method it is possible to find out which of the ids doesn't exist.
+ * Unfortunately, the server does not provide an error header or an error body for a 404 reply.
+ *
+ * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION}
+ * @param pkg the set of ids
+ * @return the {@link FetchResult} of this operation
+ * @throws OsmTransferException if an error occurs while communicating with the API server
+ */
+ protected FetchResult singleGetIdPackage(OsmPrimitiveType type, Set pkg, ProgressMonitor progressMonitor) throws OsmTransferException {
+ FetchResult result = new FetchResult(new DataSet(), new HashSet());
+ String baseUrl = OsmApi.getOsmApi().getBaseUrl();
+ for (long id : pkg) {
+ try {
+ String msg = "";
+ switch (type) {
+ case NODE: msg = tr("Fetching node with id {0} from ''{1}''", id, baseUrl); break;
+ case WAY: msg = tr("Fetching way with id {0} from ''{1}''", id, baseUrl); break;
+ case RELATION: msg = tr("Fetching relation with id {0} from ''{1}''", id, baseUrl); break;
+ }
+ progressMonitor.setCustomText(msg);
+ result.dataSet.mergeFrom(singleGetId(type, id, progressMonitor));
+ } catch (OsmApiException e) {
+ if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) {
+ System.out.println(tr("Server replied with response code 404 for id {0}. Skipping.", Long.toString(id)));
+ result.missingPrimitives.add(new SimplePrimitiveId(id, type));
+ } else {
+ throw e;
+ }
+ }
+ }
+ return result;
+ }
+ }
}