source: josm/trunk/src/org/openstreetmap/josm/io/MultiFetchServerObjectReader.java@ 5299

Last change on this file since 5299 was 5266, checked in by bastiK, 12 years ago

fixed majority of javadoc warnings by replacing "{@see" by "{@link"
  • Property svn:eol-style set to native
File size: 17.7 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.io;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5import static org.openstreetmap.josm.tools.I18n.trn;
6
7import java.io.InputStream;
8import java.net.HttpURLConnection;
9import java.util.Collection;
10import java.util.HashSet;
11import java.util.Iterator;
12import java.util.LinkedHashSet;
13import java.util.NoSuchElementException;
14import java.util.Set;
15
16import org.openstreetmap.josm.data.osm.DataSet;
17import org.openstreetmap.josm.data.osm.DataSetMerger;
18import org.openstreetmap.josm.data.osm.Node;
19import org.openstreetmap.josm.data.osm.OsmPrimitive;
20import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
21import org.openstreetmap.josm.data.osm.PrimitiveId;
22import org.openstreetmap.josm.data.osm.Relation;
23import org.openstreetmap.josm.data.osm.RelationMember;
24import org.openstreetmap.josm.data.osm.SimplePrimitiveId;
25import org.openstreetmap.josm.data.osm.Way;
26import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
27import org.openstreetmap.josm.gui.progress.ProgressMonitor;
28import org.openstreetmap.josm.tools.CheckParameterUtil;
29
30/**
31 * Retrieves a set of {@link OsmPrimitive}s from an OSM server using the so called
32 * Multi Fetch API.
33 *
34 * Usage:
35 * <pre>
36 * MultiFetchServerObjectReader reader = MultiFetchServerObjectReader()
37 * .append(2345,2334,4444)
38 * .append(new Node(72343));
39 * reader.parseOsm();
40 * if (!reader.getMissingPrimitives().isEmpty()) {
41 * System.out.println("There are missing primitives: " + reader.getMissingPrimitives());
42 * }
43 * if (!reader.getSkippedWays().isEmpty()) {
44 * System.out.println("There are skipped ways: " + reader.getMissingPrimitives());
45 * }
46 * </pre>
47 *
48 *
49 */
50public class MultiFetchServerObjectReader extends OsmServerReader{
51 /**
52 * the max. number of primitives retrieved in one step. Assuming IDs with 7 digits,
53 * this leads to a max. request URL of ~ 1600 Bytes ((7 digits + 1 Separator) * 200),
54 * which should be safe according to the
55 * <a href="http://www.boutell.com/newfaq/misc/urllength.html">WWW FAQ</a>.
56 *
57 */
58 static private int MAX_IDS_PER_REQUEST = 200;
59
60 private Set<Long> nodes;
61 private Set<Long> ways;
62 private Set<Long> relations;
63 private Set<PrimitiveId> missingPrimitives;
64 private DataSet outputDataSet;
65
66 /**
67 * constructor
68 *
69 */
70 public MultiFetchServerObjectReader() {
71 nodes = new LinkedHashSet<Long>();
72 ways = new LinkedHashSet<Long>();
73 relations = new LinkedHashSet<Long>();
74 this.outputDataSet = new DataSet();
75 this.missingPrimitives = new LinkedHashSet<PrimitiveId>();
76 }
77
78 /**
79 * Remembers an {@link OsmPrimitive}'s id. The id will
80 * later be fetched as part of a Multi Get request.
81 *
82 * Ignore the id if it represents a new primitives.
83 *
84 * @param id the id
85 */
86 protected void remember(PrimitiveId id) {
87 if (id.isNew()) return;
88 switch(id.getType()) {
89 case NODE: nodes.add(id.getUniqueId()); break;
90 case WAY: ways.add(id.getUniqueId()); break;
91 case RELATION: relations.add(id.getUniqueId()); break;
92 }
93 }
94
95 /**
96 * remembers an {@link OsmPrimitive}'s id. <code>ds</code> must include
97 * an {@link OsmPrimitive} with id=<code>id</code>. The id will
98 * later we fetched as part of a Multi Get request.
99 *
100 * Ignore the id if it id <= 0.
101 *
102 * @param ds the dataset (must not be null)
103 * @param id the id
104 * @exception IllegalArgumentException thrown, if ds is null
105 * @exception NoSuchElementException thrown, if ds doesn't include an {@link OsmPrimitive} with
106 * id=<code>id</code>
107 */
108 protected void remember(DataSet ds, long id, OsmPrimitiveType type) throws IllegalArgumentException, NoSuchElementException{
109 CheckParameterUtil.ensureParameterNotNull(ds, "ds");
110 if (id <= 0) return;
111 OsmPrimitive primitive = ds.getPrimitiveById(id, type);
112 if (primitive == null)
113 throw new NoSuchElementException(tr("No primitive with id {0} in local dataset. Cannot infer primitive type.", id));
114 remember(primitive.getPrimitiveId());
115 return;
116 }
117
118 public MultiFetchServerObjectReader append(DataSet ds, long id, OsmPrimitiveType type) {
119 switch(type) {
120 case NODE:
121 Node n = (Node)ds.getPrimitiveById(id,type);
122 appendNode(n);
123 break;
124 case WAY:
125 Way w= (Way)ds.getPrimitiveById(id,type);
126 appendWay(w);
127 break;
128 case RELATION:
129 Relation r = (Relation)ds.getPrimitiveById(id,type);
130 appendRelation(r);
131 break;
132 }
133 return this;
134 }
135
136 /**
137 * appends a {@link Node}s id to the list of ids which will be fetched from the server.
138 *
139 * @param node the node (ignored, if null)
140 * @return this
141 *
142 */
143 public MultiFetchServerObjectReader appendNode(Node node) {
144 if (node == null) return this;
145 remember(node.getPrimitiveId());
146 return this;
147 }
148
149 /**
150 * appends a {@link Way}s id and the list of ids of nodes the way refers to the list of ids which will be fetched from the server.
151 *
152 * @param way the way (ignored, if null)
153 * @return this
154 *
155 */
156 public MultiFetchServerObjectReader appendWay(Way way) {
157 if (way == null) return this;
158 if (way.isNew()) return this;
159 for (Node node: way.getNodes()) {
160 if (!node.isNew()) {
161 remember(node.getPrimitiveId());
162 }
163 }
164 remember(way.getPrimitiveId());
165 return this;
166 }
167
168 /**
169 * appends a {@link Relation}s id to the list of ids which will be fetched from the server.
170 *
171 * @param relation the relation (ignored, if null)
172 * @return this
173 *
174 */
175 protected MultiFetchServerObjectReader appendRelation(Relation relation) {
176 if (relation == null) return this;
177 if (relation.isNew()) return this;
178 remember(relation.getPrimitiveId());
179 for (RelationMember member : relation.getMembers()) {
180 if (OsmPrimitiveType.from(member.getMember()).equals(OsmPrimitiveType.RELATION)) {
181 // avoid infinite recursion in case of cyclic dependencies in relations
182 //
183 if (relations.contains(member.getMember().getId())) {
184 continue;
185 }
186 }
187 if (!member.getMember().isIncomplete()) {
188 append(member.getMember());
189 }
190 }
191 return this;
192 }
193
194 public MultiFetchServerObjectReader append(OsmPrimitive primitive) {
195 if (OsmPrimitiveType.from(primitive).equals(OsmPrimitiveType.NODE))
196 return appendNode((Node)primitive);
197 else if (OsmPrimitiveType.from(primitive).equals(OsmPrimitiveType.WAY))
198 return appendWay((Way)primitive);
199 else if (OsmPrimitiveType.from(primitive).equals(OsmPrimitiveType.RELATION))
200 return appendRelation((Relation)primitive);
201
202 return this;
203 }
204
205 /**
206 * appends a list of {@link OsmPrimitive} to the list of ids which will be fetched from the server.
207 *
208 * @param primitives the list of primitives (ignored, if null)
209 * @return this
210 *
211 * @see #append(Node)
212 * @see #append(Way)
213 * @see #append(Relation)
214 *
215 */
216 public MultiFetchServerObjectReader append(Collection<? extends OsmPrimitive> primitives) {
217 if (primitives == null) return this;
218 for (OsmPrimitive primitive : primitives) {
219 append(primitive);
220 }
221 return this;
222 }
223
224 /**
225 * extracts a subset of max {@link #MAX_IDS_PER_REQUEST} ids from <code>ids</code> and
226 * replies the subset. The extracted subset is removed from <code>ids</code>.
227 *
228 * @param ids a set of ids
229 * @return the subset of ids
230 */
231 protected Set<Long> extractIdPackage(Set<Long> ids) {
232 HashSet<Long> pkg = new HashSet<Long>();
233 if (ids.isEmpty())
234 return pkg;
235 if (ids.size() > MAX_IDS_PER_REQUEST) {
236 Iterator<Long> it = ids.iterator();
237 for (int i =0;i<MAX_IDS_PER_REQUEST;i++) {
238 pkg.add(it.next());
239 }
240 ids.removeAll(pkg);
241 } else {
242 pkg.addAll(ids);
243 ids.clear();
244 }
245 return pkg;
246 }
247
248 /**
249 * builds the Multi Get request string for a set of ids and a given
250 * {@link OsmPrimitiveType}.
251 *
252 * @param type the type
253 * @param idPackage the package of ids
254 * @return the request string
255 */
256 protected String buildRequestString(OsmPrimitiveType type, Set<Long> idPackage) {
257 StringBuilder sb = new StringBuilder();
258 sb.append(type.getAPIName()).append("s?")
259 .append(type.getAPIName()).append("s=");
260
261 Iterator<Long> it = idPackage.iterator();
262 for (int i=0; i< idPackage.size();i++) {
263 sb.append(it.next());
264 if (i < idPackage.size()-1) {
265 sb.append(",");
266 }
267 }
268 return sb.toString();
269 }
270
271 /**
272 * builds the Multi Get request string for a single id and a given
273 * {@link OsmPrimitiveType}.
274 *
275 * @param type the type
276 * @param id the id
277 * @return the request string
278 */
279 protected String buildRequestString(OsmPrimitiveType type, long id) {
280 StringBuilder sb = new StringBuilder();
281 sb.append(type.getAPIName()).append("s?")
282 .append(type.getAPIName()).append("s=")
283 .append(id);
284 return sb.toString();
285 }
286
287 /**
288 * invokes a Multi Get for a set of ids and a given {@link OsmPrimitiveType}.
289 * The retrieved primitives are merged to {@link #outputDataSet}.
290 *
291 * @param type the type
292 * @param pkg the package of ids
293 * @exception OsmTransferException thrown if an error occurs while communicating with the API server
294 *
295 */
296 protected void multiGetIdPackage(OsmPrimitiveType type, Set<Long> pkg, ProgressMonitor progressMonitor) throws OsmTransferException {
297 String request = buildRequestString(type, pkg);
298 final InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE);
299 if (in == null) return;
300 progressMonitor.subTask(tr("Downloading OSM data..."));
301 try {
302 DataSet loaded = OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(pkg.size(), false));
303 rememberNodesOfIncompleteWaysToLoad(loaded);
304 merge(loaded);
305 } catch(Exception e) {
306 throw new OsmTransferException(e);
307 }
308 }
309
310 /**
311 * invokes a Multi Get for a single id and a given {@link OsmPrimitiveType}.
312 * The retrieved primitive is merged to {@link #outputDataSet}.
313 *
314 * @param type the type
315 * @param id the id
316 * @exception OsmTransferException thrown if an error occurs while communicating with the API server
317 *
318 */
319 protected void singleGetId(OsmPrimitiveType type, long id, ProgressMonitor progressMonitor) throws OsmTransferException {
320 String request = buildRequestString(type, id);
321 final InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE);
322 if (in == null)
323 return;
324 progressMonitor.subTask(tr("Downloading OSM data..."));
325 try {
326 DataSet loaded = OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(1, false));
327 rememberNodesOfIncompleteWaysToLoad(loaded);
328 merge(loaded);
329 } catch(Exception e) {
330 throw new OsmTransferException(e);
331 }
332 }
333
334 /**
335 * invokes a sequence of Multi Gets for individual ids in a set of ids and a given {@link OsmPrimitiveType}.
336 * The retrieved primitives are merged to {@link #outputDataSet}.
337 *
338 * This method is used if one of the ids in pkg doesn't exist (the server replies with return code 404).
339 * If the set is fetched with this method it is possible to find out which of the ids doesn't exist.
340 * Unfortunatelly, the server does not provide an error header or an error body for a 404 reply.
341 *
342 * @param type the type
343 * @param pkg the set of ids
344 * @exception OsmTransferException thrown if an error occurs while communicating with the API server
345 *
346 */
347 protected void singleGetIdPackage(OsmPrimitiveType type, Set<Long> pkg, ProgressMonitor progressMonitor) throws OsmTransferException {
348 for (long id : pkg) {
349 try {
350 String msg = "";
351 switch(type) {
352 case NODE: msg = tr("Fetching node with id {0} from ''{1}''", id, OsmApi.getOsmApi().getBaseUrl()); break;
353 case WAY: msg = tr("Fetching way with id {0} from ''{1}''", id, OsmApi.getOsmApi().getBaseUrl()); break;
354 case RELATION: msg = tr("Fetching relation with id {0} from ''{1}''", id, OsmApi.getOsmApi().getBaseUrl()); break;
355 }
356 progressMonitor.setCustomText(msg);
357 singleGetId(type, id, progressMonitor);
358 } catch(OsmApiException e) {
359 if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) {
360 System.out.println(tr("Server replied with response code 404 for id {0}. Skipping.", Long.toString(id)));
361 missingPrimitives.add(new SimplePrimitiveId(id, type));
362 continue;
363 }
364 throw e;
365 }
366 }
367 }
368
369 protected void rememberNodesOfIncompleteWaysToLoad(DataSet from) {
370 for (Way w: from.getWays()) {
371 if (w.hasIncompleteNodes()) {
372 for (Node n: w.getNodes()) {
373 if (n.isIncomplete()) {
374 nodes.add(n.getId());
375 }
376 }
377 }
378 }
379 }
380
381 /**
382 * merges the dataset <code>from</code> to {@link #outputDataSet}.
383 *
384 * @param from the other dataset
385 *
386 */
387 protected void merge(DataSet from) {
388 final DataSetMerger visitor = new DataSetMerger(outputDataSet,from);
389 visitor.merge();
390 }
391
392 /**
393 * fetches a set of ids of a given {@link OsmPrimitiveType} from the server
394 *
395 * @param ids the set of ids
396 * @param type the type
397 * @exception OsmTransferException thrown if an error occurs while communicating with the API server
398 */
399 protected void fetchPrimitives(Set<Long> ids, OsmPrimitiveType type, ProgressMonitor progressMonitor) throws OsmTransferException{
400 String msg = "";
401 switch(type) {
402 case NODE: msg = tr("Fetching a package of nodes from ''{0}''", OsmApi.getOsmApi().getBaseUrl()); break;
403 case WAY: msg = tr("Fetching a package of ways from ''{0}''", OsmApi.getOsmApi().getBaseUrl()); break;
404 case RELATION: msg = tr("Fetching a package of relations from ''{0}''", OsmApi.getOsmApi().getBaseUrl()); break;
405 }
406 progressMonitor.setTicksCount(ids.size());
407 progressMonitor.setTicks(0);
408 Set<Long> toFetch = new HashSet<Long>(ids);
409 while(! toFetch.isEmpty() && !isCanceled()) {
410 Set<Long> pkg = extractIdPackage(toFetch);
411 progressMonitor.subTask(msg + "... " + progressMonitor.getTicks() + "/" + progressMonitor.getTicksCount());
412 try {
413 multiGetIdPackage(type, pkg, progressMonitor);
414 } catch(OsmApiException e) {
415 if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) {
416 System.out.println(tr("Server replied with response code 404, retrying with an individual request for each object."));
417 singleGetIdPackage(type, pkg, progressMonitor);
418 } else
419 throw e;
420 }
421 }
422 }
423
424 /**
425 * invokes one or more Multi Gets to fetch the {@link OsmPrimitive}s and replies
426 * the dataset of retrieved primitives. Note that the dataset includes non visible primitives too!
427 * In contrast to a simple Get for a node, a way, or a relation, a Multi Get always replies
428 * the latest version of the primitive (if any), even if the primitive is not visible (i.e. if
429 * visible==false).
430 *
431 * Invoke {@link #getMissingPrimitives()} to get a list of primitives which have not been
432 * found on the server (the server response code was 404)
433 *
434 * @return the parsed data
435 * @exception OsmTransferException thrown if an error occurs while communicating with the API server
436 * @see #getMissingPrimitives()
437 *
438 */
439 @Override
440 public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException {
441 int n = nodes.size() + ways.size() + relations.size();
442 progressMonitor.beginTask(trn("Downloading {0} object from ''{1}''", "Downloading {0} objects from ''{1}''", n, n, OsmApi.getOsmApi().getBaseUrl()));
443 try {
444 missingPrimitives = new HashSet<PrimitiveId>();
445 if (isCanceled())return null;
446 fetchPrimitives(ways,OsmPrimitiveType.WAY, progressMonitor);
447 if (isCanceled())return null;
448 fetchPrimitives(nodes,OsmPrimitiveType.NODE, progressMonitor);
449 if (isCanceled())return null;
450 fetchPrimitives(relations,OsmPrimitiveType.RELATION, progressMonitor);
451 if (outputDataSet != null) {
452 outputDataSet.deleteInvisible();
453 }
454 return outputDataSet;
455 } finally {
456 progressMonitor.finishTask();
457 }
458 }
459
460 /**
461 * replies the set of ids of all primitives for which a fetch request to the
462 * server was submitted but which are not available from the server (the server
463 * replied a return code of 404)
464 *
465 * @return the set of ids of missing primitives
466 */
467 public Set<PrimitiveId> getMissingPrimitives() {
468 return missingPrimitives;
469 }
470}
Note: See TracBrowser for help on using the repository browser.