source: josm/trunk/src/org/openstreetmap/josm/io/BoundingBoxDownloader.java

Last change on this file was 19406, checked in by GerdP, 7 months ago

fix #24315: Wrong status referrers-not-all-downloaded for restriction relation, Warning message when trying to delete
This adds code to set the flag for those relations in the downloaded area which have no child relations.
  • Property svn:eol-style set to native
File size: 13.4 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.io;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.IOException;
7import java.io.InputStream;
8import java.net.SocketException;
9import java.util.Collection;
10import java.util.Collections;
11import java.util.List;
12
13import org.openstreetmap.josm.data.Bounds;
14import org.openstreetmap.josm.data.DataSource;
15import org.openstreetmap.josm.data.gpx.GpxData;
16import org.openstreetmap.josm.data.gpx.IGpxTrack;
17import org.openstreetmap.josm.data.notes.Note;
18import org.openstreetmap.josm.data.osm.DataSet;
19import org.openstreetmap.josm.gui.progress.ProgressMonitor;
20import org.openstreetmap.josm.spi.preferences.Config;
21import org.openstreetmap.josm.tools.CheckParameterUtil;
22import org.openstreetmap.josm.tools.JosmRuntimeException;
23import org.openstreetmap.josm.tools.Logging;
24import org.openstreetmap.josm.tools.Utils;
25import org.xml.sax.SAXException;
26
27/**
28 * Read content from OSM server for a given bounding box
29 * @since 627
30 */
31public class BoundingBoxDownloader extends OsmServerReader {
32
33 /**
34 * The boundings of the desired map data.
35 */
36 protected final double lat1;
37 protected final double lon1;
38 protected final double lat2;
39 protected final double lon2;
40 protected final boolean crosses180th;
41
42 /**
43 * Constructs a new {@code BoundingBoxDownloader}.
44 * @param downloadArea The area to download
45 */
46 public BoundingBoxDownloader(Bounds downloadArea) {
47 CheckParameterUtil.ensureParameterNotNull(downloadArea, "downloadArea");
48 this.lat1 = downloadArea.getMinLat();
49 this.lon1 = downloadArea.getMinLon();
50 this.lat2 = downloadArea.getMaxLat();
51 this.lon2 = downloadArea.getMaxLon();
52 this.crosses180th = downloadArea.crosses180thMeridian();
53 }
54
55 private GpxData downloadRawGps(Bounds b, ProgressMonitor progressMonitor) throws IOException, OsmTransferException, SAXException {
56 boolean done = false;
57 GpxData result = null;
58 final int pointsPerPage = 5000; // see https://wiki.openstreetmap.org/wiki/API_v0.6#GPS_traces
59 final String url = getBaseUrl() + "trackpoints?bbox="+b.getMinLon()+','+b.getMinLat()+','+b.getMaxLon()+','+b.getMaxLat()+"&page=";
60 for (int i = 0; !done && !isCanceled(); ++i) {
61 progressMonitor.subTask(tr("Downloading points {0} to {1}...", i * pointsPerPage, (i + 1) * pointsPerPage));
62 try (InputStream in = getInputStream(url+i, progressMonitor.createSubTaskMonitor(1, true))) {
63 if (in == null) {
64 break;
65 }
66 progressMonitor.setTicks(0);
67 GpxReader reader = new GpxReader(in);
68 gpxParsedProperly = reader.parse(false);
69 GpxData currentGpx = reader.getGpxData();
70
71 // #21538 - Apparently track URLs are no longer complete URLs, but only paths
72 // We'll prefix the browse URL to get something to navigate to again.
73 final String browseUrl = Config.getUrls().getBaseBrowseUrl();
74 for (IGpxTrack track : currentGpx.tracks) {
75 Object trackUrl = track.get("url");
76 if (trackUrl instanceof String) {
77 String sTrackUrl = (String) trackUrl;
78 if (!Utils.isStripEmpty(sTrackUrl) && !sTrackUrl.startsWith("http")) {
79 track.put("url", browseUrl + sTrackUrl);
80 }
81 }
82 }
83
84 long count = 0;
85 if (currentGpx.hasTrackPoints()) {
86 count = currentGpx.getTrackPoints().count();
87 }
88 if (count < pointsPerPage)
89 done = true;
90 Logging.debug("got {0} gpx points", count);
91 if (result == null) {
92 result = currentGpx;
93 } else {
94 result.mergeFrom(currentGpx);
95 }
96 } catch (OsmApiException ex) {
97 throw ex; // this avoids infinite loop in case of API error such as bad request (ex: bbox too large, see #12853)
98 } catch (OsmTransferException | SocketException ex) {
99 if (isCanceled()) {
100 final OsmTransferCanceledException canceledException = new OsmTransferCanceledException("Operation canceled");
101 canceledException.initCause(ex);
102 Logging.warn(canceledException);
103 }
104 }
105 activeConnection = null;
106 }
107 if (result != null) {
108 result.fromServer = true;
109 result.dataSources.add(new DataSource(b, "OpenStreetMap server"));
110 }
111 return result;
112 }
113
114 @Override
115 public GpxData parseRawGps(ProgressMonitor progressMonitor) throws OsmTransferException {
116 progressMonitor.beginTask("", 1);
117 try {
118 progressMonitor.indeterminateSubTask(getTaskName());
119 if (crosses180th) {
120 // API 0.6 does not support requests crossing the 180th meridian, so make two requests
121 GpxData result = downloadRawGps(new Bounds(lat1, lon1, lat2, 180.0), progressMonitor);
122 if (result != null)
123 result.mergeFrom(downloadRawGps(new Bounds(lat1, -180.0, lat2, lon2), progressMonitor));
124 return result;
125 } else {
126 // Simple request
127 return downloadRawGps(new Bounds(lat1, lon1, lat2, lon2), progressMonitor);
128 }
129 } catch (IllegalArgumentException e) {
130 // caused by HttpUrlConnection in case of illegal stuff in the response
131 if (cancel)
132 return null;
133 throw new OsmTransferException("Illegal characters within the HTTP-header response.", e);
134 } catch (IOException e) {
135 if (cancel)
136 return null;
137 throw new OsmTransferException(e);
138 } catch (SAXException e) {
139 throw new OsmTransferException(e);
140 } catch (OsmTransferException e) {
141 throw e;
142 } catch (JosmRuntimeException | IllegalStateException e) {
143 if (cancel)
144 return null;
145 throw e;
146 } finally {
147 progressMonitor.finishTask();
148 }
149 }
150
151 /**
152 * Returns the name of the download task to be displayed in the {@link ProgressMonitor}.
153 * @return task name
154 */
155 protected String getTaskName() {
156 return tr("Contacting OSM Server...");
157 }
158
159 /**
160 * Builds the request part for the bounding box.
161 * @param lon1 left
162 * @param lat1 bottom
163 * @param lon2 right
164 * @param lat2 top
165 * @return "map?bbox=left,bottom,right,top"
166 */
167 protected String getRequestForBbox(double lon1, double lat1, double lon2, double lat2) {
168 return "map?bbox=" + lon1 + ',' + lat1 + ',' + lon2 + ',' + lat2;
169 }
170
171 /**
172 * Parse the given input source and return the dataset.
173 * @param source input stream
174 * @param progressMonitor progress monitor
175 * @return dataset
176 * @throws IllegalDataException if an error was found while parsing the OSM data
177 *
178 * @see OsmReader#parseDataSet(InputStream, ProgressMonitor)
179 */
180 protected DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
181 return OsmReader.parseDataSet(source, progressMonitor);
182 }
183
184 @Override
185 public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException {
186 progressMonitor.beginTask(getTaskName(), 10);
187 try {
188 DataSet ds = null;
189 progressMonitor.indeterminateSubTask(null);
190 if (crosses180th) {
191 // API 0.6 does not support requests crossing the 180th meridian, so make two requests
192 DataSet ds2 = null;
193
194 try (InputStream in = getInputStream(getRequestForBbox(lon1, lat1, 180.0, lat2),
195 progressMonitor.createSubTaskMonitor(9, false))) {
196 if (in == null)
197 return null;
198 ds = parseDataSet(in, progressMonitor.createSubTaskMonitor(1, false));
199 }
200
201 try (InputStream in = getInputStream(getRequestForBbox(-180.0, lat1, lon2, lat2),
202 progressMonitor.createSubTaskMonitor(9, false))) {
203 if (in == null)
204 return null;
205 ds2 = parseDataSet(in, progressMonitor.createSubTaskMonitor(1, false));
206 }
207 if (ds2 == null)
208 return null;
209 ds.mergeFrom(ds2);
210
211 } else {
212 // Simple request
213 try (InputStream in = getInputStream(getRequestForBbox(lon1, lat1, lon2, lat2),
214 progressMonitor.createSubTaskMonitor(9, false))) {
215 if (in == null)
216 return null;
217 ds = parseDataSet(in, progressMonitor.createSubTaskMonitor(1, false));
218 }
219 }
220 // From https://wiki.openstreetmap.org/wiki/API_v0.6#Retrieving_map_data_by_bounding_box:_GET_/api/0.6/map,
221 // relations are not recursed up, so they *may* have parent relations.
222 // Nodes inside the download area should have all relations and ways that refer to them.
223 // Ways should have all relations that refer to them and all child nodes, but those child nodes may not
224 // have their parent referrers.
225 // Relations will have the *first* parent relations downloaded, but those are not split out in the returns.
226 // So we always assume that a relation has referrers that need to be downloaded unless it has no child relations.
227 // Our "full" overpass query doesn't return the same data as a standard download, so we cannot
228 // mark relations with no child relations as fully downloaded *yet*.
229 if (this.considerAsFullDownload()) {
230 final Collection<Bounds> bounds = this.getBounds();
231 // We cannot use OsmPrimitive#isOutsideDownloadArea yet since some download methods haven't added
232 // the download bounds to the dataset yet. This is specifically the case for overpass downloads.
233 ds.getNodes().stream().filter(n -> bounds.stream().anyMatch(b -> b.contains(n)))
234 .forEach(i -> i.setReferrersDownloaded(true));
235 ds.getWays().forEach(i -> i.setReferrersDownloaded(true));
236 ds.getRelations().stream().filter(r -> r.getMembers().stream().noneMatch(rm -> rm.isRelation()))
237 .forEach(i -> i.setReferrersDownloaded(true));
238 }
239 return ds;
240 } catch (OsmTransferException e) {
241 throw e;
242 } catch (IllegalDataException | IOException e) {
243 throw new OsmTransferException(e);
244 } finally {
245 progressMonitor.finishTask();
246 activeConnection = null;
247 }
248 }
249
250 @Override
251 public List<Note> parseNotes(int noteLimit, int daysClosed, ProgressMonitor progressMonitor) throws OsmTransferException {
252 progressMonitor.beginTask(tr("Downloading notes"));
253 CheckParameterUtil.ensureThat(noteLimit > 0, "Requested note limit is less than 1.");
254 // see result_limit in https://github.com/openstreetmap/openstreetmap-website/blob/master/app/controllers/notes_controller.rb
255 CheckParameterUtil.ensureThat(noteLimit <= 10_000, "Requested note limit is over API hard limit of 10000.");
256 CheckParameterUtil.ensureThat(daysClosed >= -1, "Requested note limit is less than -1.");
257 String url = "notes?limit=" + noteLimit + "&closed=" + daysClosed + "&bbox=" + lon1 + ',' + lat1 + ',' + lon2 + ',' + lat2;
258 try (InputStream is = getInputStream(url, progressMonitor.createSubTaskMonitor(1, false))) {
259 final List<Note> notes = new NoteReader(is).parse();
260 if (notes.size() == noteLimit) {
261 throw new MoreNotesException(notes, noteLimit);
262 }
263 return notes;
264 } catch (IOException | SAXException e) {
265 throw new OsmTransferException(e);
266 } finally {
267 progressMonitor.finishTask();
268 }
269 }
270
271 /**
272 * Indicates that the number of fetched notes equals the specified limit. Thus there might be more notes to download.
273 */
274 public static class MoreNotesException extends RuntimeException {
275 /**
276 * The downloaded notes
277 */
278 public final transient List<Note> notes;
279 /**
280 * The download limit sent to the server.
281 */
282 public final int limit;
283
284 /**
285 * Constructs a {@code MoreNotesException}.
286 * @param notes downloaded notes
287 * @param limit download limit sent to the server
288 */
289 public MoreNotesException(List<Note> notes, int limit) {
290 this.notes = notes;
291 this.limit = limit;
292 }
293 }
294
295 /**
296 * Determines if download is complete for the given bounding box.
297 * @return true if download is complete for the given bounding box (not filtered)
298 */
299 public boolean considerAsFullDownload() {
300 return true;
301 }
302
303 /**
304 * Get the bounds for this downloader
305 * @return The bounds for this downloader
306 * @since 19078
307 */
308 protected Collection<Bounds> getBounds() {
309 return Collections.singleton(new Bounds(this.lat1, this.lon1, this.lat2, this.lon2));
310 }
311}
Note: See TracBrowser for help on using the repository browser.