source: josm/trunk/test/unit/org/openstreetmap/josm/gui/preferences/imagery/ImageryPreferenceTestIT.java@ 14611

Last change on this file since 14611 was 14611, checked in by Don-vip, 5 years ago

see #16073 - increase read timeout for slow WMS servers
  • Property svn:eol-style set to native
File size: 16.2 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.gui.preferences.imagery;
3
4import static org.junit.Assert.assertFalse;
5import static org.junit.Assert.assertTrue;
6
7import java.io.ByteArrayInputStream;
8import java.io.IOException;
9import java.net.URL;
10import java.nio.charset.StandardCharsets;
11import java.util.ArrayList;
12import java.util.Collections;
13import java.util.List;
14import java.util.Locale;
15import java.util.Map;
16import java.util.Objects;
17import java.util.Optional;
18import java.util.TreeMap;
19import java.util.concurrent.TimeUnit;
20import java.util.stream.Collectors;
21
22import javax.imageio.ImageIO;
23
24import org.apache.commons.jcs.access.CacheAccess;
25import org.junit.BeforeClass;
26import org.junit.ClassRule;
27import org.junit.Test;
28import org.junit.runner.Description;
29import org.junit.runner.RunWith;
30import org.junit.runners.Parameterized.Parameters;
31import org.junit.runners.model.Statement;
32import org.openstreetmap.gui.jmapviewer.Coordinate;
33import org.openstreetmap.gui.jmapviewer.TileXY;
34import org.openstreetmap.gui.jmapviewer.interfaces.ICoordinate;
35import org.openstreetmap.gui.jmapviewer.tilesources.AbstractTileSource;
36import org.openstreetmap.gui.jmapviewer.tilesources.BingAerialTileSource;
37import org.openstreetmap.gui.jmapviewer.tilesources.ScanexTileSource;
38import org.openstreetmap.gui.jmapviewer.tilesources.TemplatedTMSTileSource;
39import org.openstreetmap.josm.TestUtils;
40import org.openstreetmap.josm.actions.AddImageryLayerAction;
41import org.openstreetmap.josm.actions.AddImageryLayerAction.LayerSelection;
42import org.openstreetmap.josm.data.Bounds;
43import org.openstreetmap.josm.data.coor.LatLon;
44import org.openstreetmap.josm.data.imagery.CoordinateConversion;
45import org.openstreetmap.josm.data.imagery.ImageryInfo;
46import org.openstreetmap.josm.data.imagery.ImageryInfo.ImageryBounds;
47import org.openstreetmap.josm.data.imagery.ImageryInfo.ImageryType;
48import org.openstreetmap.josm.data.imagery.ImageryLayerInfo;
49import org.openstreetmap.josm.data.imagery.LayerDetails;
50import org.openstreetmap.josm.data.imagery.Shape;
51import org.openstreetmap.josm.data.imagery.TMSCachedTileLoaderJob;
52import org.openstreetmap.josm.data.imagery.TemplatedWMSTileSource;
53import org.openstreetmap.josm.data.imagery.TileJobOptions;
54import org.openstreetmap.josm.data.imagery.WMTSTileSource;
55import org.openstreetmap.josm.data.imagery.WMTSTileSource.WMTSGetCapabilitiesException;
56import org.openstreetmap.josm.data.projection.Projection;
57import org.openstreetmap.josm.data.projection.ProjectionRegistry;
58import org.openstreetmap.josm.data.projection.Projections;
59import org.openstreetmap.josm.io.imagery.WMSImagery.WMSGetCapabilitiesException;
60import org.openstreetmap.josm.testutils.JOSMTestRules;
61import org.openstreetmap.josm.testutils.ParallelParameterized;
62import org.openstreetmap.josm.tools.HttpClient;
63import org.openstreetmap.josm.tools.HttpClient.Response;
64import org.openstreetmap.josm.tools.Logging;
65import org.openstreetmap.josm.tools.Utils;
66
67import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
68
69/**
70 * Integration tests of {@link ImageryPreference} class.
71 */
72@RunWith(ParallelParameterized.class)
73public class ImageryPreferenceTestIT {
74
75 private static final LatLon GREENWICH = new LatLon(51.47810, -0.00170);
76 private static final int DEFAULT_ZOOM = 12;
77
78 /**
79 * Setup rule
80 */
81 @ClassRule
82 @SuppressFBWarnings(value = "URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD")
83 public static JOSMTestRules test = new JOSMTestRules().https().i18n().preferences().projection().projectionNadGrids()
84 .timeout((int) TimeUnit.MINUTES.toMillis(40));
85
86 static {
87 try {
88 test.apply(new Statement() {
89 @Override
90 public void evaluate() throws Throwable {
91 // Do nothing. Hack needed because @Parameters are computed before anything else
92 }
93 }, Description.createSuiteDescription(ImageryPreferenceTestIT.class)).evaluate();
94 } catch (Throwable e) {
95 Logging.error(e);
96 }
97 }
98
99 /** Entry to test */
100 private final ImageryInfo info;
101 private final Map<String, Map<ImageryInfo, List<String>>> errors = Collections.synchronizedMap(new TreeMap<>());
102 private static final Map<String, byte[]> workingURLs = Collections.synchronizedMap(new TreeMap<>());
103
104 private static TMSCachedTileLoaderJob helper;
105 private static List<String> ignoredErrors;
106
107 /**
108 * Setup test
109 * @throws IOException in case of I/O error
110 */
111 @BeforeClass
112 public static void beforeClass() throws IOException {
113 helper = new TMSCachedTileLoaderJob(null, null, new CacheAccess<>(null), new TileJobOptions(0, 0, null, 0), null);
114 ignoredErrors = TestUtils.getIgnoredErrorMessages(ImageryPreferenceTestIT.class);
115 }
116
117 /**
118 * Returns list of imagery entries to test.
119 * @return list of imagery entries to test
120 */
121 @Parameters(name = "{0}")
122 public static List<Object[]> data() {
123 ImageryLayerInfo.instance.load(false);
124 return ImageryLayerInfo.instance.getDefaultLayers()
125 .stream()
126 .map(x -> new Object[] {x.getId(), x})
127 .collect(Collectors.toList());
128 }
129
130 /**
131 * Constructs a new {@code ImageryPreferenceTestIT} instance.
132 * @param id entry ID, used only to name tests
133 * @param info entry to test
134 */
135 public ImageryPreferenceTestIT(String id, ImageryInfo info) {
136 this.info = Objects.requireNonNull(info);
137 }
138
139 private boolean addError(ImageryInfo info, String error) {
140 String errorMsg = error.replace('\n', ' ');
141 return !ignoredErrors.contains(errorMsg) &&
142 errors.computeIfAbsent(info.getCountryCode(), x -> Collections.synchronizedMap(new TreeMap<>()))
143 .computeIfAbsent(info, x -> Collections.synchronizedList(new ArrayList<>()))
144 .add(errorMsg);
145 }
146
147 private Optional<byte[]> checkUrl(ImageryInfo info, String url) {
148 if (url != null && !url.isEmpty()) {
149 if (workingURLs.containsKey(url)) {
150 return Optional.of(workingURLs.get(url));
151 }
152 try {
153 Response response = HttpClient.create(new URL(url))
154 .setHeaders(info.getCustomHttpHeaders())
155 .setConnectTimeout((int) TimeUnit.MINUTES.toMillis(1))
156 .setReadTimeout((int) TimeUnit.MINUTES.toMillis(3))
157 .connect();
158 if (response.getResponseCode() >= 400) {
159 addError(info, url + " -> HTTP " + response.getResponseCode());
160 } else if (response.getResponseCode() >= 300) {
161 Logging.warn(url + " -> HTTP " + response.getResponseCode());
162 }
163 try {
164 byte[] data = Utils.readBytesFromStream(response.getContent());
165 if (response.getResponseCode() < 300) {
166 workingURLs.put(url, data);
167 }
168 return Optional.of(data);
169 } catch (IOException e) {
170 if (response.getResponseCode() < 300) {
171 addError(info, url + " -> " + e);
172 }
173 } finally {
174 response.disconnect();
175 }
176 } catch (IOException e) {
177 addError(info, url + " -> " + e);
178 }
179 }
180 return Optional.empty();
181 }
182
183 private void checkLinkUrl(ImageryInfo info, String url) {
184 checkUrl(info, url).filter(x -> x.length == 0).ifPresent(x -> addError(info, url + " -> returned empty contents"));
185 }
186
187 private String checkTileUrl(ImageryInfo info, AbstractTileSource tileSource, ICoordinate center, int zoom)
188 throws IOException {
189 TileXY xy = tileSource.latLonToTileXY(center, zoom);
190 for (int i = 0; i < 3; i++) {
191 try {
192 String url = tileSource.getTileUrl(zoom, xy.getXIndex(), xy.getYIndex());
193 Optional<byte[]> optional = checkUrl(info, url);
194 String error = "";
195 if (optional.isPresent()) {
196 byte[] data = optional.get();
197 try (ByteArrayInputStream bais = new ByteArrayInputStream(data)) {
198 if (ImageIO.read(bais) == null) {
199 error = addImageError(info, url, data, zoom, "did not return an image");
200 }
201 } catch (IOException e) {
202 error = addImageError(info, url, data, zoom, e.toString());
203 Logging.trace(e);
204 }
205 }
206 return error;
207 } catch (IOException e) {
208 // Try up to three times max to allow Bing source to initialize itself
209 // and avoid random network errors
210 Logging.trace(e);
211 if (i == 2) {
212 throw e;
213 }
214 try {
215 Thread.sleep(500);
216 } catch (InterruptedException ex) {
217 Logging.warn(ex);
218 }
219 }
220 }
221 return "";
222 }
223
224 private static String zoomMarker(int zoom) {
225 return " -> zoom " + zoom + " -> ";
226 }
227
228 private String addImageError(ImageryInfo info, String url, byte[] data, int zoom, String defaultMessage) {
229 // Check if we have received an error message
230 String error = helper.detectErrorMessage(new String(data, StandardCharsets.UTF_8));
231 String errorMsg = url + zoomMarker(zoom) + (error != null ? error.split("\\n")[0] : defaultMessage);
232 addError(info, errorMsg);
233 return errorMsg;
234 }
235
236 private static LatLon getPointInShape(Shape shape) {
237 final Coordinate p1 = shape.getPoints().get(0);
238 final Bounds bounds = new Bounds(p1.getLat(), p1.getLon(), p1.getLat(), p1.getLon());
239 shape.getPoints().forEach(p -> bounds.extend(p.getLat(), p.getLon()));
240
241 final double w = bounds.getWidth();
242 final double h = bounds.getHeight();
243
244 final double x2 = bounds.getMinLon() + (w / 2.0);
245 final double y2 = bounds.getMinLat() + (h / 2.0);
246
247 final LatLon center = new LatLon(y2, x2);
248
249 // check to see if center is inside shape
250 if (shape.contains(center)) {
251 return center;
252 }
253
254 // if center position (C) is not inside shape, try naively some other positions as follows:
255 final double x1 = bounds.getMinLon() + (.25 * w);
256 final double x3 = bounds.getMinLon() + (.75 * w);
257 final double y1 = bounds.getMinLat() + (.25 * h);
258 final double y3 = bounds.getMinLat() + (.75 * h);
259 // +-----------+
260 // | 5 1 6 |
261 // | 4 C 2 |
262 // | 8 3 7 |
263 // +-----------+
264 for (LatLon candidate : new LatLon[] {
265 new LatLon(y1, x2),
266 new LatLon(y2, x3),
267 new LatLon(y3, x2),
268 new LatLon(y2, x1),
269 new LatLon(y1, x1),
270 new LatLon(y1, x3),
271 new LatLon(y3, x3),
272 new LatLon(y3, x1)
273 }) {
274 if (shape.contains(candidate)) {
275 return candidate;
276 }
277 }
278 return center;
279 }
280
281 private static LatLon getCenter(ImageryBounds bounds) {
282 List<Shape> shapes = bounds.getShapes();
283 return shapes != null && !shapes.isEmpty() ? getPointInShape(shapes.get(0)) : bounds.getCenter();
284 }
285
286 private void checkEntry(ImageryInfo info) {
287 Logging.info("Checking "+ info);
288
289 if (info.getAttributionImageRaw() != null && info.getAttributionImage() == null) {
290 addError(info, "Can't fetch attribution image: " + info.getAttributionImageRaw());
291 }
292
293 checkLinkUrl(info, info.getAttributionImageURL());
294 checkLinkUrl(info, info.getAttributionLinkURL());
295 String eula = info.getEulaAcceptanceRequired();
296 if (eula != null) {
297 checkLinkUrl(info, eula.replaceAll("\\{lang\\}", ""));
298 }
299 checkLinkUrl(info, info.getPermissionReferenceURL());
300 checkLinkUrl(info, info.getTermsOfUseURL());
301
302 try {
303 ImageryBounds bounds = info.getBounds();
304 // Some imagery sources do not define tiles at (0,0). So pickup Greenwich Royal Observatory for global sources
305 ICoordinate center = CoordinateConversion.llToCoor(bounds != null ? getCenter(bounds) : GREENWICH);
306 AbstractTileSource tileSource = getTileSource(info);
307 // test min zoom and try to detect the correct value in case of error
308 int maxZoom = info.getMaxZoom() > 0 ? Math.min(DEFAULT_ZOOM, info.getMaxZoom()) : DEFAULT_ZOOM;
309 for (int zoom = info.getMinZoom(); zoom < maxZoom; zoom++) {
310 if (!isZoomError(checkTileUrl(info, tileSource, center, zoom))) {
311 break;
312 }
313 }
314 // checking max zoom for real is complex, see https://josm.openstreetmap.de/ticket/16073#comment:27
315 if (info.getMaxZoom() > 0 && info.getImageryType() != ImageryType.SCANEX) {
316 checkTileUrl(info, tileSource, center, Utils.clamp(DEFAULT_ZOOM, info.getMinZoom() + 1, info.getMaxZoom()));
317 }
318 } catch (IOException | RuntimeException | WMSGetCapabilitiesException | WMTSGetCapabilitiesException e) {
319 addError(info, info.getUrl() + " -> " + e.toString());
320 }
321
322 for (ImageryInfo mirror : info.getMirrors()) {
323 checkEntry(mirror);
324 }
325 }
326
327 private static boolean isZoomError(String error) {
328 String[] parts = error.split(" -> ");
329 String lastPart = parts.length > 0 ? parts[parts.length - 1].toLowerCase(Locale.ENGLISH) : "";
330 return lastPart.contains("bbox")
331 || lastPart.contains("bounding box");
332 }
333
334 private static Projection getProjection(ImageryInfo info) {
335 for (String code : info.getServerProjections()) {
336 Projection proj = Projections.getProjectionByCode(code);
337 if (proj != null) {
338 return proj;
339 }
340 }
341 return ProjectionRegistry.getProjection();
342 }
343
344 @SuppressWarnings("fallthrough")
345 private static AbstractTileSource getTileSource(ImageryInfo info)
346 throws IOException, WMTSGetCapabilitiesException, WMSGetCapabilitiesException {
347 switch (info.getImageryType()) {
348 case BING:
349 return new BingAerialTileSource(info);
350 case SCANEX:
351 return new ScanexTileSource(info);
352 case TMS:
353 return new TemplatedTMSTileSource(info);
354 case WMS_ENDPOINT:
355 info = convertWmsEndpointToWms(info); // fall-through
356 case WMS:
357 return new TemplatedWMSTileSource(info, getProjection(info));
358 case WMTS:
359 return new WMTSTileSource(info, getProjection(info));
360 default:
361 throw new UnsupportedOperationException(info.toString());
362 }
363 }
364
365 private static ImageryInfo convertWmsEndpointToWms(ImageryInfo info) throws IOException, WMSGetCapabilitiesException {
366 return Optional.ofNullable(AddImageryLayerAction.getWMSLayerInfo(
367 info, wms -> new LayerSelection(firstLeafLayer(wms.getLayers()), wms.getPreferredFormat(), true)))
368 .orElseThrow(() -> new IllegalStateException("Unable to convert WMS_ENDPOINT to WMS"));
369 }
370
371 private static List<LayerDetails> firstLeafLayer(List<LayerDetails> layers) {
372 for (LayerDetails layer : layers) {
373 boolean hasNoChildren = layer.getChildren().isEmpty();
374 if (hasNoChildren && layer.getName() != null) {
375 return Collections.singletonList(layer);
376 } else if (!hasNoChildren) {
377 return firstLeafLayer(layer.getChildren());
378 }
379 }
380 throw new IllegalArgumentException("Unable to find a valid WMS layer");
381 }
382
383 /**
384 * Test that available imagery entry is valid.
385 */
386 @Test
387 public void testImageryEntryValidity() {
388 checkEntry(info);
389 assertTrue(errors.toString().replaceAll("\\}, ", "\n\\}, ").replaceAll(", ImageryInfo\\{", "\n ,ImageryInfo\\{"),
390 errors.isEmpty());
391 assertFalse(workingURLs.isEmpty());
392 }
393}
Note: See TracBrowser for help on using the repository browser.