source: josm/trunk/test/unit/org/openstreetmap/josm/gui/preferences/imagery/ImageryPreferenceTestIT.java@ 17474

Last change on this file since 17474 was 17474, checked in by Don-vip, 3 years ago

test all WMS projections
  • Property svn:eol-style set to native
File size: 18.9 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.gui.preferences.imagery;
3
4import static java.util.Collections.singletonList;
5import static java.util.Collections.synchronizedList;
6import static java.util.Collections.synchronizedMap;
7import static java.util.stream.Collectors.toList;
8import static org.junit.jupiter.api.Assertions.assertFalse;
9import static org.junit.jupiter.api.Assertions.assertTrue;
10import static org.junit.jupiter.api.Assumptions.assumeTrue;
11
12import java.io.ByteArrayInputStream;
13import java.io.IOException;
14import java.net.URL;
15import java.nio.charset.StandardCharsets;
16import java.util.ArrayList;
17import java.util.List;
18import java.util.Locale;
19import java.util.Map;
20import java.util.Objects;
21import java.util.Optional;
22import java.util.TreeMap;
23import java.util.concurrent.TimeUnit;
24import java.util.stream.Stream;
25
26import javax.imageio.ImageIO;
27
28import org.apache.commons.jcs3.access.CacheAccess;
29import org.junit.jupiter.api.AfterAll;
30import org.junit.jupiter.api.BeforeAll;
31import org.junit.jupiter.api.extension.RegisterExtension;
32import org.junit.jupiter.params.ParameterizedTest;
33import org.junit.jupiter.params.provider.Arguments;
34import org.junit.jupiter.params.provider.MethodSource;
35import org.openstreetmap.gui.jmapviewer.Coordinate;
36import org.openstreetmap.gui.jmapviewer.FeatureAdapter;
37import org.openstreetmap.gui.jmapviewer.TileXY;
38import org.openstreetmap.gui.jmapviewer.interfaces.ICoordinate;
39import org.openstreetmap.gui.jmapviewer.tilesources.AbstractTileSource;
40import org.openstreetmap.gui.jmapviewer.tilesources.BingAerialTileSource;
41import org.openstreetmap.gui.jmapviewer.tilesources.ScanexTileSource;
42import org.openstreetmap.josm.TestUtils;
43import org.openstreetmap.josm.actions.AddImageryLayerAction;
44import org.openstreetmap.josm.actions.AddImageryLayerAction.LayerSelection;
45import org.openstreetmap.josm.data.Bounds;
46import org.openstreetmap.josm.data.coor.LatLon;
47import org.openstreetmap.josm.data.imagery.CoordinateConversion;
48import org.openstreetmap.josm.data.imagery.ImageryInfo;
49import org.openstreetmap.josm.data.imagery.ImageryInfo.ImageryBounds;
50import org.openstreetmap.josm.data.imagery.ImageryInfo.ImageryType;
51import org.openstreetmap.josm.data.imagery.ImageryLayerInfo;
52import org.openstreetmap.josm.data.imagery.JosmTemplatedTMSTileSource;
53import org.openstreetmap.josm.data.imagery.LayerDetails;
54import org.openstreetmap.josm.data.imagery.Shape;
55import org.openstreetmap.josm.data.imagery.TMSCachedTileLoaderJob;
56import org.openstreetmap.josm.data.imagery.TemplatedWMSTileSource;
57import org.openstreetmap.josm.data.imagery.TileJobOptions;
58import org.openstreetmap.josm.data.imagery.WMTSTileSource;
59import org.openstreetmap.josm.data.imagery.WMTSTileSource.WMTSGetCapabilitiesException;
60import org.openstreetmap.josm.data.projection.Projection;
61import org.openstreetmap.josm.data.projection.ProjectionRegistry;
62import org.openstreetmap.josm.data.projection.Projections;
63import org.openstreetmap.josm.io.imagery.ApiKeyProvider;
64import org.openstreetmap.josm.io.imagery.WMSImagery.WMSGetCapabilitiesException;
65import org.openstreetmap.josm.testutils.JOSMTestRules;
66import org.openstreetmap.josm.tools.HttpClient;
67import org.openstreetmap.josm.tools.HttpClient.Response;
68import org.openstreetmap.josm.tools.Logging;
69import org.openstreetmap.josm.tools.Utils;
70
71import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
72
73/**
74 * Integration tests of {@link ImageryPreference} class.
75 */
76public class ImageryPreferenceTestIT {
77
78 private static final String ERROR_SEP = " -> ";
79 private static final LatLon GREENWICH = new LatLon(51.47810, -0.00170);
80 private static final int DEFAULT_ZOOM = 12;
81
82 /**
83 * Setup rule
84 */
85 @RegisterExtension
86 @SuppressFBWarnings(value = "URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD")
87 static JOSMTestRules test = new JOSMTestRules().https().i18n().preferences().projection().projectionNadGrids()
88 .timeout((int) TimeUnit.MINUTES.toMillis(40));
89
90 /** Entry to test */
91 private final Map<String, Map<ImageryInfo, List<String>>> errors = synchronizedMap(new TreeMap<>());
92 private final Map<String, Map<ImageryInfo, List<String>>> ignoredErrors = synchronizedMap(new TreeMap<>());
93 private static final Map<String, byte[]> workingURLs = synchronizedMap(new TreeMap<>());
94
95 private static TMSCachedTileLoaderJob helper;
96 private static final List<String> errorsToIgnore = new ArrayList<>();
97 private static final List<String> notIgnoredErrors = new ArrayList<>();
98
99 /**
100 * Setup test
101 * @throws IOException in case of I/O error
102 */
103 @BeforeAll
104 public static void beforeClass() throws IOException {
105 FeatureAdapter.registerApiKeyAdapter(ApiKeyProvider::retrieveApiKey);
106 helper = new TMSCachedTileLoaderJob(null, null, new CacheAccess<>(null), new TileJobOptions(0, 0, null, 0), null);
107 errorsToIgnore.addAll(TestUtils.getIgnoredErrorMessages(ImageryPreferenceTestIT.class));
108 notIgnoredErrors.addAll(errorsToIgnore);
109 }
110
111 /**
112 * Cleanup test
113 */
114 @AfterAll
115 public static void afterClass() {
116 for (String e : notIgnoredErrors) {
117 Logging.warn("Ignore line unused: " + e);
118 }
119 }
120
121 /**
122 * Returns list of imagery entries to test.
123 * @return list of imagery entries to test
124 */
125 public static List<Arguments> data() {
126 ImageryLayerInfo.instance.load(false);
127 return ImageryLayerInfo.instance.getDefaultLayers()
128 .stream()
129 //.filter(i -> "OGDLidarZH-DOM-2017".equals(i.getId())) // enable to test one specific entry
130 .map(i -> Arguments.of(i.getCountryCode().isEmpty() ? i.getId() : i.getCountryCode() + '-' + i.getId(), i))
131 .collect(toList());
132 }
133
134 private boolean addError(ImageryInfo info, String error) {
135 String errorMsg = error.replace('\n', ' ');
136 notIgnoredErrors.remove(errorMsg);
137 return addError(isIgnoredError(errorMsg) ? ignoredErrors : errors, info, errorMsg);
138 }
139
140 private static boolean isIgnoredError(String errorMsg) {
141 int idx = errorMsg.lastIndexOf(ERROR_SEP);
142 return isIgnoredSubstring(errorMsg) || (idx > -1 && isIgnoredSubstring(errorMsg.substring(idx + ERROR_SEP.length())));
143 }
144
145 private static boolean isIgnoredSubstring(String substring) {
146 return errorsToIgnore.parallelStream().anyMatch(x -> substring.contains(x));
147 }
148
149 private static boolean addError(Map<String, Map<ImageryInfo, List<String>>> map, ImageryInfo info, String errorMsg) {
150 return map.computeIfAbsent(info.getCountryCode(), x -> synchronizedMap(new TreeMap<>()))
151 .computeIfAbsent(info, x -> synchronizedList(new ArrayList<>()))
152 .add(errorMsg);
153 }
154
155 private Optional<byte[]> checkUrl(ImageryInfo info, String url) {
156 if (url != null && !url.isEmpty()) {
157 if (workingURLs.containsKey(url)) {
158 return Optional.of(workingURLs.get(url));
159 }
160 try {
161 Response response = HttpClient.create(new URL(url))
162 .setHeaders(info.getCustomHttpHeaders())
163 .setConnectTimeout((int) TimeUnit.MINUTES.toMillis(1))
164 .setReadTimeout((int) TimeUnit.MINUTES.toMillis(5))
165 .connect();
166 if (response.getResponseCode() >= 400) {
167 addError(info, url + " -> HTTP " + response.getResponseCode());
168 } else if (response.getResponseCode() >= 300) {
169 Logging.warn(url + " -> HTTP " + response.getResponseCode());
170 }
171 try {
172 byte[] data = Utils.readBytesFromStream(response.getContent());
173 if (response.getResponseCode() < 300) {
174 workingURLs.put(url, data);
175 }
176 return Optional.of(data);
177 } catch (IOException e) {
178 if (response.getResponseCode() < 300) {
179 addError(info, url + ERROR_SEP + e);
180 }
181 } finally {
182 response.disconnect();
183 }
184 } catch (IOException e) {
185 addError(info, url + ERROR_SEP + e);
186 }
187 }
188 return Optional.empty();
189 }
190
191 private void checkLinkUrl(ImageryInfo info, String url) {
192 checkUrl(info, url).filter(x -> x.length == 0).ifPresent(x -> addError(info, url + " -> returned empty contents"));
193 }
194
195 private List<String> checkTileUrls(ImageryInfo info, List<AbstractTileSource> tileSources, ICoordinate center, int zoom)
196 throws IOException {
197 List<String> errors = new ArrayList<>();
198 for (AbstractTileSource tileSource : tileSources) {
199 String error = checkTileUrl(info, tileSource, center, zoom);
200 if (!error.isEmpty()) {
201 errors.add(error);
202 }
203 }
204 return errors;
205 }
206
207 private String checkTileUrl(ImageryInfo info, AbstractTileSource tileSource, ICoordinate center, int zoom)
208 throws IOException {
209 TileXY xy = tileSource.latLonToTileXY(center, zoom);
210 for (int i = 0; i < 3; i++) {
211 try {
212 String url = tileSource.getTileUrl(zoom, xy.getXIndex(), xy.getYIndex());
213 Optional<byte[]> optional = checkUrl(info, url);
214 String error = "";
215 if (optional.isPresent()) {
216 byte[] data = optional.get();
217 try (ByteArrayInputStream bais = new ByteArrayInputStream(data)) {
218 if (ImageIO.read(bais) == null) {
219 error = addImageError(info, url, data, zoom, "did not return an image");
220 }
221 } catch (IOException e) {
222 error = addImageError(info, url, data, zoom, e.toString());
223 Logging.trace(e);
224 }
225 }
226 return error;
227 } catch (IOException e) {
228 // Try up to three times max to allow Bing source to initialize itself
229 // and avoid random network errors
230 Logging.trace(e);
231 if (i == 2) {
232 throw e;
233 }
234 try {
235 Thread.sleep(500);
236 } catch (InterruptedException ex) {
237 Logging.warn(ex);
238 }
239 }
240 }
241 return "";
242 }
243
244 private static String zoomMarker(int zoom) {
245 return " -> zoom " + zoom + ERROR_SEP;
246 }
247
248 private String addImageError(ImageryInfo info, String url, byte[] data, int zoom, String defaultMessage) {
249 // Check if we have received an error message
250 String error = helper.detectErrorMessage(new String(data, StandardCharsets.UTF_8));
251 String errorMsg = url + zoomMarker(zoom) + (error != null ? error.split("\\n", -1)[0] : defaultMessage);
252 addError(info, errorMsg);
253 return errorMsg;
254 }
255
256 private static LatLon getPointInShape(Shape shape) {
257 final Coordinate p1 = shape.getPoints().get(0);
258 final Bounds bounds = new Bounds(p1.getLat(), p1.getLon(), p1.getLat(), p1.getLon());
259 shape.getPoints().forEach(p -> bounds.extend(p.getLat(), p.getLon()));
260
261 final double w = bounds.getWidth();
262 final double h = bounds.getHeight();
263
264 final double x2 = bounds.getMinLon() + (w / 2.0);
265 final double y2 = bounds.getMinLat() + (h / 2.0);
266
267 final LatLon center = new LatLon(y2, x2);
268
269 // check to see if center is inside shape
270 if (shape.contains(center)) {
271 return center;
272 }
273
274 // if center position (C) is not inside shape, try naively some other positions as follows:
275 final double x1 = bounds.getMinLon() + (.25 * w);
276 final double x3 = bounds.getMinLon() + (.75 * w);
277 final double y1 = bounds.getMinLat() + (.25 * h);
278 final double y3 = bounds.getMinLat() + (.75 * h);
279 // +-----------+
280 // | 5 1 6 |
281 // | 4 C 2 |
282 // | 8 3 7 |
283 // +-----------+
284 return Stream.of(
285 new LatLon(y1, x2),
286 new LatLon(y2, x3),
287 new LatLon(y3, x2),
288 new LatLon(y2, x1),
289 new LatLon(y1, x1),
290 new LatLon(y1, x3),
291 new LatLon(y3, x3),
292 new LatLon(y3, x1)
293 ).filter(shape::contains).findFirst().orElse(center);
294 }
295
296 private static LatLon getCenter(ImageryBounds bounds) {
297 List<Shape> shapes = bounds.getShapes();
298 return shapes != null && !shapes.isEmpty() ? getPointInShape(shapes.get(0)) : bounds.getCenter();
299 }
300
301 private void checkEntry(ImageryInfo info) {
302 Logging.info("Checking "+ info);
303
304 if (info.getAttributionImageRaw() != null && info.getAttributionImage() == null) {
305 addError(info, "Can't fetch attribution image: " + info.getAttributionImageRaw());
306 }
307
308 checkLinkUrl(info, info.getAttributionImageURL());
309 checkLinkUrl(info, info.getAttributionLinkURL());
310 String eula = info.getEulaAcceptanceRequired();
311 if (eula != null) {
312 checkLinkUrl(info, eula.replaceAll("\\{lang\\}", ""));
313 }
314 checkLinkUrl(info, info.getPrivacyPolicyURL());
315 checkLinkUrl(info, info.getPermissionReferenceURL());
316 checkLinkUrl(info, info.getTermsOfUseURL());
317 if (info.getUrl().contains("{time}")) {
318 info.setDate("2020-01-01T00:00:00Z/2020-01-02T00:00:00Z");
319 }
320
321 try {
322 ImageryBounds bounds = info.getBounds();
323 // Some imagery sources do not define tiles at (0,0). So pickup Greenwich Royal Observatory for global sources
324 ICoordinate center = CoordinateConversion.llToCoor(bounds != null ? getCenter(bounds) : GREENWICH);
325 List<AbstractTileSource> tileSources = getTileSources(info);
326 // test min zoom and try to detect the correct value in case of error
327 int maxZoom = info.getMaxZoom() > 0 ? Math.min(DEFAULT_ZOOM, info.getMaxZoom()) : DEFAULT_ZOOM;
328 for (int zoom = info.getMinZoom(); zoom < maxZoom; zoom++) {
329 if (!isZoomError(checkTileUrls(info, tileSources, center, zoom))) {
330 break;
331 }
332 }
333 // checking max zoom for real is complex, see https://josm.openstreetmap.de/ticket/16073#comment:27
334 if (info.getMaxZoom() > 0 && info.getImageryType() != ImageryType.SCANEX) {
335 checkTileUrls(info, tileSources, center, Utils.clamp(DEFAULT_ZOOM, info.getMinZoom() + 1, info.getMaxZoom()));
336 }
337 } catch (IOException | RuntimeException | WMSGetCapabilitiesException e) {
338 addError(info, info.getUrl() + ERROR_SEP + e.toString());
339 }
340
341 for (ImageryInfo mirror : info.getMirrors()) {
342 checkEntry(mirror);
343 }
344 }
345
346 private static boolean isZoomError(List<String> errors) {
347 return errors.stream().anyMatch(error -> {
348 String[] parts = error.split(ERROR_SEP, -1);
349 String lastPart = parts.length > 0 ? parts[parts.length - 1].toLowerCase(Locale.ENGLISH) : "";
350 return lastPart.contains("bbox")
351 || lastPart.contains("bounding box");
352 });
353 }
354
355 private static List<Projection> getProjections(ImageryInfo info) {
356 List<Projection> projs = info.getServerProjections().stream()
357 .map(Projections::getProjectionByCode).filter(Objects::nonNull).collect(toList());
358 return projs.isEmpty() ? singletonList(ProjectionRegistry.getProjection()) : projs;
359 }
360
361 private List<AbstractTileSource> getTileSources(ImageryInfo info)
362 throws IOException, WMSGetCapabilitiesException {
363 switch (info.getImageryType()) {
364 case BING:
365 return singletonList(new BingAerialTileSource(info));
366 case SCANEX:
367 return singletonList(new ScanexTileSource(info));
368 case TMS:
369 return singletonList(new JosmTemplatedTMSTileSource(info));
370 case WMS_ENDPOINT:
371 return getWmsTileSources(convertWmsEndpointToWms(info));
372 case WMS:
373 return getWmsTileSources(info);
374 case WMTS:
375 return getWmtsTileSources(info);
376 default:
377 throw new UnsupportedOperationException(info.toString());
378 }
379 }
380
381 private static List<AbstractTileSource> getWmsTileSources(ImageryInfo info) {
382 return getProjections(info).stream().map(proj -> new TemplatedWMSTileSource(info, proj)).collect(toList());
383 }
384
385 private List<AbstractTileSource> getWmtsTileSources(ImageryInfo info) {
386 return getProjections(info).stream().map(proj -> {
387 try {
388 return new WMTSTileSource(info, proj);
389 } catch (IOException | WMTSGetCapabilitiesException e) {
390 addError(info, info.getUrl() + ERROR_SEP + e.toString());
391 return null;
392 }
393 }).filter(Objects::nonNull).collect(toList());
394 }
395
396 private static ImageryInfo convertWmsEndpointToWms(ImageryInfo info) throws IOException, WMSGetCapabilitiesException {
397 return Optional.ofNullable(AddImageryLayerAction.getWMSLayerInfo(
398 info, wms -> new LayerSelection(firstLeafLayer(wms.getLayers()), wms.getPreferredFormat(), true)))
399 .orElseThrow(() -> new IllegalStateException("Unable to convert WMS_ENDPOINT to WMS"));
400 }
401
402 private static List<LayerDetails> firstLeafLayer(List<LayerDetails> layers) {
403 for (LayerDetails layer : layers) {
404 boolean hasNoChildren = layer.getChildren().isEmpty();
405 if (hasNoChildren && layer.getName() != null) {
406 return singletonList(layer);
407 } else if (!hasNoChildren) {
408 return firstLeafLayer(layer.getChildren());
409 }
410 }
411 throw new IllegalArgumentException("Unable to find a valid WMS layer");
412 }
413
414 private static String format(String id, Map<String, Map<ImageryInfo, List<String>>> map) {
415 // #16567 - Shouldn't be necessary to print id if Ant worked properly
416 // See https://josm.openstreetmap.de/ticket/16567#comment:53
417 // See https://bz.apache.org/bugzilla/show_bug.cgi?id=64564
418 // See https://github.com/apache/ant/pull/121
419 return id + " => " + map.toString().replaceAll("\\}, ", "\n\\}, ").replaceAll(", ImageryInfo\\{", "\n ,ImageryInfo\\{");
420 }
421
422 /**
423 * Test that available imagery entry is valid.
424 *
425 * @param id The id of the imagery info to show as the test name
426 * @param info The imagery info to test
427 */
428 @ParameterizedTest(name = "{0}")
429 @MethodSource("data")
430 void testImageryEntryValidity(String id, ImageryInfo info) {
431 checkEntry(info);
432 assertTrue(errors.isEmpty(), format(id, errors));
433 assertFalse(workingURLs.isEmpty());
434 assumeTrue(ignoredErrors.isEmpty(), format(id, ignoredErrors));
435 }
436}
Note: See TracBrowser for help on using the repository browser.