source: josm/trunk/test/unit/org/openstreetmap/josm/gui/preferences/imagery/ImageryPreferenceTestIT.java@ 17275

Last change on this file since 17275 was 17275, checked in by Don-vip, 4 years ago

see #16567 - upgrade almost all tests to JUnit 5, except those depending on WiremockRule

See ​https://github.com/tomakehurst/wiremock/issues/684
  • Property svn:eol-style set to native
File size: 17.3 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.gui.preferences.imagery;
3
4import static org.junit.jupiter.api.Assertions.assertFalse;
5import static org.junit.jupiter.api.Assertions.assertTrue;
6import static org.junit.jupiter.api.Assumptions.assumeTrue;
7
8import java.io.ByteArrayInputStream;
9import java.io.IOException;
10import java.net.URL;
11import java.nio.charset.StandardCharsets;
12import java.util.ArrayList;
13import java.util.Collections;
14import java.util.List;
15import java.util.Locale;
16import java.util.Map;
17import java.util.Optional;
18import java.util.TreeMap;
19import java.util.concurrent.TimeUnit;
20import java.util.stream.Collectors;
21import java.util.stream.Stream;
22
23import javax.imageio.ImageIO;
24
25import org.apache.commons.jcs3.access.CacheAccess;
26import org.junit.jupiter.api.AfterAll;
27import org.junit.jupiter.api.BeforeAll;
28import org.junit.jupiter.api.extension.RegisterExtension;
29import org.junit.jupiter.params.ParameterizedTest;
30import org.junit.jupiter.params.provider.Arguments;
31import org.junit.jupiter.params.provider.MethodSource;
32import org.openstreetmap.gui.jmapviewer.Coordinate;
33import org.openstreetmap.gui.jmapviewer.FeatureAdapter;
34import org.openstreetmap.gui.jmapviewer.TileXY;
35import org.openstreetmap.gui.jmapviewer.interfaces.ICoordinate;
36import org.openstreetmap.gui.jmapviewer.tilesources.AbstractTileSource;
37import org.openstreetmap.gui.jmapviewer.tilesources.BingAerialTileSource;
38import org.openstreetmap.gui.jmapviewer.tilesources.ScanexTileSource;
39import org.openstreetmap.josm.TestUtils;
40import org.openstreetmap.josm.actions.AddImageryLayerAction;
41import org.openstreetmap.josm.actions.AddImageryLayerAction.LayerSelection;
42import org.openstreetmap.josm.data.Bounds;
43import org.openstreetmap.josm.data.coor.LatLon;
44import org.openstreetmap.josm.data.imagery.CoordinateConversion;
45import org.openstreetmap.josm.data.imagery.ImageryInfo;
46import org.openstreetmap.josm.data.imagery.ImageryInfo.ImageryBounds;
47import org.openstreetmap.josm.data.imagery.ImageryInfo.ImageryType;
48import org.openstreetmap.josm.data.imagery.ImageryLayerInfo;
49import org.openstreetmap.josm.data.imagery.JosmTemplatedTMSTileSource;
50import org.openstreetmap.josm.data.imagery.LayerDetails;
51import org.openstreetmap.josm.data.imagery.Shape;
52import org.openstreetmap.josm.data.imagery.TMSCachedTileLoaderJob;
53import org.openstreetmap.josm.data.imagery.TemplatedWMSTileSource;
54import org.openstreetmap.josm.data.imagery.TileJobOptions;
55import org.openstreetmap.josm.data.imagery.WMTSTileSource;
56import org.openstreetmap.josm.data.imagery.WMTSTileSource.WMTSGetCapabilitiesException;
57import org.openstreetmap.josm.data.projection.Projection;
58import org.openstreetmap.josm.data.projection.ProjectionRegistry;
59import org.openstreetmap.josm.data.projection.Projections;
60import org.openstreetmap.josm.io.imagery.ApiKeyProvider;
61import org.openstreetmap.josm.io.imagery.WMSImagery.WMSGetCapabilitiesException;
62import org.openstreetmap.josm.testutils.JOSMTestRules;
63import org.openstreetmap.josm.tools.HttpClient;
64import org.openstreetmap.josm.tools.HttpClient.Response;
65import org.openstreetmap.josm.tools.Logging;
66import org.openstreetmap.josm.tools.Utils;
67
68import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
69
70/**
71 * Integration tests of {@link ImageryPreference} class.
72 */
73public class ImageryPreferenceTestIT {
74
75 private static final String ERROR_SEP = " -> ";
76 private static final LatLon GREENWICH = new LatLon(51.47810, -0.00170);
77 private static final int DEFAULT_ZOOM = 12;
78
79 /**
80 * Setup rule
81 */
82 @RegisterExtension
83 @SuppressFBWarnings(value = "URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD")
84 static JOSMTestRules test = new JOSMTestRules().https().i18n().preferences().projection().projectionNadGrids()
85 .timeout((int) TimeUnit.MINUTES.toMillis(40));
86
87 /** Entry to test */
88 private final Map<String, Map<ImageryInfo, List<String>>> errors = Collections.synchronizedMap(new TreeMap<>());
89 private final Map<String, Map<ImageryInfo, List<String>>> ignoredErrors = Collections.synchronizedMap(new TreeMap<>());
90 private static final Map<String, byte[]> workingURLs = Collections.synchronizedMap(new TreeMap<>());
91
92 private static TMSCachedTileLoaderJob helper;
93 private static final List<String> errorsToIgnore = new ArrayList<>();
94 private static final List<String> notIgnoredErrors = new ArrayList<>();
95
96 /**
97 * Setup test
98 * @throws IOException in case of I/O error
99 */
100 @BeforeAll
101 public static void beforeClass() throws IOException {
102 FeatureAdapter.registerApiKeyAdapter(ApiKeyProvider::retrieveApiKey);
103 helper = new TMSCachedTileLoaderJob(null, null, new CacheAccess<>(null), new TileJobOptions(0, 0, null, 0), null);
104 errorsToIgnore.addAll(TestUtils.getIgnoredErrorMessages(ImageryPreferenceTestIT.class));
105 notIgnoredErrors.addAll(errorsToIgnore);
106 }
107
108 /**
109 * Cleanup test
110 */
111 @AfterAll
112 public static void afterClass() {
113 for (String e : notIgnoredErrors) {
114 Logging.warn("Ignore line unused: " + e);
115 }
116 }
117
118 /**
119 * Returns list of imagery entries to test.
120 * @return list of imagery entries to test
121 */
122 public static List<Arguments> data() {
123 ImageryLayerInfo.instance.load(false);
124 return ImageryLayerInfo.instance.getDefaultLayers()
125 .stream()
126 .map(i -> Arguments.of(i.getId(), i))
127 .collect(Collectors.toList());
128 }
129
130 private boolean addError(ImageryInfo info, String error) {
131 String errorMsg = error.replace('\n', ' ');
132 notIgnoredErrors.remove(errorMsg);
133 return addError(isIgnoredError(errorMsg) ? ignoredErrors : errors, info, errorMsg);
134 }
135
136 private static boolean isIgnoredError(String errorMsg) {
137 int idx = errorMsg.lastIndexOf(ERROR_SEP);
138 return isIgnoredSubstring(errorMsg) || (idx > -1 && isIgnoredSubstring(errorMsg.substring(idx + ERROR_SEP.length())));
139 }
140
141 private static boolean isIgnoredSubstring(String substring) {
142 return errorsToIgnore.parallelStream().anyMatch(x -> substring.contains(x));
143 }
144
145 private static boolean addError(Map<String, Map<ImageryInfo, List<String>>> map, ImageryInfo info, String errorMsg) {
146 return map.computeIfAbsent(info.getCountryCode(), x -> Collections.synchronizedMap(new TreeMap<>()))
147 .computeIfAbsent(info, x -> Collections.synchronizedList(new ArrayList<>()))
148 .add(errorMsg);
149 }
150
151 private Optional<byte[]> checkUrl(ImageryInfo info, String url) {
152 if (url != null && !url.isEmpty()) {
153 if (workingURLs.containsKey(url)) {
154 return Optional.of(workingURLs.get(url));
155 }
156 try {
157 Response response = HttpClient.create(new URL(url))
158 .setHeaders(info.getCustomHttpHeaders())
159 .setConnectTimeout((int) TimeUnit.MINUTES.toMillis(1))
160 .setReadTimeout((int) TimeUnit.MINUTES.toMillis(5))
161 .connect();
162 if (response.getResponseCode() >= 400) {
163 addError(info, url + " -> HTTP " + response.getResponseCode());
164 } else if (response.getResponseCode() >= 300) {
165 Logging.warn(url + " -> HTTP " + response.getResponseCode());
166 }
167 try {
168 byte[] data = Utils.readBytesFromStream(response.getContent());
169 if (response.getResponseCode() < 300) {
170 workingURLs.put(url, data);
171 }
172 return Optional.of(data);
173 } catch (IOException e) {
174 if (response.getResponseCode() < 300) {
175 addError(info, url + ERROR_SEP + e);
176 }
177 } finally {
178 response.disconnect();
179 }
180 } catch (IOException e) {
181 addError(info, url + ERROR_SEP + e);
182 }
183 }
184 return Optional.empty();
185 }
186
187 private void checkLinkUrl(ImageryInfo info, String url) {
188 checkUrl(info, url).filter(x -> x.length == 0).ifPresent(x -> addError(info, url + " -> returned empty contents"));
189 }
190
191 private String checkTileUrl(ImageryInfo info, AbstractTileSource tileSource, ICoordinate center, int zoom)
192 throws IOException {
193 TileXY xy = tileSource.latLonToTileXY(center, zoom);
194 for (int i = 0; i < 3; i++) {
195 try {
196 String url = tileSource.getTileUrl(zoom, xy.getXIndex(), xy.getYIndex());
197 Optional<byte[]> optional = checkUrl(info, url);
198 String error = "";
199 if (optional.isPresent()) {
200 byte[] data = optional.get();
201 try (ByteArrayInputStream bais = new ByteArrayInputStream(data)) {
202 if (ImageIO.read(bais) == null) {
203 error = addImageError(info, url, data, zoom, "did not return an image");
204 }
205 } catch (IOException e) {
206 error = addImageError(info, url, data, zoom, e.toString());
207 Logging.trace(e);
208 }
209 }
210 return error;
211 } catch (IOException e) {
212 // Try up to three times max to allow Bing source to initialize itself
213 // and avoid random network errors
214 Logging.trace(e);
215 if (i == 2) {
216 throw e;
217 }
218 try {
219 Thread.sleep(500);
220 } catch (InterruptedException ex) {
221 Logging.warn(ex);
222 }
223 }
224 }
225 return "";
226 }
227
228 private static String zoomMarker(int zoom) {
229 return " -> zoom " + zoom + ERROR_SEP;
230 }
231
232 private String addImageError(ImageryInfo info, String url, byte[] data, int zoom, String defaultMessage) {
233 // Check if we have received an error message
234 String error = helper.detectErrorMessage(new String(data, StandardCharsets.UTF_8));
235 String errorMsg = url + zoomMarker(zoom) + (error != null ? error.split("\\n", -1)[0] : defaultMessage);
236 addError(info, errorMsg);
237 return errorMsg;
238 }
239
240 private static LatLon getPointInShape(Shape shape) {
241 final Coordinate p1 = shape.getPoints().get(0);
242 final Bounds bounds = new Bounds(p1.getLat(), p1.getLon(), p1.getLat(), p1.getLon());
243 shape.getPoints().forEach(p -> bounds.extend(p.getLat(), p.getLon()));
244
245 final double w = bounds.getWidth();
246 final double h = bounds.getHeight();
247
248 final double x2 = bounds.getMinLon() + (w / 2.0);
249 final double y2 = bounds.getMinLat() + (h / 2.0);
250
251 final LatLon center = new LatLon(y2, x2);
252
253 // check to see if center is inside shape
254 if (shape.contains(center)) {
255 return center;
256 }
257
258 // if center position (C) is not inside shape, try naively some other positions as follows:
259 final double x1 = bounds.getMinLon() + (.25 * w);
260 final double x3 = bounds.getMinLon() + (.75 * w);
261 final double y1 = bounds.getMinLat() + (.25 * h);
262 final double y3 = bounds.getMinLat() + (.75 * h);
263 // +-----------+
264 // | 5 1 6 |
265 // | 4 C 2 |
266 // | 8 3 7 |
267 // +-----------+
268 return Stream.of(
269 new LatLon(y1, x2),
270 new LatLon(y2, x3),
271 new LatLon(y3, x2),
272 new LatLon(y2, x1),
273 new LatLon(y1, x1),
274 new LatLon(y1, x3),
275 new LatLon(y3, x3),
276 new LatLon(y3, x1)
277 ).filter(shape::contains).findFirst().orElse(center);
278 }
279
280 private static LatLon getCenter(ImageryBounds bounds) {
281 List<Shape> shapes = bounds.getShapes();
282 return shapes != null && !shapes.isEmpty() ? getPointInShape(shapes.get(0)) : bounds.getCenter();
283 }
284
285 private void checkEntry(ImageryInfo info) {
286 Logging.info("Checking "+ info);
287
288 if (info.getAttributionImageRaw() != null && info.getAttributionImage() == null) {
289 addError(info, "Can't fetch attribution image: " + info.getAttributionImageRaw());
290 }
291
292 checkLinkUrl(info, info.getAttributionImageURL());
293 checkLinkUrl(info, info.getAttributionLinkURL());
294 String eula = info.getEulaAcceptanceRequired();
295 if (eula != null) {
296 checkLinkUrl(info, eula.replaceAll("\\{lang\\}", ""));
297 }
298 checkLinkUrl(info, info.getPrivacyPolicyURL());
299 checkLinkUrl(info, info.getPermissionReferenceURL());
300 checkLinkUrl(info, info.getTermsOfUseURL());
301 if (info.getUrl().contains("{time}")) {
302 info.setDate("2020-01-01T00:00:00Z/2020-01-02T00:00:00Z");
303 }
304
305 try {
306 ImageryBounds bounds = info.getBounds();
307 // Some imagery sources do not define tiles at (0,0). So pickup Greenwich Royal Observatory for global sources
308 ICoordinate center = CoordinateConversion.llToCoor(bounds != null ? getCenter(bounds) : GREENWICH);
309 AbstractTileSource tileSource = getTileSource(info);
310 // test min zoom and try to detect the correct value in case of error
311 int maxZoom = info.getMaxZoom() > 0 ? Math.min(DEFAULT_ZOOM, info.getMaxZoom()) : DEFAULT_ZOOM;
312 for (int zoom = info.getMinZoom(); zoom < maxZoom; zoom++) {
313 if (!isZoomError(checkTileUrl(info, tileSource, center, zoom))) {
314 break;
315 }
316 }
317 // checking max zoom for real is complex, see https://josm.openstreetmap.de/ticket/16073#comment:27
318 if (info.getMaxZoom() > 0 && info.getImageryType() != ImageryType.SCANEX) {
319 checkTileUrl(info, tileSource, center, Utils.clamp(DEFAULT_ZOOM, info.getMinZoom() + 1, info.getMaxZoom()));
320 }
321 } catch (IOException | RuntimeException | WMSGetCapabilitiesException | WMTSGetCapabilitiesException e) {
322 addError(info, info.getUrl() + ERROR_SEP + e.toString());
323 }
324
325 for (ImageryInfo mirror : info.getMirrors()) {
326 checkEntry(mirror);
327 }
328 }
329
330 private static boolean isZoomError(String error) {
331 String[] parts = error.split(ERROR_SEP, -1);
332 String lastPart = parts.length > 0 ? parts[parts.length - 1].toLowerCase(Locale.ENGLISH) : "";
333 return lastPart.contains("bbox")
334 || lastPart.contains("bounding box");
335 }
336
337 private static Projection getProjection(ImageryInfo info) {
338 for (String code : info.getServerProjections()) {
339 Projection proj = Projections.getProjectionByCode(code);
340 if (proj != null) {
341 return proj;
342 }
343 }
344 return ProjectionRegistry.getProjection();
345 }
346
347 @SuppressWarnings("fallthrough")
348 private static AbstractTileSource getTileSource(ImageryInfo info)
349 throws IOException, WMTSGetCapabilitiesException, WMSGetCapabilitiesException {
350 switch (info.getImageryType()) {
351 case BING:
352 return new BingAerialTileSource(info);
353 case SCANEX:
354 return new ScanexTileSource(info);
355 case TMS:
356 return new JosmTemplatedTMSTileSource(info);
357 case WMS_ENDPOINT:
358 info = convertWmsEndpointToWms(info); // fall-through
359 case WMS:
360 return new TemplatedWMSTileSource(info, getProjection(info));
361 case WMTS:
362 return new WMTSTileSource(info, getProjection(info));
363 default:
364 throw new UnsupportedOperationException(info.toString());
365 }
366 }
367
368 private static ImageryInfo convertWmsEndpointToWms(ImageryInfo info) throws IOException, WMSGetCapabilitiesException {
369 return Optional.ofNullable(AddImageryLayerAction.getWMSLayerInfo(
370 info, wms -> new LayerSelection(firstLeafLayer(wms.getLayers()), wms.getPreferredFormat(), true)))
371 .orElseThrow(() -> new IllegalStateException("Unable to convert WMS_ENDPOINT to WMS"));
372 }
373
374 private static List<LayerDetails> firstLeafLayer(List<LayerDetails> layers) {
375 for (LayerDetails layer : layers) {
376 boolean hasNoChildren = layer.getChildren().isEmpty();
377 if (hasNoChildren && layer.getName() != null) {
378 return Collections.singletonList(layer);
379 } else if (!hasNoChildren) {
380 return firstLeafLayer(layer.getChildren());
381 }
382 }
383 throw new IllegalArgumentException("Unable to find a valid WMS layer");
384 }
385
386 private static String format(Map<String, Map<ImageryInfo, List<String>>> map) {
387 return map.toString().replaceAll("\\}, ", "\n\\}, ").replaceAll(", ImageryInfo\\{", "\n ,ImageryInfo\\{");
388 }
389
390 /**
391 * Test that available imagery entry is valid.
392 *
393 * @param id The id of the imagery info to show as the test name
394 * @param info The imagery info to test
395 */
396 @ParameterizedTest(name = "{0}")
397 @MethodSource("data")
398 void testImageryEntryValidity(String id, ImageryInfo info) {
399 checkEntry(info);
400 assertTrue(errors.isEmpty(), format(errors));
401 assertFalse(workingURLs.isEmpty());
402 assumeTrue(ignoredErrors.isEmpty(), format(ignoredErrors));
403 }
404}
Note: See TracBrowser for help on using the repository browser.