source: josm/trunk/test/unit/org/openstreetmap/josm/gui/preferences/imagery/ImageryPreferenceTestIT.java@ 16128

Last change on this file since 16128 was 16128, checked in by Don-vip, 4 years ago

see #17285 - add privacy-policy-url in mirror, check links in integration test
  • Property svn:eol-style set to native
File size: 17.5 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.gui.preferences.imagery;
3
4import static org.junit.Assert.assertFalse;
5import static org.junit.Assert.assertTrue;
6import static org.junit.Assume.assumeTrue;
7
8import java.io.ByteArrayInputStream;
9import java.io.IOException;
10import java.net.URL;
11import java.nio.charset.StandardCharsets;
12import java.util.ArrayList;
13import java.util.Collections;
14import java.util.List;
15import java.util.Locale;
16import java.util.Map;
17import java.util.Objects;
18import java.util.Optional;
19import java.util.TreeMap;
20import java.util.concurrent.TimeUnit;
21import java.util.stream.Collectors;
22
23import javax.imageio.ImageIO;
24
25import org.apache.commons.jcs.access.CacheAccess;
26import org.junit.AfterClass;
27import org.junit.BeforeClass;
28import org.junit.ClassRule;
29import org.junit.Test;
30import org.junit.runner.RunWith;
31import org.junit.runners.Parameterized.Parameters;
32import org.openstreetmap.gui.jmapviewer.Coordinate;
33import org.openstreetmap.gui.jmapviewer.FeatureAdapter;
34import org.openstreetmap.gui.jmapviewer.TileXY;
35import org.openstreetmap.gui.jmapviewer.interfaces.ICoordinate;
36import org.openstreetmap.gui.jmapviewer.tilesources.AbstractTileSource;
37import org.openstreetmap.gui.jmapviewer.tilesources.BingAerialTileSource;
38import org.openstreetmap.gui.jmapviewer.tilesources.ScanexTileSource;
39import org.openstreetmap.josm.TestUtils;
40import org.openstreetmap.josm.actions.AddImageryLayerAction;
41import org.openstreetmap.josm.actions.AddImageryLayerAction.LayerSelection;
42import org.openstreetmap.josm.data.Bounds;
43import org.openstreetmap.josm.data.coor.LatLon;
44import org.openstreetmap.josm.data.imagery.CoordinateConversion;
45import org.openstreetmap.josm.data.imagery.ImageryInfo;
46import org.openstreetmap.josm.data.imagery.ImageryInfo.ImageryBounds;
47import org.openstreetmap.josm.data.imagery.ImageryInfo.ImageryType;
48import org.openstreetmap.josm.data.imagery.ImageryLayerInfo;
49import org.openstreetmap.josm.data.imagery.JosmTemplatedTMSTileSource;
50import org.openstreetmap.josm.data.imagery.LayerDetails;
51import org.openstreetmap.josm.data.imagery.Shape;
52import org.openstreetmap.josm.data.imagery.TMSCachedTileLoaderJob;
53import org.openstreetmap.josm.data.imagery.TemplatedWMSTileSource;
54import org.openstreetmap.josm.data.imagery.TileJobOptions;
55import org.openstreetmap.josm.data.imagery.WMTSTileSource;
56import org.openstreetmap.josm.data.imagery.WMTSTileSource.WMTSGetCapabilitiesException;
57import org.openstreetmap.josm.data.projection.Projection;
58import org.openstreetmap.josm.data.projection.ProjectionRegistry;
59import org.openstreetmap.josm.data.projection.Projections;
60import org.openstreetmap.josm.io.imagery.ApiKeyProvider;
61import org.openstreetmap.josm.io.imagery.WMSImagery.WMSGetCapabilitiesException;
62import org.openstreetmap.josm.testutils.JOSMTestRules;
63import org.openstreetmap.josm.testutils.ParallelParameterized;
64import org.openstreetmap.josm.tools.HttpClient;
65import org.openstreetmap.josm.tools.HttpClient.Response;
66import org.openstreetmap.josm.tools.Logging;
67import org.openstreetmap.josm.tools.Utils;
68
69import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
70
71/**
72 * Integration tests of {@link ImageryPreference} class.
73 */
74@RunWith(ParallelParameterized.class)
75public class ImageryPreferenceTestIT {
76
77 private static final String ERROR_SEP = " -> ";
78 private static final LatLon GREENWICH = new LatLon(51.47810, -0.00170);
79 private static final int DEFAULT_ZOOM = 12;
80
81 /**
82 * Setup rule
83 */
84 @ClassRule
85 @SuppressFBWarnings(value = "URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD")
86 public static JOSMTestRules test = new JOSMTestRules().https().i18n().preferences().projection().projectionNadGrids()
87 .timeout((int) TimeUnit.MINUTES.toMillis(40)).parameters();
88
89 /** Entry to test */
90 private final ImageryInfo info;
91 private final Map<String, Map<ImageryInfo, List<String>>> errors = Collections.synchronizedMap(new TreeMap<>());
92 private final Map<String, Map<ImageryInfo, List<String>>> ignoredErrors = Collections.synchronizedMap(new TreeMap<>());
93 private static final Map<String, byte[]> workingURLs = Collections.synchronizedMap(new TreeMap<>());
94
95 private static TMSCachedTileLoaderJob helper;
96 private static final List<String> errorsToIgnore = new ArrayList<>();
97 private static final List<String> notIgnoredErrors = new ArrayList<>();
98
99 /**
100 * Setup test
101 * @throws IOException in case of I/O error
102 */
103 @BeforeClass
104 public static void beforeClass() throws IOException {
105 FeatureAdapter.registerApiKeyAdapter(ApiKeyProvider::retrieveApiKey);
106 helper = new TMSCachedTileLoaderJob(null, null, new CacheAccess<>(null), new TileJobOptions(0, 0, null, 0), null);
107 errorsToIgnore.addAll(TestUtils.getIgnoredErrorMessages(ImageryPreferenceTestIT.class));
108 notIgnoredErrors.addAll(errorsToIgnore);
109 }
110
111 /**
112 * Cleanup test
113 */
114 @AfterClass
115 public static void afterClass() {
116 for (String e : notIgnoredErrors) {
117 Logging.warn("Ignore line unused: " + e);
118 }
119 }
120
121 /**
122 * Returns list of imagery entries to test.
123 * @return list of imagery entries to test
124 */
125 @Parameters(name = "{0}")
126 public static List<Object[]> data() {
127 ImageryLayerInfo.instance.load(false);
128 return ImageryLayerInfo.instance.getDefaultLayers()
129 .stream()
130 .map(x -> new Object[] {x.getId(), x})
131 .collect(Collectors.toList());
132 }
133
134 /**
135 * Constructs a new {@code ImageryPreferenceTestIT} instance.
136 * @param id entry ID, used only to name tests
137 * @param info entry to test
138 */
139 public ImageryPreferenceTestIT(String id, ImageryInfo info) {
140 this.info = Objects.requireNonNull(info);
141 }
142
143 private boolean addError(ImageryInfo info, String error) {
144 String errorMsg = error.replace('\n', ' ');
145 if (notIgnoredErrors.contains(errorMsg))
146 notIgnoredErrors.remove(errorMsg);
147 return addError(isIgnoredError(errorMsg) ? ignoredErrors : errors, info, errorMsg);
148 }
149
150 private static boolean isIgnoredError(String errorMsg) {
151 int idx = errorMsg.lastIndexOf(ERROR_SEP);
152 return isIgnoredSubstring(errorMsg) || (idx > -1 && isIgnoredSubstring(errorMsg.substring(idx + ERROR_SEP.length())));
153 }
154
155 private static boolean isIgnoredSubstring(String substring) {
156 return errorsToIgnore.parallelStream().anyMatch(x -> substring.contains(x));
157 }
158
159 private static boolean addError(Map<String, Map<ImageryInfo, List<String>>> map, ImageryInfo info, String errorMsg) {
160 return map.computeIfAbsent(info.getCountryCode(), x -> Collections.synchronizedMap(new TreeMap<>()))
161 .computeIfAbsent(info, x -> Collections.synchronizedList(new ArrayList<>()))
162 .add(errorMsg);
163 }
164
165 private Optional<byte[]> checkUrl(ImageryInfo info, String url) {
166 if (url != null && !url.isEmpty()) {
167 if (workingURLs.containsKey(url)) {
168 return Optional.of(workingURLs.get(url));
169 }
170 try {
171 Response response = HttpClient.create(new URL(url))
172 .setHeaders(info.getCustomHttpHeaders())
173 .setConnectTimeout((int) TimeUnit.MINUTES.toMillis(1))
174 .setReadTimeout((int) TimeUnit.MINUTES.toMillis(5))
175 .connect();
176 if (response.getResponseCode() >= 400) {
177 addError(info, url + " -> HTTP " + response.getResponseCode());
178 } else if (response.getResponseCode() >= 300) {
179 Logging.warn(url + " -> HTTP " + response.getResponseCode());
180 }
181 try {
182 byte[] data = Utils.readBytesFromStream(response.getContent());
183 if (response.getResponseCode() < 300) {
184 workingURLs.put(url, data);
185 }
186 return Optional.of(data);
187 } catch (IOException e) {
188 if (response.getResponseCode() < 300) {
189 addError(info, url + ERROR_SEP + e);
190 }
191 } finally {
192 response.disconnect();
193 }
194 } catch (IOException e) {
195 addError(info, url + ERROR_SEP + e);
196 }
197 }
198 return Optional.empty();
199 }
200
201 private void checkLinkUrl(ImageryInfo info, String url) {
202 checkUrl(info, url).filter(x -> x.length == 0).ifPresent(x -> addError(info, url + " -> returned empty contents"));
203 }
204
205 private String checkTileUrl(ImageryInfo info, AbstractTileSource tileSource, ICoordinate center, int zoom)
206 throws IOException {
207 TileXY xy = tileSource.latLonToTileXY(center, zoom);
208 for (int i = 0; i < 3; i++) {
209 try {
210 String url = tileSource.getTileUrl(zoom, xy.getXIndex(), xy.getYIndex());
211 Optional<byte[]> optional = checkUrl(info, url);
212 String error = "";
213 if (optional.isPresent()) {
214 byte[] data = optional.get();
215 try (ByteArrayInputStream bais = new ByteArrayInputStream(data)) {
216 if (ImageIO.read(bais) == null) {
217 error = addImageError(info, url, data, zoom, "did not return an image");
218 }
219 } catch (IOException e) {
220 error = addImageError(info, url, data, zoom, e.toString());
221 Logging.trace(e);
222 }
223 }
224 return error;
225 } catch (IOException e) {
226 // Try up to three times max to allow Bing source to initialize itself
227 // and avoid random network errors
228 Logging.trace(e);
229 if (i == 2) {
230 throw e;
231 }
232 try {
233 Thread.sleep(500);
234 } catch (InterruptedException ex) {
235 Logging.warn(ex);
236 }
237 }
238 }
239 return "";
240 }
241
242 private static String zoomMarker(int zoom) {
243 return " -> zoom " + zoom + ERROR_SEP;
244 }
245
246 private String addImageError(ImageryInfo info, String url, byte[] data, int zoom, String defaultMessage) {
247 // Check if we have received an error message
248 String error = helper.detectErrorMessage(new String(data, StandardCharsets.UTF_8));
249 String errorMsg = url + zoomMarker(zoom) + (error != null ? error.split("\\n")[0] : defaultMessage);
250 addError(info, errorMsg);
251 return errorMsg;
252 }
253
254 private static LatLon getPointInShape(Shape shape) {
255 final Coordinate p1 = shape.getPoints().get(0);
256 final Bounds bounds = new Bounds(p1.getLat(), p1.getLon(), p1.getLat(), p1.getLon());
257 shape.getPoints().forEach(p -> bounds.extend(p.getLat(), p.getLon()));
258
259 final double w = bounds.getWidth();
260 final double h = bounds.getHeight();
261
262 final double x2 = bounds.getMinLon() + (w / 2.0);
263 final double y2 = bounds.getMinLat() + (h / 2.0);
264
265 final LatLon center = new LatLon(y2, x2);
266
267 // check to see if center is inside shape
268 if (shape.contains(center)) {
269 return center;
270 }
271
272 // if center position (C) is not inside shape, try naively some other positions as follows:
273 final double x1 = bounds.getMinLon() + (.25 * w);
274 final double x3 = bounds.getMinLon() + (.75 * w);
275 final double y1 = bounds.getMinLat() + (.25 * h);
276 final double y3 = bounds.getMinLat() + (.75 * h);
277 // +-----------+
278 // | 5 1 6 |
279 // | 4 C 2 |
280 // | 8 3 7 |
281 // +-----------+
282 for (LatLon candidate : new LatLon[] {
283 new LatLon(y1, x2),
284 new LatLon(y2, x3),
285 new LatLon(y3, x2),
286 new LatLon(y2, x1),
287 new LatLon(y1, x1),
288 new LatLon(y1, x3),
289 new LatLon(y3, x3),
290 new LatLon(y3, x1)
291 }) {
292 if (shape.contains(candidate)) {
293 return candidate;
294 }
295 }
296 return center;
297 }
298
299 private static LatLon getCenter(ImageryBounds bounds) {
300 List<Shape> shapes = bounds.getShapes();
301 return shapes != null && !shapes.isEmpty() ? getPointInShape(shapes.get(0)) : bounds.getCenter();
302 }
303
304 private void checkEntry(ImageryInfo info) {
305 Logging.info("Checking "+ info);
306
307 if (info.getAttributionImageRaw() != null && info.getAttributionImage() == null) {
308 addError(info, "Can't fetch attribution image: " + info.getAttributionImageRaw());
309 }
310
311 checkLinkUrl(info, info.getAttributionImageURL());
312 checkLinkUrl(info, info.getAttributionLinkURL());
313 String eula = info.getEulaAcceptanceRequired();
314 if (eula != null) {
315 checkLinkUrl(info, eula.replaceAll("\\{lang\\}", ""));
316 }
317 checkLinkUrl(info, info.getPrivacyPolicyURL());
318 checkLinkUrl(info, info.getPermissionReferenceURL());
319 checkLinkUrl(info, info.getTermsOfUseURL());
320 if (info.getUrl().contains("{time}")) {
321 info.setDate("2020-01-01T00:00:00Z/2020-01-02T00:00:00Z");
322 }
323
324 try {
325 ImageryBounds bounds = info.getBounds();
326 // Some imagery sources do not define tiles at (0,0). So pickup Greenwich Royal Observatory for global sources
327 ICoordinate center = CoordinateConversion.llToCoor(bounds != null ? getCenter(bounds) : GREENWICH);
328 AbstractTileSource tileSource = getTileSource(info);
329 // test min zoom and try to detect the correct value in case of error
330 int maxZoom = info.getMaxZoom() > 0 ? Math.min(DEFAULT_ZOOM, info.getMaxZoom()) : DEFAULT_ZOOM;
331 for (int zoom = info.getMinZoom(); zoom < maxZoom; zoom++) {
332 if (!isZoomError(checkTileUrl(info, tileSource, center, zoom))) {
333 break;
334 }
335 }
336 // checking max zoom for real is complex, see https://josm.openstreetmap.de/ticket/16073#comment:27
337 if (info.getMaxZoom() > 0 && info.getImageryType() != ImageryType.SCANEX) {
338 checkTileUrl(info, tileSource, center, Utils.clamp(DEFAULT_ZOOM, info.getMinZoom() + 1, info.getMaxZoom()));
339 }
340 } catch (IOException | RuntimeException | WMSGetCapabilitiesException | WMTSGetCapabilitiesException e) {
341 addError(info, info.getUrl() + ERROR_SEP + e.toString());
342 }
343
344 for (ImageryInfo mirror : info.getMirrors()) {
345 checkEntry(mirror);
346 }
347 }
348
349 private static boolean isZoomError(String error) {
350 String[] parts = error.split(ERROR_SEP);
351 String lastPart = parts.length > 0 ? parts[parts.length - 1].toLowerCase(Locale.ENGLISH) : "";
352 return lastPart.contains("bbox")
353 || lastPart.contains("bounding box");
354 }
355
356 private static Projection getProjection(ImageryInfo info) {
357 for (String code : info.getServerProjections()) {
358 Projection proj = Projections.getProjectionByCode(code);
359 if (proj != null) {
360 return proj;
361 }
362 }
363 return ProjectionRegistry.getProjection();
364 }
365
366 @SuppressWarnings("fallthrough")
367 private static AbstractTileSource getTileSource(ImageryInfo info)
368 throws IOException, WMTSGetCapabilitiesException, WMSGetCapabilitiesException {
369 switch (info.getImageryType()) {
370 case BING:
371 return new BingAerialTileSource(info);
372 case SCANEX:
373 return new ScanexTileSource(info);
374 case TMS:
375 return new JosmTemplatedTMSTileSource(info);
376 case WMS_ENDPOINT:
377 info = convertWmsEndpointToWms(info); // fall-through
378 case WMS:
379 return new TemplatedWMSTileSource(info, getProjection(info));
380 case WMTS:
381 return new WMTSTileSource(info, getProjection(info));
382 default:
383 throw new UnsupportedOperationException(info.toString());
384 }
385 }
386
387 private static ImageryInfo convertWmsEndpointToWms(ImageryInfo info) throws IOException, WMSGetCapabilitiesException {
388 return Optional.ofNullable(AddImageryLayerAction.getWMSLayerInfo(
389 info, wms -> new LayerSelection(firstLeafLayer(wms.getLayers()), wms.getPreferredFormat(), true)))
390 .orElseThrow(() -> new IllegalStateException("Unable to convert WMS_ENDPOINT to WMS"));
391 }
392
393 private static List<LayerDetails> firstLeafLayer(List<LayerDetails> layers) {
394 for (LayerDetails layer : layers) {
395 boolean hasNoChildren = layer.getChildren().isEmpty();
396 if (hasNoChildren && layer.getName() != null) {
397 return Collections.singletonList(layer);
398 } else if (!hasNoChildren) {
399 return firstLeafLayer(layer.getChildren());
400 }
401 }
402 throw new IllegalArgumentException("Unable to find a valid WMS layer");
403 }
404
405 private static String format(Map<String, Map<ImageryInfo, List<String>>> map) {
406 return map.toString().replaceAll("\\}, ", "\n\\}, ").replaceAll(", ImageryInfo\\{", "\n ,ImageryInfo\\{");
407 }
408
409 /**
410 * Test that available imagery entry is valid.
411 */
412 @Test
413 public void testImageryEntryValidity() {
414 checkEntry(info);
415 assertTrue(format(errors), errors.isEmpty());
416 assertFalse(workingURLs.isEmpty());
417 assumeTrue(format(ignoredErrors), ignoredErrors.isEmpty());
418 }
419}
Note: See TracBrowser for help on using the repository browser.