source: josm/trunk/test/unit/org/openstreetmap/josm/gui/preferences/imagery/ImageryPreferenceTestIT.java@ 16438

Last change on this file since 16438 was 16438, checked in by simon04, 4 years ago

see #19251 - Java 8: use Stream
  • Property svn:eol-style set to native
File size: 17.5 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.gui.preferences.imagery;
3
4import static org.junit.Assert.assertFalse;
5import static org.junit.Assert.assertTrue;
6import static org.junit.Assume.assumeTrue;
7
8import java.io.ByteArrayInputStream;
9import java.io.IOException;
10import java.net.URL;
11import java.nio.charset.StandardCharsets;
12import java.util.ArrayList;
13import java.util.Collections;
14import java.util.List;
15import java.util.Locale;
16import java.util.Map;
17import java.util.Objects;
18import java.util.Optional;
19import java.util.TreeMap;
20import java.util.concurrent.TimeUnit;
21import java.util.stream.Collectors;
22import java.util.stream.Stream;
23
24import javax.imageio.ImageIO;
25
26import org.apache.commons.jcs3.access.CacheAccess;
27import org.junit.AfterClass;
28import org.junit.BeforeClass;
29import org.junit.ClassRule;
30import org.junit.Test;
31import org.junit.runner.RunWith;
32import org.junit.runners.Parameterized.Parameters;
33import org.openstreetmap.gui.jmapviewer.Coordinate;
34import org.openstreetmap.gui.jmapviewer.FeatureAdapter;
35import org.openstreetmap.gui.jmapviewer.TileXY;
36import org.openstreetmap.gui.jmapviewer.interfaces.ICoordinate;
37import org.openstreetmap.gui.jmapviewer.tilesources.AbstractTileSource;
38import org.openstreetmap.gui.jmapviewer.tilesources.BingAerialTileSource;
39import org.openstreetmap.gui.jmapviewer.tilesources.ScanexTileSource;
40import org.openstreetmap.josm.TestUtils;
41import org.openstreetmap.josm.actions.AddImageryLayerAction;
42import org.openstreetmap.josm.actions.AddImageryLayerAction.LayerSelection;
43import org.openstreetmap.josm.data.Bounds;
44import org.openstreetmap.josm.data.coor.LatLon;
45import org.openstreetmap.josm.data.imagery.CoordinateConversion;
46import org.openstreetmap.josm.data.imagery.ImageryInfo;
47import org.openstreetmap.josm.data.imagery.ImageryInfo.ImageryBounds;
48import org.openstreetmap.josm.data.imagery.ImageryInfo.ImageryType;
49import org.openstreetmap.josm.data.imagery.ImageryLayerInfo;
50import org.openstreetmap.josm.data.imagery.JosmTemplatedTMSTileSource;
51import org.openstreetmap.josm.data.imagery.LayerDetails;
52import org.openstreetmap.josm.data.imagery.Shape;
53import org.openstreetmap.josm.data.imagery.TMSCachedTileLoaderJob;
54import org.openstreetmap.josm.data.imagery.TemplatedWMSTileSource;
55import org.openstreetmap.josm.data.imagery.TileJobOptions;
56import org.openstreetmap.josm.data.imagery.WMTSTileSource;
57import org.openstreetmap.josm.data.imagery.WMTSTileSource.WMTSGetCapabilitiesException;
58import org.openstreetmap.josm.data.projection.Projection;
59import org.openstreetmap.josm.data.projection.ProjectionRegistry;
60import org.openstreetmap.josm.data.projection.Projections;
61import org.openstreetmap.josm.io.imagery.ApiKeyProvider;
62import org.openstreetmap.josm.io.imagery.WMSImagery.WMSGetCapabilitiesException;
63import org.openstreetmap.josm.testutils.JOSMTestRules;
64import org.openstreetmap.josm.testutils.ParallelParameterized;
65import org.openstreetmap.josm.tools.HttpClient;
66import org.openstreetmap.josm.tools.HttpClient.Response;
67import org.openstreetmap.josm.tools.Logging;
68import org.openstreetmap.josm.tools.Utils;
69
70import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
71
72/**
73 * Integration tests of {@link ImageryPreference} class.
74 */
75@RunWith(ParallelParameterized.class)
76public class ImageryPreferenceTestIT {
77
78 private static final String ERROR_SEP = " -> ";
79 private static final LatLon GREENWICH = new LatLon(51.47810, -0.00170);
80 private static final int DEFAULT_ZOOM = 12;
81
82 /**
83 * Setup rule
84 */
85 @ClassRule
86 @SuppressFBWarnings(value = "URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD")
87 public static JOSMTestRules test = new JOSMTestRules().https().i18n().preferences().projection().projectionNadGrids()
88 .timeout((int) TimeUnit.MINUTES.toMillis(40)).parameters();
89
90 /** Entry to test */
91 private final ImageryInfo info;
92 private final Map<String, Map<ImageryInfo, List<String>>> errors = Collections.synchronizedMap(new TreeMap<>());
93 private final Map<String, Map<ImageryInfo, List<String>>> ignoredErrors = Collections.synchronizedMap(new TreeMap<>());
94 private static final Map<String, byte[]> workingURLs = Collections.synchronizedMap(new TreeMap<>());
95
96 private static TMSCachedTileLoaderJob helper;
97 private static final List<String> errorsToIgnore = new ArrayList<>();
98 private static final List<String> notIgnoredErrors = new ArrayList<>();
99
100 /**
101 * Setup test
102 * @throws IOException in case of I/O error
103 */
104 @BeforeClass
105 public static void beforeClass() throws IOException {
106 FeatureAdapter.registerApiKeyAdapter(ApiKeyProvider::retrieveApiKey);
107 helper = new TMSCachedTileLoaderJob(null, null, new CacheAccess<>(null), new TileJobOptions(0, 0, null, 0), null);
108 errorsToIgnore.addAll(TestUtils.getIgnoredErrorMessages(ImageryPreferenceTestIT.class));
109 notIgnoredErrors.addAll(errorsToIgnore);
110 }
111
112 /**
113 * Cleanup test
114 */
115 @AfterClass
116 public static void afterClass() {
117 for (String e : notIgnoredErrors) {
118 Logging.warn("Ignore line unused: " + e);
119 }
120 }
121
122 /**
123 * Returns list of imagery entries to test.
124 * @return list of imagery entries to test
125 */
126 @Parameters(name = "{0}")
127 public static List<Object[]> data() {
128 ImageryLayerInfo.instance.load(false);
129 return ImageryLayerInfo.instance.getDefaultLayers()
130 .stream()
131 .map(x -> new Object[] {x.getId(), x})
132 .collect(Collectors.toList());
133 }
134
135 /**
136 * Constructs a new {@code ImageryPreferenceTestIT} instance.
137 * @param id entry ID, used only to name tests
138 * @param info entry to test
139 */
140 public ImageryPreferenceTestIT(String id, ImageryInfo info) {
141 this.info = Objects.requireNonNull(info);
142 }
143
144 private boolean addError(ImageryInfo info, String error) {
145 String errorMsg = error.replace('\n', ' ');
146 if (notIgnoredErrors.contains(errorMsg))
147 notIgnoredErrors.remove(errorMsg);
148 return addError(isIgnoredError(errorMsg) ? ignoredErrors : errors, info, errorMsg);
149 }
150
151 private static boolean isIgnoredError(String errorMsg) {
152 int idx = errorMsg.lastIndexOf(ERROR_SEP);
153 return isIgnoredSubstring(errorMsg) || (idx > -1 && isIgnoredSubstring(errorMsg.substring(idx + ERROR_SEP.length())));
154 }
155
156 private static boolean isIgnoredSubstring(String substring) {
157 return errorsToIgnore.parallelStream().anyMatch(x -> substring.contains(x));
158 }
159
160 private static boolean addError(Map<String, Map<ImageryInfo, List<String>>> map, ImageryInfo info, String errorMsg) {
161 return map.computeIfAbsent(info.getCountryCode(), x -> Collections.synchronizedMap(new TreeMap<>()))
162 .computeIfAbsent(info, x -> Collections.synchronizedList(new ArrayList<>()))
163 .add(errorMsg);
164 }
165
166 private Optional<byte[]> checkUrl(ImageryInfo info, String url) {
167 if (url != null && !url.isEmpty()) {
168 if (workingURLs.containsKey(url)) {
169 return Optional.of(workingURLs.get(url));
170 }
171 try {
172 Response response = HttpClient.create(new URL(url))
173 .setHeaders(info.getCustomHttpHeaders())
174 .setConnectTimeout((int) TimeUnit.MINUTES.toMillis(1))
175 .setReadTimeout((int) TimeUnit.MINUTES.toMillis(5))
176 .connect();
177 if (response.getResponseCode() >= 400) {
178 addError(info, url + " -> HTTP " + response.getResponseCode());
179 } else if (response.getResponseCode() >= 300) {
180 Logging.warn(url + " -> HTTP " + response.getResponseCode());
181 }
182 try {
183 byte[] data = Utils.readBytesFromStream(response.getContent());
184 if (response.getResponseCode() < 300) {
185 workingURLs.put(url, data);
186 }
187 return Optional.of(data);
188 } catch (IOException e) {
189 if (response.getResponseCode() < 300) {
190 addError(info, url + ERROR_SEP + e);
191 }
192 } finally {
193 response.disconnect();
194 }
195 } catch (IOException e) {
196 addError(info, url + ERROR_SEP + e);
197 }
198 }
199 return Optional.empty();
200 }
201
202 private void checkLinkUrl(ImageryInfo info, String url) {
203 checkUrl(info, url).filter(x -> x.length == 0).ifPresent(x -> addError(info, url + " -> returned empty contents"));
204 }
205
206 private String checkTileUrl(ImageryInfo info, AbstractTileSource tileSource, ICoordinate center, int zoom)
207 throws IOException {
208 TileXY xy = tileSource.latLonToTileXY(center, zoom);
209 for (int i = 0; i < 3; i++) {
210 try {
211 String url = tileSource.getTileUrl(zoom, xy.getXIndex(), xy.getYIndex());
212 Optional<byte[]> optional = checkUrl(info, url);
213 String error = "";
214 if (optional.isPresent()) {
215 byte[] data = optional.get();
216 try (ByteArrayInputStream bais = new ByteArrayInputStream(data)) {
217 if (ImageIO.read(bais) == null) {
218 error = addImageError(info, url, data, zoom, "did not return an image");
219 }
220 } catch (IOException e) {
221 error = addImageError(info, url, data, zoom, e.toString());
222 Logging.trace(e);
223 }
224 }
225 return error;
226 } catch (IOException e) {
227 // Try up to three times max to allow Bing source to initialize itself
228 // and avoid random network errors
229 Logging.trace(e);
230 if (i == 2) {
231 throw e;
232 }
233 try {
234 Thread.sleep(500);
235 } catch (InterruptedException ex) {
236 Logging.warn(ex);
237 }
238 }
239 }
240 return "";
241 }
242
243 private static String zoomMarker(int zoom) {
244 return " -> zoom " + zoom + ERROR_SEP;
245 }
246
247 private String addImageError(ImageryInfo info, String url, byte[] data, int zoom, String defaultMessage) {
248 // Check if we have received an error message
249 String error = helper.detectErrorMessage(new String(data, StandardCharsets.UTF_8));
250 String errorMsg = url + zoomMarker(zoom) + (error != null ? error.split("\\n")[0] : defaultMessage);
251 addError(info, errorMsg);
252 return errorMsg;
253 }
254
255 private static LatLon getPointInShape(Shape shape) {
256 final Coordinate p1 = shape.getPoints().get(0);
257 final Bounds bounds = new Bounds(p1.getLat(), p1.getLon(), p1.getLat(), p1.getLon());
258 shape.getPoints().forEach(p -> bounds.extend(p.getLat(), p.getLon()));
259
260 final double w = bounds.getWidth();
261 final double h = bounds.getHeight();
262
263 final double x2 = bounds.getMinLon() + (w / 2.0);
264 final double y2 = bounds.getMinLat() + (h / 2.0);
265
266 final LatLon center = new LatLon(y2, x2);
267
268 // check to see if center is inside shape
269 if (shape.contains(center)) {
270 return center;
271 }
272
273 // if center position (C) is not inside shape, try naively some other positions as follows:
274 final double x1 = bounds.getMinLon() + (.25 * w);
275 final double x3 = bounds.getMinLon() + (.75 * w);
276 final double y1 = bounds.getMinLat() + (.25 * h);
277 final double y3 = bounds.getMinLat() + (.75 * h);
278 // +-----------+
279 // | 5 1 6 |
280 // | 4 C 2 |
281 // | 8 3 7 |
282 // +-----------+
283 return Stream.of(
284 new LatLon(y1, x2),
285 new LatLon(y2, x3),
286 new LatLon(y3, x2),
287 new LatLon(y2, x1),
288 new LatLon(y1, x1),
289 new LatLon(y1, x3),
290 new LatLon(y3, x3),
291 new LatLon(y3, x1)
292 ).filter(shape::contains).findFirst().orElse(center);
293 }
294
295 private static LatLon getCenter(ImageryBounds bounds) {
296 List<Shape> shapes = bounds.getShapes();
297 return shapes != null && !shapes.isEmpty() ? getPointInShape(shapes.get(0)) : bounds.getCenter();
298 }
299
300 private void checkEntry(ImageryInfo info) {
301 Logging.info("Checking "+ info);
302
303 if (info.getAttributionImageRaw() != null && info.getAttributionImage() == null) {
304 addError(info, "Can't fetch attribution image: " + info.getAttributionImageRaw());
305 }
306
307 checkLinkUrl(info, info.getAttributionImageURL());
308 checkLinkUrl(info, info.getAttributionLinkURL());
309 String eula = info.getEulaAcceptanceRequired();
310 if (eula != null) {
311 checkLinkUrl(info, eula.replaceAll("\\{lang\\}", ""));
312 }
313 checkLinkUrl(info, info.getPrivacyPolicyURL());
314 checkLinkUrl(info, info.getPermissionReferenceURL());
315 checkLinkUrl(info, info.getTermsOfUseURL());
316 if (info.getUrl().contains("{time}")) {
317 info.setDate("2020-01-01T00:00:00Z/2020-01-02T00:00:00Z");
318 }
319
320 try {
321 ImageryBounds bounds = info.getBounds();
322 // Some imagery sources do not define tiles at (0,0). So pickup Greenwich Royal Observatory for global sources
323 ICoordinate center = CoordinateConversion.llToCoor(bounds != null ? getCenter(bounds) : GREENWICH);
324 AbstractTileSource tileSource = getTileSource(info);
325 // test min zoom and try to detect the correct value in case of error
326 int maxZoom = info.getMaxZoom() > 0 ? Math.min(DEFAULT_ZOOM, info.getMaxZoom()) : DEFAULT_ZOOM;
327 for (int zoom = info.getMinZoom(); zoom < maxZoom; zoom++) {
328 if (!isZoomError(checkTileUrl(info, tileSource, center, zoom))) {
329 break;
330 }
331 }
332 // checking max zoom for real is complex, see https://josm.openstreetmap.de/ticket/16073#comment:27
333 if (info.getMaxZoom() > 0 && info.getImageryType() != ImageryType.SCANEX) {
334 checkTileUrl(info, tileSource, center, Utils.clamp(DEFAULT_ZOOM, info.getMinZoom() + 1, info.getMaxZoom()));
335 }
336 } catch (IOException | RuntimeException | WMSGetCapabilitiesException | WMTSGetCapabilitiesException e) {
337 addError(info, info.getUrl() + ERROR_SEP + e.toString());
338 }
339
340 for (ImageryInfo mirror : info.getMirrors()) {
341 checkEntry(mirror);
342 }
343 }
344
345 private static boolean isZoomError(String error) {
346 String[] parts = error.split(ERROR_SEP);
347 String lastPart = parts.length > 0 ? parts[parts.length - 1].toLowerCase(Locale.ENGLISH) : "";
348 return lastPart.contains("bbox")
349 || lastPart.contains("bounding box");
350 }
351
352 private static Projection getProjection(ImageryInfo info) {
353 for (String code : info.getServerProjections()) {
354 Projection proj = Projections.getProjectionByCode(code);
355 if (proj != null) {
356 return proj;
357 }
358 }
359 return ProjectionRegistry.getProjection();
360 }
361
362 @SuppressWarnings("fallthrough")
363 private static AbstractTileSource getTileSource(ImageryInfo info)
364 throws IOException, WMTSGetCapabilitiesException, WMSGetCapabilitiesException {
365 switch (info.getImageryType()) {
366 case BING:
367 return new BingAerialTileSource(info);
368 case SCANEX:
369 return new ScanexTileSource(info);
370 case TMS:
371 return new JosmTemplatedTMSTileSource(info);
372 case WMS_ENDPOINT:
373 info = convertWmsEndpointToWms(info); // fall-through
374 case WMS:
375 return new TemplatedWMSTileSource(info, getProjection(info));
376 case WMTS:
377 return new WMTSTileSource(info, getProjection(info));
378 default:
379 throw new UnsupportedOperationException(info.toString());
380 }
381 }
382
383 private static ImageryInfo convertWmsEndpointToWms(ImageryInfo info) throws IOException, WMSGetCapabilitiesException {
384 return Optional.ofNullable(AddImageryLayerAction.getWMSLayerInfo(
385 info, wms -> new LayerSelection(firstLeafLayer(wms.getLayers()), wms.getPreferredFormat(), true)))
386 .orElseThrow(() -> new IllegalStateException("Unable to convert WMS_ENDPOINT to WMS"));
387 }
388
389 private static List<LayerDetails> firstLeafLayer(List<LayerDetails> layers) {
390 for (LayerDetails layer : layers) {
391 boolean hasNoChildren = layer.getChildren().isEmpty();
392 if (hasNoChildren && layer.getName() != null) {
393 return Collections.singletonList(layer);
394 } else if (!hasNoChildren) {
395 return firstLeafLayer(layer.getChildren());
396 }
397 }
398 throw new IllegalArgumentException("Unable to find a valid WMS layer");
399 }
400
401 private static String format(Map<String, Map<ImageryInfo, List<String>>> map) {
402 return map.toString().replaceAll("\\}, ", "\n\\}, ").replaceAll(", ImageryInfo\\{", "\n ,ImageryInfo\\{");
403 }
404
405 /**
406 * Test that available imagery entry is valid.
407 */
408 @Test
409 public void testImageryEntryValidity() {
410 checkEntry(info);
411 assertTrue(format(errors), errors.isEmpty());
412 assertFalse(workingURLs.isEmpty());
413 assumeTrue(format(ignoredErrors), ignoredErrors.isEmpty());
414 }
415}
Note: See TracBrowser for help on using the repository browser.