source: josm/trunk/test/unit/org/openstreetmap/josm/gui/preferences/imagery/ImageryPreferenceTestIT.java@ 14605

Last change on this file since 14605 was 14605, checked in by Don-vip, 5 years ago

see #16073 - convert single global test to one test per imagery entry

Slower, but this improves readbility and allows to focus on entries failing for several consecutive tries.
  • Property svn:eol-style set to native
File size: 16.2 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.gui.preferences.imagery;
3
4import static org.junit.Assert.assertFalse;
5import static org.junit.Assert.assertTrue;
6
7import java.io.ByteArrayInputStream;
8import java.io.IOException;
9import java.net.URL;
10import java.nio.charset.StandardCharsets;
11import java.util.ArrayList;
12import java.util.Collections;
13import java.util.List;
14import java.util.Locale;
15import java.util.Map;
16import java.util.Objects;
17import java.util.Optional;
18import java.util.TreeMap;
19import java.util.concurrent.TimeUnit;
20import java.util.stream.Collectors;
21
22import javax.imageio.ImageIO;
23
24import org.apache.commons.jcs.access.CacheAccess;
25import org.junit.BeforeClass;
26import org.junit.ClassRule;
27import org.junit.Test;
28import org.junit.runner.Description;
29import org.junit.runner.RunWith;
30import org.junit.runners.Parameterized.Parameters;
31import org.junit.runners.model.Statement;
32import org.openstreetmap.gui.jmapviewer.Coordinate;
33import org.openstreetmap.gui.jmapviewer.TileXY;
34import org.openstreetmap.gui.jmapviewer.interfaces.ICoordinate;
35import org.openstreetmap.gui.jmapviewer.tilesources.AbstractTileSource;
36import org.openstreetmap.gui.jmapviewer.tilesources.BingAerialTileSource;
37import org.openstreetmap.gui.jmapviewer.tilesources.ScanexTileSource;
38import org.openstreetmap.gui.jmapviewer.tilesources.TemplatedTMSTileSource;
39import org.openstreetmap.josm.TestUtils;
40import org.openstreetmap.josm.actions.AddImageryLayerAction;
41import org.openstreetmap.josm.actions.AddImageryLayerAction.LayerSelection;
42import org.openstreetmap.josm.data.Bounds;
43import org.openstreetmap.josm.data.coor.LatLon;
44import org.openstreetmap.josm.data.imagery.CoordinateConversion;
45import org.openstreetmap.josm.data.imagery.ImageryInfo;
46import org.openstreetmap.josm.data.imagery.ImageryInfo.ImageryBounds;
47import org.openstreetmap.josm.data.imagery.ImageryInfo.ImageryType;
48import org.openstreetmap.josm.data.imagery.ImageryLayerInfo;
49import org.openstreetmap.josm.data.imagery.LayerDetails;
50import org.openstreetmap.josm.data.imagery.Shape;
51import org.openstreetmap.josm.data.imagery.TMSCachedTileLoaderJob;
52import org.openstreetmap.josm.data.imagery.TemplatedWMSTileSource;
53import org.openstreetmap.josm.data.imagery.TileJobOptions;
54import org.openstreetmap.josm.data.imagery.WMTSTileSource;
55import org.openstreetmap.josm.data.imagery.WMTSTileSource.WMTSGetCapabilitiesException;
56import org.openstreetmap.josm.data.projection.Projection;
57import org.openstreetmap.josm.data.projection.ProjectionRegistry;
58import org.openstreetmap.josm.data.projection.Projections;
59import org.openstreetmap.josm.io.imagery.WMSImagery.WMSGetCapabilitiesException;
60import org.openstreetmap.josm.testutils.JOSMTestRules;
61import org.openstreetmap.josm.testutils.ParallelParameterized;
62import org.openstreetmap.josm.tools.HttpClient;
63import org.openstreetmap.josm.tools.HttpClient.Response;
64import org.openstreetmap.josm.tools.Logging;
65import org.openstreetmap.josm.tools.Utils;
66
67import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
68
69/**
70 * Integration tests of {@link ImageryPreference} class.
71 */
72@RunWith(ParallelParameterized.class)
73public class ImageryPreferenceTestIT {
74
75 private static final LatLon GREENWICH = new LatLon(51.47810, -0.00170);
76 private static final int DEFAULT_ZOOM = 12;
77
78 /**
79 * Setup rule
80 */
81 @ClassRule
82 @SuppressFBWarnings(value = "URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD")
83 public static JOSMTestRules test = new JOSMTestRules().https().preferences().projection().projectionNadGrids()
84 .timeout((int) TimeUnit.MINUTES.toMillis(40));
85
86 static {
87 try {
88 test.apply(new Statement() {
89 @Override
90 public void evaluate() throws Throwable {
91 // Do nothing. Hack needed because @Parameters are computed before anything else
92 }
93 }, Description.createSuiteDescription(ImageryPreferenceTestIT.class)).evaluate();
94 } catch (Throwable e) {
95 Logging.error(e);
96 }
97 }
98
99 /** Entry to test */
100 private final ImageryInfo info;
101 private final Map<String, Map<ImageryInfo, List<String>>> errors = Collections.synchronizedMap(new TreeMap<>());
102 private static final Map<String, byte[]> workingURLs = Collections.synchronizedMap(new TreeMap<>());
103
104 private static TMSCachedTileLoaderJob helper;
105 private static List<String> ignoredErrors;
106
107 /**
108 * Setup test
109 * @throws IOException in case of I/O error
110 */
111 @BeforeClass
112 public static void beforeClass() throws IOException {
113 helper = new TMSCachedTileLoaderJob(null, null, new CacheAccess<>(null), new TileJobOptions(0, 0, null, 0), null);
114 ignoredErrors = TestUtils.getIgnoredErrorMessages(ImageryPreferenceTestIT.class);
115 }
116
117 /**
118 * Returns list of imagery entries to test.
119 * @return list of imagery entries to test
120 */
121 @Parameters(name = "{0}")
122 public static List<Object[]> data() {
123 ImageryLayerInfo.instance.load(false);
124 return ImageryLayerInfo.instance.getDefaultLayers()
125 .stream().map(x -> new Object[] {x.getId(), x})
126 .collect(Collectors.toList());
127 }
128
129 /**
130 * Constructs a new {@code ImageryPreferenceTestIT} instance.
131 * @param id entry ID, used only to name tests
132 * @param info entry to test
133 */
134 public ImageryPreferenceTestIT(String id, ImageryInfo info) {
135 this.info = Objects.requireNonNull(info);
136 }
137
138 private boolean addError(ImageryInfo info, String error) {
139 return !ignoredErrors.contains(error) &&
140 errors.computeIfAbsent(info.getCountryCode(), x -> Collections.synchronizedMap(new TreeMap<>()))
141 .computeIfAbsent(info, x -> Collections.synchronizedList(new ArrayList<>()))
142 .add(error);
143 }
144
145 private Optional<byte[]> checkUrl(ImageryInfo info, String url) {
146 if (url != null && !url.isEmpty()) {
147 if (workingURLs.containsKey(url)) {
148 return Optional.of(workingURLs.get(url));
149 }
150 try {
151 Response response = HttpClient.create(new URL(url))
152 .setHeaders(info.getCustomHttpHeaders())
153 .setConnectTimeout((int) TimeUnit.MINUTES.toMillis(1))
154 .setReadTimeout((int) TimeUnit.MINUTES.toMillis(2))
155 .connect();
156 if (response.getResponseCode() >= 400) {
157 addError(info, url + " -> HTTP " + response.getResponseCode());
158 } else if (response.getResponseCode() >= 300) {
159 Logging.warn(url + " -> HTTP " + response.getResponseCode());
160 }
161 try {
162 byte[] data = Utils.readBytesFromStream(response.getContent());
163 if (response.getResponseCode() < 300) {
164 workingURLs.put(url, data);
165 }
166 return Optional.of(data);
167 } catch (IOException e) {
168 if (response.getResponseCode() < 300) {
169 addError(info, url + " -> " + e);
170 }
171 } finally {
172 response.disconnect();
173 }
174 } catch (IOException e) {
175 addError(info, url + " -> " + e);
176 }
177 }
178 return Optional.empty();
179 }
180
181 private void checkLinkUrl(ImageryInfo info, String url) {
182 checkUrl(info, url).filter(x -> x.length == 0).ifPresent(x -> addError(info, url + " -> returned empty contents"));
183 }
184
185 private String checkTileUrl(ImageryInfo info, AbstractTileSource tileSource, ICoordinate center, int zoom)
186 throws IOException {
187 TileXY xy = tileSource.latLonToTileXY(center, zoom);
188 for (int i = 0; i < 3; i++) {
189 try {
190 String url = tileSource.getTileUrl(zoom, xy.getXIndex(), xy.getYIndex());
191 Optional<byte[]> optional = checkUrl(info, url);
192 String error = "";
193 if (optional.isPresent()) {
194 byte[] data = optional.get();
195 try (ByteArrayInputStream bais = new ByteArrayInputStream(data)) {
196 if (ImageIO.read(bais) == null) {
197 error = addImageError(info, url, data, zoom, "did not return an image");
198 }
199 } catch (IOException e) {
200 error = addImageError(info, url, data, zoom, e.toString());
201 Logging.trace(e);
202 }
203 }
204 return error;
205 } catch (IOException e) {
206 // Try up to three times max to allow Bing source to initialize itself
207 // and avoid random network errors
208 Logging.trace(e);
209 if (i == 2) {
210 throw e;
211 }
212 try {
213 Thread.sleep(500);
214 } catch (InterruptedException ex) {
215 Logging.warn(ex);
216 }
217 }
218 }
219 return "";
220 }
221
222 private static String zoomMarker(int zoom) {
223 return " -> zoom " + zoom + " -> ";
224 }
225
226 private String addImageError(ImageryInfo info, String url, byte[] data, int zoom, String defaultMessage) {
227 // Check if we have received an error message
228 String error = helper.detectErrorMessage(new String(data, StandardCharsets.UTF_8));
229 String errorMsg = url + zoomMarker(zoom) + (error != null ? error.split("\\n")[0] : defaultMessage);
230 addError(info, errorMsg);
231 return errorMsg;
232 }
233
234 private static LatLon getPointInShape(Shape shape) {
235 final Coordinate p1 = shape.getPoints().get(0);
236 final Bounds bounds = new Bounds(p1.getLat(), p1.getLon(), p1.getLat(), p1.getLon());
237 shape.getPoints().forEach(p -> bounds.extend(p.getLat(), p.getLon()));
238
239 final double w = bounds.getWidth();
240 final double h = bounds.getHeight();
241
242 final double x2 = bounds.getMinLon() + (w / 2.0);
243 final double y2 = bounds.getMinLat() + (h / 2.0);
244
245 final LatLon center = new LatLon(y2, x2);
246
247 // check to see if center is inside shape
248 if (shape.contains(center)) {
249 return center;
250 }
251
252 // if center position (C) is not inside shape, try naively some other positions as follows:
253 final double x1 = bounds.getMinLon() + (.25 * w);
254 final double x3 = bounds.getMinLon() + (.75 * w);
255 final double y1 = bounds.getMinLat() + (.25 * h);
256 final double y3 = bounds.getMinLat() + (.75 * h);
257 // +-----------+
258 // | 5 1 6 |
259 // | 4 C 2 |
260 // | 8 3 7 |
261 // +-----------+
262 for (LatLon candidate : new LatLon[] {
263 new LatLon(y1, x2),
264 new LatLon(y2, x3),
265 new LatLon(y3, x2),
266 new LatLon(y2, x1),
267 new LatLon(y1, x1),
268 new LatLon(y1, x3),
269 new LatLon(y3, x3),
270 new LatLon(y3, x1)
271 }) {
272 if (shape.contains(candidate)) {
273 return candidate;
274 }
275 }
276 return center;
277 }
278
279 private static LatLon getCenter(ImageryBounds bounds) {
280 List<Shape> shapes = bounds.getShapes();
281 return shapes != null && !shapes.isEmpty() ? getPointInShape(shapes.get(0)) : bounds.getCenter();
282 }
283
284 private void checkEntry(ImageryInfo info) {
285 Logging.info("Checking "+ info);
286
287 if (info.getAttributionImageRaw() != null && info.getAttributionImage() == null) {
288 addError(info, "Can't fetch attribution image: " + info.getAttributionImageRaw());
289 }
290
291 checkLinkUrl(info, info.getAttributionImageURL());
292 checkLinkUrl(info, info.getAttributionLinkURL());
293 String eula = info.getEulaAcceptanceRequired();
294 if (eula != null) {
295 checkLinkUrl(info, eula.replaceAll("\\{lang\\}", ""));
296 }
297 checkLinkUrl(info, info.getPermissionReferenceURL());
298 checkLinkUrl(info, info.getTermsOfUseURL());
299
300 try {
301 ImageryBounds bounds = info.getBounds();
302 // Some imagery sources do not define tiles at (0,0). So pickup Greenwich Royal Observatory for global sources
303 ICoordinate center = CoordinateConversion.llToCoor(bounds != null ? getCenter(bounds) : GREENWICH);
304 AbstractTileSource tileSource = getTileSource(info);
305 // test min zoom and try to detect the correct value in case of error
306 int maxZoom = info.getMaxZoom() > 0 ? Math.min(DEFAULT_ZOOM, info.getMaxZoom()) : DEFAULT_ZOOM;
307 for (int zoom = info.getMinZoom(); zoom < maxZoom; zoom++) {
308 if (!isZoomError(checkTileUrl(info, tileSource, center, zoom))) {
309 break;
310 }
311 }
312 // checking max zoom for real is complex, see https://josm.openstreetmap.de/ticket/16073#comment:27
313 if (info.getMaxZoom() > 0 && info.getImageryType() != ImageryType.SCANEX) {
314 checkTileUrl(info, tileSource, center, Utils.clamp(DEFAULT_ZOOM, info.getMinZoom() + 1, info.getMaxZoom()));
315 }
316 } catch (IOException | RuntimeException | WMSGetCapabilitiesException | WMTSGetCapabilitiesException e) {
317 addError(info, info.getUrl() + " -> " + e.toString());
318 }
319
320 for (ImageryInfo mirror : info.getMirrors()) {
321 checkEntry(mirror);
322 }
323 }
324
325 private static boolean isZoomError(String error) {
326 String[] parts = error.split(" -> ");
327 String lastPart = parts.length > 0 ? parts[parts.length - 1].toLowerCase(Locale.ENGLISH) : "";
328 return lastPart.contains("bbox")
329 || lastPart.contains("bounding box");
330 }
331
332 private static Projection getProjection(ImageryInfo info) {
333 for (String code : info.getServerProjections()) {
334 Projection proj = Projections.getProjectionByCode(code);
335 if (proj != null) {
336 return proj;
337 }
338 }
339 return ProjectionRegistry.getProjection();
340 }
341
342 @SuppressWarnings("fallthrough")
343 private static AbstractTileSource getTileSource(ImageryInfo info)
344 throws IOException, WMTSGetCapabilitiesException, WMSGetCapabilitiesException {
345 switch (info.getImageryType()) {
346 case BING:
347 return new BingAerialTileSource(info);
348 case SCANEX:
349 return new ScanexTileSource(info);
350 case TMS:
351 return new TemplatedTMSTileSource(info);
352 case WMS_ENDPOINT:
353 info = convertWmsEndpointToWms(info); // fall-through
354 case WMS:
355 return new TemplatedWMSTileSource(info, getProjection(info));
356 case WMTS:
357 return new WMTSTileSource(info, getProjection(info));
358 default:
359 throw new UnsupportedOperationException(info.toString());
360 }
361 }
362
363 private static ImageryInfo convertWmsEndpointToWms(ImageryInfo info) throws IOException, WMSGetCapabilitiesException {
364 return Optional.ofNullable(AddImageryLayerAction.getWMSLayerInfo(
365 info, wms -> new LayerSelection(firstLeafLayer(wms.getLayers()), wms.getPreferredFormat(), true)))
366 .orElseThrow(() -> new IllegalStateException("Unable to convert WMS_ENDPOINT to WMS"));
367 }
368
369 private static List<LayerDetails> firstLeafLayer(List<LayerDetails> layers) {
370 for (LayerDetails layer : layers) {
371 boolean hasNoChildren = layer.getChildren().isEmpty();
372 if (hasNoChildren && layer.getName() != null) {
373 return Collections.singletonList(layer);
374 } else if (!hasNoChildren) {
375 return firstLeafLayer(layer.getChildren());
376 }
377 }
378 throw new IllegalArgumentException("Unable to find a valid WMS layer");
379 }
380
381 /**
382 * Test that available imagery entry is valid.
383 */
384 @Test
385 public void testValidityOfAvailableImageryEntry() {
386 checkEntry(info);
387 assertTrue(errors.toString().replaceAll("\\}, ", "\n\\}, ").replaceAll(", ImageryInfo\\{", "\n ,ImageryInfo\\{"),
388 errors.isEmpty());
389 assertFalse(workingURLs.isEmpty());
390 }
391}
Note: See TracBrowser for help on using the repository browser.