source: josm/trunk/test/unit/org/openstreetmap/josm/gui/preferences/imagery/ImageryPreferenceTestIT.java@ 14694

Last change on this file since 14694 was 14694, checked in by Don-vip, 5 years ago

see #16073 - allow to ignore error only based on its last part
  • Property svn:eol-style set to native
File size: 17.5 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.gui.preferences.imagery;
3
4import static org.junit.Assert.assertFalse;
5import static org.junit.Assert.assertTrue;
6import static org.junit.Assume.assumeTrue;
7
8import java.io.ByteArrayInputStream;
9import java.io.IOException;
10import java.net.URL;
11import java.nio.charset.StandardCharsets;
12import java.util.ArrayList;
13import java.util.Collections;
14import java.util.LinkedList;
15import java.util.List;
16import java.util.Locale;
17import java.util.Map;
18import java.util.Objects;
19import java.util.Optional;
20import java.util.TreeMap;
21import java.util.concurrent.TimeUnit;
22import java.util.stream.Collectors;
23
24import javax.imageio.ImageIO;
25
26import org.apache.commons.jcs.access.CacheAccess;
27import org.junit.AfterClass;
28import org.junit.BeforeClass;
29import org.junit.ClassRule;
30import org.junit.Test;
31import org.junit.runner.Description;
32import org.junit.runner.RunWith;
33import org.junit.runners.Parameterized.Parameters;
34import org.junit.runners.model.Statement;
35import org.openstreetmap.gui.jmapviewer.Coordinate;
36import org.openstreetmap.gui.jmapviewer.TileXY;
37import org.openstreetmap.gui.jmapviewer.interfaces.ICoordinate;
38import org.openstreetmap.gui.jmapviewer.tilesources.AbstractTileSource;
39import org.openstreetmap.gui.jmapviewer.tilesources.BingAerialTileSource;
40import org.openstreetmap.gui.jmapviewer.tilesources.ScanexTileSource;
41import org.openstreetmap.gui.jmapviewer.tilesources.TemplatedTMSTileSource;
42import org.openstreetmap.josm.TestUtils;
43import org.openstreetmap.josm.actions.AddImageryLayerAction;
44import org.openstreetmap.josm.actions.AddImageryLayerAction.LayerSelection;
45import org.openstreetmap.josm.data.Bounds;
46import org.openstreetmap.josm.data.coor.LatLon;
47import org.openstreetmap.josm.data.imagery.CoordinateConversion;
48import org.openstreetmap.josm.data.imagery.ImageryInfo;
49import org.openstreetmap.josm.data.imagery.ImageryInfo.ImageryBounds;
50import org.openstreetmap.josm.data.imagery.ImageryInfo.ImageryType;
51import org.openstreetmap.josm.data.imagery.ImageryLayerInfo;
52import org.openstreetmap.josm.data.imagery.LayerDetails;
53import org.openstreetmap.josm.data.imagery.Shape;
54import org.openstreetmap.josm.data.imagery.TMSCachedTileLoaderJob;
55import org.openstreetmap.josm.data.imagery.TemplatedWMSTileSource;
56import org.openstreetmap.josm.data.imagery.TileJobOptions;
57import org.openstreetmap.josm.data.imagery.WMTSTileSource;
58import org.openstreetmap.josm.data.imagery.WMTSTileSource.WMTSGetCapabilitiesException;
59import org.openstreetmap.josm.data.projection.Projection;
60import org.openstreetmap.josm.data.projection.ProjectionRegistry;
61import org.openstreetmap.josm.data.projection.Projections;
62import org.openstreetmap.josm.io.imagery.WMSImagery.WMSGetCapabilitiesException;
63import org.openstreetmap.josm.testutils.JOSMTestRules;
64import org.openstreetmap.josm.testutils.ParallelParameterized;
65import org.openstreetmap.josm.tools.HttpClient;
66import org.openstreetmap.josm.tools.HttpClient.Response;
67import org.openstreetmap.josm.tools.Logging;
68import org.openstreetmap.josm.tools.Utils;
69
70import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
71
72/**
73 * Integration tests of {@link ImageryPreference} class.
74 */
75@RunWith(ParallelParameterized.class)
76public class ImageryPreferenceTestIT {
77
78 private static final String ERROR_SEP = " -> ";
79 private static final LatLon GREENWICH = new LatLon(51.47810, -0.00170);
80 private static final int DEFAULT_ZOOM = 12;
81
82 /**
83 * Setup rule
84 */
85 @ClassRule
86 @SuppressFBWarnings(value = "URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD")
87 public static JOSMTestRules test = new JOSMTestRules().https().i18n().preferences().projection().projectionNadGrids()
88 .timeout((int) TimeUnit.MINUTES.toMillis(40));
89
90 static {
91 try {
92 test.apply(new Statement() {
93 @Override
94 public void evaluate() throws Throwable {
95 // Do nothing. Hack needed because @Parameters are computed before anything else
96 }
97 }, Description.createSuiteDescription(ImageryPreferenceTestIT.class)).evaluate();
98 } catch (Throwable e) {
99 Logging.error(e);
100 }
101 }
102
103 /** Entry to test */
104 private final ImageryInfo info;
105 private final Map<String, Map<ImageryInfo, List<String>>> errors = Collections.synchronizedMap(new TreeMap<>());
106 private final Map<String, Map<ImageryInfo, List<String>>> ignoredErrors = Collections.synchronizedMap(new TreeMap<>());
107 private static final Map<String, byte[]> workingURLs = Collections.synchronizedMap(new TreeMap<>());
108
109 private static TMSCachedTileLoaderJob helper;
110 private static List<String> errorsToIgnore;
111 private static List<String> notIgnoredErrors;
112
113 /**
114 * Setup test
115 * @throws IOException in case of I/O error
116 */
117 @BeforeClass
118 public static void beforeClass() throws IOException {
119 helper = new TMSCachedTileLoaderJob(null, null, new CacheAccess<>(null), new TileJobOptions(0, 0, null, 0), null);
120 errorsToIgnore = TestUtils.getIgnoredErrorMessages(ImageryPreferenceTestIT.class);
121 notIgnoredErrors = new LinkedList<>(errorsToIgnore);
122 }
123
124 /**
125 * Cleanup test
126 */
127 @AfterClass
128 public static void afterClass() {
129 for (String e : notIgnoredErrors) {
130 Logging.warn("Ignore line unused: " + e);
131 }
132 }
133
134 /**
135 * Returns list of imagery entries to test.
136 * @return list of imagery entries to test
137 */
138 @Parameters(name = "{0}")
139 public static List<Object[]> data() {
140 ImageryLayerInfo.instance.load(false);
141 return ImageryLayerInfo.instance.getDefaultLayers()
142 .stream()
143 .map(x -> new Object[] {x.getId(), x})
144 .collect(Collectors.toList());
145 }
146
147 /**
148 * Constructs a new {@code ImageryPreferenceTestIT} instance.
149 * @param id entry ID, used only to name tests
150 * @param info entry to test
151 */
152 public ImageryPreferenceTestIT(String id, ImageryInfo info) {
153 this.info = Objects.requireNonNull(info);
154 }
155
156 private boolean addError(ImageryInfo info, String error) {
157 String errorMsg = error.replace('\n', ' ');
158 if (notIgnoredErrors.contains(errorMsg))
159 notIgnoredErrors.remove(errorMsg);
160 return addError(isIgnoredError(errorMsg) ? ignoredErrors : errors, info, errorMsg);
161 }
162
163 private boolean isIgnoredError(String errorMsg) {
164 int idx = errorMsg.lastIndexOf(ERROR_SEP);
165 return errorsToIgnore.contains(errorMsg) || (idx > -1 && errorsToIgnore.contains(errorMsg.substring(idx + ERROR_SEP.length())));
166 }
167
168 private static boolean addError(Map<String, Map<ImageryInfo, List<String>>> map, ImageryInfo info, String errorMsg) {
169 return map.computeIfAbsent(info.getCountryCode(), x -> Collections.synchronizedMap(new TreeMap<>()))
170 .computeIfAbsent(info, x -> Collections.synchronizedList(new ArrayList<>()))
171 .add(errorMsg);
172 }
173
174 private Optional<byte[]> checkUrl(ImageryInfo info, String url) {
175 if (url != null && !url.isEmpty()) {
176 if (workingURLs.containsKey(url)) {
177 return Optional.of(workingURLs.get(url));
178 }
179 try {
180 Response response = HttpClient.create(new URL(url))
181 .setHeaders(info.getCustomHttpHeaders())
182 .setConnectTimeout((int) TimeUnit.MINUTES.toMillis(1))
183 .setReadTimeout((int) TimeUnit.MINUTES.toMillis(5))
184 .connect();
185 if (response.getResponseCode() >= 400) {
186 addError(info, url + " -> HTTP " + response.getResponseCode());
187 } else if (response.getResponseCode() >= 300) {
188 Logging.warn(url + " -> HTTP " + response.getResponseCode());
189 }
190 try {
191 byte[] data = Utils.readBytesFromStream(response.getContent());
192 if (response.getResponseCode() < 300) {
193 workingURLs.put(url, data);
194 }
195 return Optional.of(data);
196 } catch (IOException e) {
197 if (response.getResponseCode() < 300) {
198 addError(info, url + ERROR_SEP + e);
199 }
200 } finally {
201 response.disconnect();
202 }
203 } catch (IOException e) {
204 addError(info, url + ERROR_SEP + e);
205 }
206 }
207 return Optional.empty();
208 }
209
210 private void checkLinkUrl(ImageryInfo info, String url) {
211 checkUrl(info, url).filter(x -> x.length == 0).ifPresent(x -> addError(info, url + " -> returned empty contents"));
212 }
213
214 private String checkTileUrl(ImageryInfo info, AbstractTileSource tileSource, ICoordinate center, int zoom)
215 throws IOException {
216 TileXY xy = tileSource.latLonToTileXY(center, zoom);
217 for (int i = 0; i < 3; i++) {
218 try {
219 String url = tileSource.getTileUrl(zoom, xy.getXIndex(), xy.getYIndex());
220 Optional<byte[]> optional = checkUrl(info, url);
221 String error = "";
222 if (optional.isPresent()) {
223 byte[] data = optional.get();
224 try (ByteArrayInputStream bais = new ByteArrayInputStream(data)) {
225 if (ImageIO.read(bais) == null) {
226 error = addImageError(info, url, data, zoom, "did not return an image");
227 }
228 } catch (IOException e) {
229 error = addImageError(info, url, data, zoom, e.toString());
230 Logging.trace(e);
231 }
232 }
233 return error;
234 } catch (IOException e) {
235 // Try up to three times max to allow Bing source to initialize itself
236 // and avoid random network errors
237 Logging.trace(e);
238 if (i == 2) {
239 throw e;
240 }
241 try {
242 Thread.sleep(500);
243 } catch (InterruptedException ex) {
244 Logging.warn(ex);
245 }
246 }
247 }
248 return "";
249 }
250
251 private static String zoomMarker(int zoom) {
252 return " -> zoom " + zoom + ERROR_SEP;
253 }
254
255 private String addImageError(ImageryInfo info, String url, byte[] data, int zoom, String defaultMessage) {
256 // Check if we have received an error message
257 String error = helper.detectErrorMessage(new String(data, StandardCharsets.UTF_8));
258 String errorMsg = url + zoomMarker(zoom) + (error != null ? error.split("\\n")[0] : defaultMessage);
259 addError(info, errorMsg);
260 return errorMsg;
261 }
262
263 private static LatLon getPointInShape(Shape shape) {
264 final Coordinate p1 = shape.getPoints().get(0);
265 final Bounds bounds = new Bounds(p1.getLat(), p1.getLon(), p1.getLat(), p1.getLon());
266 shape.getPoints().forEach(p -> bounds.extend(p.getLat(), p.getLon()));
267
268 final double w = bounds.getWidth();
269 final double h = bounds.getHeight();
270
271 final double x2 = bounds.getMinLon() + (w / 2.0);
272 final double y2 = bounds.getMinLat() + (h / 2.0);
273
274 final LatLon center = new LatLon(y2, x2);
275
276 // check to see if center is inside shape
277 if (shape.contains(center)) {
278 return center;
279 }
280
281 // if center position (C) is not inside shape, try naively some other positions as follows:
282 final double x1 = bounds.getMinLon() + (.25 * w);
283 final double x3 = bounds.getMinLon() + (.75 * w);
284 final double y1 = bounds.getMinLat() + (.25 * h);
285 final double y3 = bounds.getMinLat() + (.75 * h);
286 // +-----------+
287 // | 5 1 6 |
288 // | 4 C 2 |
289 // | 8 3 7 |
290 // +-----------+
291 for (LatLon candidate : new LatLon[] {
292 new LatLon(y1, x2),
293 new LatLon(y2, x3),
294 new LatLon(y3, x2),
295 new LatLon(y2, x1),
296 new LatLon(y1, x1),
297 new LatLon(y1, x3),
298 new LatLon(y3, x3),
299 new LatLon(y3, x1)
300 }) {
301 if (shape.contains(candidate)) {
302 return candidate;
303 }
304 }
305 return center;
306 }
307
308 private static LatLon getCenter(ImageryBounds bounds) {
309 List<Shape> shapes = bounds.getShapes();
310 return shapes != null && !shapes.isEmpty() ? getPointInShape(shapes.get(0)) : bounds.getCenter();
311 }
312
313 private void checkEntry(ImageryInfo info) {
314 Logging.info("Checking "+ info);
315
316 if (info.getAttributionImageRaw() != null && info.getAttributionImage() == null) {
317 addError(info, "Can't fetch attribution image: " + info.getAttributionImageRaw());
318 }
319
320 checkLinkUrl(info, info.getAttributionImageURL());
321 checkLinkUrl(info, info.getAttributionLinkURL());
322 String eula = info.getEulaAcceptanceRequired();
323 if (eula != null) {
324 checkLinkUrl(info, eula.replaceAll("\\{lang\\}", ""));
325 }
326 checkLinkUrl(info, info.getPermissionReferenceURL());
327 checkLinkUrl(info, info.getTermsOfUseURL());
328
329 try {
330 ImageryBounds bounds = info.getBounds();
331 // Some imagery sources do not define tiles at (0,0). So pickup Greenwich Royal Observatory for global sources
332 ICoordinate center = CoordinateConversion.llToCoor(bounds != null ? getCenter(bounds) : GREENWICH);
333 AbstractTileSource tileSource = getTileSource(info);
334 // test min zoom and try to detect the correct value in case of error
335 int maxZoom = info.getMaxZoom() > 0 ? Math.min(DEFAULT_ZOOM, info.getMaxZoom()) : DEFAULT_ZOOM;
336 for (int zoom = info.getMinZoom(); zoom < maxZoom; zoom++) {
337 if (!isZoomError(checkTileUrl(info, tileSource, center, zoom))) {
338 break;
339 }
340 }
341 // checking max zoom for real is complex, see https://josm.openstreetmap.de/ticket/16073#comment:27
342 if (info.getMaxZoom() > 0 && info.getImageryType() != ImageryType.SCANEX) {
343 checkTileUrl(info, tileSource, center, Utils.clamp(DEFAULT_ZOOM, info.getMinZoom() + 1, info.getMaxZoom()));
344 }
345 } catch (IOException | RuntimeException | WMSGetCapabilitiesException | WMTSGetCapabilitiesException e) {
346 addError(info, info.getUrl() + ERROR_SEP + e.toString());
347 }
348
349 for (ImageryInfo mirror : info.getMirrors()) {
350 checkEntry(mirror);
351 }
352 }
353
354 private static boolean isZoomError(String error) {
355 String[] parts = error.split(ERROR_SEP);
356 String lastPart = parts.length > 0 ? parts[parts.length - 1].toLowerCase(Locale.ENGLISH) : "";
357 return lastPart.contains("bbox")
358 || lastPart.contains("bounding box");
359 }
360
361 private static Projection getProjection(ImageryInfo info) {
362 for (String code : info.getServerProjections()) {
363 Projection proj = Projections.getProjectionByCode(code);
364 if (proj != null) {
365 return proj;
366 }
367 }
368 return ProjectionRegistry.getProjection();
369 }
370
371 @SuppressWarnings("fallthrough")
372 private static AbstractTileSource getTileSource(ImageryInfo info)
373 throws IOException, WMTSGetCapabilitiesException, WMSGetCapabilitiesException {
374 switch (info.getImageryType()) {
375 case BING:
376 return new BingAerialTileSource(info);
377 case SCANEX:
378 return new ScanexTileSource(info);
379 case TMS:
380 return new TemplatedTMSTileSource(info);
381 case WMS_ENDPOINT:
382 info = convertWmsEndpointToWms(info); // fall-through
383 case WMS:
384 return new TemplatedWMSTileSource(info, getProjection(info));
385 case WMTS:
386 return new WMTSTileSource(info, getProjection(info));
387 default:
388 throw new UnsupportedOperationException(info.toString());
389 }
390 }
391
392 private static ImageryInfo convertWmsEndpointToWms(ImageryInfo info) throws IOException, WMSGetCapabilitiesException {
393 return Optional.ofNullable(AddImageryLayerAction.getWMSLayerInfo(
394 info, wms -> new LayerSelection(firstLeafLayer(wms.getLayers()), wms.getPreferredFormat(), true)))
395 .orElseThrow(() -> new IllegalStateException("Unable to convert WMS_ENDPOINT to WMS"));
396 }
397
398 private static List<LayerDetails> firstLeafLayer(List<LayerDetails> layers) {
399 for (LayerDetails layer : layers) {
400 boolean hasNoChildren = layer.getChildren().isEmpty();
401 if (hasNoChildren && layer.getName() != null) {
402 return Collections.singletonList(layer);
403 } else if (!hasNoChildren) {
404 return firstLeafLayer(layer.getChildren());
405 }
406 }
407 throw new IllegalArgumentException("Unable to find a valid WMS layer");
408 }
409
410 private static String format(Map<String, Map<ImageryInfo, List<String>>> map) {
411 return map.toString().replaceAll("\\}, ", "\n\\}, ").replaceAll(", ImageryInfo\\{", "\n ,ImageryInfo\\{");
412 }
413
414 /**
415 * Test that available imagery entry is valid.
416 */
417 @Test
418 public void testImageryEntryValidity() {
419 checkEntry(info);
420 assertTrue(format(errors), errors.isEmpty());
421 assertFalse(workingURLs.isEmpty());
422 assumeTrue(format(ignoredErrors), ignoredErrors.isEmpty());
423 }
424}
Note: See TracBrowser for help on using the repository browser.