Ignore:
Timestamp:
2018-05-12T14:18:57+02:00 (6 years ago)
Author:
wiktorn
Message:

Imagery definition refactor

Extend imagery definitions by:

  • allowing setting default layers for WMS_ENDPOINT and WMTS
  • allowing setting minimum expires time for tile for this imagery
  • allowing setting custom headers that will be sent for all requests

(get map, get capabilities) for this imagery

Additional changes in code:

  • use TileJobOptions to pass miscellaneous options to loaders
  • refactor WMSImagery to use SAX parser

See: #15981, #7953, #16224, #15940, #16249

Location:
trunk/test/unit/org/openstreetmap/josm/data/cache
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • trunk/test/unit/org/openstreetmap/josm/data/cache/HostLimitQueueTest.java

    r12620 r13733  
    1414import org.junit.Rule;
    1515import org.junit.Test;
     16import org.openstreetmap.josm.data.imagery.TileJobOptions;
    1617import org.openstreetmap.josm.testutils.JOSMTestRules;
    1718import org.openstreetmap.josm.tools.Logging;
    … …  
    5455
    5556        Task(ICacheAccess<String, CacheEntry> cache, URL url, AtomicInteger counter) {
    56             super(cache, 1, 1, null);
     57            super(cache, new TileJobOptions(1, 1, null, 10));
    5758            this.url = url;
    5859            this.counter = counter;

  • trunk/test/unit/org/openstreetmap/josm/data/cache/JCSCachedTileLoaderJobTest.java

    r13358 r13733  
    22package org.openstreetmap.josm.data.cache;
    33
     4import static org.junit.Assert.assertArrayEquals;
    45import static org.junit.Assert.assertEquals;
    56import static org.junit.Assert.assertFalse;
     7import static org.junit.Assert.assertTrue;
    68
    79import java.io.IOException;
    … …  
    911import java.net.URL;
    1012import java.nio.charset.StandardCharsets;
     13import java.util.concurrent.TimeUnit;
    1114
    1215import org.apache.commons.jcs.access.behavior.ICacheAccess;
     16import org.apache.commons.jcs.engine.behavior.ICacheElement;
    1317import org.junit.Before;
    1418import org.junit.Rule;
    1519import org.junit.Test;
     20import org.openstreetmap.josm.TestUtils;
    1621import org.openstreetmap.josm.data.cache.ICachedLoaderListener.LoadResult;
     22import org.openstreetmap.josm.data.imagery.TileJobOptions;
    1723import org.openstreetmap.josm.testutils.JOSMTestRules;
    1824import org.openstreetmap.josm.tools.Logging;
     25
     26import com.github.tomakehurst.wiremock.client.WireMock;
     27import com.github.tomakehurst.wiremock.core.WireMockConfiguration;
     28import com.github.tomakehurst.wiremock.junit.WireMockRule;
     29import com.github.tomakehurst.wiremock.matching.UrlPattern;
    1930
    2031import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
    … …  
    2536public class JCSCachedTileLoaderJobTest {
    2637
     38    /**
     39     * mocked tile server
     40     */
     41    @Rule
     42    public WireMockRule tileServer = new WireMockRule(WireMockConfiguration.options()
     43            .dynamicPort());
     44
    2745    private static class TestCachedTileLoaderJob extends JCSCachedTileLoaderJob<String, CacheEntry> {
    2846        private String url;
    2947        private String key;
    3048
    31         TestCachedTileLoaderJob(String url, String key) throws IOException {
    32             super(getCache(), 30000, 30000, null);
     49        TestCachedTileLoaderJob(String url, String key)  {
     50            this(url, key, (int) TimeUnit.DAYS.toSeconds(1));
     51        }
     52
     53        TestCachedTileLoaderJob(String url, String key, int minimumExpiry)  {
     54            super(getCache(), new TileJobOptions(30000, 30000, null, minimumExpiry));
    3355
    3456            this.url = url;
    3557            this.key = key;
    3658        }
     59
    3760
    3861        @Override
    … …  
    5275        @Override
    5376        protected CacheEntry createCacheEntry(byte[] content) {
    54             return new CacheEntry("dummy".getBytes(StandardCharsets.UTF_8));
     77            return new CacheEntry(content);
    5578        }
    5679    }
    … …  
    6083        private boolean ready;
    6184        private LoadResult result;
     85        private byte[] data;
    6286
    6387        @Override
    … …  
    6690            this.ready = true;
    6791            this.result = result;
     92            if (data != null) {
     93                this.data = data.content;
     94            }
    6895            this.notifyAll();
    6996        }
    … …  
    113140        String key = "key_unknown_host";
    114141        TestCachedTileLoaderJob job = new TestCachedTileLoaderJob("http://unkownhost.unkownhost/unkown", key);
    115         Listener listener = new Listener();
    116         job.submit(listener, true);
    117         synchronized (listener) {
    118             while (!listener.ready) {
    119                 try {
    120                     listener.wait();
    121                 } catch (InterruptedException e1) {
    122                     // do nothing, still wait
    123                     Logging.trace(e1);
    124                 }
    125             }
    126         }
     142        Listener listener = submitJob(job);
    127143        assertEquals(LoadResult.FAILURE, listener.result); // because response will be cached, and that is checked below
    128144        assertEquals("java.net.UnknownHostException: unkownhost.unkownhost", listener.attributes.getErrorMessage());
    … …  
    135151
    136152        job = new TestCachedTileLoaderJob("http://unkownhost.unkownhost/unkown", key);
    137         listener = new Listener();
    138         job.submit(listener, true);
     153        listener = submitJob(job);
     154        assertEquals(LoadResult.SUCCESS, listener.result);
     155        assertFalse(job.isCacheElementValid());
     156    }
     157
     158    private void doTestStatusCode(int responseCode) throws IOException {
     159        TestCachedTileLoaderJob job = getStatusLoaderJob(responseCode);
     160        Listener listener = submitJob(job);
     161        assertEquals(responseCode, listener.attributes.getResponseCode());
     162    }
     163
     164    private Listener submitJob(TestCachedTileLoaderJob job) throws IOException {
     165        return submitJob(job, true);
     166    }
     167
     168    private Listener submitJob(TestCachedTileLoaderJob job, boolean force) throws IOException {
     169        Listener listener = new Listener();
     170        job.submit(listener, force);
    139171        synchronized (listener) {
    140172            while (!listener.ready) {
    141173                try {
    142174                    listener.wait();
    143                 } catch (InterruptedException e1) {
     175                } catch (InterruptedException e) {
    144176                    // do nothing, wait
    145                     Logging.trace(e1);
     177                    Logging.trace(e);
    146178                }
    147179            }
    148180        }
    149         assertEquals(LoadResult.SUCCESS, listener.result);
    150         assertFalse(job.isCacheElementValid());
    151     }
    152 
    153     @SuppressFBWarnings(value = "WA_NOT_IN_LOOP")
    154     private void doTestStatusCode(int responseCode) throws IOException, InterruptedException {
    155         TestCachedTileLoaderJob job = getStatusLoaderJob(responseCode);
    156         Listener listener = new Listener();
    157         job.submit(listener, true);
    158         synchronized (listener) {
    159             if (!listener.ready) {
    160                 listener.wait();
    161             }
    162         }
    163         assertEquals(responseCode, listener.attributes.getResponseCode());
    164     }
    165 
    166     private static TestCachedTileLoaderJob getStatusLoaderJob(int responseCode) throws IOException {
     181        return listener;
     182    }
     183
     184    /**
     185     * That no requst is made when entry is in cache and force == false
     186     * @throws IOException
     187     */
     188    @Test
     189    public void testNoRequestMadeWhenEntryInCache() throws IOException {
     190        ICacheAccess<String, CacheEntry> cache = getCache();
     191        long expires = TimeUnit.DAYS.toMillis(1);
     192        long testStart = System.currentTimeMillis();
     193        cache.put("test",
     194                new CacheEntry("cached entry".getBytes(StandardCharsets.UTF_8)),
     195                createEntryAttributes(expires, 200, testStart, "eTag")
     196                );
     197        createHeadGetStub(WireMock.urlEqualTo("/test"), expires, testStart, "eTag", "mock entry");
     198
     199        TestCachedTileLoaderJob job = new TestCachedTileLoaderJob(tileServer.url("/test"), "test");
     200        Listener listener = submitJob(job, false);
     201        tileServer.verify(0, WireMock.getRequestedFor(WireMock.anyUrl()));
     202        assertArrayEquals("cached entry".getBytes(StandardCharsets.UTF_8), listener.data);
     203    }
     204
     205    /**
     206     * that request is made, when object is in cache, but force mode is used
     207     * @throws IOException
     208     */
     209    @Test
     210    public void testRequestMadeWhenEntryInCacheAndForce() throws IOException {
     211        ICacheAccess<String, CacheEntry> cache = getCache();
     212        long expires =  TimeUnit.DAYS.toMillis(1);
     213        long testStart = System.currentTimeMillis();
     214        cache.put("test",
     215                new CacheEntry("cached dummy".getBytes(StandardCharsets.UTF_8)),
     216                createEntryAttributes(expires, 200, testStart + expires, "eTag")
     217                );
     218        createHeadGetStub(WireMock.urlEqualTo("/test"), expires, testStart, "eTag", "mock entry");
     219
     220        TestCachedTileLoaderJob job = new TestCachedTileLoaderJob(tileServer.url("/test"), "test");
     221        Listener listener = submitJob(job, true);
     222        tileServer.verify(1, WireMock.getRequestedFor(WireMock.urlEqualTo("/test")));
     223        assertArrayEquals("mock entry".getBytes(StandardCharsets.UTF_8), listener.data);
     224    }
     225
     226    /**
     227     * Mock returns no cache-control / expires headers
     228     * Expire time should be set to DEFAULT_EXPIRE_TIME
     229     * @throws IOException
     230     */
     231    @Test
     232    public void testSettingMinimumExpiryWhenNoExpires() throws IOException {
     233        long testStart = System.currentTimeMillis();
     234        tileServer.stubFor(
     235                WireMock.get(WireMock.urlEqualTo("/test"))
     236                .willReturn(WireMock.aResponse()
     237                        .withBody("mock entry")
     238                        )
     239                );
     240
     241        TestCachedTileLoaderJob job = new TestCachedTileLoaderJob(tileServer.url("/test"), "test");
     242        Listener listener = submitJob(job, false);
     243        tileServer.verify(1, WireMock.getRequestedFor(WireMock.urlEqualTo("/test")));
     244
     245        assertTrue("Cache entry expiration is " + (listener.attributes.getExpirationTime() - testStart) + " which is not larger than " +
     246                JCSCachedTileLoaderJob.DEFAULT_EXPIRE_TIME + " (DEFAULT_EXPIRE_TIME)",
     247                listener.attributes.getExpirationTime() >= testStart + JCSCachedTileLoaderJob.DEFAULT_EXPIRE_TIME);
     248
     249        assertTrue("Cache entry expiration is " + (listener.attributes.getExpirationTime() - System.currentTimeMillis()) + " which is not less than " +
     250                JCSCachedTileLoaderJob.DEFAULT_EXPIRE_TIME + " (DEFAULT_EXPIRE_TIME)",
     251                listener.attributes.getExpirationTime() <= System.currentTimeMillis() + JCSCachedTileLoaderJob.DEFAULT_EXPIRE_TIME);
     252
     253        assertArrayEquals("mock entry".getBytes(StandardCharsets.UTF_8), listener.data);
     254    }
     255
     256    /**
     257     * Mock returns expires headers, but Cache-Control
     258     * Expire time should be set to max-age
     259     * @throws IOException
     260     */
     261    @Test
     262    public void testSettingExpireByMaxAge() throws IOException {
     263        long testStart = System.currentTimeMillis();
     264        long expires =  TimeUnit.DAYS.toSeconds(1);
     265        tileServer.stubFor(
     266                WireMock.get(WireMock.urlEqualTo("/test"))
     267                .willReturn(WireMock.aResponse()
     268                        .withHeader("Cache-control", "max-age=" + expires)
     269                        .withBody("mock entry")
     270                        )
     271                );
     272
     273        TestCachedTileLoaderJob job = new TestCachedTileLoaderJob(tileServer.url("/test"), "test");
     274        Listener listener = submitJob(job, false);
     275        tileServer.verify(1, WireMock.getRequestedFor(WireMock.urlEqualTo("/test")));
     276
     277        assertTrue("Cache entry expiration is " + (listener.attributes.getExpirationTime() - testStart) + " which is not larger than " +
     278                TimeUnit.SECONDS.toMillis(expires) + " (max-age)",
     279                listener.attributes.getExpirationTime() >= testStart + TimeUnit.SECONDS.toMillis(expires));
     280
     281        assertTrue("Cache entry expiration is " + (listener.attributes.getExpirationTime() - System.currentTimeMillis()) + " which is not less than " +
     282                TimeUnit.SECONDS.toMillis(expires) + " (max-age)",
     283                listener.attributes.getExpirationTime() <= System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(expires));
     284
     285        assertArrayEquals("mock entry".getBytes(StandardCharsets.UTF_8), listener.data);
     286    }
     287
     288    /**
     289     * mock returns expiration: JCSCachedTileLoaderJob.DEFAULT_EXPIRE_TIME / 10
     290     * minimum expire time: JCSCachedTileLoaderJob.DEFAULT_EXPIRE_TIME / 2
     291     * @throws IOException
     292     */
     293    @Test
     294    public void testSettingMinimumExpiryByMinimumExpiryTimeLessThanDefault() throws IOException {
     295        long testStart = System.currentTimeMillis();
     296        int minimumExpiryTimeSeconds = (int)(JCSCachedTileLoaderJob.DEFAULT_EXPIRE_TIME / 2);
     297
     298        createHeadGetStub(WireMock.urlEqualTo("/test"), (JCSCachedTileLoaderJob.DEFAULT_EXPIRE_TIME / 10), testStart, "eTag", "mock entry");
     299
     300        TestCachedTileLoaderJob job = new TestCachedTileLoaderJob(tileServer.url("/test"), "test", minimumExpiryTimeSeconds);
     301        Listener listener = submitJob(job, false);
     302        tileServer.verify(1, WireMock.getRequestedFor(WireMock.urlEqualTo("/test")));
     303        assertArrayEquals("mock entry".getBytes(StandardCharsets.UTF_8), listener.data);
     304
     305
     306        assertTrue("Cache entry expiration is " + (listener.attributes.getExpirationTime() - testStart) + " which is not larger than " +
     307                TimeUnit.SECONDS.toMillis(minimumExpiryTimeSeconds) + " (minimumExpireTime)",
     308                listener.attributes.getExpirationTime() >= testStart + TimeUnit.SECONDS.toMillis(minimumExpiryTimeSeconds) );
     309
     310        assertTrue("Cache entry expiration is " + (listener.attributes.getExpirationTime() - System.currentTimeMillis()) + " which is not less than " +
     311                TimeUnit.SECONDS.toMillis(minimumExpiryTimeSeconds) + " (minimumExpireTime)",
     312                listener.attributes.getExpirationTime() <= System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(minimumExpiryTimeSeconds));
     313    }
     314
     315    /**
     316     * mock returns expiration: JCSCachedTileLoaderJob.DEFAULT_EXPIRE_TIME / 10
     317     * minimum expire time: JCSCachedTileLoaderJob.DEFAULT_EXPIRE_TIME * 2
     318     * @throws IOException
     319     */
     320
     321    @Test
     322    public void testSettingMinimumExpiryByMinimumExpiryTimeGreaterThanDefault() throws IOException {
     323        long testStart = System.currentTimeMillis();
     324        int minimumExpiryTimeSeconds = (int)(JCSCachedTileLoaderJob.DEFAULT_EXPIRE_TIME * 2);
     325
     326        createHeadGetStub(WireMock.urlEqualTo("/test"), (JCSCachedTileLoaderJob.DEFAULT_EXPIRE_TIME / 10), testStart, "eTag", "mock entry");
     327
     328        TestCachedTileLoaderJob job = new TestCachedTileLoaderJob(tileServer.url("/test"), "test", minimumExpiryTimeSeconds);
     329        Listener listener = submitJob(job, false);
     330        tileServer.verify(1, WireMock.getRequestedFor(WireMock.urlEqualTo("/test")));
     331        assertArrayEquals("mock entry".getBytes(StandardCharsets.UTF_8), listener.data);
     332
     333
     334        assertTrue("Cache entry expiration is " + (listener.attributes.getExpirationTime() - testStart) + " which is not larger than " +
     335                TimeUnit.SECONDS.toMillis(minimumExpiryTimeSeconds) + " (minimumExpireTime)",
     336                listener.attributes.getExpirationTime() >= testStart + TimeUnit.SECONDS.toMillis(minimumExpiryTimeSeconds) );
     337
     338        assertTrue("Cache entry expiration is " + (listener.attributes.getExpirationTime() - System.currentTimeMillis()) + " which is not less than " +
     339                TimeUnit.SECONDS.toMillis(minimumExpiryTimeSeconds) + " (minimumExpireTime)",
     340                listener.attributes.getExpirationTime() <= System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(minimumExpiryTimeSeconds));
     341    }
     342
     343    /**
     344     * Check if verifying cache entries using HEAD requests work properly
     345     * @throws IOException
     346     */
     347    @Test
     348    public void testCheckUsingHead() throws IOException {
     349        ICacheAccess<String, CacheEntry> cache = getCache();
     350        long expires = TimeUnit.DAYS.toMillis(1);
     351        long testStart = System.currentTimeMillis();
     352        cache.put("test",
     353                new CacheEntry("cached dummy".getBytes(StandardCharsets.UTF_8)),
     354                createEntryAttributes(-1 * expires, 200, testStart, "eTag--gzip") // Jetty adds --gzip to etags when compressing output
     355                );
     356
     357        tileServer.stubFor(
     358                WireMock.get(WireMock.urlEqualTo("/test"))
     359                .willReturn(WireMock.aResponse()
     360                        .withHeader("Expires", TestUtils.getHTTPDate(testStart + expires))
     361                        .withHeader("Last-Modified", Long.toString(testStart))
     362                        .withHeader("ETag", "eTag") // Jetty adds "--gzip" suffix for compressed content
     363                        .withBody("mock entry")
     364                        )
     365                );
     366        tileServer.stubFor(
     367                WireMock.head(WireMock.urlEqualTo("/test"))
     368                .willReturn(WireMock.aResponse()
     369                        .withHeader("Expires", TestUtils.getHTTPDate(testStart + expires))
     370                        .withHeader("Last-Modified", Long.toString(testStart))
     371                        .withHeader("ETag", "eTag--gzip") // but doesn't add to uncompressed
     372                        )
     373                );
     374
     375        TestCachedTileLoaderJob job = new TestCachedTileLoaderJob(tileServer.url("/test"), "test");
     376        Listener listener = submitJob(job, false); // cache entry is expired, no need to force refetch
     377        tileServer.verify(1, WireMock.getRequestedFor(WireMock.urlEqualTo("/test")));
     378        assertArrayEquals("mock entry".getBytes(StandardCharsets.UTF_8), listener.data);
     379
     380        // cache entry should be retrieved from cache
     381        listener = submitJob(job, false);
     382        tileServer.verify(1, WireMock.getRequestedFor(WireMock.urlEqualTo("/test")));
     383        assertArrayEquals("mock entry".getBytes(StandardCharsets.UTF_8), listener.data);
     384
     385        // invalidate entry in cache
     386        ICacheElement<String, CacheEntry> cacheEntry = cache.getCacheElement("test");
     387        CacheEntryAttributes attributes = (CacheEntryAttributes)cacheEntry.getElementAttributes();
     388        attributes.setExpirationTime(testStart - TimeUnit.DAYS.toMillis(1));
     389        cache.put("test", cacheEntry.getVal(), attributes);
     390
     391        // because cache entry is invalid - HEAD request shall be made
     392        tileServer.verify(0, WireMock.headRequestedFor(WireMock.urlEqualTo("/test"))); // no head requests were made until now
     393        listener = submitJob(job, false);
     394        tileServer.verify(1, WireMock.headRequestedFor(WireMock.urlEqualTo("/test"))); // verify head requests were made
     395        tileServer.verify(1, WireMock.getRequestedFor(WireMock.urlEqualTo("/test"))); // verify no more get requests were made
     396        assertArrayEquals("mock entry".getBytes(StandardCharsets.UTF_8), listener.data);
     397        assertTrue(listener.attributes.getExpirationTime() >= testStart + expires);
     398
     399        // cache entry should be retrieved from cache
     400        listener = submitJob(job, false); // cache entry is expired, no need to force refetch
     401        tileServer.verify(1, WireMock.getRequestedFor(WireMock.urlEqualTo("/test")));
     402        tileServer.verify(1, WireMock.getRequestedFor(WireMock.urlEqualTo("/test")));
     403        assertArrayEquals("mock entry".getBytes(StandardCharsets.UTF_8), listener.data);
     404    }
     405
     406    /**
     407     * Check if server returns 304 - it will update cache attributes and not ask again for it
     408     * @throws IOException
     409     */
     410    @Test
     411    public void testCheckUsing304() throws IOException {
     412        ICacheAccess<String, CacheEntry> cache = getCache();
     413        long expires = TimeUnit.DAYS.toMillis(1);
     414        long testStart = System.currentTimeMillis();
     415        cache.put("test",
     416                new CacheEntry("cached dummy".getBytes(StandardCharsets.UTF_8)),
     417                createEntryAttributes(-1 * expires, 200, testStart, "eTag")
     418                );
     419
     420        tileServer.stubFor(
     421                WireMock.get(WireMock.urlEqualTo("/test"))
     422                .willReturn(WireMock.status(304)
     423                        .withHeader("Expires", TestUtils.getHTTPDate(testStart + expires))
     424                        .withHeader("Last-Modified", Long.toString(testStart))
     425                        .withHeader("ETag", "eTag")
     426                        )
     427                );
     428
     429        TestCachedTileLoaderJob job = new TestCachedTileLoaderJob(tileServer.url("/test"), "test");
     430        Listener listener = submitJob(job, false);
     431        tileServer.verify(1, WireMock.getRequestedFor(WireMock.urlEqualTo("/test")));
     432        assertArrayEquals("cached dummy".getBytes(StandardCharsets.UTF_8), listener.data);
     433        assertTrue(testStart + expires <= listener.attributes.getExpirationTime());
     434        listener = submitJob(job, false);
     435        tileServer.verify(1, WireMock.getRequestedFor(WireMock.urlEqualTo("/test"))); // no more requests were made
     436    }
     437
     438    private void createHeadGetStub(UrlPattern url, long expires, long lastModified, String eTag, String body) {
     439        tileServer.stubFor(
     440                WireMock.get(url)
     441                .willReturn(WireMock.aResponse()
     442                        .withHeader("Expires", TestUtils.getHTTPDate(lastModified + expires))
     443                        .withHeader("Last-Modified", Long.toString(lastModified))
     444                        .withHeader("ETag", eTag)
     445                        .withBody(body)
     446                        )
     447                );
     448        tileServer.stubFor(
     449                WireMock.head(url)
     450                .willReturn(WireMock.aResponse()
     451                        .withHeader("Expires", TestUtils.getHTTPDate(lastModified + expires))
     452                        .withHeader("Last-Modified", Long.toString(lastModified))
     453                        .withHeader("ETag", eTag)
     454                        )
     455                );
     456    }
     457
     458    private CacheEntryAttributes createEntryAttributes(long maxAge, int responseCode, String eTag) {
     459        long validTo = maxAge + System.currentTimeMillis();
     460        return createEntryAttributes(maxAge, responseCode, validTo, eTag);
     461    }
     462
     463    private CacheEntryAttributes createEntryAttributes(long expirationTime, int responseCode, long lastModification, String eTag) {
     464        CacheEntryAttributes entryAttributes = new CacheEntryAttributes();
     465        entryAttributes.setExpirationTime(lastModification + expirationTime);
     466        entryAttributes.setResponseCode(responseCode);
     467        entryAttributes.setLastModification(lastModification);
     468        entryAttributes.setEtag(eTag);
     469        return entryAttributes;
     470    }
     471
     472    private static TestCachedTileLoaderJob getStatusLoaderJob(int responseCode)  {
    167473        return new TestCachedTileLoaderJob("http://httpstat.us/" + responseCode, "key_" + responseCode);
    168474    }
    169475
    170     private static ICacheAccess<String, CacheEntry> getCache() throws IOException {
     476    private static ICacheAccess<String, CacheEntry> getCache() {
    171477        return JCSCacheManager.getCache("test");
    172478    }
Note: See TracChangeset for help on using the changeset viewer.