Changeset 8397 in josm


Ignore:
Timestamp:
2015-05-19T20:54:26+02:00 (6 years ago)
Author:
wiktorn
Message:

addresses #11437 - introduce infinite queue for tile loading and clear the queue when user pans the map or changes the zoom. Fixing hosts limit is last problem, thay may incur additional load

Location:
trunk/src/org/openstreetmap/josm
Files:
4 edited

Legend:

Unmodified
Added
Removed
  • trunk/src/org/openstreetmap/josm/data/cache/JCSCachedTileLoaderJob.java

    r8389 r8397  
    6363
    6464    public static class LIFOQueue extends LinkedBlockingDeque<Runnable> {
     65        public LIFOQueue() {
     66            super();
     67        }
     68
    6569        public LIFOQueue(int capacity) {
    6670            super(capacity);
     
    9195     * and performance (we do want to have something to offer to worker threads before tasks will be resubmitted by class consumer)
    9296     */
    93     private static Executor DOWNLOAD_JOB_DISPATCHER = new ThreadPoolExecutor(
     97    private static Executor DEFAULT_DOWNLOAD_JOB_DISPATCHER = new ThreadPoolExecutor(
    9498            2, // we have a small queue, so threads will be quickly started (threads are started only, when queue is full)
    9599            THREAD_LIMIT.get().intValue(), // do not this number of threads
     
    114118    private int readTimeout;
    115119    private Map<String, String> headers;
     120    private Executor downloadJobExecutor;
    116121
    117122    /**
     
    120125     * @param readTimeout
    121126     * @param connectTimeout
     127     * @param downloadJobExecutor
    122128     */
    123129    public JCSCachedTileLoaderJob(ICacheAccess<K,V> cache,
    124130            int connectTimeout, int readTimeout,
    125             Map<String, String> headers) {
     131            Map<String, String> headers,
     132            Executor downloadJobExecutor) {
    126133
    127134        this.cache = cache;
     
    130137        this.readTimeout = readTimeout;
    131138        this.headers = headers;
     139        this.downloadJobExecutor = downloadJobExecutor;
     140    }
     141
     142    /**
     143     * @param cache cache instance that we will work on
     144     * @param headers
     145     * @param readTimeout
     146     * @param connectTimeout
     147     */
     148    public JCSCachedTileLoaderJob(ICacheAccess<K, V> cache,
     149            int connectTimeout, int readTimeout,
     150            Map<String, String> headers) {
     151        this(cache, connectTimeout, readTimeout,
     152                headers, DEFAULT_DOWNLOAD_JOB_DISPATCHER);
    132153    }
    133154
     
    240261     */
    241262    protected Executor getDownloadExecutor() {
    242         return DOWNLOAD_JOB_DISPATCHER;
    243     }
    244 
     263        return downloadJobExecutor;
     264    }
    245265
    246266    public void run() {
     
    330350                return true;
    331351            }
     352
    332353            HttpURLConnection urlConn = getURLConnection();
    333354
  • trunk/src/org/openstreetmap/josm/data/imagery/TMSCachedTileLoader.java

    r8327 r8397  
    44import java.io.IOException;
    55import java.util.Map;
     6import java.util.concurrent.ThreadPoolExecutor;
     7import java.util.concurrent.TimeUnit;
    68
    79import org.apache.commons.jcs.access.behavior.ICacheAccess;
     
    1517import org.openstreetmap.josm.data.cache.BufferedImageCacheEntry;
    1618import org.openstreetmap.josm.data.cache.JCSCacheManager;
     19import org.openstreetmap.josm.data.cache.JCSCachedTileLoaderJob.LIFOQueue;
    1720import org.openstreetmap.josm.data.preferences.IntegerProperty;
    1821
     
    3033    private Map<String, String> headers;
    3134    private TileLoaderListener listener;
    32     public static final String PREFERENCE_PREFIX   = "imagery.tms.cache.";
     35    private static final String PREFERENCE_PREFIX   = "imagery.tms.cache.";
    3336    // average tile size is about 20kb
    3437    public static final IntegerProperty MAX_OBJECTS_ON_DISK = new IntegerProperty(PREFERENCE_PREFIX + "max_objects_disk", 25000); // 25000 is around 500MB under this assumptions
    3538
     39    /**
     40     * overrides the THREAD_LIMIT in superclass, as we want to have separate limit and pool for TMS
     41     */
     42    public static final IntegerProperty THREAD_LIMIT = new IntegerProperty("imagery.tms.tmsloader.maxjobs", 25);
     43
     44    /**
     45     * separate from JCS thread pool for TMS loader, so we can have different thread pools for default JCS
     46     * and for TMS imagery
     47     */
     48    private static ThreadPoolExecutor DEFAULT_DOWNLOAD_JOB_DISPATCHER = getThreadPoolExecutor();
     49
     50    private static ThreadPoolExecutor getThreadPoolExecutor() {
     51        return new ThreadPoolExecutor(
     52                THREAD_LIMIT.get().intValue(), // keep the thread number constant
     53                THREAD_LIMIT.get().intValue(), // do not this number of threads
     54                30, // keepalive for thread
     55                TimeUnit.SECONDS,
     56                // make queue of LIFO type - so recently requested tiles will be loaded first (assuming that these are which user is waiting to see)
     57                new LIFOQueue()
     58                    /* keep the queue size fairly small, we do not want to
     59                     download a lot of tiles, that user is not seeing anyway */
     60                );
     61    }
     62
     63    private ThreadPoolExecutor downloadExecutor = DEFAULT_DOWNLOAD_JOB_DISPATCHER;
    3664
    3765    /**
     
    5886    @Override
    5987    public TileJob createTileLoaderJob(Tile tile) {
    60         return new TMSCachedTileLoaderJob(listener, tile, cache, connectTimeout, readTimeout, headers);
     88        return new TMSCachedTileLoaderJob(listener, tile, cache,
     89                connectTimeout, readTimeout, headers, downloadExecutor);
    6190    }
    6291
     
    86115    }
    87116
     117    /**
     118     * @return cache statistics as string
     119     */
    88120    public String getStats() {
    89121        return cache.getStats();
    90122    }
     123
     124    /**
     125     * Sets the download executor for this tile loader factory. Enables to use different queuing method
     126     * for this factory.
     127     * @param downloadExecutor
     128     */
     129    public void setDownloadExecutor(ThreadPoolExecutor downloadExecutor) {
     130        this.downloadExecutor = downloadExecutor;
     131    }
     132
     133    /**
     134     * @return Executor that handles the jobs for this tile loader
     135     */
     136    public ThreadPoolExecutor getDownloadExecutor() {
     137        return downloadExecutor;
     138    }
     139
     140    /**
     141     * cancels all outstanding tasks in the queue. This rollbacks the state of the tiles in the queue
     142     * to loading = false / loaded = false
     143     */
     144    public void cancelOutstandingTasks() {
     145        for(Runnable elem: downloadExecutor.getQueue()) {
     146            if (elem instanceof TMSCachedTileLoaderJob) {
     147                TMSCachedTileLoaderJob loaderJob = (TMSCachedTileLoaderJob) elem;
     148                if (downloadExecutor.remove(loaderJob)) {
     149                    Tile t = loaderJob.getTile();
     150                    t.finishLoading();
     151                    t.setLoaded(false);
     152                }
     153            }
     154        }
     155    }
     156
    91157}
  • trunk/src/org/openstreetmap/josm/data/imagery/TMSCachedTileLoaderJob.java

    r8389 r8397  
    1515import java.util.concurrent.Executor;
    1616import java.util.concurrent.Semaphore;
    17 import java.util.concurrent.ThreadPoolExecutor;
    18 import java.util.concurrent.TimeUnit;
    1917import java.util.logging.Level;
    2018import java.util.logging.Logger;
     
    9997
    10098    /**
    101      * overrides the THREAD_LIMIT in superclass, as we want to have separate limit and pool for TMS
    102      */
    103     public static final IntegerProperty THREAD_LIMIT = new IntegerProperty("imagery.tms.tmsloader.maxjobs", 25);
    104 
    105     /**
    106      * separate from JCS thread pool for TMS loader, so we can have different thread pools for default JCS
    107      * and for TMS imagery
    108      */
    109     private static ThreadPoolExecutor DOWNLOAD_JOB_DISPATCHER = getThreadPoolExecutor();
    110 
    111     private static ThreadPoolExecutor getThreadPoolExecutor() {
    112         return new ThreadPoolExecutor(
    113                 THREAD_LIMIT.get().intValue(), // keep the thread number constant
    114                 THREAD_LIMIT.get().intValue(), // do not this number of threads
    115                 30, // keepalive for thread
    116                 TimeUnit.SECONDS,
    117                 // make queue of LIFO type - so recently requested tiles will be loaded first (assuming that these are which user is waiting to see)
    118                 new LIFOQueue(5)
    119                     /* keep the queue size fairly small, we do not want to
    120                      download a lot of tiles, that user is not seeing anyway */
    121                 );
    122     }
    123 
    124     /**
    12599     * Reconfigures download dispatcher using current values of THREAD_LIMIT and HOST_LIMIT
    126100     */
    127101    public static final void reconfigureDownloadDispatcher() {
    128102        HOST_LIMITS = new ConcurrentHashMap<>();
    129         DOWNLOAD_JOB_DISPATCHER = getThreadPoolExecutor();
    130103    }
    131104
     
    139112     * @param headers to be sent together with request
    140113     */
    141     public TMSCachedTileLoaderJob(TileLoaderListener listener, Tile tile, ICacheAccess<String, BufferedImageCacheEntry> cache, int connectTimeout, int readTimeout,
    142             Map<String, String> headers) {
    143         super(cache, connectTimeout, readTimeout, headers);
     114    public TMSCachedTileLoaderJob(TileLoaderListener listener, Tile tile,
     115            ICacheAccess<String, BufferedImageCacheEntry> cache,
     116            int connectTimeout, int readTimeout, Map<String, String> headers,
     117            Executor downloadExecutor) {
     118        super(cache, connectTimeout, readTimeout, headers, downloadExecutor);
    144119        this.tile = tile;
    145120        if (listener != null) {
     
    230205        }
    231206        return false;
    232     }
    233 
    234     @Override
    235     protected Executor getDownloadExecutor() {
    236         return DOWNLOAD_JOB_DISPATCHER;
    237207    }
    238208
  • trunk/src/org/openstreetmap/josm/gui/layer/TMSLayer.java

    r8393 r8397  
    7070import org.openstreetmap.josm.gui.MapView;
    7171import org.openstreetmap.josm.gui.MapView.LayerChangeListener;
     72import org.openstreetmap.josm.gui.NavigatableComponent.ZoomChangeListener;
    7273import org.openstreetmap.josm.gui.PleaseWaitRunnable;
    7374import org.openstreetmap.josm.gui.dialogs.LayerListDialog;
     
    9192 *
    9293 */
    93 public class TMSLayer extends ImageryLayer implements ImageObserver, TileLoaderListener {
     94public class TMSLayer extends ImageryLayer implements ImageObserver, TileLoaderListener, ZoomChangeListener {
    9495    public static final String PREFERENCE_PREFIX   = "imagery.tms";
    9596
     
    104105    public static final IntegerProperty PROP_MIN_ZOOM_LVL = new IntegerProperty(PREFERENCE_PREFIX + ".min_zoom_lvl", DEFAULT_MIN_ZOOM);
    105106    public static final IntegerProperty PROP_MAX_ZOOM_LVL = new IntegerProperty(PREFERENCE_PREFIX + ".max_zoom_lvl", DEFAULT_MAX_ZOOM);
    106     //public static final BooleanProperty PROP_DRAW_DEBUG = new BooleanProperty(PREFERENCE_PREFIX + ".draw_debug", false);
    107     public static final BooleanProperty PROP_ADD_TO_SLIPPYMAP_CHOOSER = new BooleanProperty(PREFERENCE_PREFIX + ".add_to_slippymap_chooser", true);
     107    public static final BooleanProperty PROP_ADD_TO_SLIPPYMAP_CHOOSER = new BooleanProperty(PREFERENCE_PREFIX +
     108            ".add_to_slippymap_chooser", true);
    108109    public static final StringProperty PROP_TILECACHE_DIR;
    109110
     
    118119    }
    119120
     121    /**
     122     * Interface for creating TileLoaders, ie. classes responsible for loading tiles on map
     123     *
     124     */
    120125    public interface TileLoaderFactory {
     126        /**
     127         * @param listener object that will be notified, when tile has finished loading
     128         * @return TileLoader that will notify the listener
     129         */
    121130        TileLoader makeTileLoader(TileLoaderListener listener);
     131
     132        /**
     133         * @param listener object that will be notified, when tile has finished loading
     134         * @param headers HTTP headers that should be sent by TileLoader to tile server
     135         * @return TileLoader that will notify the listener
     136         */
    122137        TileLoader makeTileLoader(TileLoaderListener listener, Map<String, String> headers);
    123138    }
     
    189204     * method.
    190205     *
    191      * @param monitor
    192      * @see OsmFileCacheTileLoader#clearCache(org.openstreetmap.gui.jmapviewer.interfaces.TileSource, org.openstreetmap.gui.jmapviewer.interfaces.TileClearController)
    193      */
    194     void clearTileCache(ProgressMonitor monitor) {
     206     * @param monitor not used in this implementation - as cache clear is instaneus
     207     */
     208    public void clearTileCache(ProgressMonitor monitor) {
    195209        tileCache.clear();
    196210        if (tileLoader instanceof CachedTileLoader) {
    197211            ((CachedTileLoader)tileLoader).clearCache(tileSource);
    198212        }
     213        redraw();
    199214    }
    200215
     
    225240     * @see MapFrame#repaint()
    226241     */
    227     void redraw() {
     242    protected void redraw() {
    228243        needRedraw = true;
    229244        Main.map.repaint();
    230245    }
    231246
    232     static int checkMaxZoomLvl(int maxZoomLvl, TileSource ts) {
     247    protected static int checkMaxZoomLvl(int maxZoomLvl, TileSource ts) {
    233248        if(maxZoomLvl > MAX_ZOOM) {
    234249            maxZoomLvl = MAX_ZOOM;
     
    248263
    249264    public static void setMaxZoomLvl(int maxZoomLvl) {
    250         maxZoomLvl = checkMaxZoomLvl(maxZoomLvl, null);
    251         PROP_MAX_ZOOM_LVL.put(maxZoomLvl);
     265        Integer newMaxZoom = Integer.valueOf(checkMaxZoomLvl(maxZoomLvl, null));
     266        PROP_MAX_ZOOM_LVL.put(newMaxZoom);
    252267    }
    253268
     
    342357            return t;
    343358        } else if (info.getImageryType() == ImageryType.BING) {
    344             //return new CachedAttributionBingAerialTileSource(info.getId());
    345359            return new CachedAttributionBingAerialTileSource(info);
    346360        } else if (info.getImageryType() == ImageryType.SCANEX) {
    347             //return new ScanexTileSource(info.getName(), info.getUrl(), info.getId(), info.getMaxZoom());
    348361            return new ScanexTileSource(info);
    349362        }
     
    385398        }
    386399
    387         // FIXME: tileCache = new MemoryTileCache();
    388400        tileLoader = loaderFactory.makeTileLoader(this, headers);
    389401        if (tileLoader instanceof TMSCachedTileLoader) {
     
    464476            throw new IllegalStateException("Cannot create TMSLayer with non-TMS ImageryInfo");
    465477        initTileSource(source);
     478
     479        MapView.addZoomChangeListener(this);
    466480    }
    467481
     
    538552        }));
    539553
    540         /* FIXME
    541554        tileOptionMenu.add(new JMenuItem(new AbstractAction(
    542555                tr("Request Update")) {
    543556            public void actionPerformed(ActionEvent ae) {
    544557                if (clickedTile != null) {
    545                     clickedTile.requestUpdate();
    546                     redraw();
    547                 }
    548             }
    549         }));*/
     558                    clickedTile.setLoaded(false);
     559                    tileLoader.createTileLoaderJob(clickedTile).submit();
     560                }
     561            }
     562        }));
    550563
    551564        tileOptionMenu.add(new JMenuItem(new AbstractAction(
     
    590603            @Override
    591604            public void actionPerformed(ActionEvent ae) {
    592                 double new_factor = Math.sqrt(getScaleFactor(currentZoomLevel));
    593                 Main.map.mapView.zoomToFactor(new_factor);
     605                double newFactor = Math.sqrt(getScaleFactor(currentZoomLevel));
     606                Main.map.mapView.zoomToFactor(newFactor);
    594607                redraw();
    595608            }
     
    647660                if (oldLayer == TMSLayer.this) {
    648661                    Main.map.mapView.removeMouseListener(adapter);
     662                    MapView.removeZoomChangeListener(TMSLayer.this);
    649663                    MapView.removeLayerChangeListener(this);
    650664                }
     
    653667    }
    654668
    655     void zoomChanged() {
     669    /**
     670     * This fires every time the user changes the zoom, but also (due to ZoomChangeListener) - on all
     671     * changes to visible map (panning/zooming)
     672     */
     673    @Override
     674    public void zoomChanged() {
    656675        if (Main.isDebugEnabled()) {
    657676            Main.debug("zoomChanged(): " + currentZoomLevel);
    658677        }
    659678        needRedraw = true;
    660     }
    661 
    662     int getMaxZoomLvl() {
     679        if (tileLoader instanceof TMSCachedTileLoader) {
     680            ((TMSCachedTileLoader) tileLoader).cancelOutstandingTasks();
     681        }
     682    }
     683
     684    protected int getMaxZoomLvl() {
    663685        if (info.getMaxZoom() != 0)
    664686            return checkMaxZoomLvl(info.getMaxZoom(), tileSource);
     
    667689    }
    668690
    669     int getMinZoomLvl() {
     691    protected int getMinZoomLvl() {
    670692        return getMinZoomLvl(tileSource);
    671693    }
     
    674696     * Zoom in, go closer to map.
    675697     *
    676      * @return    true, if zoom increasing was successfull, false othervise
     698     * @return    true, if zoom increasing was successful, false otherwise
    677699     */
    678700    public boolean zoomIncreaseAllowed() {
     
    742764     * into the tileCache.
    743765     */
    744     Tile tempCornerTile(Tile t) {
     766    private Tile tempCornerTile(Tile t) {
    745767        int x = t.getXtile() + 1;
    746768        int y = t.getYtile() + 1;
     
    752774    }
    753775
    754     Tile getOrCreateTile(int x, int y, int zoom) {
     776    private Tile getOrCreateTile(int x, int y, int zoom) {
    755777        Tile tile = getTile(x, y, zoom);
    756778        if (tile == null) {
     
    766788     * already in the cache.
    767789     */
    768     Tile getTile(int x, int y, int zoom) {
     790    private Tile getTile(int x, int y, int zoom) {
    769791        int max = 1 << zoom;
    770792        if (x < 0 || x >= max || y < 0 || y >= max)
     
    773795    }
    774796
    775     boolean loadTile(Tile tile, boolean force) {
     797    private boolean loadTile(Tile tile, boolean force) {
    776798        if (tile == null)
    777799            return false;
     
    784806    }
    785807
    786     void loadAllTiles(boolean force) {
     808    private void loadAllTiles(boolean force) {
    787809        MapView mv = Main.map.mapView;
    788810        EastNorth topLeft = mv.getEastNorth(0, 0);
     
    800822    }
    801823
    802     void loadAllErrorTiles(boolean force) {
     824    private void loadAllErrorTiles(boolean force) {
    803825        MapView mv = Main.map.mapView;
    804826        EastNorth topLeft = mv.getEastNorth(0, 0);
     
    821843    }
    822844
    823     boolean imageLoaded(Image i) {
     845    private boolean imageLoaded(Image i) {
    824846        if (i == null)
    825847            return false;
     
    837859     * @return  the image of the tile or null.
    838860     */
    839     Image getLoadedTileImage(Tile tile) {
     861    private Image getLoadedTileImage(Tile tile) {
    840862        if (!tile.isLoaded())
    841863            return null;
     
    846868    }
    847869
    848     LatLon tileLatLon(Tile t) {
     870    private LatLon tileLatLon(Tile t) {
    849871        int zoom = t.getZoom();
    850872        return new LatLon(tileSource.tileYToLat(t.getYtile(), zoom),
     
    852874    }
    853875
    854     Rectangle tileToRect(Tile t1) {
     876    private Rectangle tileToRect(Tile t1) {
    855877        /*
    856878         * We need to get a box in which to draw, so advance by one tile in
     
    870892    // 'border' is the screen cordinates that need to be drawn.
    871893    //  We must not draw outside of it.
    872     void drawImageInside(Graphics g, Image sourceImg, Rectangle source, Rectangle border) {
     894    private void drawImageInside(Graphics g, Image sourceImg, Rectangle source, Rectangle border) {
    873895        Rectangle target = source;
    874896
     
    931953    // drawn currently.  If drawing the displayZoomLevel,
    932954    // border is null and we draw the entire tile set.
    933     List<Tile> paintTileImages(Graphics g, TileSet ts, int zoom, Tile border) {
     955    private List<Tile> paintTileImages(Graphics g, TileSet ts, int zoom, Tile border) {
    934956        if (zoom <= 0) return Collections.emptyList();
    935957        Rectangle borderRect = null;
     
    962984    }
    963985
    964     void myDrawString(Graphics g, String text, int x, int y) {
     986    private void myDrawString(Graphics g, String text, int x, int y) {
    965987        Color oldColor = g.getColor();
    966988        g.setColor(Color.black);
     
    970992    }
    971993
    972     void paintTileText(TileSet ts, Tile tile, Graphics g, MapView mv, int zoom, Tile t) {
     994    private void paintTileText(TileSet ts, Tile tile, Graphics g, MapView mv, int zoom, Tile t) {
    973995        int fontHeight = g.getFontMetrics().getHeight();
    974996        if (tile == null)
     
    10661088         * Create a TileSet by EastNorth bbox taking a layer shift in account
    10671089         */
    1068         TileSet(EastNorth topLeft, EastNorth botRight, int zoom) {
     1090        private TileSet(EastNorth topLeft, EastNorth botRight, int zoom) {
    10691091            this(getShiftedLatLon(topLeft), getShiftedLatLon(botRight),zoom);
    10701092        }
     
    10731095         * Create a TileSet by known LatLon bbox without layer shift correction
    10741096         */
    1075         TileSet(LatLon topLeft, LatLon botRight, int zoom) {
     1097        private TileSet(LatLon topLeft, LatLon botRight, int zoom) {
    10761098            this.zoom = zoom;
    10771099            if (zoom == 0)
     
    11071129        }
    11081130
    1109         boolean tooSmall() {
     1131        private boolean tooSmall() {
    11101132            return this.tilesSpanned() < 2.1;
    11111133        }
    11121134
    1113         boolean tooLarge() {
     1135        private boolean tooLarge() {
    11141136            return this.tilesSpanned() > 10;
    11151137        }
    11161138
    1117         boolean insane() {
     1139        private boolean insane() {
    11181140            return this.tilesSpanned() > 100;
    11191141        }
    11201142
    1121         double tilesSpanned() {
     1143        private double tilesSpanned() {
    11221144            return Math.sqrt(1.0 * this.size());
    11231145        }
    11241146
    1125         int size() {
     1147        private int size() {
    11261148            int x_span = x1 - x0 + 1;
    11271149            int y_span = y1 - y0 + 1;
     
    11331155         * already in the tileCache.
    11341156         */
    1135         List<Tile> allExistingTiles() {
     1157        private List<Tile> allExistingTiles() {
    11361158            return this.__allTiles(false);
    11371159        }
    11381160
    1139         List<Tile> allTilesCreate() {
     1161        private List<Tile> allTilesCreate() {
    11401162            return this.__allTiles(true);
    11411163        }
     
    11711193        }
    11721194
    1173         void loadAllTiles(boolean force) {
     1195        private void loadAllTiles(boolean force) {
    11741196            if (!autoLoad && !force)
    11751197                return;
     
    11791201        }
    11801202
    1181         void loadAllErrorTiles(boolean force) {
     1203        private void loadAllErrorTiles(boolean force) {
    11821204            if (!autoLoad && !force)
    11831205                return;
     
    14181440     * user right-clicks on the map.
    14191441     */
    1420     Tile getTileForPixelpos(int px, int py) {
     1442    private Tile getTileForPixelpos(int px, int py) {
    14211443        if (Main.isDebugEnabled()) {
    14221444            Main.debug("getTileForPixelpos("+px+", "+py+")");
Note: See TracChangeset for help on using the changeset viewer.