Ticket #10902: TMS_simultaneus_connections.patch

File TMS_simultaneus_connections.patch, 9.2 KB (added by wiktorn, 9 years ago)

  • src/org/openstreetmap/josm/data/imagery/TMSCachedTileLoaderJob.java 

    diff --git src/org/openstreetmap/josm/data/imagery/TMSCachedTileLoaderJob.java src/org/openstreetmap/josm/data/imagery/TMSCachedTileLoaderJob.java
index c6a22a2..ad837ca 100644
    import java.io.ByteArrayInputStream;  
    55import java.io.IOException;
    66import java.net.URL;
    77import java.util.Map;
     8import java.util.concurrent.ConcurrentHashMap;
    89import java.util.concurrent.Executor;
    910import java.util.concurrent.LinkedBlockingDeque;
     11import java.util.concurrent.Semaphore;
    1012import java.util.concurrent.ThreadPoolExecutor;
    1113import java.util.concurrent.TimeUnit;
    1214import java.util.logging.Level;
    … … import org.openstreetmap.gui.jmapviewer.interfaces.TileJob;  
    1921import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener;
    2022import org.openstreetmap.gui.jmapviewer.interfaces.TileSource;
    2123import org.openstreetmap.gui.jmapviewer.tilesources.AbstractTMSTileSource;
     24import org.openstreetmap.josm.Main;
    2225import org.openstreetmap.josm.data.cache.BufferedImageCacheEntry;
    2326import org.openstreetmap.josm.data.cache.CacheEntry;
    2427import org.openstreetmap.josm.data.cache.ICachedLoaderListener;
    … … public class TMSCachedTileLoaderJob extends JCSCachedTileLoaderJob<String, Buffe  
    4144     * overrides the THREAD_LIMIT in superclass, as we want to have separate limit and pool for TMS
    4245     */
    4346    public static final IntegerProperty THREAD_LIMIT = new IntegerProperty("imagery.tms.tmsloader.maxjobs", 25);
     47
     48    /**
     49     * Limit definition for per host concurrent connections
     50     */
     51    public static final IntegerProperty HOST_LIMIT = new IntegerProperty("imagery.tms.tmsloader.maxjobsperhost", 6);
     52
    4453    /**
    4554     * separate from JCS thread pool for TMS loader, so we can have different thread pools for default JCS
    4655     * and for TMS imagery
    4756     */
    48     private static ThreadPoolExecutor DOWNLOAD_JOB_DISPATCHER = new ThreadPoolExecutor(
    49             THREAD_LIMIT.get().intValue(), // keep the thread number constant
    50             THREAD_LIMIT.get().intValue(), // do not this number of threads
    51             30, // keepalive for thread
    52             TimeUnit.SECONDS,
    53             // make queue of LIFO type - so recently requested tiles will be loaded first (assuming that these are which user is waiting to see)
    54             new LinkedBlockingDeque<Runnable>(5) {
    55                 /* keep the queue size fairly small, we do not want to
    56                  download a lot of tiles, that user is not seeing anyway */
    57                 @Override
    58                 public boolean offer(Runnable t) {
    59                     return super.offerFirst(t);
     57
     58    private static class LIFOQueue extends LinkedBlockingDeque<Runnable> {
     59        public LIFOQueue(int capacity) {
     60            super(capacity);
     61        }
     62
     63        private final static Semaphore getSemaphore(Runnable r) {
     64            if (!(r instanceof TMSCachedTileLoaderJob))
     65                return null;
     66            TMSCachedTileLoaderJob cachedJob = (TMSCachedTileLoaderJob) r;
     67            Semaphore limit = HOST_LIMITS.get(cachedJob.getUrl().getHost());
     68            if (limit == null) {
     69                synchronized(HOST_LIMITS) {
     70                    limit = HOST_LIMITS.get(cachedJob.getUrl().getHost());
     71                    if (limit == null) {
     72                        limit = new Semaphore(HOST_LIMIT.get().intValue());
     73                        HOST_LIMITS.put(cachedJob.getUrl().getHost(), limit);
     74                    }
    6075                }
     76            }
     77            return limit;
     78        }
    6179
    62                 @Override
    63                 public Runnable remove() {
    64                     return super.removeFirst();
     80        private boolean acquireSemaphore(Runnable r) {
     81            boolean ret = true;
     82            Semaphore limit = getSemaphore(r);
     83            if (limit != null) {
     84                ret = limit.tryAcquire();
     85                if (!ret) {
     86                    Main.debug("rejecting job because of per host limit");
    6587                }
    6688            }
    67             );
     89            return ret;
     90
     91        }
     92        @Override
     93        public boolean offer(Runnable t) {
     94            return acquireSemaphore(t) && super.offerFirst(t);
     95        }
     96
     97        private Runnable releaseSemaphore(Runnable r) {
     98            Semaphore limit = getSemaphore(r);
     99            if (limit != null)
     100                limit.release();
     101            return r;
     102        }
     103
     104        @Override
     105        public Runnable remove() {
     106            return releaseSemaphore(super.removeFirst());
     107        }
     108
     109        @Override
     110        public Runnable poll(long timeout, TimeUnit unit) throws InterruptedException {
     111            return releaseSemaphore(super.poll(timeout, unit));
     112        }
     113
     114        @Override
     115        public Runnable take() throws InterruptedException {
     116            return releaseSemaphore(super.take());
     117        }
     118    }
     119
     120    private static Map<String, Semaphore> HOST_LIMITS = new ConcurrentHashMap<>();
     121
     122    private static ThreadPoolExecutor DOWNLOAD_JOB_DISPATCHER = getThreadPoolExecutor();
     123
     124    private static ThreadPoolExecutor getThreadPoolExecutor() {
     125        return new ThreadPoolExecutor(
     126                THREAD_LIMIT.get().intValue(), // keep the thread number constant
     127                THREAD_LIMIT.get().intValue(), // do not this number of threads
     128                30, // keepalive for thread
     129                TimeUnit.SECONDS,
     130                // make queue of LIFO type - so recently requested tiles will be loaded first (assuming that these are which user is waiting to see)
     131                new LIFOQueue(5)
     132                    /* keep the queue size fairly small, we do not want to
     133                     download a lot of tiles, that user is not seeing anyway */
     134                );
     135    }
     136
     137    /**
     138     * Reconfigures download dispatcher using current values of THREAD_LIMIT and HOST_LIMIT
     139     */
     140    public static final void reconfigureDownloadDispatcher() {
     141        HOST_LIMITS = new ConcurrentHashMap<>();
     142        DOWNLOAD_JOB_DISPATCHER = getThreadPoolExecutor();
     143    }
     144
    68145
    69146    /**
    70147     * Constructor for creating a job, to get a specific tile from cache

  • src/org/openstreetmap/josm/gui/preferences/imagery/TMSSettingsPanel.java 

    diff --git src/org/openstreetmap/josm/gui/preferences/imagery/TMSSettingsPanel.java src/org/openstreetmap/josm/gui/preferences/imagery/TMSSettingsPanel.java
index 75ce084..0a7b89d 100644
    public class TMSSettingsPanel extends JPanel {  
    3232    private final JosmTextField tilecacheDir = new JosmTextField();
    3333    private final JSpinner maxElementsOnDisk;
    3434    private final JSpinner maxConcurrentDownloads;
     35    private final JSpinner maxDownloadsPerHost;
    3536
    3637
    3738    /**
    … … public class TMSSettingsPanel extends JPanel {  
    4344        maxZoomLvl = new JSpinner(new SpinnerNumberModel(TMSLayer.DEFAULT_MAX_ZOOM, TMSLayer.MIN_ZOOM, TMSLayer.MAX_ZOOM, 1));
    4445        maxElementsOnDisk = new JSpinner(new SpinnerNumberModel(TMSCachedTileLoader.MAX_OBJECTS_ON_DISK.get().intValue(), 0, Integer.MAX_VALUE, 1));
    4546        maxConcurrentDownloads = new JSpinner(new SpinnerNumberModel(TMSCachedTileLoaderJob.THREAD_LIMIT.get().intValue(), 0, Integer.MAX_VALUE, 1));
     47        maxDownloadsPerHost = new JSpinner(new SpinnerNumberModel(TMSCachedTileLoaderJob.HOST_LIMIT.get().intValue(), 0, Integer.MAX_VALUE, 1));
    4648
    4749        add(new JLabel(tr("Auto zoom by default: ")), GBC.std());
    4850        add(GBC.glue(5, 0), GBC.std());
    … … public class TMSSettingsPanel extends JPanel {  
    7274        add(GBC.glue(5, 0), GBC.std());
    7375        add(maxConcurrentDownloads, GBC.eol());
    7476
     77        add(new JLabel(tr("Maximum concurrent downloads per host: ")), GBC.std());
     78        add(GBC.glue(5, 0), GBC.std());
     79        add(maxDownloadsPerHost, GBC.eol());
     80
     81
    7582        add(new JLabel(tr("Maximum elements in disk cache: ")), GBC.std());
    7683        add(GBC.glue(5, 0), GBC.std());
    7784        add(this.maxElementsOnDisk, GBC.eol());
    … … public class TMSSettingsPanel extends JPanel {  
    9097        this.tilecacheDir.setText(TMSLayer.PROP_TILECACHE_DIR.get());
    9198        this.maxElementsOnDisk.setValue(TMSCachedTileLoader.MAX_OBJECTS_ON_DISK.get());
    9299        this.maxConcurrentDownloads.setValue(TMSCachedTileLoaderJob.THREAD_LIMIT.get());
     100        this.maxDownloadsPerHost.setValue(TMSCachedTileLoaderJob.HOST_LIMIT.get());
    93101    }
    94102
    95103    /**
    … … public class TMSSettingsPanel extends JPanel {  
    110118
    111119        TMSCachedTileLoader.MAX_OBJECTS_ON_DISK.put((Integer) this.maxElementsOnDisk.getValue());
    112120
    113         if (!TMSCachedTileLoaderJob.THREAD_LIMIT.get().equals(this.maxConcurrentDownloads.getValue())) {
    114             restartRequired = true;
    115             TMSCachedTileLoaderJob.THREAD_LIMIT.put((Integer) this.maxConcurrentDownloads.getValue());
    116         }
     121        TMSCachedTileLoaderJob.THREAD_LIMIT.put((Integer) this.maxConcurrentDownloads.getValue());
     122        TMSCachedTileLoaderJob.HOST_LIMIT.put((Integer) this.maxDownloadsPerHost.getValue());
     123        TMSCachedTileLoaderJob.reconfigureDownloadDispatcher();
    117124
    118125        if (!TMSLayer.PROP_TILECACHE_DIR.get().equals(this.tilecacheDir.getText())) {
    119126            restartRequired = true;