Changeset 15787 in josm


Ignore:
Timestamp:
2020-01-27T09:17:42+01:00 (2 months ago)
Author:
GerdP
Message:

see #18624: Download parent ways/relations downloads way nodes twice (or more)
Improve performance, esp. when multiple nodes of the same way are selected

  • DownloadReferrersTask: download all incomplete nodes of all parent ways with one MultiFetchServerObjectReader instead of repeating this step for each parent way. Saves lots of FULL api calls for the parent ways (/full)
  • updated parent way nodes are recognized
  • DownloadPrimitivesWithReferrersTask creates only one DownloadReferrersTask for all primitives to profit from the above changes
Location:
trunk/src/org/openstreetmap/josm
Files:
2 edited

Legend:

Unmodified
Added
Removed
  • trunk/src/org/openstreetmap/josm/actions/downloadtasks/DownloadReferrersTask.java

    r15152 r15787  
    88import java.text.MessageFormat;
    99import java.util.Collection;
    10 import java.util.HashMap;
    11 import java.util.Map;
    12 import java.util.Map.Entry;
     10import java.util.LinkedHashSet;
    1311import java.util.Set;
    1412import java.util.stream.Collectors;
     
    4846    private final OsmDataLayer targetLayer;
    4947    /** the collection of child primitives */
    50     private final Map<Long, OsmPrimitiveType> children;
     48    private final Set<PrimitiveId> children;
    5149    /** the parents */
    5250    private final DataSet parents;
     
    5755     * @param targetLayer  the target layer for the downloaded primitives. Must not be null.
    5856     * @param children the collection of child primitives for which parents are to be downloaded
     57     * @since 15787 (modified interface)
    5958     */
    60     public DownloadReferrersTask(OsmDataLayer targetLayer, Collection<OsmPrimitive> children) {
     59    public DownloadReferrersTask(OsmDataLayer targetLayer, Collection<? extends PrimitiveId> children) {
    6160        super("Download referrers", false /* don't ignore exception*/);
    6261        CheckParameterUtil.ensureParameterNotNull(targetLayer, "targetLayer");
     
    6564        }
    6665        canceled = false;
    67         this.children = new HashMap<>();
     66        this.children = new LinkedHashSet<>();
    6867        if (children != null) {
    69             for (OsmPrimitive p: children) {
    70                 if (!p.isNew()) {
    71                     this.children.put(p.getId(), OsmPrimitiveType.from(p));
    72                 }
    73             }
     68            children.stream().filter(p -> !p.isNew()).forEach(this.children::add);
    7469        }
     70
    7571        this.targetLayer = targetLayer;
    7672        parents = new DataSet();
     
    9490                    "Cannot download referrers for new primitives (ID {0})", primitiveId.getUniqueId()));
    9591        canceled = false;
    96         this.children = new HashMap<>();
    97         this.children.put(primitiveId.getUniqueId(), primitiveId.getType());
     92        this.children = new LinkedHashSet<>();
     93        this.children.add(primitiveId);
    9894        this.targetLayer = targetLayer;
    9995        parents = new DataSet();
     
    141137
    142138    protected void downloadParents(long id, OsmPrimitiveType type, ProgressMonitor progressMonitor) throws OsmTransferException {
    143         reader = new OsmServerBackreferenceReader(id, type);
     139        reader = new OsmServerBackreferenceReader(id, type, false).setAllowIncompleteParentWays(true);
     140
    144141        DataSet ds = reader.parseOsm(progressMonitor.createSubTaskMonitor(1, false));
    145142        synchronized (this) { // avoid race condition in cancel()
    146143            reader = null;
    147         }
    148         Collection<Way> ways = ds.getWays();
    149 
    150         if (!ways.isEmpty()) {
    151             // Ensure each node is only listed once
    152             Set<Node> nodes = ways.stream().flatMap(w -> w.getNodes().stream()).collect(Collectors.toSet());
    153             // Don't retrieve any nodes we've already grabbed
    154             nodes.removeAll(targetLayer.data.getNodes());
    155             if (!nodes.isEmpty()) {
    156                 reader = MultiFetchServerObjectReader.create();
    157                 ((MultiFetchServerObjectReader) reader).append(nodes);
    158                 DataSet wayNodes = reader.parseOsm(progressMonitor.createSubTaskMonitor(1, false));
    159                 synchronized (this) { // avoid race condition in cancel()
    160                     reader = null;
    161                 }
    162                 new DataSetMerger(ds, wayNodes).merge();
    163             }
     144            if (canceled)
     145                return;
    164146        }
    165147        new DataSetMerger(parents, ds).merge();
     
    171153            progressMonitor.setTicksCount(children.size());
    172154            int i = 1;
    173             for (Entry<Long, OsmPrimitiveType> entry: children.entrySet()) {
     155            for (PrimitiveId p : children) {
    174156                if (canceled)
    175157                    return;
    176158                String msg;
    177                 switch(entry.getValue()) {
    178                 case NODE: msg = tr("({0}/{1}) Loading parents of node {2}", i+1, children.size(), entry.getKey()); break;
    179                 case WAY: msg = tr("({0}/{1}) Loading parents of way {2}", i+1, children.size(), entry.getKey()); break;
    180                 case RELATION: msg = tr("({0}/{1}) Loading parents of relation {2}", i+1, children.size(), entry.getKey()); break;
     159                String id = Long.toString(p.getUniqueId());
     160                switch(p.getType()) {
     161                case NODE: msg = tr("({0}/{1}) Loading parents of node {2}", i, children.size(), id); break;
     162                case WAY: msg = tr("({0}/{1}) Loading parents of way {2}", i, children.size(), id); break;
     163                case RELATION: msg = tr("({0}/{1}) Loading parents of relation {2}", i, children.size(), id); break;
    181164                default: throw new AssertionError();
    182165                }
    183166                progressMonitor.subTask(msg);
    184                 downloadParents(entry.getKey(), entry.getValue(), progressMonitor);
     167                downloadParents(p.getUniqueId(), p.getType(), progressMonitor);
    185168                i++;
     169            }
     170            Collection<Way> ways = parents.getWays();
     171
     172            if (!ways.isEmpty()) {
     173                // Collect incomplete nodes of parent ways
     174                Set<Node> nodes = ways.stream().flatMap(w -> w.getNodes().stream().filter(OsmPrimitive::isIncomplete))
     175                        .collect(Collectors.toSet());
     176                if (!nodes.isEmpty()) {
     177                    reader = MultiFetchServerObjectReader.create();
     178                    ((MultiFetchServerObjectReader) reader).append(nodes);
     179                    DataSet wayNodes = reader.parseOsm(progressMonitor.createSubTaskMonitor(1, false));
     180                    synchronized (this) { // avoid race condition in cancel()
     181                        reader = null;
     182                    }
     183                    new DataSetMerger(parents, wayNodes).merge();
     184                }
    186185            }
    187186        } catch (OsmTransferException e) {
  • trunk/src/org/openstreetmap/josm/gui/io/DownloadPrimitivesWithReferrersTask.java

    r15746 r15787  
    114114        currentTask.run();
    115115        // Then, download referrers for each primitive
    116         if (downloadReferrers)
    117             for (PrimitiveId id : ids) {
    118                 synchronized (this) {
    119                     if (canceled) {
    120                         currentTask = null;
    121                         return;
    122                     }
    123                     currentTask = new DownloadReferrersTask(
    124                             tmpLayer, id, getProgressMonitor().createSubTaskMonitor(1, false));
    125                 }
    126                 currentTask.run();
    127             }
     116        if (downloadReferrers) {
     117            currentTask = new DownloadReferrersTask(tmpLayer, ids);
     118            currentTask.run();
     119            synchronized (this) {
     120                if (currentTask.getProgressMonitor().isCanceled())
     121                    cancel();
     122            }
     123        }
    128124        currentTask = null;
    129125    }
Note: See TracChangeset for help on using the changeset viewer.