Changeset 35078 in osm for applications/editors/josm

Timestamp:

2019-08-11T02:08:18+02:00 (5 years ago)

Author:

upliner

Message:

Reverter: Update MultiOsmReader to support null data after redaction

Location:

applications/editors/josm/plugins/reverter/src/reverter

Files:

• ChangesetReverter.java (modified) (3 diffs)
• MultiOsmReader.java (modified) (2 diffs)
• OsmServerMultiObjectReader.java (modified) (2 diffs)

applications/editors/josm/plugins/reverter/src/reverter/ChangesetReverter.java

@@ -14 +14 @@
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map.Entry;
 
 import org.openstreetmap.josm.command.Command;
@@ -222 +223 @@
                         trn("Downloading history for {0} object", "Downloading history for {0} objects", num, num)), num + 1);
         try {
+            HashMap<Long,Integer> nodeList = new HashMap<>(),
+                    wayList = new HashMap<>(),
+                    relationList = new HashMap<>();
+
             for (HashSet<HistoryOsmPrimitive> collection : Arrays.asList(updated, deleted)) {
                 for (HistoryOsmPrimitive entry : collection) {
@@ -228 +233 @@
                     if (earliestVersion == null || earliestVersion <= 1)
                         throw new OsmTransferException(tr("Unexpected data in changeset #{1}", String.valueOf(changesetId)));
-                    readObjectVersion(rdr, id, earliestVersion - 1, progressMonitor);
-                    if (progressMonitor.isCanceled()) return;
+                    switch (id.getType()) {
+                    case NODE: nodeList.put(id.getUniqueId(), earliestVersion - 1); break;
+                    case WAY: wayList.put(id.getUniqueId(), earliestVersion - 1); break;
+                    case RELATION: relationList.put(id.getUniqueId(), earliestVersion - 1); break;
+                    default: throw new AssertionError();
+                    }
                 }
             }
+            Logging.info("New fetcher is started");
+            rdr.readMultiObjects(OsmPrimitiveType.NODE, nodeList, progressMonitor);
+            rdr.readMultiObjects(OsmPrimitiveType.WAY, wayList, progressMonitor);
+            rdr.readMultiObjects(OsmPrimitiveType.RELATION, relationList, progressMonitor);
+            if (progressMonitor.isCanceled()) return;
+            // If multi-read failed, retry with regular read
+            for (Entry<Long,Integer> entry : nodeList.entrySet()) {
+                if (progressMonitor.isCanceled()) return;
+                readObjectVersion(rdr, new SimplePrimitiveId(entry.getKey(),OsmPrimitiveType.NODE), entry.getValue(), progressMonitor);
+            }
+            for (Entry<Long,Integer> entry : wayList.entrySet()) {
+                if (progressMonitor.isCanceled()) return;
+                readObjectVersion(rdr, new SimplePrimitiveId(entry.getKey(),OsmPrimitiveType.WAY), entry.getValue(), progressMonitor);
+            }
+            for (Entry<Long,Integer> entry : relationList.entrySet()) {
+                if (progressMonitor.isCanceled()) return;
+                readObjectVersion(rdr, new SimplePrimitiveId(entry.getKey(),OsmPrimitiveType.RELATION), entry.getValue(), progressMonitor);
+            }
+            if (progressMonitor.isCanceled()) return;
             nds = rdr.parseOsm(progressMonitor.createSubTaskMonitor(1, true));
             for (OsmPrimitive p : nds.allPrimitives()) {

applications/editors/josm/plugins/reverter/src/reverter/MultiOsmReader.java

@@ -5 +5 @@
 import java.io.InputStreamReader;
 
+import javax.xml.stream.XMLStreamException;
+
+import org.openstreetmap.josm.data.osm.Node;
+import org.openstreetmap.josm.data.osm.Relation;
+import org.openstreetmap.josm.data.osm.Way;
 import org.openstreetmap.josm.io.IllegalDataException;
 import org.openstreetmap.josm.io.OsmReader;
@@ -33 +38 @@
         prepareDataSet();
     }
+
+    public ParseCallback callback;
+
+    @Override
+    protected Node parseNode() throws XMLStreamException  {
+        Node node = super.parseNode();
+        if (callback != null) {
+            callback.primitiveParsed(node.getPrimitiveId());
+        }
+        return node;
+    }
+    @Override
+    protected Way parseWay() throws XMLStreamException  {
+        Way way = super.parseWay();
+        if (callback != null) {
+            callback.primitiveParsed(way.getPrimitiveId());
+        }
+        return way;
+    }
+    @Override
+    protected Relation parseRelation() throws XMLStreamException  {
+        Relation relation = super.parseRelation();
+        if (callback != null) {
+            callback.primitiveParsed(relation.getPrimitiveId());
+        }
+        return relation;
+    }
 }

applications/editors/josm/plugins/reverter/src/reverter/OsmServerMultiObjectReader.java

@@ -6 +6 @@
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
 
 import org.openstreetmap.josm.data.osm.DataSet;
 import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
 import org.openstreetmap.josm.data.osm.PrimitiveId;
+import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
 import org.openstreetmap.josm.gui.progress.ProgressMonitor;
 import org.openstreetmap.josm.io.IllegalDataException;
 import org.openstreetmap.josm.io.OsmServerReader;
 import org.openstreetmap.josm.io.OsmTransferException;
+import org.openstreetmap.josm.tools.Logging;
 
 public class OsmServerMultiObjectReader extends OsmServerReader {
@@ -38 +44 @@
         }
     }
+    private List<String> makeQueryStrings(OsmPrimitiveType type, Map<Long,Integer> list) {
+        List<String> result = new ArrayList<>((list.size()+maxQueryIds-1)/maxQueryIds);
+        StringBuilder sb = new StringBuilder();
+        int cnt=0;
+        for (Entry<Long,Integer> entry : list.entrySet()) {
+            if (cnt == 0) {
+                sb.append(type.getAPIName());
+                sb.append("s?");
+                sb.append(type.getAPIName());
+                sb.append("s=");
+            } else {
+                sb.append(",");
+            }
+            sb.append(entry.getKey());
+            sb.append("v");
+            sb.append(entry.getValue());
+            cnt++;
+            if (cnt >=maxQueryIds) {
+                result.add(sb.toString());
+                sb.setLength(0);
+                cnt = 0;
+            }
+        }
+        if (cnt>0) {
+            result.add(sb.toString());
+        }
+        return result;
+    }
+
+    protected static final int maxQueryIds = 128;
+    public void readMultiObjects(OsmPrimitiveType type, Map<Long,Integer> list, ProgressMonitor progressMonitor) {
+        for (String query : makeQueryStrings(type,list)) {
+            if (progressMonitor.isCanceled()) {
+                return;
+            }
+            rdr.callback = new ParseCallback() {
+                @Override
+                public void primitiveParsed(PrimitiveId id) {
+                    if (id.getType() == type && list.remove(id.getUniqueId()) != null) {
+                        progressMonitor.worked(1);
+                    }
+                }
+            };
+            try (InputStream in = getInputStream(query, NullProgressMonitor.INSTANCE)) {
+                rdr.addData(in);
+            } catch (IOException | IllegalDataException | OsmTransferException e) {
+                Logging.error(e);
+            } finally {
+                rdr.callback = null;
+            }
+        }
+    }
+
 
     /**