Changeset 14086 in josm for trunk/src

Timestamp:

2018-08-05T14:09:31+02:00 (6 years ago)

Author:

Don-vip

Message:

fix #16546 - Support Overpass API JSON format

Location:

trunk/src/org/openstreetmap/josm/io

Files:

• AbstractReader.java (modified) (4 diffs)
• OsmJsonReader.java (added)
• OsmReader.java (modified) (14 diffs)
• OverpassDownloadReader.java (modified) (3 diffs)

trunk/src/org/openstreetmap/josm/io/AbstractReader.java

@@ -4 +4 @@
 import static org.openstreetmap.josm.tools.I18n.tr;
 
+import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -11 +14 @@
 import java.util.Map;
 import java.util.Map.Entry;
-
+import java.util.function.Consumer;
+
+import org.openstreetmap.josm.data.Bounds;
+import org.openstreetmap.josm.data.DataSource;
+import org.openstreetmap.josm.data.coor.LatLon;
+import org.openstreetmap.josm.data.osm.AbstractPrimitive;
 import org.openstreetmap.josm.data.osm.Changeset;
 import org.openstreetmap.josm.data.osm.DataSet;
+import org.openstreetmap.josm.data.osm.DownloadPolicy;
 import org.openstreetmap.josm.data.osm.Node;
+import org.openstreetmap.josm.data.osm.NodeData;
 import org.openstreetmap.josm.data.osm.OsmPrimitive;
 import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
+import org.openstreetmap.josm.data.osm.PrimitiveData;
 import org.openstreetmap.josm.data.osm.PrimitiveId;
 import org.openstreetmap.josm.data.osm.Relation;
+import org.openstreetmap.josm.data.osm.RelationData;
 import org.openstreetmap.josm.data.osm.RelationMember;
 import org.openstreetmap.josm.data.osm.RelationMemberData;
 import org.openstreetmap.josm.data.osm.SimplePrimitiveId;
+import org.openstreetmap.josm.data.osm.Tagged;
+import org.openstreetmap.josm.data.osm.UploadPolicy;
+import org.openstreetmap.josm.data.osm.User;
 import org.openstreetmap.josm.data.osm.Way;
+import org.openstreetmap.josm.data.osm.WayData;
+import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
 import org.openstreetmap.josm.gui.progress.ProgressMonitor;
+import org.openstreetmap.josm.tools.CheckParameterUtil;
 import org.openstreetmap.josm.tools.Logging;
+import org.openstreetmap.josm.tools.Utils;
+import org.openstreetmap.josm.tools.date.DateUtils;
 
 /**
  * Abstract Reader, allowing other implementations than OsmReader (PbfReader in PBF plugin for example)
  * @author Vincent
- *
+ * @since 4490
  */
 public abstract class AbstractReader {
+
+    /** Used by plugins to register themselves as data postprocessors. */
+    private static volatile List<OsmServerReadPostprocessor> postprocessors;
+
+    protected boolean cancel;
+
+    /**
+     * Register a new postprocessor.
+     * @param pp postprocessor
+     * @see #deregisterPostprocessor
+     * @since xxx (moved from OsmReader)
+     */
+    public static void registerPostprocessor(OsmServerReadPostprocessor pp) {
+        if (postprocessors == null) {
+            postprocessors = new ArrayList<>();
+        }
+        postprocessors.add(pp);
+    }
+
+    /**
+     * Deregister a postprocessor previously registered with {@link #registerPostprocessor}.
+     * @param pp postprocessor
+     * @see #registerPostprocessor
+     * @since xxx (moved from OsmReader)
+     */
+    public static void deregisterPostprocessor(OsmServerReadPostprocessor pp) {
+        if (postprocessors != null) {
+            postprocessors.remove(pp);
+        }
+    }
 
     /**
@@ -63 +113 @@
     public DataSet getDataSet() {
         return ds;
+    }
+
+    /**
+     * Iterate over registered postprocessors and give them each a chance to modify the dataset we have just loaded.
+     * @param progressMonitor Progress monitor
+     */
+    protected void callPostProcessors(ProgressMonitor progressMonitor) {
+        if (postprocessors != null) {
+            for (OsmServerReadPostprocessor pp : postprocessors) {
+                pp.postprocessDataSet(getDataSet(), progressMonitor);
+            }
+        }
     }
 
@@ -207 +269 @@
 
     protected abstract DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException;
+
+    @FunctionalInterface
+    protected interface ParserWorker {
+       void accept(InputStreamReader ir) throws IllegalDataException, IOException;
+    }
+
+    protected final DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor, ParserWorker parserWorker)
+            throws IllegalDataException {
+        if (progressMonitor == null) {
+            progressMonitor = NullProgressMonitor.INSTANCE;
+        }
+        ProgressMonitor.CancelListener cancelListener = () -> cancel = true;
+        progressMonitor.addCancelListener(cancelListener);
+        CheckParameterUtil.ensureParameterNotNull(source, "source");
+        try {
+            progressMonitor.beginTask(tr("Prepare OSM data...", 2));
+            progressMonitor.indeterminateSubTask(tr("Parsing OSM data..."));
+
+            try (InputStreamReader ir = UTFInputStreamReader.create(source)) {
+                parserWorker.accept(ir);
+            }
+            progressMonitor.worked(1);
+
+            boolean readOnly = getDataSet().isLocked();
+
+            progressMonitor.indeterminateSubTask(tr("Preparing data set..."));
+            if (readOnly) {
+                getDataSet().unlock();
+            }
+            prepareDataSet();
+            if (readOnly) {
+                getDataSet().lock();
+            }
+            progressMonitor.worked(1);
+
+            // iterate over registered postprocessors and give them each a chance
+            // to modify the dataset we have just loaded.
+            callPostProcessors(progressMonitor);
+            // Make sure postprocessors did not change the read-only state
+            if (readOnly && !getDataSet().isLocked()) {
+                getDataSet().lock();
+            }
+            return getDataSet();
+        } catch (IllegalDataException e) {
+            throw e;
+        } catch (IOException e) {
+            throw new IllegalDataException(e);
+        } finally {
+            progressMonitor.finishTask();
+            progressMonitor.removeCancelListener(cancelListener);
+        }
+    }
+
+    protected final long getLong(String name, String value) throws IllegalDataException {
+        if (value == null) {
+            throw new IllegalDataException(tr("Missing required attribute ''{0}''.", name));
+        }
+        try {
+            return Long.parseLong(value);
+        } catch (NumberFormatException e) {
+            throw new IllegalDataException(tr("Illegal long value for attribute ''{0}''. Got ''{1}''.", name, value), e);
+        }
+    }
+
+    protected final void parseVersion(String version) throws IllegalDataException {
+        validateVersion(version);
+        ds.setVersion(version);
+    }
+
+    private static void validateVersion(String version) throws IllegalDataException {
+        if (version == null) {
+            throw new IllegalDataException(tr("Missing mandatory attribute ''{0}''.", "version"));
+        }
+        if (!"0.6".equals(version)) {
+            throw new IllegalDataException(tr("Unsupported version: {0}", version));
+        }
+    }
+
+    protected final void parseDownloadPolicy(String key, String downloadPolicy) throws IllegalDataException {
+        parsePolicy(key, downloadPolicy, policy -> ds.setDownloadPolicy(DownloadPolicy.of(policy)));
+    }
+
+    protected final void parseUploadPolicy(String key, String uploadPolicy) throws IllegalDataException {
+        parsePolicy(key, uploadPolicy, policy -> ds.setUploadPolicy(UploadPolicy.of(policy)));
+    }
+
+    private static void parsePolicy(String key, String policy, Consumer<String> consumer) throws IllegalDataException {
+        if (policy != null) {
+            try {
+                consumer.accept(policy);
+            } catch (IllegalArgumentException e) {
+                throw new IllegalDataException(MessageFormat.format(
+                        "Illegal value for attribute ''{0}''. Got ''{1}''.", key, policy), e);
+            }
+        }
+    }
+
+    protected final void parseLocked(String locked) {
+        if ("true".equalsIgnoreCase(locked)) {
+            ds.lock();
+        }
+    }
+
+    protected final void parseBounds(String generator, String minlon, String minlat, String maxlon, String maxlat, String origin)
+            throws IllegalDataException {
+        if (minlon != null && maxlon != null && minlat != null && maxlat != null) {
+            if (origin == null) {
+                origin = generator;
+            }
+            Bounds bounds = new Bounds(
+                    Double.parseDouble(minlat), Double.parseDouble(minlon),
+                    Double.parseDouble(maxlat), Double.parseDouble(maxlon));
+            if (bounds.isOutOfTheWorld()) {
+                Bounds copy = new Bounds(bounds);
+                bounds.normalize();
+                Logging.info("Bbox " + copy + " is out of the world, normalized to " + bounds);
+            }
+            ds.addDataSource(new DataSource(bounds, origin));
+        } else {
+            throw new IllegalDataException(tr("Missing mandatory attributes on element ''bounds''. " +
+                    "Got minlon=''{0}'',minlat=''{1}'',maxlon=''{2}'',maxlat=''{3}'', origin=''{4}''.",
+                    minlon, minlat, maxlon, maxlat, origin
+            ));
+        }
+    }
+
+    protected final void parseId(PrimitiveData current, long id) throws IllegalDataException {
+        current.setId(id);
+        if (current.getUniqueId() == 0) {
+            throw new IllegalDataException(tr("Illegal object with ID=0."));
+        }
+    }
+
+    protected final void parseTimestamp(PrimitiveData current, String time) {
+        if (time != null && !time.isEmpty()) {
+            current.setRawTimestamp((int) (DateUtils.tsFromString(time)/1000));
+        }
+    }
+
+    private static User createUser(String uid, String name) throws IllegalDataException {
+        if (uid == null) {
+            if (name == null)
+                return null;
+            return User.createLocalUser(name);
+        }
+        try {
+            return User.createOsmUser(Long.parseLong(uid), name);
+        } catch (NumberFormatException e) {
+            throw new IllegalDataException(MessageFormat.format("Illegal value for attribute ''uid''. Got ''{0}''.", uid), e);
+        }
+    }
+
+    protected final void parseUser(PrimitiveData current, String user, long uid) {
+        current.setUser(User.createOsmUser(uid, user));
+    }
+
+    protected final void parseUser(PrimitiveData current, String user, String uid) throws IllegalDataException {
+        current.setUser(createUser(uid, user));
+    }
+
+    protected final void parseVisible(PrimitiveData current, String visible) {
+        if (visible != null) {
+            current.setVisible(Boolean.parseBoolean(visible));
+        }
+    }
+
+    protected final void parseVersion(PrimitiveData current, String versionString) throws IllegalDataException {
+        int version = 0;
+        if (versionString != null) {
+            try {
+                version = Integer.parseInt(versionString);
+            } catch (NumberFormatException e) {
+                throw new IllegalDataException(
+                        tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.",
+                        Long.toString(current.getUniqueId()), versionString), e);
+            }
+            parseVersion(current, version);
+        } else {
+            // version expected for OSM primitives with an id assigned by the server (id > 0), since API 0.6
+            if (!current.isNew() && ds.getVersion() != null && "0.6".equals(ds.getVersion())) {
+                throw new IllegalDataException(
+                        tr("Missing attribute ''version'' on OSM primitive with ID {0}.", Long.toString(current.getUniqueId())));
+            }
+        }
+    }
+
+    protected final void parseVersion(PrimitiveData current, int version) throws IllegalDataException {
+        switch (ds.getVersion()) {
+        case "0.6":
+            if (version <= 0 && !current.isNew()) {
+                throw new IllegalDataException(
+                        tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.",
+                        Long.toString(current.getUniqueId()), version));
+            } else if (version < 0 && current.isNew()) {
+                Logging.warn(tr("Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.",
+                        current.getUniqueId(), version, 0, "0.6"));
+                version = 0;
+            }
+            break;
+        default:
+            // should not happen. API version has been checked before
+            throw new IllegalDataException(tr("Unknown or unsupported API version. Got {0}.", ds.getVersion()));
+        }
+        current.setVersion(version);
+    }
+
+    protected final void parseAction(PrimitiveData current, String action) {
+        if (action == null) {
+            // do nothing
+        } else if ("delete".equals(action)) {
+            current.setDeleted(true);
+            current.setModified(current.isVisible());
+        } else if ("modify".equals(action)) {
+            current.setModified(true);
+        }
+    }
+
+    private static void handleIllegalChangeset(PrimitiveData current, IllegalArgumentException e, Object v)
+            throws IllegalDataException {
+        Logging.debug(e.getMessage());
+        if (current.isNew()) {
+            // for a new primitive we just log a warning
+            Logging.info(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.",
+                    v, current.getUniqueId()));
+            current.setChangesetId(0);
+        } else {
+            // for an existing primitive this is a problem
+            throw new IllegalDataException(tr("Illegal value for attribute ''changeset''. Got {0}.", v), e);
+        }
+    }
+
+    protected final void parseChangeset(PrimitiveData current, String v) throws IllegalDataException {
+        if (v == null) {
+            current.setChangesetId(0);
+        } else {
+            try {
+                parseChangeset(current, Integer.parseInt(v));
+            } catch (NumberFormatException e) {
+                handleIllegalChangeset(current, e, v);
+            }
+        }
+    }
+
+    protected final void parseChangeset(PrimitiveData current, int v) throws IllegalDataException {
+        try {
+            current.setChangesetId(v);
+        } catch (IllegalArgumentException e) {
+            handleIllegalChangeset(current, e, v);
+        } catch (IllegalStateException e) {
+            // thrown for positive changeset id on new primitives
+            Logging.debug(e);
+            Logging.info(e.getMessage());
+            current.setChangesetId(0);
+        }
+        if (current.getChangesetId() <= 0) {
+            if (current.isNew()) {
+                // for a new primitive we just log a warning
+                Logging.info(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.",
+                        v, current.getUniqueId()));
+                current.setChangesetId(0);
+            } else if (current.getChangesetId() < 0) {
+                // for an existing primitive this is a problem only for negative ids (GDPR extracts are set to 0)
+                throw new IllegalDataException(tr("Illegal value for attribute ''changeset''. Got {0}.", v));
+            }
+        }
+    }
+
+    protected final void parseTag(Tagged t, String key, String value) throws IllegalDataException {
+        if (key == null || value == null) {
+            throw new IllegalDataException(tr("Missing key or value attribute in tag."));
+        } else if (Utils.isStripEmpty(key) && t instanceof AbstractPrimitive) {
+            // #14199: Empty keys as ignored by AbstractPrimitive#put, but it causes problems to fix existing data
+            // Drop the tag on import, but flag the primitive as modified
+            ((AbstractPrimitive) t).setModified(true);
+        } else {
+            t.put(key.intern(), value.intern());
+        }
+    }
+
+    @FunctionalInterface
+    protected interface CommonReader {
+       void accept(PrimitiveData pd) throws IllegalDataException;
+    }
+
+    @FunctionalInterface
+    protected interface NodeReader {
+       void accept(Node n) throws IllegalDataException;
+    }
+
+    @FunctionalInterface
+    protected interface WayReader {
+       void accept(Way w, Collection<Long> nodeIds) throws IllegalDataException;
+    }
+
+    @FunctionalInterface
+    protected interface RelationReader {
+       void accept(Relation r, Collection<RelationMemberData> members) throws IllegalDataException;
+    }
+
+    private static boolean areLatLonDefined(String lat, String lon) {
+        return lat != null && lon != null;
+    }
+
+    private static boolean areLatLonDefined(double lat, double lon) {
+        return lat != Double.NaN && lon != Double.NaN;
+    }
+
+    private Node addNode(NodeData nd, NodeReader nodeReader) throws IllegalDataException {
+        Node n = new Node(nd.getId(), nd.getVersion());
+        n.setVisible(nd.isVisible());
+        n.load(nd);
+        nodeReader.accept(n);
+        externalIdMap.put(nd.getPrimitiveId(), n);
+        return n;
+    }
+
+    protected final Node parseNode(double lat, double lon, CommonReader commonReader, NodeReader nodeReader)
+            throws IllegalDataException {
+        NodeData nd = new NodeData();
+        LatLon ll = null;
+        if (areLatLonDefined(lat, lon)) {
+            try {
+                ll = new LatLon(lat, lon);
+                nd.setCoor(ll);
+            } catch (NumberFormatException e) {
+                Logging.trace(e);
+            }
+        }
+        commonReader.accept(nd);
+        if (areLatLonDefined(lat, lon) && (ll == null || !ll.isValid())) {
+            throw new IllegalDataException(tr("Illegal value for attributes ''lat'', ''lon'' on node with ID {0}. Got ''{1}'', ''{2}''.",
+                    Long.toString(nd.getId()), lat, lon));
+        }
+        return addNode(nd, nodeReader);
+    }
+
+    protected final Node parseNode(String lat, String lon, CommonReader commonReader, NodeReader nodeReader)
+            throws IllegalDataException {
+        NodeData nd = new NodeData();
+        LatLon ll = null;
+        if (areLatLonDefined(lat, lon)) {
+            try {
+                ll = new LatLon(Double.parseDouble(lat), Double.parseDouble(lon));
+                nd.setCoor(ll);
+            } catch (NumberFormatException e) {
+                Logging.trace(e);
+            }
+        }
+        commonReader.accept(nd);
+        if (areLatLonDefined(lat, lon) && (ll == null || !ll.isValid())) {
+            throw new IllegalDataException(tr("Illegal value for attributes ''lat'', ''lon'' on node with ID {0}. Got ''{1}'', ''{2}''.",
+                    Long.toString(nd.getId()), lat, lon));
+        }
+        return addNode(nd, nodeReader);
+    }
+
+    protected final Way parseWay(CommonReader commonReader, WayReader wayReader) throws IllegalDataException {
+        WayData wd = new WayData();
+        commonReader.accept(wd);
+        Way w = new Way(wd.getId(), wd.getVersion());
+        w.setVisible(wd.isVisible());
+        w.load(wd);
+        externalIdMap.put(wd.getPrimitiveId(), w);
+
+        Collection<Long> nodeIds = new ArrayList<>();
+        wayReader.accept(w, nodeIds);
+        if (w.isDeleted() && !nodeIds.isEmpty()) {
+            Logging.info(tr("Deleted way {0} contains nodes", Long.toString(w.getUniqueId())));
+            nodeIds = new ArrayList<>();
+        }
+        ways.put(wd.getUniqueId(), nodeIds);
+        return w;
+    }
+
+    protected final Relation parseRelation(CommonReader commonReader, RelationReader relationReader) throws IllegalDataException {
+        RelationData rd = new RelationData();
+        commonReader.accept(rd);
+        Relation r = new Relation(rd.getId(), rd.getVersion());
+        r.setVisible(rd.isVisible());
+        r.load(rd);
+        externalIdMap.put(rd.getPrimitiveId(), r);
+
+        Collection<RelationMemberData> members = new ArrayList<>();
+        relationReader.accept(r, members);
+        if (r.isDeleted() && !members.isEmpty()) {
+            Logging.info(tr("Deleted relation {0} contains members", Long.toString(r.getUniqueId())));
+            members = new ArrayList<>();
+        }
+        relations.put(rd.getUniqueId(), members);
+        return r;
+    }
+
+    protected final RelationMemberData parseRelationMember(Relation r, String ref, String type, String role) throws IllegalDataException {
+        if (ref == null) {
+            throw new IllegalDataException(tr("Missing attribute ''ref'' on member in relation {0}.",
+                    Long.toString(r.getUniqueId())));
+        }
+        try {
+            return parseRelationMember(r, Long.parseLong(ref), type, role);
+        } catch (NumberFormatException e) {
+            throw new IllegalDataException(tr("Illegal value for attribute ''ref'' on member in relation {0}. Got {1}",
+                    Long.toString(r.getUniqueId()), ref), e);
+        }
+    }
+
+    protected final RelationMemberData parseRelationMember(Relation r, long id, String type, String role) throws IllegalDataException {
+        if (id == 0) {
+            throw new IllegalDataException(tr("Incomplete <member> specification with ref=0"));
+        }
+        if (type == null) {
+            throw new IllegalDataException(tr("Missing attribute ''type'' on member {0} in relation {1}.",
+                    Long.toString(id), Long.toString(r.getUniqueId())));
+        }
+        try {
+            return new RelationMemberData(role, OsmPrimitiveType.fromApiTypeName(type), id);
+        } catch (IllegalArgumentException e) {
+            throw new IllegalDataException(tr("Illegal value for attribute ''type'' on member {0} in relation {1}. Got {2}.",
+                    Long.toString(id), Long.toString(r.getUniqueId()), type), e);
+        }
+    }
 }

trunk/src/org/openstreetmap/josm/io/OsmReader.java

@@ -4 +4 @@
 import static org.openstreetmap.josm.tools.I18n.tr;
 
-import java.io.IOException;
 import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.text.MessageFormat;
-import java.util.ArrayList;
 import java.util.Collection;
-import java.util.List;
 import java.util.Objects;
-import java.util.function.Consumer;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -21 +15 @@
 import javax.xml.stream.XMLStreamReader;
 
-import org.openstreetmap.josm.data.Bounds;
-import org.openstreetmap.josm.data.DataSource;
-import org.openstreetmap.josm.data.coor.LatLon;
-import org.openstreetmap.josm.data.osm.AbstractPrimitive;
 import org.openstreetmap.josm.data.osm.Changeset;
 import org.openstreetmap.josm.data.osm.DataSet;
-import org.openstreetmap.josm.data.osm.DownloadPolicy;
 import org.openstreetmap.josm.data.osm.Node;
-import org.openstreetmap.josm.data.osm.NodeData;
-import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
 import org.openstreetmap.josm.data.osm.PrimitiveData;
 import org.openstreetmap.josm.data.osm.Relation;
-import org.openstreetmap.josm.data.osm.RelationData;
 import org.openstreetmap.josm.data.osm.RelationMemberData;
 import org.openstreetmap.josm.data.osm.Tagged;
-import org.openstreetmap.josm.data.osm.UploadPolicy;
-import org.openstreetmap.josm.data.osm.User;
 import org.openstreetmap.josm.data.osm.Way;
-import org.openstreetmap.josm.data.osm.WayData;
 import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
 import org.openstreetmap.josm.gui.progress.ProgressMonitor;
-import org.openstreetmap.josm.tools.CheckParameterUtil;
 import org.openstreetmap.josm.tools.Logging;
 import org.openstreetmap.josm.tools.UncheckedParseException;
-import org.openstreetmap.josm.tools.Utils;
 import org.openstreetmap.josm.tools.XmlUtils;
-import org.openstreetmap.josm.tools.date.DateUtils;
 
 /**
- * Parser for the Osm Api. Read from an input stream and construct a dataset out of it.
+ * Parser for the Osm API (XML output). Read from an input stream and construct a dataset out of it.
  *
  * For each xml element, there is a dedicated method.
@@ -59 +39 @@
 
     protected XMLStreamReader parser;
-
-    protected boolean cancel;
-
-    /** Used by plugins to register themselves as data postprocessors. */
-    private static volatile List<OsmServerReadPostprocessor> postprocessors;
-
-    /** Register a new postprocessor.
-     * @param pp postprocessor
-     * @see #deregisterPostprocessor
-     */
-    public static void registerPostprocessor(OsmServerReadPostprocessor pp) {
-        if (postprocessors == null) {
-            postprocessors = new ArrayList<>();
-        }
-        postprocessors.add(pp);
-    }
-
-    /**
-     * Deregister a postprocessor previously registered with {@link #registerPostprocessor}.
-     * @param pp postprocessor
-     * @see #registerPostprocessor
-     */
-    public static void deregisterPostprocessor(OsmServerReadPostprocessor pp) {
-        if (postprocessors != null) {
-            postprocessors.remove(pp);
-        }
-    }
 
     /**
@@ -98 +51 @@
     protected void setParser(XMLStreamReader parser) {
         this.parser = parser;
+    }
+
+    protected void throwException(Throwable th) throws XMLStreamException {
+        throw new XmlStreamParsingException(th.getMessage(), parser.getLocation(), th);
     }
 
@@ -133 +90 @@
 
     private void parseOsm() throws XMLStreamException {
-        String v = parser.getAttributeValue(null, "version");
-        if (v == null) {
-            throwException(tr("Missing mandatory attribute ''{0}''.", "version"));
-        }
-        if (!"0.6".equals(v)) {
-            throwException(tr("Unsupported version: {0}", v));
-        }
-        ds.setVersion(v);
-        parsePolicy("download", policy -> ds.setDownloadPolicy(DownloadPolicy.of(policy)));
-        parsePolicy("upload", policy -> ds.setUploadPolicy(UploadPolicy.of(policy)));
-        if ("true".equalsIgnoreCase(parser.getAttributeValue(null, "locked"))) {
-            ds.lock();
+        try {
+            parseVersion(parser.getAttributeValue(null, "version"));
+            parseDownloadPolicy("download", parser.getAttributeValue(null, "download"));
+            parseUploadPolicy("upload", parser.getAttributeValue(null, "upload"));
+            parseLocked(parser.getAttributeValue(null, "locked"));
+        } catch (IllegalDataException e) {
+            throwException(e);
         }
         String generator = parser.getAttributeValue(null, "generator");
@@ -181 +133 @@
             } else if (event == XMLStreamConstants.END_ELEMENT) {
                 return;
-            }
-        }
-    }
-
-    private void parsePolicy(String key, Consumer<String> consumer) throws XMLStreamException {
-        String policy = parser.getAttributeValue(null, key);
-        if (policy != null) {
-            try {
-                consumer.accept(policy);
-            } catch (IllegalArgumentException e) {
-                throwException(MessageFormat.format("Illegal value for attribute ''{0}''. Got ''{1}''.", key, policy), e);
             }
         }
@@ -202 +143 @@
         String maxlat = parser.getAttributeValue(null, "maxlat");
         String origin = parser.getAttributeValue(null, "origin");
-        if (minlon != null && maxlon != null && minlat != null && maxlat != null) {
-            if (origin == null) {
-                origin = generator;
-            }
-            Bounds bounds = new Bounds(
-                    Double.parseDouble(minlat), Double.parseDouble(minlon),
-                    Double.parseDouble(maxlat), Double.parseDouble(maxlon));
-            if (bounds.isOutOfTheWorld()) {
-                Bounds copy = new Bounds(bounds);
-                bounds.normalize();
-                Logging.info("Bbox " + copy + " is out of the world, normalized to " + bounds);
-            }
-            DataSource src = new DataSource(bounds, origin);
-            ds.addDataSource(src);
-        } else {
-            throwException(tr("Missing mandatory attributes on element ''bounds''. " +
-                    "Got minlon=''{0}'',minlat=''{1}'',maxlon=''{2}'',maxlat=''{3}'', origin=''{4}''.",
-                    minlon, minlat, maxlon, maxlat, origin
-            ));
+        try {
+            parseBounds(generator, minlon, minlat, maxlon, maxlat, origin);
+        } catch (IllegalDataException e) {
+            throwException(e);
         }
         jumpToEnd();
@@ -226 +152 @@
 
     protected Node parseNode() throws XMLStreamException {
-        NodeData nd = new NodeData();
         String lat = parser.getAttributeValue(null, "lat");
         String lon = parser.getAttributeValue(null, "lon");
-        LatLon ll = null;
-        if (lat != null && lon != null) {
-            try {
-                ll = new LatLon(Double.parseDouble(lat), Double.parseDouble(lon));
-                nd.setCoor(ll);
-            } catch (NumberFormatException e) {
-                Logging.trace(e);
-            }
-        }
-        readCommon(nd);
-        if (lat != null && lon != null && (ll == null || !ll.isValid())) {
-            throwException(tr("Illegal value for attributes ''lat'', ''lon'' on node with ID {0}. Got ''{1}'', ''{2}''.",
-                    Long.toString(nd.getId()), lat, lon));
-        }
-        Node n = new Node(nd.getId(), nd.getVersion());
-        n.setVisible(nd.isVisible());
-        n.load(nd);
-        externalIdMap.put(nd.getPrimitiveId(), n);
-        while (true) {
-            int event = parser.next();
-            if (event == XMLStreamConstants.START_ELEMENT) {
-                if ("tag".equals(parser.getLocalName())) {
-                    parseTag(n);
-                } else {
-                    parseUnknown();
+        try {
+            return parseNode(lat, lon, this::readCommon, this::parseNodeTags);
+        } catch (IllegalDataException e) {
+            throwException(e);
+        }
+        return null;
+    }
+
+    private void parseNodeTags(Node n) throws IllegalDataException {
+        try {
+            while (parser.hasNext()) {
+                int event = parser.next();
+                if (event == XMLStreamConstants.START_ELEMENT) {
+                    if ("tag".equals(parser.getLocalName())) {
+                        parseTag(n);
+                    } else {
+                        parseUnknown();
+                    }
+                } else if (event == XMLStreamConstants.END_ELEMENT) {
+                    return;
                 }
-            } else if (event == XMLStreamConstants.END_ELEMENT)
-                return n;
+            }
+        } catch (XMLStreamException e) {
+            throw new IllegalDataException(e);
         }
     }
 
     protected Way parseWay() throws XMLStreamException {
-        WayData wd = new WayData();
-        readCommon(wd);
-        Way w = new Way(wd.getId(), wd.getVersion());
-        w.setVisible(wd.isVisible());
-        w.load(wd);
-        externalIdMap.put(wd.getPrimitiveId(), w);
-
-        Collection<Long> nodeIds = new ArrayList<>();
-        while (true) {
-            int event = parser.next();
-            if (event == XMLStreamConstants.START_ELEMENT) {
-                switch (parser.getLocalName()) {
-                case "nd":
-                    nodeIds.add(parseWayNode(w));
-                    break;
-                case "tag":
-                    parseTag(w);
-                    break;
-                default:
-                    parseUnknown();
+        try {
+            return parseWay(this::readCommon, this::parseWayNodesAndTags);
+        } catch (IllegalDataException e) {
+            throwException(e);
+        }
+        return null;
+    }
+
+    private void parseWayNodesAndTags(Way w, Collection<Long> nodeIds) throws IllegalDataException {
+        try {
+            while (parser.hasNext()) {
+                int event = parser.next();
+                if (event == XMLStreamConstants.START_ELEMENT) {
+                    switch (parser.getLocalName()) {
+                    case "nd":
+                        nodeIds.add(parseWayNode(w));
+                        break;
+                    case "tag":
+                        parseTag(w);
+                        break;
+                    default:
+                        parseUnknown();
+                    }
+                } else if (event == XMLStreamConstants.END_ELEMENT) {
+                    break;
                 }
-            } else if (event == XMLStreamConstants.END_ELEMENT) {
-                break;
-            }
-        }
-        if (w.isDeleted() && !nodeIds.isEmpty()) {
-            Logging.info(tr("Deleted way {0} contains nodes", Long.toString(w.getUniqueId())));
-            nodeIds = new ArrayList<>();
-        }
-        ways.put(wd.getUniqueId(), nodeIds);
-        return w;
+            }
+        } catch (XMLStreamException e) {
+            throw new IllegalDataException(e);
+        }
     }
 
@@ -311 +231 @@
 
     protected Relation parseRelation() throws XMLStreamException {
-        RelationData rd = new RelationData();
-        readCommon(rd);
-        Relation r = new Relation(rd.getId(), rd.getVersion());
-        r.setVisible(rd.isVisible());
-        r.load(rd);
-        externalIdMap.put(rd.getPrimitiveId(), r);
-
-        Collection<RelationMemberData> members = new ArrayList<>();
-        while (true) {
-            int event = parser.next();
-            if (event == XMLStreamConstants.START_ELEMENT) {
-                switch (parser.getLocalName()) {
-                case "member":
-                    members.add(parseRelationMember(r));
-                    break;
-                case "tag":
-                    parseTag(r);
-                    break;
-                default:
-                    parseUnknown();
+        try {
+            return parseRelation(this::readCommon, this::parseRelationMembersAndTags);
+        } catch (IllegalDataException e) {
+            throw new XMLStreamException(e);
+        }
+    }
+
+    private void parseRelationMembersAndTags(Relation r, Collection<RelationMemberData> members) throws IllegalDataException {
+        try {
+            while (parser.hasNext()) {
+                int event = parser.next();
+                if (event == XMLStreamConstants.START_ELEMENT) {
+                    switch (parser.getLocalName()) {
+                    case "member":
+                        members.add(parseRelationMember(r));
+                        break;
+                    case "tag":
+                        parseTag(r);
+                        break;
+                    default:
+                        parseUnknown();
+                    }
+                } else if (event == XMLStreamConstants.END_ELEMENT) {
+                    break;
                 }
-            } else if (event == XMLStreamConstants.END_ELEMENT) {
-                break;
-            }
-        }
-        if (r.isDeleted() && !members.isEmpty()) {
-            Logging.info(tr("Deleted relation {0} contains members", Long.toString(r.getUniqueId())));
-            members = new ArrayList<>();
-        }
-        relations.put(rd.getUniqueId(), members);
-        return r;
+            }
+        } catch (XMLStreamException e) {
+            throw new IllegalDataException(e);
+        }
     }
 
     private RelationMemberData parseRelationMember(Relation r) throws XMLStreamException {
-        OsmPrimitiveType type = null;
-        long id = 0;
-        String value = parser.getAttributeValue(null, "ref");
-        if (value == null) {
-            throwException(tr("Missing attribute ''ref'' on member in relation {0}.", Long.toString(r.getUniqueId())));
-        }
-        try {
-            id = Long.parseLong(value);
-        } catch (NumberFormatException e) {
-            throwException(tr("Illegal value for attribute ''ref'' on member in relation {0}. Got {1}", Long.toString(r.getUniqueId()),
-                    value), e);
-        }
-        value = parser.getAttributeValue(null, "type");
-        if (value == null) {
-            throwException(tr("Missing attribute ''type'' on member {0} in relation {1}.", Long.toString(id), Long.toString(r.getUniqueId())));
-        }
-        try {
-            type = OsmPrimitiveType.fromApiTypeName(value);
-        } catch (IllegalArgumentException e) {
-            throwException(tr("Illegal value for attribute ''type'' on member {0} in relation {1}. Got {2}.",
-                    Long.toString(id), Long.toString(r.getUniqueId()), value), e);
-        }
-        String role = parser.getAttributeValue(null, "role");
-
-        if (id == 0) {
-            throwException(tr("Incomplete <member> specification with ref=0"));
-        }
-        jumpToEnd();
-        return new RelationMemberData(role, type, id);
+        RelationMemberData result = null;
+        try {
+            String ref = parser.getAttributeValue(null, "ref");
+            String type = parser.getAttributeValue(null, "type");
+            String role = parser.getAttributeValue(null, "role");
+            result = parseRelationMember(r, ref, type, role);
+            jumpToEnd();
+        } catch (IllegalDataException e) {
+            throwException(e);
+        }
+        return result;
     }
 
@@ -404 +304 @@
         String key = parser.getAttributeValue(null, "k");
         String value = parser.getAttributeValue(null, "v");
-        if (key == null || value == null) {
-            throwException(tr("Missing key or value attribute in tag."));
-        } else if (Utils.isStripEmpty(key) && t instanceof AbstractPrimitive) {
-            // #14199: Empty keys as ignored by AbstractPrimitive#put, but it causes problems to fix existing data
-            // Drop the tag on import, but flag the primitive as modified
-            ((AbstractPrimitive) t).setModified(true);
-        } else {
-            t.put(key.intern(), value.intern());
+        try {
+            parseTag(t, key, value);
+        } catch (IllegalDataException e) {
+            throwException(e);
         }
         jumpToEnd();
@@ -460 +356 @@
     }
 
-    private User createUser(String uid, String name) throws XMLStreamException {
-        if (uid == null) {
-            if (name == null)
-                return null;
-            return User.createLocalUser(name);
-        }
-        try {
-            long id = Long.parseLong(uid);
-            return User.createOsmUser(id, name);
-        } catch (NumberFormatException e) {
-            throwException(MessageFormat.format("Illegal value for attribute ''uid''. Got ''{0}''.", uid), e);
-        }
-        return null;
-    }
-
     /**
      * Read out the common attributes and put them into current OsmPrimitive.
      * @param current primitive to update
-     * @throws XMLStreamException if there is an error processing the underlying XML source
+     * @throws IllegalDataException if there is an error processing the underlying XML source
      */
-    private void readCommon(PrimitiveData current) throws XMLStreamException {
-        current.setId(getLong("id"));
-        if (current.getUniqueId() == 0) {
-            throwException(tr("Illegal object with ID=0."));
-        }
-
-        String time = parser.getAttributeValue(null, "timestamp");
-        if (time != null && !time.isEmpty()) {
-            current.setRawTimestamp((int) (DateUtils.tsFromString(time)/1000));
-        }
-
-        String user = parser.getAttributeValue(null, "user");
-        String uid = parser.getAttributeValue(null, "uid");
-        current.setUser(createUser(uid, user));
-
-        String visible = parser.getAttributeValue(null, "visible");
-        if (visible != null) {
-            current.setVisible(Boolean.parseBoolean(visible));
-        }
-
-        String versionString = parser.getAttributeValue(null, "version");
-        int version = 0;
-        if (versionString != null) {
-            try {
-                version = Integer.parseInt(versionString);
-            } catch (NumberFormatException e) {
-                throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.",
-                        Long.toString(current.getUniqueId()), versionString), e);
-            }
-            switch (ds.getVersion()) {
-            case "0.6":
-                if (version <= 0 && !current.isNew()) {
-                    throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.",
-                            Long.toString(current.getUniqueId()), versionString));
-                } else if (version < 0 && current.isNew()) {
-                    Logging.warn(tr("Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.",
-                            current.getUniqueId(), version, 0, "0.6"));
-                    version = 0;
-                }
-                break;
-            default:
-                // should not happen. API version has been checked before
-                throwException(tr("Unknown or unsupported API version. Got {0}.", ds.getVersion()));
-            }
-        } else {
-            // version expected for OSM primitives with an id assigned by the server (id > 0), since API 0.6
-            if (!current.isNew() && ds.getVersion() != null && "0.6".equals(ds.getVersion())) {
-                throwException(tr("Missing attribute ''version'' on OSM primitive with ID {0}.", Long.toString(current.getUniqueId())));
-            }
-        }
-        current.setVersion(version);
-
-        String action = parser.getAttributeValue(null, "action");
-        if (action == null) {
-            // do nothing
-        } else if ("delete".equals(action)) {
-            current.setDeleted(true);
-            current.setModified(current.isVisible());
-        } else if ("modify".equals(action)) {
-            current.setModified(true);
-        }
-
-        String v = parser.getAttributeValue(null, "changeset");
-        if (v == null) {
-            current.setChangesetId(0);
-        } else {
-            try {
-                current.setChangesetId(Integer.parseInt(v));
-            } catch (IllegalArgumentException e) {
-                Logging.debug(e.getMessage());
-                if (current.isNew()) {
-                    // for a new primitive we just log a warning
-                    Logging.info(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.",
-                            v, current.getUniqueId()));
-                    current.setChangesetId(0);
-                } else {
-                    // for an existing primitive this is a problem
-                    throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v), e);
-                }
-            } catch (IllegalStateException e) {
-                // thrown for positive changeset id on new primitives
-                Logging.debug(e);
-                Logging.info(e.getMessage());
-                current.setChangesetId(0);
-            }
-            if (current.getChangesetId() <= 0) {
-                if (current.isNew()) {
-                    // for a new primitive we just log a warning
-                    Logging.info(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.",
-                            v, current.getUniqueId()));
-                    current.setChangesetId(0);
-                } else if (current.getChangesetId() < 0) {
-                    // for an existing primitive this is a problem only for negative ids (GDPR extracts are set to 0)
-                    throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v));
-                }
-            }
+    private void readCommon(PrimitiveData current) throws IllegalDataException {
+        try {
+            parseId(current, getLong("id"));
+            parseTimestamp(current, parser.getAttributeValue(null, "timestamp"));
+            parseUser(current, parser.getAttributeValue(null, "user"), parser.getAttributeValue(null, "uid"));
+            parseVisible(current, parser.getAttributeValue(null, "visible"));
+            parseVersion(current, parser.getAttributeValue(null, "version"));
+            parseAction(current, parser.getAttributeValue(null, "action"));
+            parseChangeset(current, parser.getAttributeValue(null, "changeset"));
+        } catch (UncheckedParseException | XMLStreamException e) {
+            throw new IllegalDataException(e);
         }
     }
@@ -581 +377 @@
     private long getLong(String name) throws XMLStreamException {
         String value = parser.getAttributeValue(null, name);
-        if (value == null) {
-            throwException(tr("Missing required attribute ''{0}''.", name));
-        }
-        try {
-            return Long.parseLong(value);
-        } catch (NumberFormatException e) {
-            throwException(tr("Illegal long value for attribute ''{0}''. Got ''{1}''.", name, value), e);
+        try {
+            return getLong(name, value);
+        } catch (IllegalDataException e) {
+            throwException(e);
         }
         return 0; // should not happen
@@ -608 +401 @@
     @Override
     protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
-        if (progressMonitor == null) {
-            progressMonitor = NullProgressMonitor.INSTANCE;
-        }
-        ProgressMonitor.CancelListener cancelListener = () -> cancel = true;
-        progressMonitor.addCancelListener(cancelListener);
-        CheckParameterUtil.ensureParameterNotNull(source, "source");
-        try {
-            progressMonitor.beginTask(tr("Prepare OSM data...", 2));
-            progressMonitor.indeterminateSubTask(tr("Parsing OSM data..."));
-
-            try (InputStreamReader ir = UTFInputStreamReader.create(source)) {
+        return doParseDataSet(source, progressMonitor, ir -> {
+            try {
                 setParser(XmlUtils.newSafeXMLInputFactory().createXMLStreamReader(ir));
                 parse();
-            }
-            progressMonitor.worked(1);
-
-            boolean readOnly = getDataSet().isLocked();
-
-            progressMonitor.indeterminateSubTask(tr("Preparing data set..."));
-            if (readOnly) {
-                getDataSet().unlock();
-            }
-            prepareDataSet();
-            if (readOnly) {
-                getDataSet().lock();
-            }
-            progressMonitor.worked(1);
-
-            // iterate over registered postprocessors and give them each a chance
-            // to modify the dataset we have just loaded.
-            if (postprocessors != null) {
-                for (OsmServerReadPostprocessor pp : postprocessors) {
-                    pp.postprocessDataSet(getDataSet(), progressMonitor);
+            } catch (XmlStreamParsingException | UncheckedParseException e) {
+                throw new IllegalDataException(e.getMessage(), e);
+            } catch (XMLStreamException e) {
+                String msg = e.getMessage();
+                Pattern p = Pattern.compile("Message: (.+)");
+                Matcher m = p.matcher(msg);
+                if (m.find()) {
+                    msg = m.group(1);
                 }
-            }
-            // Make sure postprocessors did not change the read-only state
-            if (readOnly && !getDataSet().isLocked()) {
-                getDataSet().lock();
-            }
-            return getDataSet();
-        } catch (IllegalDataException e) {
-            throw e;
-        } catch (XmlStreamParsingException | UncheckedParseException e) {
-            throw new IllegalDataException(e.getMessage(), e);
-        } catch (XMLStreamException e) {
-            String msg = e.getMessage();
-            Pattern p = Pattern.compile("Message: (.+)");
-            Matcher m = p.matcher(msg);
-            if (m.find()) {
-                msg = m.group(1);
-            }
-            if (e.getLocation() != null)
-                throw new IllegalDataException(tr("Line {0} column {1}: ",
-                        e.getLocation().getLineNumber(), e.getLocation().getColumnNumber()) + msg, e);
-            else
-                throw new IllegalDataException(msg, e);
-        } catch (IOException e) {
-            throw new IllegalDataException(e);
-        } finally {
-            progressMonitor.finishTask();
-            progressMonitor.removeCancelListener(cancelListener);
-        }
+                if (e.getLocation() != null)
+                    throw new IllegalDataException(tr("Line {0} column {1}: ",
+                            e.getLocation().getLineNumber(), e.getLocation().getColumnNumber()) + msg, e);
+                else
+                    throw new IllegalDataException(msg, e);
+            }
+        });
     }
 
@@ -676 +427 @@
      *
      * @param source the source input stream. Must not be null.
-     * @param progressMonitor  the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed
+     * @param progressMonitor the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed
      *
      * @return the dataset with the parsed data

trunk/src/org/openstreetmap/josm/io/OverpassDownloadReader.java

@@ -83 +83 @@
     }
 
+    static final class OverpassOsmJsonReader extends OsmJsonReader {
+
+    }
+
     /**
      * Possible Overpass API output format, with the {@code [out:<directive>]} statement.
@@ -165 +169 @@
     static {
         registerOverpassOutpoutFormatReader(OverpassOutpoutFormat.OSM_XML, OverpassOsmReader.class);
+        registerOverpassOutpoutFormatReader(OverpassOutpoutFormat.OSM_JSON, OverpassOsmJsonReader.class);
     }
 
@@ -410 +415 @@
         return query == null ? query : query
                 .replaceAll("out( body| skel| ids)?( id| qt)?;", "out meta$2;")
-                .replaceAll("(?s)\\[out:(json|csv)[^\\]]*\\]", "[out:xml]");
+                .replaceAll("(?s)\\[out:(csv)[^\\]]*\\]", "[out:xml]");
     }
 }