Index: trunk/src/org/openstreetmap/josm/io/AbstractReader.java =================================================================== --- trunk/src/org/openstreetmap/josm/io/AbstractReader.java (revision 14085) +++ trunk/src/org/openstreetmap/josm/io/AbstractReader.java (revision 14086) @@ -4,5 +4,8 @@ import static org.openstreetmap.josm.tools.I18n.tr; +import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; +import java.text.MessageFormat; import java.util.ArrayList; import java.util.Collection; @@ -11,25 +14,72 @@ import java.util.Map; import java.util.Map.Entry; - +import java.util.function.Consumer; + +import org.openstreetmap.josm.data.Bounds; +import org.openstreetmap.josm.data.DataSource; +import org.openstreetmap.josm.data.coor.LatLon; +import org.openstreetmap.josm.data.osm.AbstractPrimitive; import org.openstreetmap.josm.data.osm.Changeset; import org.openstreetmap.josm.data.osm.DataSet; +import org.openstreetmap.josm.data.osm.DownloadPolicy; import org.openstreetmap.josm.data.osm.Node; +import org.openstreetmap.josm.data.osm.NodeData; import org.openstreetmap.josm.data.osm.OsmPrimitive; import org.openstreetmap.josm.data.osm.OsmPrimitiveType; +import org.openstreetmap.josm.data.osm.PrimitiveData; import org.openstreetmap.josm.data.osm.PrimitiveId; import org.openstreetmap.josm.data.osm.Relation; +import org.openstreetmap.josm.data.osm.RelationData; import org.openstreetmap.josm.data.osm.RelationMember; import org.openstreetmap.josm.data.osm.RelationMemberData; import org.openstreetmap.josm.data.osm.SimplePrimitiveId; +import org.openstreetmap.josm.data.osm.Tagged; +import org.openstreetmap.josm.data.osm.UploadPolicy; +import org.openstreetmap.josm.data.osm.User; import org.openstreetmap.josm.data.osm.Way; +import org.openstreetmap.josm.data.osm.WayData; +import org.openstreetmap.josm.gui.progress.NullProgressMonitor; import org.openstreetmap.josm.gui.progress.ProgressMonitor; +import org.openstreetmap.josm.tools.CheckParameterUtil; import org.openstreetmap.josm.tools.Logging; +import org.openstreetmap.josm.tools.Utils; +import org.openstreetmap.josm.tools.date.DateUtils; /** * Abstract Reader, allowing other implementations than OsmReader (PbfReader in PBF plugin for example) * @author Vincent - * + * @since 4490 */ public abstract class AbstractReader { + + /** Used by plugins to register themselves as data postprocessors. */ + private static volatile List postprocessors; + + protected boolean cancel; + + /** + * Register a new postprocessor. + * @param pp postprocessor + * @see #deregisterPostprocessor + * @since xxx (moved from OsmReader) + */ + public static void registerPostprocessor(OsmServerReadPostprocessor pp) { + if (postprocessors == null) { + postprocessors = new ArrayList<>(); + } + postprocessors.add(pp); + } + + /** + * Deregister a postprocessor previously registered with {@link #registerPostprocessor}. + * @param pp postprocessor + * @see #registerPostprocessor + * @since xxx (moved from OsmReader) + */ + public static void deregisterPostprocessor(OsmServerReadPostprocessor pp) { + if (postprocessors != null) { + postprocessors.remove(pp); + } + } /** @@ -63,4 +113,16 @@ public DataSet getDataSet() { return ds; + } + + /** + * Iterate over registered postprocessors and give them each a chance to modify the dataset we have just loaded. + * @param progressMonitor Progress monitor + */ + protected void callPostProcessors(ProgressMonitor progressMonitor) { + if (postprocessors != null) { + for (OsmServerReadPostprocessor pp : postprocessors) { + pp.postprocessDataSet(getDataSet(), progressMonitor); + } + } } @@ -207,3 +269,423 @@ protected abstract DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException; + + @FunctionalInterface + protected interface ParserWorker { + void accept(InputStreamReader ir) throws IllegalDataException, IOException; + } + + protected final DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor, ParserWorker parserWorker) + throws IllegalDataException { + if (progressMonitor == null) { + progressMonitor = NullProgressMonitor.INSTANCE; + } + ProgressMonitor.CancelListener cancelListener = () -> cancel = true; + progressMonitor.addCancelListener(cancelListener); + CheckParameterUtil.ensureParameterNotNull(source, "source"); + try { + progressMonitor.beginTask(tr("Prepare OSM data...", 2)); + progressMonitor.indeterminateSubTask(tr("Parsing OSM data...")); + + try (InputStreamReader ir = UTFInputStreamReader.create(source)) { + parserWorker.accept(ir); + } + progressMonitor.worked(1); + + boolean readOnly = getDataSet().isLocked(); + + progressMonitor.indeterminateSubTask(tr("Preparing data set...")); + if (readOnly) { + getDataSet().unlock(); + } + prepareDataSet(); + if (readOnly) { + getDataSet().lock(); + } + progressMonitor.worked(1); + + // iterate over registered postprocessors and give them each a chance + // to modify the dataset we have just loaded. + callPostProcessors(progressMonitor); + // Make sure postprocessors did not change the read-only state + if (readOnly && !getDataSet().isLocked()) { + getDataSet().lock(); + } + return getDataSet(); + } catch (IllegalDataException e) { + throw e; + } catch (IOException e) { + throw new IllegalDataException(e); + } finally { + progressMonitor.finishTask(); + progressMonitor.removeCancelListener(cancelListener); + } + } + + protected final long getLong(String name, String value) throws IllegalDataException { + if (value == null) { + throw new IllegalDataException(tr("Missing required attribute ''{0}''.", name)); + } + try { + return Long.parseLong(value); + } catch (NumberFormatException e) { + throw new IllegalDataException(tr("Illegal long value for attribute ''{0}''. Got ''{1}''.", name, value), e); + } + } + + protected final void parseVersion(String version) throws IllegalDataException { + validateVersion(version); + ds.setVersion(version); + } + + private static void validateVersion(String version) throws IllegalDataException { + if (version == null) { + throw new IllegalDataException(tr("Missing mandatory attribute ''{0}''.", "version")); + } + if (!"0.6".equals(version)) { + throw new IllegalDataException(tr("Unsupported version: {0}", version)); + } + } + + protected final void parseDownloadPolicy(String key, String downloadPolicy) throws IllegalDataException { + parsePolicy(key, downloadPolicy, policy -> ds.setDownloadPolicy(DownloadPolicy.of(policy))); + } + + protected final void parseUploadPolicy(String key, String uploadPolicy) throws IllegalDataException { + parsePolicy(key, uploadPolicy, policy -> ds.setUploadPolicy(UploadPolicy.of(policy))); + } + + private static void parsePolicy(String key, String policy, Consumer consumer) throws IllegalDataException { + if (policy != null) { + try { + consumer.accept(policy); + } catch (IllegalArgumentException e) { + throw new IllegalDataException(MessageFormat.format( + "Illegal value for attribute ''{0}''. Got ''{1}''.", key, policy), e); + } + } + } + + protected final void parseLocked(String locked) { + if ("true".equalsIgnoreCase(locked)) { + ds.lock(); + } + } + + protected final void parseBounds(String generator, String minlon, String minlat, String maxlon, String maxlat, String origin) + throws IllegalDataException { + if (minlon != null && maxlon != null && minlat != null && maxlat != null) { + if (origin == null) { + origin = generator; + } + Bounds bounds = new Bounds( + Double.parseDouble(minlat), Double.parseDouble(minlon), + Double.parseDouble(maxlat), Double.parseDouble(maxlon)); + if (bounds.isOutOfTheWorld()) { + Bounds copy = new Bounds(bounds); + bounds.normalize(); + Logging.info("Bbox " + copy + " is out of the world, normalized to " + bounds); + } + ds.addDataSource(new DataSource(bounds, origin)); + } else { + throw new IllegalDataException(tr("Missing mandatory attributes on element ''bounds''. " + + "Got minlon=''{0}'',minlat=''{1}'',maxlon=''{2}'',maxlat=''{3}'', origin=''{4}''.", + minlon, minlat, maxlon, maxlat, origin + )); + } + } + + protected final void parseId(PrimitiveData current, long id) throws IllegalDataException { + current.setId(id); + if (current.getUniqueId() == 0) { + throw new IllegalDataException(tr("Illegal object with ID=0.")); + } + } + + protected final void parseTimestamp(PrimitiveData current, String time) { + if (time != null && !time.isEmpty()) { + current.setRawTimestamp((int) (DateUtils.tsFromString(time)/1000)); + } + } + + private static User createUser(String uid, String name) throws IllegalDataException { + if (uid == null) { + if (name == null) + return null; + return User.createLocalUser(name); + } + try { + return User.createOsmUser(Long.parseLong(uid), name); + } catch (NumberFormatException e) { + throw new IllegalDataException(MessageFormat.format("Illegal value for attribute ''uid''. Got ''{0}''.", uid), e); + } + } + + protected final void parseUser(PrimitiveData current, String user, long uid) { + current.setUser(User.createOsmUser(uid, user)); + } + + protected final void parseUser(PrimitiveData current, String user, String uid) throws IllegalDataException { + current.setUser(createUser(uid, user)); + } + + protected final void parseVisible(PrimitiveData current, String visible) { + if (visible != null) { + current.setVisible(Boolean.parseBoolean(visible)); + } + } + + protected final void parseVersion(PrimitiveData current, String versionString) throws IllegalDataException { + int version = 0; + if (versionString != null) { + try { + version = Integer.parseInt(versionString); + } catch (NumberFormatException e) { + throw new IllegalDataException( + tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.", + Long.toString(current.getUniqueId()), versionString), e); + } + parseVersion(current, version); + } else { + // version expected for OSM primitives with an id assigned by the server (id > 0), since API 0.6 + if (!current.isNew() && ds.getVersion() != null && "0.6".equals(ds.getVersion())) { + throw new IllegalDataException( + tr("Missing attribute ''version'' on OSM primitive with ID {0}.", Long.toString(current.getUniqueId()))); + } + } + } + + protected final void parseVersion(PrimitiveData current, int version) throws IllegalDataException { + switch (ds.getVersion()) { + case "0.6": + if (version <= 0 && !current.isNew()) { + throw new IllegalDataException( + tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.", + Long.toString(current.getUniqueId()), version)); + } else if (version < 0 && current.isNew()) { + Logging.warn(tr("Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", + current.getUniqueId(), version, 0, "0.6")); + version = 0; + } + break; + default: + // should not happen. API version has been checked before + throw new IllegalDataException(tr("Unknown or unsupported API version. Got {0}.", ds.getVersion())); + } + current.setVersion(version); + } + + protected final void parseAction(PrimitiveData current, String action) { + if (action == null) { + // do nothing + } else if ("delete".equals(action)) { + current.setDeleted(true); + current.setModified(current.isVisible()); + } else if ("modify".equals(action)) { + current.setModified(true); + } + } + + private static void handleIllegalChangeset(PrimitiveData current, IllegalArgumentException e, Object v) + throws IllegalDataException { + Logging.debug(e.getMessage()); + if (current.isNew()) { + // for a new primitive we just log a warning + Logging.info(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", + v, current.getUniqueId())); + current.setChangesetId(0); + } else { + // for an existing primitive this is a problem + throw new IllegalDataException(tr("Illegal value for attribute ''changeset''. Got {0}.", v), e); + } + } + + protected final void parseChangeset(PrimitiveData current, String v) throws IllegalDataException { + if (v == null) { + current.setChangesetId(0); + } else { + try { + parseChangeset(current, Integer.parseInt(v)); + } catch (NumberFormatException e) { + handleIllegalChangeset(current, e, v); + } + } + } + + protected final void parseChangeset(PrimitiveData current, int v) throws IllegalDataException { + try { + current.setChangesetId(v); + } catch (IllegalArgumentException e) { + handleIllegalChangeset(current, e, v); + } catch (IllegalStateException e) { + // thrown for positive changeset id on new primitives + Logging.debug(e); + Logging.info(e.getMessage()); + current.setChangesetId(0); + } + if (current.getChangesetId() <= 0) { + if (current.isNew()) { + // for a new primitive we just log a warning + Logging.info(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", + v, current.getUniqueId())); + current.setChangesetId(0); + } else if (current.getChangesetId() < 0) { + // for an existing primitive this is a problem only for negative ids (GDPR extracts are set to 0) + throw new IllegalDataException(tr("Illegal value for attribute ''changeset''. Got {0}.", v)); + } + } + } + + protected final void parseTag(Tagged t, String key, String value) throws IllegalDataException { + if (key == null || value == null) { + throw new IllegalDataException(tr("Missing key or value attribute in tag.")); + } else if (Utils.isStripEmpty(key) && t instanceof AbstractPrimitive) { + // #14199: Empty keys as ignored by AbstractPrimitive#put, but it causes problems to fix existing data + // Drop the tag on import, but flag the primitive as modified + ((AbstractPrimitive) t).setModified(true); + } else { + t.put(key.intern(), value.intern()); + } + } + + @FunctionalInterface + protected interface CommonReader { + void accept(PrimitiveData pd) throws IllegalDataException; + } + + @FunctionalInterface + protected interface NodeReader { + void accept(Node n) throws IllegalDataException; + } + + @FunctionalInterface + protected interface WayReader { + void accept(Way w, Collection nodeIds) throws IllegalDataException; + } + + @FunctionalInterface + protected interface RelationReader { + void accept(Relation r, Collection members) throws IllegalDataException; + } + + private static boolean areLatLonDefined(String lat, String lon) { + return lat != null && lon != null; + } + + private static boolean areLatLonDefined(double lat, double lon) { + return lat != Double.NaN && lon != Double.NaN; + } + + private Node addNode(NodeData nd, NodeReader nodeReader) throws IllegalDataException { + Node n = new Node(nd.getId(), nd.getVersion()); + n.setVisible(nd.isVisible()); + n.load(nd); + nodeReader.accept(n); + externalIdMap.put(nd.getPrimitiveId(), n); + return n; + } + + protected final Node parseNode(double lat, double lon, CommonReader commonReader, NodeReader nodeReader) + throws IllegalDataException { + NodeData nd = new NodeData(); + LatLon ll = null; + if (areLatLonDefined(lat, lon)) { + try { + ll = new LatLon(lat, lon); + nd.setCoor(ll); + } catch (NumberFormatException e) { + Logging.trace(e); + } + } + commonReader.accept(nd); + if (areLatLonDefined(lat, lon) && (ll == null || !ll.isValid())) { + throw new IllegalDataException(tr("Illegal value for attributes ''lat'', ''lon'' on node with ID {0}. Got ''{1}'', ''{2}''.", + Long.toString(nd.getId()), lat, lon)); + } + return addNode(nd, nodeReader); + } + + protected final Node parseNode(String lat, String lon, CommonReader commonReader, NodeReader nodeReader) + throws IllegalDataException { + NodeData nd = new NodeData(); + LatLon ll = null; + if (areLatLonDefined(lat, lon)) { + try { + ll = new LatLon(Double.parseDouble(lat), Double.parseDouble(lon)); + nd.setCoor(ll); + } catch (NumberFormatException e) { + Logging.trace(e); + } + } + commonReader.accept(nd); + if (areLatLonDefined(lat, lon) && (ll == null || !ll.isValid())) { + throw new IllegalDataException(tr("Illegal value for attributes ''lat'', ''lon'' on node with ID {0}. Got ''{1}'', ''{2}''.", + Long.toString(nd.getId()), lat, lon)); + } + return addNode(nd, nodeReader); + } + + protected final Way parseWay(CommonReader commonReader, WayReader wayReader) throws IllegalDataException { + WayData wd = new WayData(); + commonReader.accept(wd); + Way w = new Way(wd.getId(), wd.getVersion()); + w.setVisible(wd.isVisible()); + w.load(wd); + externalIdMap.put(wd.getPrimitiveId(), w); + + Collection nodeIds = new ArrayList<>(); + wayReader.accept(w, nodeIds); + if (w.isDeleted() && !nodeIds.isEmpty()) { + Logging.info(tr("Deleted way {0} contains nodes", Long.toString(w.getUniqueId()))); + nodeIds = new ArrayList<>(); + } + ways.put(wd.getUniqueId(), nodeIds); + return w; + } + + protected final Relation parseRelation(CommonReader commonReader, RelationReader relationReader) throws IllegalDataException { + RelationData rd = new RelationData(); + commonReader.accept(rd); + Relation r = new Relation(rd.getId(), rd.getVersion()); + r.setVisible(rd.isVisible()); + r.load(rd); + externalIdMap.put(rd.getPrimitiveId(), r); + + Collection members = new ArrayList<>(); + relationReader.accept(r, members); + if (r.isDeleted() && !members.isEmpty()) { + Logging.info(tr("Deleted relation {0} contains members", Long.toString(r.getUniqueId()))); + members = new ArrayList<>(); + } + relations.put(rd.getUniqueId(), members); + return r; + } + + protected final RelationMemberData parseRelationMember(Relation r, String ref, String type, String role) throws IllegalDataException { + if (ref == null) { + throw new IllegalDataException(tr("Missing attribute ''ref'' on member in relation {0}.", + Long.toString(r.getUniqueId()))); + } + try { + return parseRelationMember(r, Long.parseLong(ref), type, role); + } catch (NumberFormatException e) { + throw new IllegalDataException(tr("Illegal value for attribute ''ref'' on member in relation {0}. Got {1}", + Long.toString(r.getUniqueId()), ref), e); + } + } + + protected final RelationMemberData parseRelationMember(Relation r, long id, String type, String role) throws IllegalDataException { + if (id == 0) { + throw new IllegalDataException(tr("Incomplete specification with ref=0")); + } + if (type == null) { + throw new IllegalDataException(tr("Missing attribute ''type'' on member {0} in relation {1}.", + Long.toString(id), Long.toString(r.getUniqueId()))); + } + try { + return new RelationMemberData(role, OsmPrimitiveType.fromApiTypeName(type), id); + } catch (IllegalArgumentException e) { + throw new IllegalDataException(tr("Illegal value for attribute ''type'' on member {0} in relation {1}. Got {2}.", + Long.toString(id), Long.toString(r.getUniqueId()), type), e); + } + } } Index: trunk/src/org/openstreetmap/josm/io/OsmJsonReader.java =================================================================== --- trunk/src/org/openstreetmap/josm/io/OsmJsonReader.java (revision 14086) +++ trunk/src/org/openstreetmap/josm/io/OsmJsonReader.java (revision 14086) @@ -0,0 +1,192 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.io; + +import static org.openstreetmap.josm.tools.I18n.tr; + +import java.io.InputStream; +import java.util.Collection; +import java.util.Map.Entry; + +import javax.json.Json; +import javax.json.JsonArray; +import javax.json.JsonNumber; +import javax.json.JsonObject; +import javax.json.JsonString; +import javax.json.JsonValue; +import javax.json.stream.JsonParser; +import javax.json.stream.JsonParser.Event; + +import org.openstreetmap.josm.data.osm.DataSet; +import org.openstreetmap.josm.data.osm.PrimitiveData; +import org.openstreetmap.josm.data.osm.Relation; +import org.openstreetmap.josm.data.osm.RelationMemberData; +import org.openstreetmap.josm.data.osm.Tagged; +import org.openstreetmap.josm.data.osm.Way; +import org.openstreetmap.josm.gui.progress.NullProgressMonitor; +import org.openstreetmap.josm.gui.progress.ProgressMonitor; +import org.openstreetmap.josm.tools.Logging; +import org.openstreetmap.josm.tools.UncheckedParseException; + +/** + * Parser for the Osm API (JSON output). Read from an input stream and construct a dataset out of it. + * + * For each json element, there is a dedicated method. + * @since 14086 + */ +public class OsmJsonReader extends AbstractReader { + + protected JsonParser parser; + + /** + * constructor (for private and subclasses use only) + * + * @see #parseDataSet(InputStream, ProgressMonitor) + */ + protected OsmJsonReader() { + // Restricts visibility + } + + protected void setParser(JsonParser parser) { + this.parser = parser; + } + + protected void parse() throws IllegalDataException { + while (parser.hasNext()) { + Event event = parser.next(); + if (event == Event.START_OBJECT) { + parseRoot(parser.getObject()); + } + } + parser.close(); + } + + private void parseRoot(JsonObject object) throws IllegalDataException { + parseVersion(object.get("version").toString()); + parseDownloadPolicy("download", object.getString("download", null)); + parseUploadPolicy("upload", object.getString("upload", null)); + parseLocked(object.getString("locked", null)); + parseElements(object.getJsonArray("elements")); + } + + private void parseElements(JsonArray jsonArray) throws IllegalDataException { + for (JsonValue value : jsonArray) { + if (value instanceof JsonObject) { + JsonObject item = (JsonObject) value; + switch (item.getString("type")) { + case "node": + parseNode(item); + break; + case "way": + parseWay(item); + break; + case "relation": + parseRelation(item); + break; + default: + parseUnknown(item); + } + } else { + throw new IllegalDataException("Unexpected JSON item: " + value); + } + } + } + + /** + * Read out the common attributes and put them into current OsmPrimitive. + * @param item current JSON object + * @param current primitive to update + * @throws IllegalDataException if there is an error processing the underlying JSON source + */ + private void readCommon(JsonObject item, PrimitiveData current) throws IllegalDataException { + try { + parseId(current, item.getJsonNumber("id").longValue()); + parseTimestamp(current, item.getString("timestamp", null)); + JsonNumber uid = item.getJsonNumber("uid"); + if (uid != null) { + parseUser(current, item.getString("user", null), uid.longValue()); + } + parseVisible(current, item.getString("visible", null)); + JsonNumber version = item.getJsonNumber("version"); + if (version != null) { + parseVersion(current, version.intValue()); + } + parseAction(current, item.getString("action", null)); + JsonNumber changeset = item.getJsonNumber("changeset"); + if (changeset != null) { + parseChangeset(current, changeset.intValue()); + } + } catch (UncheckedParseException e) { + throw new IllegalDataException(e); + } + } + + private void readTags(JsonObject item, Tagged t) { + JsonObject tags = item.getJsonObject("tags"); + if (tags != null) { + for (Entry entry : tags.entrySet()) { + t.put(entry.getKey(), ((JsonString) entry.getValue()).getString()); + } + } + } + + private void parseNode(JsonObject item) throws IllegalDataException { + parseNode(item.getJsonNumber("lat").doubleValue(), + item.getJsonNumber("lon").doubleValue(), nd -> readCommon(item, nd), n -> readTags(item, n)); + } + + private void parseWay(JsonObject item) throws IllegalDataException { + parseWay(wd -> readCommon(item, wd), (w, nodeIds) -> readWayNodesAndTags(item, w, nodeIds)); + } + + private void readWayNodesAndTags(JsonObject item, Way w, Collection nodeIds) { + for (JsonValue v : item.getJsonArray("nodes")) { + nodeIds.add(((JsonNumber) v).longValue()); + } + readTags(item, w); + } + + private void parseRelation(JsonObject item) throws IllegalDataException { + parseRelation(rd -> readCommon(item, rd), (r, members) -> readRelationMembersAndTags(item, r, members)); + } + + private void readRelationMembersAndTags(JsonObject item, Relation r, Collection members) + throws IllegalDataException { + for (JsonValue v : item.getJsonArray("members")) { + JsonObject o = v.asJsonObject(); + members.add(parseRelationMember(r, ((JsonNumber) o.get("ref")).longValue(), o.getString("type"), o.getString("role"))); + } + readTags(item, r); + } + + protected void parseUnknown(JsonObject element, boolean printWarning) { + if (printWarning) { + Logging.info(tr("Undefined element ''{0}'' found in input stream. Skipping.", element)); + } + } + + private void parseUnknown(JsonObject element) { + parseUnknown(element, true); + } + + @Override + protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException { + return doParseDataSet(source, progressMonitor, ir -> { + setParser(Json.createParser(ir)); + parse(); + }); + } + + /** + * Parse the given input source and return the dataset. + * + * @param source the source input stream. Must not be null. + * @param progressMonitor the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed + * + * @return the dataset with the parsed data + * @throws IllegalDataException if an error was found while parsing the data from the source + * @throws IllegalArgumentException if source is null + */ + public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException { + return new OsmJsonReader().doParseDataSet(source, progressMonitor); + } +} Index: trunk/src/org/openstreetmap/josm/io/OsmReader.java =================================================================== --- trunk/src/org/openstreetmap/josm/io/OsmReader.java (revision 14085) +++ trunk/src/org/openstreetmap/josm/io/OsmReader.java (revision 14086) @@ -4,13 +4,7 @@ import static org.openstreetmap.josm.tools.I18n.tr; -import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; -import java.text.MessageFormat; -import java.util.ArrayList; import java.util.Collection; -import java.util.List; import java.util.Objects; -import java.util.function.Consumer; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -21,34 +15,20 @@ import javax.xml.stream.XMLStreamReader; -import org.openstreetmap.josm.data.Bounds; -import org.openstreetmap.josm.data.DataSource; -import org.openstreetmap.josm.data.coor.LatLon; -import org.openstreetmap.josm.data.osm.AbstractPrimitive; import org.openstreetmap.josm.data.osm.Changeset; import org.openstreetmap.josm.data.osm.DataSet; -import org.openstreetmap.josm.data.osm.DownloadPolicy; import org.openstreetmap.josm.data.osm.Node; -import org.openstreetmap.josm.data.osm.NodeData; -import org.openstreetmap.josm.data.osm.OsmPrimitiveType; import org.openstreetmap.josm.data.osm.PrimitiveData; import org.openstreetmap.josm.data.osm.Relation; -import org.openstreetmap.josm.data.osm.RelationData; import org.openstreetmap.josm.data.osm.RelationMemberData; import org.openstreetmap.josm.data.osm.Tagged; -import org.openstreetmap.josm.data.osm.UploadPolicy; -import org.openstreetmap.josm.data.osm.User; import org.openstreetmap.josm.data.osm.Way; -import org.openstreetmap.josm.data.osm.WayData; import org.openstreetmap.josm.gui.progress.NullProgressMonitor; import org.openstreetmap.josm.gui.progress.ProgressMonitor; -import org.openstreetmap.josm.tools.CheckParameterUtil; import org.openstreetmap.josm.tools.Logging; import org.openstreetmap.josm.tools.UncheckedParseException; -import org.openstreetmap.josm.tools.Utils; import org.openstreetmap.josm.tools.XmlUtils; -import org.openstreetmap.josm.tools.date.DateUtils; /** - * Parser for the Osm Api. Read from an input stream and construct a dataset out of it. + * Parser for the Osm API (XML output). Read from an input stream and construct a dataset out of it. * * For each xml element, there is a dedicated method. @@ -59,31 +39,4 @@ protected XMLStreamReader parser; - - protected boolean cancel; - - /** Used by plugins to register themselves as data postprocessors. */ - private static volatile List postprocessors; - - /** Register a new postprocessor. - * @param pp postprocessor - * @see #deregisterPostprocessor - */ - public static void registerPostprocessor(OsmServerReadPostprocessor pp) { - if (postprocessors == null) { - postprocessors = new ArrayList<>(); - } - postprocessors.add(pp); - } - - /** - * Deregister a postprocessor previously registered with {@link #registerPostprocessor}. - * @param pp postprocessor - * @see #registerPostprocessor - */ - public static void deregisterPostprocessor(OsmServerReadPostprocessor pp) { - if (postprocessors != null) { - postprocessors.remove(pp); - } - } /** @@ -98,4 +51,8 @@ protected void setParser(XMLStreamReader parser) { this.parser = parser; + } + + protected void throwException(Throwable th) throws XMLStreamException { + throw new XmlStreamParsingException(th.getMessage(), parser.getLocation(), th); } @@ -133,16 +90,11 @@ private void parseOsm() throws XMLStreamException { - String v = parser.getAttributeValue(null, "version"); - if (v == null) { - throwException(tr("Missing mandatory attribute ''{0}''.", "version")); - } - if (!"0.6".equals(v)) { - throwException(tr("Unsupported version: {0}", v)); - } - ds.setVersion(v); - parsePolicy("download", policy -> ds.setDownloadPolicy(DownloadPolicy.of(policy))); - parsePolicy("upload", policy -> ds.setUploadPolicy(UploadPolicy.of(policy))); - if ("true".equalsIgnoreCase(parser.getAttributeValue(null, "locked"))) { - ds.lock(); + try { + parseVersion(parser.getAttributeValue(null, "version")); + parseDownloadPolicy("download", parser.getAttributeValue(null, "download")); + parseUploadPolicy("upload", parser.getAttributeValue(null, "upload")); + parseLocked(parser.getAttributeValue(null, "locked")); + } catch (IllegalDataException e) { + throwException(e); } String generator = parser.getAttributeValue(null, "generator"); @@ -181,15 +133,4 @@ } else if (event == XMLStreamConstants.END_ELEMENT) { return; - } - } - } - - private void parsePolicy(String key, Consumer consumer) throws XMLStreamException { - String policy = parser.getAttributeValue(null, key); - if (policy != null) { - try { - consumer.accept(policy); - } catch (IllegalArgumentException e) { - throwException(MessageFormat.format("Illegal value for attribute ''{0}''. Got ''{1}''.", key, policy), e); } } @@ -202,23 +143,8 @@ String maxlat = parser.getAttributeValue(null, "maxlat"); String origin = parser.getAttributeValue(null, "origin"); - if (minlon != null && maxlon != null && minlat != null && maxlat != null) { - if (origin == null) { - origin = generator; - } - Bounds bounds = new Bounds( - Double.parseDouble(minlat), Double.parseDouble(minlon), - Double.parseDouble(maxlat), Double.parseDouble(maxlon)); - if (bounds.isOutOfTheWorld()) { - Bounds copy = new Bounds(bounds); - bounds.normalize(); - Logging.info("Bbox " + copy + " is out of the world, normalized to " + bounds); - } - DataSource src = new DataSource(bounds, origin); - ds.addDataSource(src); - } else { - throwException(tr("Missing mandatory attributes on element ''bounds''. " + - "Got minlon=''{0}'',minlat=''{1}'',maxlon=''{2}'',maxlat=''{3}'', origin=''{4}''.", - minlon, minlat, maxlon, maxlat, origin - )); + try { + parseBounds(generator, minlon, minlat, maxlon, maxlat, origin); + } catch (IllegalDataException e) { + throwException(e); } jumpToEnd(); @@ -226,70 +152,64 @@ protected Node parseNode() throws XMLStreamException { - NodeData nd = new NodeData(); String lat = parser.getAttributeValue(null, "lat"); String lon = parser.getAttributeValue(null, "lon"); - LatLon ll = null; - if (lat != null && lon != null) { - try { - ll = new LatLon(Double.parseDouble(lat), Double.parseDouble(lon)); - nd.setCoor(ll); - } catch (NumberFormatException e) { - Logging.trace(e); - } - } - readCommon(nd); - if (lat != null && lon != null && (ll == null || !ll.isValid())) { - throwException(tr("Illegal value for attributes ''lat'', ''lon'' on node with ID {0}. Got ''{1}'', ''{2}''.", - Long.toString(nd.getId()), lat, lon)); - } - Node n = new Node(nd.getId(), nd.getVersion()); - n.setVisible(nd.isVisible()); - n.load(nd); - externalIdMap.put(nd.getPrimitiveId(), n); - while (true) { - int event = parser.next(); - if (event == XMLStreamConstants.START_ELEMENT) { - if ("tag".equals(parser.getLocalName())) { - parseTag(n); - } else { - parseUnknown(); + try { + return parseNode(lat, lon, this::readCommon, this::parseNodeTags); + } catch (IllegalDataException e) { + throwException(e); + } + return null; + } + + private void parseNodeTags(Node n) throws IllegalDataException { + try { + while (parser.hasNext()) { + int event = parser.next(); + if (event == XMLStreamConstants.START_ELEMENT) { + if ("tag".equals(parser.getLocalName())) { + parseTag(n); + } else { + parseUnknown(); + } + } else if (event == XMLStreamConstants.END_ELEMENT) { + return; } - } else if (event == XMLStreamConstants.END_ELEMENT) - return n; + } + } catch (XMLStreamException e) { + throw new IllegalDataException(e); } } protected Way parseWay() throws XMLStreamException { - WayData wd = new WayData(); - readCommon(wd); - Way w = new Way(wd.getId(), wd.getVersion()); - w.setVisible(wd.isVisible()); - w.load(wd); - externalIdMap.put(wd.getPrimitiveId(), w); - - Collection nodeIds = new ArrayList<>(); - while (true) { - int event = parser.next(); - if (event == XMLStreamConstants.START_ELEMENT) { - switch (parser.getLocalName()) { - case "nd": - nodeIds.add(parseWayNode(w)); - break; - case "tag": - parseTag(w); - break; - default: - parseUnknown(); + try { + return parseWay(this::readCommon, this::parseWayNodesAndTags); + } catch (IllegalDataException e) { + throwException(e); + } + return null; + } + + private void parseWayNodesAndTags(Way w, Collection nodeIds) throws IllegalDataException { + try { + while (parser.hasNext()) { + int event = parser.next(); + if (event == XMLStreamConstants.START_ELEMENT) { + switch (parser.getLocalName()) { + case "nd": + nodeIds.add(parseWayNode(w)); + break; + case "tag": + parseTag(w); + break; + default: + parseUnknown(); + } + } else if (event == XMLStreamConstants.END_ELEMENT) { + break; } - } else if (event == XMLStreamConstants.END_ELEMENT) { - break; - } - } - if (w.isDeleted() && !nodeIds.isEmpty()) { - Logging.info(tr("Deleted way {0} contains nodes", Long.toString(w.getUniqueId()))); - nodeIds = new ArrayList<>(); - } - ways.put(wd.getUniqueId(), nodeIds); - return w; + } + } catch (XMLStreamException e) { + throw new IllegalDataException(e); + } } @@ -311,67 +231,47 @@ protected Relation parseRelation() throws XMLStreamException { - RelationData rd = new RelationData(); - readCommon(rd); - Relation r = new Relation(rd.getId(), rd.getVersion()); - r.setVisible(rd.isVisible()); - r.load(rd); - externalIdMap.put(rd.getPrimitiveId(), r); - - Collection members = new ArrayList<>(); - while (true) { - int event = parser.next(); - if (event == XMLStreamConstants.START_ELEMENT) { - switch (parser.getLocalName()) { - case "member": - members.add(parseRelationMember(r)); - break; - case "tag": - parseTag(r); - break; - default: - parseUnknown(); + try { + return parseRelation(this::readCommon, this::parseRelationMembersAndTags); + } catch (IllegalDataException e) { + throw new XMLStreamException(e); + } + } + + private void parseRelationMembersAndTags(Relation r, Collection members) throws IllegalDataException { + try { + while (parser.hasNext()) { + int event = parser.next(); + if (event == XMLStreamConstants.START_ELEMENT) { + switch (parser.getLocalName()) { + case "member": + members.add(parseRelationMember(r)); + break; + case "tag": + parseTag(r); + break; + default: + parseUnknown(); + } + } else if (event == XMLStreamConstants.END_ELEMENT) { + break; } - } else if (event == XMLStreamConstants.END_ELEMENT) { - break; - } - } - if (r.isDeleted() && !members.isEmpty()) { - Logging.info(tr("Deleted relation {0} contains members", Long.toString(r.getUniqueId()))); - members = new ArrayList<>(); - } - relations.put(rd.getUniqueId(), members); - return r; + } + } catch (XMLStreamException e) { + throw new IllegalDataException(e); + } } private RelationMemberData parseRelationMember(Relation r) throws XMLStreamException { - OsmPrimitiveType type = null; - long id = 0; - String value = parser.getAttributeValue(null, "ref"); - if (value == null) { - throwException(tr("Missing attribute ''ref'' on member in relation {0}.", Long.toString(r.getUniqueId()))); - } - try { - id = Long.parseLong(value); - } catch (NumberFormatException e) { - throwException(tr("Illegal value for attribute ''ref'' on member in relation {0}. Got {1}", Long.toString(r.getUniqueId()), - value), e); - } - value = parser.getAttributeValue(null, "type"); - if (value == null) { - throwException(tr("Missing attribute ''type'' on member {0} in relation {1}.", Long.toString(id), Long.toString(r.getUniqueId()))); - } - try { - type = OsmPrimitiveType.fromApiTypeName(value); - } catch (IllegalArgumentException e) { - throwException(tr("Illegal value for attribute ''type'' on member {0} in relation {1}. Got {2}.", - Long.toString(id), Long.toString(r.getUniqueId()), value), e); - } - String role = parser.getAttributeValue(null, "role"); - - if (id == 0) { - throwException(tr("Incomplete specification with ref=0")); - } - jumpToEnd(); - return new RelationMemberData(role, type, id); + RelationMemberData result = null; + try { + String ref = parser.getAttributeValue(null, "ref"); + String type = parser.getAttributeValue(null, "type"); + String role = parser.getAttributeValue(null, "role"); + result = parseRelationMember(r, ref, type, role); + jumpToEnd(); + } catch (IllegalDataException e) { + throwException(e); + } + return result; } @@ -404,12 +304,8 @@ String key = parser.getAttributeValue(null, "k"); String value = parser.getAttributeValue(null, "v"); - if (key == null || value == null) { - throwException(tr("Missing key or value attribute in tag.")); - } else if (Utils.isStripEmpty(key) && t instanceof AbstractPrimitive) { - // #14199: Empty keys as ignored by AbstractPrimitive#put, but it causes problems to fix existing data - // Drop the tag on import, but flag the primitive as modified - ((AbstractPrimitive) t).setModified(true); - } else { - t.put(key.intern(), value.intern()); + try { + parseTag(t, key, value); + } catch (IllegalDataException e) { + throwException(e); } jumpToEnd(); @@ -460,120 +356,20 @@ } - private User createUser(String uid, String name) throws XMLStreamException { - if (uid == null) { - if (name == null) - return null; - return User.createLocalUser(name); - } - try { - long id = Long.parseLong(uid); - return User.createOsmUser(id, name); - } catch (NumberFormatException e) { - throwException(MessageFormat.format("Illegal value for attribute ''uid''. Got ''{0}''.", uid), e); - } - return null; - } - /** * Read out the common attributes and put them into current OsmPrimitive. * @param current primitive to update - * @throws XMLStreamException if there is an error processing the underlying XML source + * @throws IllegalDataException if there is an error processing the underlying XML source */ - private void readCommon(PrimitiveData current) throws XMLStreamException { - current.setId(getLong("id")); - if (current.getUniqueId() == 0) { - throwException(tr("Illegal object with ID=0.")); - } - - String time = parser.getAttributeValue(null, "timestamp"); - if (time != null && !time.isEmpty()) { - current.setRawTimestamp((int) (DateUtils.tsFromString(time)/1000)); - } - - String user = parser.getAttributeValue(null, "user"); - String uid = parser.getAttributeValue(null, "uid"); - current.setUser(createUser(uid, user)); - - String visible = parser.getAttributeValue(null, "visible"); - if (visible != null) { - current.setVisible(Boolean.parseBoolean(visible)); - } - - String versionString = parser.getAttributeValue(null, "version"); - int version = 0; - if (versionString != null) { - try { - version = Integer.parseInt(versionString); - } catch (NumberFormatException e) { - throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.", - Long.toString(current.getUniqueId()), versionString), e); - } - switch (ds.getVersion()) { - case "0.6": - if (version <= 0 && !current.isNew()) { - throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.", - Long.toString(current.getUniqueId()), versionString)); - } else if (version < 0 && current.isNew()) { - Logging.warn(tr("Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", - current.getUniqueId(), version, 0, "0.6")); - version = 0; - } - break; - default: - // should not happen. API version has been checked before - throwException(tr("Unknown or unsupported API version. Got {0}.", ds.getVersion())); - } - } else { - // version expected for OSM primitives with an id assigned by the server (id > 0), since API 0.6 - if (!current.isNew() && ds.getVersion() != null && "0.6".equals(ds.getVersion())) { - throwException(tr("Missing attribute ''version'' on OSM primitive with ID {0}.", Long.toString(current.getUniqueId()))); - } - } - current.setVersion(version); - - String action = parser.getAttributeValue(null, "action"); - if (action == null) { - // do nothing - } else if ("delete".equals(action)) { - current.setDeleted(true); - current.setModified(current.isVisible()); - } else if ("modify".equals(action)) { - current.setModified(true); - } - - String v = parser.getAttributeValue(null, "changeset"); - if (v == null) { - current.setChangesetId(0); - } else { - try { - current.setChangesetId(Integer.parseInt(v)); - } catch (IllegalArgumentException e) { - Logging.debug(e.getMessage()); - if (current.isNew()) { - // for a new primitive we just log a warning - Logging.info(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", - v, current.getUniqueId())); - current.setChangesetId(0); - } else { - // for an existing primitive this is a problem - throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v), e); - } - } catch (IllegalStateException e) { - // thrown for positive changeset id on new primitives - Logging.debug(e); - Logging.info(e.getMessage()); - current.setChangesetId(0); - } - if (current.getChangesetId() <= 0) { - if (current.isNew()) { - // for a new primitive we just log a warning - Logging.info(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", - v, current.getUniqueId())); - current.setChangesetId(0); - } else if (current.getChangesetId() < 0) { - // for an existing primitive this is a problem only for negative ids (GDPR extracts are set to 0) - throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v)); - } - } + private void readCommon(PrimitiveData current) throws IllegalDataException { + try { + parseId(current, getLong("id")); + parseTimestamp(current, parser.getAttributeValue(null, "timestamp")); + parseUser(current, parser.getAttributeValue(null, "user"), parser.getAttributeValue(null, "uid")); + parseVisible(current, parser.getAttributeValue(null, "visible")); + parseVersion(current, parser.getAttributeValue(null, "version")); + parseAction(current, parser.getAttributeValue(null, "action")); + parseChangeset(current, parser.getAttributeValue(null, "changeset")); + } catch (UncheckedParseException | XMLStreamException e) { + throw new IllegalDataException(e); } } @@ -581,11 +377,8 @@ private long getLong(String name) throws XMLStreamException { String value = parser.getAttributeValue(null, name); - if (value == null) { - throwException(tr("Missing required attribute ''{0}''.", name)); - } - try { - return Long.parseLong(value); - } catch (NumberFormatException e) { - throwException(tr("Illegal long value for attribute ''{0}''. Got ''{1}''.", name, value), e); + try { + return getLong(name, value); + } catch (IllegalDataException e) { + throwException(e); } return 0; // should not happen @@ -608,66 +401,24 @@ @Override protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException { - if (progressMonitor == null) { - progressMonitor = NullProgressMonitor.INSTANCE; - } - ProgressMonitor.CancelListener cancelListener = () -> cancel = true; - progressMonitor.addCancelListener(cancelListener); - CheckParameterUtil.ensureParameterNotNull(source, "source"); - try { - progressMonitor.beginTask(tr("Prepare OSM data...", 2)); - progressMonitor.indeterminateSubTask(tr("Parsing OSM data...")); - - try (InputStreamReader ir = UTFInputStreamReader.create(source)) { + return doParseDataSet(source, progressMonitor, ir -> { + try { setParser(XmlUtils.newSafeXMLInputFactory().createXMLStreamReader(ir)); parse(); - } - progressMonitor.worked(1); - - boolean readOnly = getDataSet().isLocked(); - - progressMonitor.indeterminateSubTask(tr("Preparing data set...")); - if (readOnly) { - getDataSet().unlock(); - } - prepareDataSet(); - if (readOnly) { - getDataSet().lock(); - } - progressMonitor.worked(1); - - // iterate over registered postprocessors and give them each a chance - // to modify the dataset we have just loaded. - if (postprocessors != null) { - for (OsmServerReadPostprocessor pp : postprocessors) { - pp.postprocessDataSet(getDataSet(), progressMonitor); + } catch (XmlStreamParsingException | UncheckedParseException e) { + throw new IllegalDataException(e.getMessage(), e); + } catch (XMLStreamException e) { + String msg = e.getMessage(); + Pattern p = Pattern.compile("Message: (.+)"); + Matcher m = p.matcher(msg); + if (m.find()) { + msg = m.group(1); } - } - // Make sure postprocessors did not change the read-only state - if (readOnly && !getDataSet().isLocked()) { - getDataSet().lock(); - } - return getDataSet(); - } catch (IllegalDataException e) { - throw e; - } catch (XmlStreamParsingException | UncheckedParseException e) { - throw new IllegalDataException(e.getMessage(), e); - } catch (XMLStreamException e) { - String msg = e.getMessage(); - Pattern p = Pattern.compile("Message: (.+)"); - Matcher m = p.matcher(msg); - if (m.find()) { - msg = m.group(1); - } - if (e.getLocation() != null) - throw new IllegalDataException(tr("Line {0} column {1}: ", - e.getLocation().getLineNumber(), e.getLocation().getColumnNumber()) + msg, e); - else - throw new IllegalDataException(msg, e); - } catch (IOException e) { - throw new IllegalDataException(e); - } finally { - progressMonitor.finishTask(); - progressMonitor.removeCancelListener(cancelListener); - } + if (e.getLocation() != null) + throw new IllegalDataException(tr("Line {0} column {1}: ", + e.getLocation().getLineNumber(), e.getLocation().getColumnNumber()) + msg, e); + else + throw new IllegalDataException(msg, e); + } + }); } @@ -676,5 +427,5 @@ * * @param source the source input stream. Must not be null. - * @param progressMonitor the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed + * @param progressMonitor the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed * * @return the dataset with the parsed data Index: trunk/src/org/openstreetmap/josm/io/OverpassDownloadReader.java =================================================================== --- trunk/src/org/openstreetmap/josm/io/OverpassDownloadReader.java (revision 14085) +++ trunk/src/org/openstreetmap/josm/io/OverpassDownloadReader.java (revision 14086) @@ -83,4 +83,8 @@ } + static final class OverpassOsmJsonReader extends OsmJsonReader { + + } + /** * Possible Overpass API output format, with the {@code [out:]} statement. @@ -165,4 +169,5 @@ static { registerOverpassOutpoutFormatReader(OverpassOutpoutFormat.OSM_XML, OverpassOsmReader.class); + registerOverpassOutpoutFormatReader(OverpassOutpoutFormat.OSM_JSON, OverpassOsmJsonReader.class); } @@ -410,5 +415,5 @@ return query == null ? query : query .replaceAll("out( body| skel| ids)?( id| qt)?;", "out meta$2;") - .replaceAll("(?s)\\[out:(json|csv)[^\\]]*\\]", "[out:xml]"); + .replaceAll("(?s)\\[out:(csv)[^\\]]*\\]", "[out:xml]"); } } Index: trunk/test/unit/org/openstreetmap/josm/io/OsmJsonReaderTest.java =================================================================== --- trunk/test/unit/org/openstreetmap/josm/io/OsmJsonReaderTest.java (revision 14086) +++ trunk/test/unit/org/openstreetmap/josm/io/OsmJsonReaderTest.java (revision 14086) @@ -0,0 +1,230 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.io; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.text.SimpleDateFormat; +import java.util.Iterator; + +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.openstreetmap.josm.data.coor.LatLon; +import org.openstreetmap.josm.data.osm.DataSet; +import org.openstreetmap.josm.data.osm.Node; +import org.openstreetmap.josm.data.osm.Relation; +import org.openstreetmap.josm.data.osm.RelationMember; +import org.openstreetmap.josm.data.osm.Way; +import org.openstreetmap.josm.gui.progress.NullProgressMonitor; +import org.openstreetmap.josm.testutils.JOSMTestRules; +import org.openstreetmap.josm.tools.date.DateUtils; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + +/** + * Unit tests of {@link OsmReader} class. + */ +public class OsmJsonReaderTest { + + /** + * Setup rule + */ + @Rule + @SuppressFBWarnings(value = "URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD") + public JOSMTestRules test = new JOSMTestRules(); + + private static final SimpleDateFormat iso8601 = DateUtils.newIsoDateTimeFormat(); + + /** + * Setup test + */ + @BeforeClass + public static void setUp() { + iso8601.setTimeZone(DateUtils.UTC); + } + + /** + * Parse JSON. + * @param osm OSM data in JSON format, without header/footer + * @return data set + * @throws Exception if any error occurs + */ + private static DataSet parse(String osm) throws Exception { + try (InputStream in = new ByteArrayInputStream(( + "{\n" + + " \"version\": 0.6,\n" + + " \"generator\": \"Overpass API\",\n" + + " \"osm3s\": {\n" + + " \"timestamp_osm_base\": \"date\",\n" + + " \"copyright\": \"The data included in this document is from www.openstreetmap.org. " + + "It has there been collected by a large group of contributors. " + + "For individual attribution of each item please refer to " + + "http://www.openstreetmap.org/api/0.6/[node|way|relation]/#id/history\"\n" + + " },\n" + + " \"elements\": [" + osm + "]\n" + + "}") + .getBytes(StandardCharsets.UTF_8))) { + return OsmJsonReader.parseDataSet(in, NullProgressMonitor.INSTANCE); + } + } + + + /** + * Test an example without data. + * @throws Exception never + */ + @Test + public void testHeader() throws Exception { + DataSet ds = parse(""); + assertEquals("0.6", ds.getVersion()); + } + + /** + * Test an example with the spatial data only. + * @throws Exception never + */ + @Test + public void testNodeSpatialData() throws Exception { + DataSet ds = parse("{\n" + + " \"type\": \"node\",\n" + + " \"id\": 1,\n" + + " \"lat\": 2.0,\n" + + " \"lon\": -3.0\n" + + "}"); + Node n = ds.getNodes().iterator().next(); + assertEquals(1, n.getUniqueId()); + assertEquals(new LatLon(2.0, -3.0), n.getCoor()); + } + + /** + * Test an example with the meta data. + * @throws Exception never + */ + @Test + public void testNodeMetaData() throws Exception { + DataSet ds = parse("{\n" + + " \"type\": \"node\",\n" + + " \"id\": 1,\n" + + " \"lat\": 2.0,\n" + + " \"lon\": -3.0,\n" + + " \"timestamp\": \"2018-01-01T00:00:00Z\",\n" + + " \"version\": 4,\n" + + " \"changeset\": 5,\n" + + " \"user\": \"somebody\",\n" + + " \"uid\": 6\n" + + "}"); + Node n = ds.getNodes().iterator().next(); + assertEquals(1, n.getUniqueId()); + assertEquals(new LatLon(2.0, -3.0), n.getCoor()); + assertEquals("2018-01-01T00:00:00Z", iso8601.format(n.getTimestamp())); + assertEquals(4, n.getVersion()); + assertEquals(5, n.getChangesetId()); + assertEquals(6, n.getUser().getId()); + assertEquals("somebody", n.getUser().getName()); + } + + /** + * Test an example with tags. + * @throws Exception never + */ + @Test + public void testNodeTags() throws Exception { + DataSet ds = parse("{\n" + + " \"type\": \"node\",\n" + + " \"id\": 1,\n" + + " \"lat\": 2.0,\n" + + " \"lon\": -3.0,\n" + + " \"tags\": {\n" + + " \"highway\": \"bus_stop\",\n" + + " \"name\": \"Main Street\"\n" + + " }" + + "}"); + Node n = ds.getNodes().iterator().next(); + assertEquals(1, n.getUniqueId()); + assertEquals(new LatLon(2.0, -3.0), n.getCoor()); + assertTrue(n.isTagged()); + assertEquals("bus_stop", n.get("highway")); + assertEquals("Main Street", n.get("name")); + } + + /** + * Test a way example. + * @throws Exception never + */ + @Test + public void testWay() throws Exception { + DataSet ds = parse("{\n" + + " \"type\": \"way\",\n" + + " \"id\": 1,\n" + + " \"nodes\": [\n" + + " 10,\n" + + " 11,\n" + + " 12\n" + + " ],\n" + + " \"tags\": {\n" + + " \"highway\": \"tertiary\",\n" + + " \"name\": \"Main Street\"\n" + + " }\n" + + "}"); + Way w = ds.getWays().iterator().next(); + assertEquals(1, w.getUniqueId()); + assertEquals(3, w.getNodesCount()); + Iterator it = w.getNodes().iterator(); + assertEquals(10, it.next().getUniqueId()); + assertEquals(11, it.next().getUniqueId()); + assertEquals(12, it.next().getUniqueId()); + assertFalse(it.hasNext()); + assertTrue(w.isTagged()); + assertEquals("tertiary", w.get("highway")); + assertEquals("Main Street", w.get("name")); + } + + /** + * Test a relation example. + * @throws Exception never + */ + @Test + public void testRelation() throws Exception { + DataSet ds = parse("{\n" + + " \"type\": \"relation\",\n" + + " \"id\": 1,\n" + + " \"members\": [\n" + + " {\n" + + " \"type\": \"way\",\n" + + " \"ref\": 1745069,\n" + + " \"role\": \"\"\n" + + " },\n" + + " {\n" + + " \"type\": \"way\",\n" + + " \"ref\": 172789,\n" + + " \"role\": \"\"\n" + + " }\n" + + " ],\n" + + " \"tags\": {\n" + + " \"from\": \"Konrad-Adenauer-Platz\",\n" + + " \"name\": \"VRS 636\",\n" + + " \"network\": \"VRS\",\n" + + " \"operator\": \"SWB\",\n" + + " \"ref\": \"636\",\n" + + " \"route\": \"bus\",\n" + + " \"to\": \"Gielgen\",\n" + + " \"type\": \"route\",\n" + + " \"via\": \"Ramersdorf\"\n" + + " }\n" + + "}"); + Relation r = ds.getRelations().iterator().next(); + assertEquals(1, r.getUniqueId()); + assertEquals(2, r.getMembersCount()); + Iterator it = r.getMembers().iterator(); + assertEquals(1745069, it.next().getUniqueId()); + assertEquals(172789, it.next().getUniqueId()); + assertFalse(it.hasNext()); + assertTrue(r.isTagged()); + assertEquals("route", r.get("type")); + } +}