Changeset 16933 in josm


Ignore:
Timestamp:
2020-08-26T20:24:32+02:00 (2 years ago)
Author:
simon04
Message:

fix #19624 - Support reading line-delimited GeoJSON (RFC 7464; patch by taylor.smock)

Location:
trunk
Files:
1 added
2 edited

Legend:

Unmodified
Added
Removed
  • trunk/src/org/openstreetmap/josm/io/GeoJSONReader.java

    r16311 r16933  
    44import static org.openstreetmap.josm.tools.I18n.tr;
    55
     6import java.io.BufferedInputStream;
     7import java.io.BufferedReader;
     8import java.io.ByteArrayInputStream;
     9import java.io.IOException;
    610import java.io.InputStream;
     11import java.io.InputStreamReader;
     12import java.nio.charset.StandardCharsets;
    713import java.util.List;
    814import java.util.Map;
     
    5258    private static final String GEOMETRY = "geometry";
    5359    private static final String TYPE = "type";
     60    /** The record separator is 0x1E per RFC 7464 */
     61    private static final byte RECORD_SEPARATOR_BYTE = 0x1E;
    5462    private JsonParser parser;
    5563    private Projection projection = Projections.getProjectionByCode("EPSG:4326"); // WGS 84
     
    351359    }
    352360
     361    /**
     362     * Check if the inputstream follows RFC 7464
     363     * @param source The source to check (should be at the beginning)
     364     * @return {@code true} if the initial character is {@link GeoJSONReader#RECORD_SEPARATOR_BYTE}.
     365     */
     366    private static boolean isLineDelimited(InputStream source) {
     367        source.mark(2);
     368        try {
     369            int start = source.read();
     370            if (RECORD_SEPARATOR_BYTE == start) {
     371                return true;
     372            }
     373            source.reset();
     374        } catch (IOException e) {
     375            Logging.error(e);
     376        }
     377        return false;
     378    }
     379
    353380    @Override
    354381    protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
    355         setParser(Json.createParser(source));
     382        InputStream markSupported = source.markSupported() ? source : new BufferedInputStream(source);
    356383        ds.setUploadPolicy(UploadPolicy.DISCOURAGED);
    357         try {
    358             parse();
    359         } catch (JsonParsingException e) {
    360             throw new IllegalDataException(e);
     384        if (isLineDelimited(markSupported)) {
     385            BufferedReader reader = new BufferedReader(new InputStreamReader(markSupported));
     386            String line = null;
     387            String rs = new String(new byte[]{RECORD_SEPARATOR_BYTE}, StandardCharsets.US_ASCII);
     388            try {
     389                while ((line = reader.readLine()) != null) {
     390                    line = line.replaceFirst(rs, "");
     391                    try (InputStream is = new ByteArrayInputStream(line.getBytes())) {
     392                        setParser(Json.createParser(is));
     393                        parse();
     394                    } catch (JsonParsingException e) {
     395                        throw new IllegalDataException(e);
     396                    } finally {
     397                        parser.close();
     398                    }
     399                }
     400            } catch (IOException e) {
     401                throw new IllegalDataException(e);
     402            }
     403        } else {
     404            setParser(Json.createParser(markSupported));
     405            try {
     406                parse();
     407            } catch (JsonParsingException e) {
     408                throw new IllegalDataException(e);
     409            }
    361410        }
    362411        return getDataSet();
  • trunk/test/unit/org/openstreetmap/josm/io/GeoJSONReaderTest.java

    r16436 r16933  
    66import static org.junit.Assert.assertNull;
    77import static org.junit.Assert.assertTrue;
     8import static org.junit.jupiter.api.Assertions.assertThrows;
    89
    910import java.io.ByteArrayInputStream;
     
    1314import java.nio.file.Paths;
    1415import java.util.ArrayList;
     16import java.util.Collection;
    1517import java.util.List;
    1618import java.util.Optional;
     
    4749                .doParseDataSet(in, null)
    4850                .getPrimitives(it -> true));
    49             assertEquals(20, primitives.size());
    5051
    51             final Node node1 = new Node(new LatLon(0.5, 102.0));
    52             final Optional<OsmPrimitive> foundNode1 = primitives.stream()
    53                 .filter(it -> areEqualNodes(it, node1))
    54                 .findFirst();
    55             assertTrue(foundNode1.isPresent());
    56             assertEquals("valueA", foundNode1.get().get("propA"));
     52            assertExpectedGeoPrimitives(primitives);
     53        }
     54    }
    5755
    58             final Way way1 = new Way();
    59             way1.addNode(new Node(new LatLon(0.5, 102.0)));
    60             way1.addNode(new Node(new LatLon(1, 103)));
    61             way1.addNode(new Node(new LatLon(0, 104)));
    62             way1.addNode(new Node(new LatLon(1, 105)));
    63             final Optional<OsmPrimitive> foundWay1 = primitives.stream()
    64                 .filter(it -> areEqualWays(it, way1))
    65                 .findFirst();
    66             assertTrue(foundWay1.isPresent());
    67             assertEquals("valueB", foundWay1.get().get("propB"));
    68             assertEquals("0.0", foundWay1.get().get("propB2"));
    69             assertEquals(foundNode1.get(), ((Way) foundWay1.get()).firstNode());
    70             assertEquals("valueA", ((Way) foundWay1.get()).firstNode().get("propA"));
     56    /**
     57     * Tests reading a GeoJSON file that is line by line separated, per RFC 7464
     58     * @throws Exception in case of an error
     59     */
     60    @Test
     61    public void testReadLineByLineGeoJSON() throws Exception {
     62        try (InputStream in = Files.newInputStream(Paths.get(TestUtils.getTestDataRoot(), "geoLineByLine.json"))) {
     63            final List<OsmPrimitive> primitives = new ArrayList<>(new GeoJSONReader()
     64                .doParseDataSet(in, null)
     65                .getPrimitives(it -> true));
    7166
    72             final Way way2 = new Way();
    73             way2.addNode(new Node(new LatLon(40, 180)));
    74             way2.addNode(new Node(new LatLon(50, 180)));
    75             way2.addNode(new Node(new LatLon(50, 170)));
    76             way2.addNode(new Node(new LatLon(40, 170)));
    77             way2.addNode(new Node(new LatLon(40, 180)));
    78             final Optional<OsmPrimitive> foundWay2 = primitives.stream()
    79                 .filter(it -> areEqualWays(it, way2))
    80                 .findFirst();
    81             assertTrue(foundWay2.isPresent());
    82             assertEquals(
    83                 ((Way) foundWay2.get()).getNode(0),
    84                 ((Way) foundWay2.get()).getNode(((Way) foundWay2.get()).getNodesCount() - 1)
    85             );
     67            assertExpectedGeoPrimitives(primitives);
     68        }
     69    }
    8670
    87             final Way way3 = new Way();
    88             way3.addNode(new Node(new LatLon(40, -170)));
    89             way3.addNode(new Node(new LatLon(50, -170)));
    90             way3.addNode(new Node(new LatLon(50, -180)));
    91             way3.addNode(new Node(new LatLon(40, -180)));
    92             way3.addNode(new Node(new LatLon(40, -170)));
    93             final Optional<OsmPrimitive> foundWay3 = primitives.stream()
    94                 .filter(it -> areEqualWays(it, way3))
    95                 .findFirst();
    96             assertTrue(foundWay3.isPresent());
    97             assertEquals(
    98                 ((Way) foundWay3.get()).getNode(0),
    99                 ((Way) foundWay3.get()).getNode(((Way) foundWay3.get()).getNodesCount() - 1)
    100             );
     71    private void assertExpectedGeoPrimitives(Collection<OsmPrimitive> primitives) {
     72        assertEquals(20, primitives.size());
    10173
    102             final Way way4 = new Way();
    103             way4.addNode(new Node(new LatLon(0, 100)));
    104             way4.addNode(new Node(new LatLon(0, 101)));
    105             way4.addNode(new Node(new LatLon(1, 101)));
    106             way4.addNode(new Node(new LatLon(1, 100)));
    107             way4.addNode(new Node(new LatLon(0, 100)));
    108             final Optional<OsmPrimitive> foundWay4 = primitives.stream()
    109                 .filter(it -> areEqualWays(it, way4))
    110                 .findFirst();
    111             assertTrue(foundWay4.isPresent());
    112             assertEquals(
    113                 ((Way) foundWay4.get()).getNode(0),
    114                 ((Way) foundWay4.get()).getNode(((Way) foundWay4.get()).getNodesCount() - 1)
    115             );
    116             assertEquals("valueD", foundWay4.get().get("propD"));
    117             assertFalse(foundWay4.get().hasTag("propD2"));
    118             assertEquals("true", foundWay4.get().get("propD3"));
    119             assertFalse(foundWay4.get().hasKey("propD4"));
    120             assertNull(foundWay4.get().get("propD4"));
    121         }
     74        final Node node1 = new Node(new LatLon(0.5, 102.0));
     75        final Optional<OsmPrimitive> foundNode1 = primitives.stream()
     76            .filter(it -> areEqualNodes(it, node1))
     77            .findFirst();
     78        assertTrue(foundNode1.isPresent());
     79        assertEquals("valueA", foundNode1.get().get("propA"));
     80
     81        final Way way1 = new Way();
     82        way1.addNode(new Node(new LatLon(0.5, 102.0)));
     83        way1.addNode(new Node(new LatLon(1, 103)));
     84        way1.addNode(new Node(new LatLon(0, 104)));
     85        way1.addNode(new Node(new LatLon(1, 105)));
     86        final Optional<OsmPrimitive> foundWay1 = primitives.stream()
     87            .filter(it -> areEqualWays(it, way1))
     88            .findFirst();
     89        assertTrue(foundWay1.isPresent());
     90        assertEquals("valueB", foundWay1.get().get("propB"));
     91        assertEquals("0.0", foundWay1.get().get("propB2"));
     92        assertEquals(foundNode1.get(), ((Way) foundWay1.get()).firstNode());
     93        assertEquals("valueA", ((Way) foundWay1.get()).firstNode().get("propA"));
     94
     95        final Way way2 = new Way();
     96        way2.addNode(new Node(new LatLon(40, 180)));
     97        way2.addNode(new Node(new LatLon(50, 180)));
     98        way2.addNode(new Node(new LatLon(50, 170)));
     99        way2.addNode(new Node(new LatLon(40, 170)));
     100        way2.addNode(new Node(new LatLon(40, 180)));
     101        final Optional<OsmPrimitive> foundWay2 = primitives.stream()
     102            .filter(it -> areEqualWays(it, way2))
     103            .findFirst();
     104        assertTrue(foundWay2.isPresent());
     105        assertEquals(
     106            ((Way) foundWay2.get()).getNode(0),
     107            ((Way) foundWay2.get()).getNode(((Way) foundWay2.get()).getNodesCount() - 1)
     108        );
     109
     110        final Way way3 = new Way();
     111        way3.addNode(new Node(new LatLon(40, -170)));
     112        way3.addNode(new Node(new LatLon(50, -170)));
     113        way3.addNode(new Node(new LatLon(50, -180)));
     114        way3.addNode(new Node(new LatLon(40, -180)));
     115        way3.addNode(new Node(new LatLon(40, -170)));
     116        final Optional<OsmPrimitive> foundWay3 = primitives.stream()
     117            .filter(it -> areEqualWays(it, way3))
     118            .findFirst();
     119        assertTrue(foundWay3.isPresent());
     120        assertEquals(
     121            ((Way) foundWay3.get()).getNode(0),
     122            ((Way) foundWay3.get()).getNode(((Way) foundWay3.get()).getNodesCount() - 1)
     123        );
     124
     125        final Way way4 = new Way();
     126        way4.addNode(new Node(new LatLon(0, 100)));
     127        way4.addNode(new Node(new LatLon(0, 101)));
     128        way4.addNode(new Node(new LatLon(1, 101)));
     129        way4.addNode(new Node(new LatLon(1, 100)));
     130        way4.addNode(new Node(new LatLon(0, 100)));
     131        final Optional<OsmPrimitive> foundWay4 = primitives.stream()
     132            .filter(it -> areEqualWays(it, way4))
     133            .findFirst();
     134        assertTrue(foundWay4.isPresent());
     135        assertEquals(
     136            ((Way) foundWay4.get()).getNode(0),
     137            ((Way) foundWay4.get()).getNode(((Way) foundWay4.get()).getNodesCount() - 1)
     138        );
     139        assertEquals("valueD", foundWay4.get().get("propD"));
     140        assertFalse(foundWay4.get().hasTag("propD2"));
     141        assertEquals("true", foundWay4.get().get("propD3"));
     142        assertFalse(foundWay4.get().hasKey("propD4"));
     143        assertNull(foundWay4.get().get("propD4"));
    122144    }
    123145
     
    140162    /**
    141163     * Test reading a JSON file which is not a proper GeoJSON (type missing).
    142      * @throws IllegalDataException always
    143164     */
    144     @Test(expected = IllegalDataException.class)
    145     public void testReadGeoJsonWithoutType() throws IllegalDataException {
    146         new GeoJSONReader().doParseDataSet(new ByteArrayInputStream("{}".getBytes(StandardCharsets.UTF_8)), null);
     165    public void testReadGeoJsonWithoutType() {
     166        assertThrows(IllegalDataException.class, () ->
     167                new GeoJSONReader().doParseDataSet(new ByteArrayInputStream("{}".getBytes(StandardCharsets.UTF_8)), null));
    147168    }
    148169
Note: See TracChangeset for help on using the changeset viewer.