Changeset 18695 in josm for trunk


Ignore:
Timestamp:
2023-03-20T21:52:18+01:00 (13 months ago)
Author:
taylor.smock
Message:

Fix #22603: Add PBF reading support

Location:
trunk
Files:
11 added
11 edited

Legend:

Unmodified
Added
Removed

  • trunk/src/org/openstreetmap/josm/actions/ExtensionFileFilter.java

    r18179 r18695  
    2828import org.openstreetmap.josm.gui.io.importexport.OsmChangeImporter;
    2929import org.openstreetmap.josm.gui.io.importexport.OsmImporter;
     30import org.openstreetmap.josm.gui.io.importexport.OsmPbfImporter;
    3031import org.openstreetmap.josm.gui.io.importexport.OziWptImporter;
    3132import org.openstreetmap.josm.gui.io.importexport.RtkLibImporter;
    … …  
    6667                OsmImporter.class,
    6768                OsmChangeImporter.class,
     69                OsmPbfImporter.class,
    6870                GeoJSONImporter.class,
    6971                GpxImporter.class,

  • trunk/src/org/openstreetmap/josm/data/Bounds.java

    r17703 r18695  
    77import java.text.DecimalFormat;
    88import java.text.MessageFormat;
    9 import java.util.Objects;
     9import java.util.Arrays;
    1010
    1111import org.openstreetmap.josm.data.coor.ILatLon;
    … …  
    593593    @Override
    594594    public int hashCode() {
    595         return Objects.hash(minLat, minLon, maxLat, maxLon);
     595        return Arrays.hashCode(new double[] {minLat, minLon, maxLat, maxLon});
    596596    }
    597597

  • trunk/src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/CommandInteger.java

    r18431 r18695  
    5151
    5252    /**
     53     * Add a parameter
     54     * @param parameterInteger The parameter to add (converted to {@code short}).
     55     * @since xxx
     56     */
     57    public void addParameter(long parameterInteger) {
     58        this.parameters[added++] = (short) parameterInteger;
     59    }
     60
     61    /**
    5362     * Get the operations for the command
    5463     * @return The operations

  • trunk/src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/Feature.java

    r18485 r18695  
    2828    /**
    2929     * The number format instance to use (using a static instance gets rid of quite o few allocations)
    30      * Doing this reduced the allocations of {@link #parseTagValue(String, Layer, Number, List)} from 22.79% of parent to
     30     * Doing this reduced the allocations of {@link #parseTagValue(String, Layer, int, List)} from 22.79% of parent to
    3131     * 12.2% of parent.
    3232     */
    … …  
    7575                    if (next.getField() == TAG_FIELD) {
    7676                        // This is packed in v1 and v2
    77                         ProtobufPacked packed = new ProtobufPacked(byteArrayOutputStream, next.getBytes());
     77                        ProtobufPacked packed = new ProtobufPacked(next.getBytes());
    7878                        if (tagList == null) {
    7979                            tagList = new ArrayList<>(packed.getArray().length);
    … …  
    8181                            tagList.ensureCapacity(tagList.size() + packed.getArray().length);
    8282                        }
    83                         for (Number number : packed.getArray()) {
    84                             key = parseTagValue(key, layer, number, tagList);
     83                        for (long number : packed.getArray()) {
     84                            key = parseTagValue(key, layer, (int) number, tagList);
    8585                        }
    8686                    } else if (next.getField() == GEOMETRY_FIELD) {
    8787                        // This is packed in v1 and v2
    88                         ProtobufPacked packed = new ProtobufPacked(byteArrayOutputStream, next.getBytes());
     88                        ProtobufPacked packed = new ProtobufPacked(next.getBytes());
    8989                        CommandInteger currentCommand = null;
    90                         for (Number number : packed.getArray()) {
     90                        for (long number : packed.getArray()) {
    9191                            if (currentCommand != null && currentCommand.hasAllExpectedParameters()) {
    9292                                currentCommand = null;
    9393                            }
    9494                            if (currentCommand == null) {
    95                                 currentCommand = new CommandInteger(number.intValue());
     95                                currentCommand = new CommandInteger(Math.toIntExact(number));
    9696                                this.geometry.add(currentCommand);
    9797                            } else {
    … …  
    128128     * @return The new key (if {@code null}, then a value was parsed and added to tags)
    129129     */
    130     private static String parseTagValue(String key, Layer layer, Number number, List<String> tagList) {
     130    private static String parseTagValue(String key, Layer layer, int number, List<String> tagList) {
    131131        if (key == null) {
    132             key = layer.getKey(number.intValue());
     132            key = layer.getKey(number);
    133133        } else {
    134134            tagList.add(key);
    135             Object value = layer.getValue(number.intValue());
     135            Object value = layer.getValue(number);
    136136            if (value instanceof Double || value instanceof Float) {
    137137                // reset grouping if the instance is a singleton

  • trunk/src/org/openstreetmap/josm/data/protobuf/ProtobufPacked.java

    r18473 r18695  
    33
    44import java.io.ByteArrayOutputStream;
    5 import java.util.ArrayList;
    6 import java.util.List;
     5import java.util.Arrays;
    76
    87/**
    … …  
    1312 */
    1413public class ProtobufPacked {
    15     private static final Number[] NO_NUMBERS = new Number[0];
    1614    private final byte[] bytes;
    17     private final Number[] numbers;
     15    private final long[] numbers;
    1816    private int location;
    1917
    … …  
    2119     * Create a new ProtobufPacked object
    2220     *
    23      * @param byteArrayOutputStream A reusable ByteArrayOutputStream (helps to reduce memory allocations)
     21     * @param ignored A reusable ByteArrayOutputStream (no longer used)
    2422     * @param bytes The packed bytes
     23     * @deprecated since we aren't using the output stream anymore
    2524     */
    26     public ProtobufPacked(ByteArrayOutputStream byteArrayOutputStream, byte[] bytes) {
     25    @Deprecated
     26    public ProtobufPacked(ByteArrayOutputStream ignored, byte[] bytes) {
     27        this(bytes);
     28    }
     29
     30    /**
     31     * Create a new ProtobufPacked object
     32     *
     33     * @param bytes The packed bytes
     34     * @since 18695
     35     */
     36    public ProtobufPacked(byte[] bytes) {
    2737        this.location = 0;
    2838        this.bytes = bytes;
    … …  
    3141        // only adds 3.7 MB to the ArrayList#init calls. Note that the real-world test case (Mapillary vector tiles)
    3242        // primarily created Shorts.
    33         List<Number> numbersT = new ArrayList<>(bytes.length);
     43        long[] numbersT = new long[bytes.length];
     44        int index = 0;
    3445        // By reusing a ByteArrayOutputStream, we can reduce allocations in nextVarInt from 230 MB to 74 MB.
    3546        while (this.location < bytes.length) {
    36             numbersT.add(ProtobufParser.convertByteArray(this.nextVarInt(byteArrayOutputStream), ProtobufParser.VAR_INT_BYTE_SIZE));
     47            int start = this.location;
     48            numbersT[index] = ProtobufParser.convertByteArray(this.bytes, ProtobufParser.VAR_INT_BYTE_SIZE,
     49                    start, this.nextVarInt());
     50            index++;
    3751        }
    3852
    39         this.numbers = numbersT.toArray(NO_NUMBERS);
     53        if (numbersT.length == index) {
     54            this.numbers = numbersT;
     55        } else {
     56            this.numbers = Arrays.copyOf(numbersT, index);
     57        }
    4058    }
    4159
    … …  
    4563     * @return The number array
    4664     */
    47     public Number[] getArray() {
     65    public long[] getArray() {
    4866        return this.numbers;
    4967    }
    5068
    51     private byte[] nextVarInt(final ByteArrayOutputStream byteArrayOutputStream) {
    52         // In a real world test, the largest List<Byte> seen had 3 elements. Use 4 to avoid most new array allocations.
    53         // Memory allocations went from 368 MB to 280 MB by using an initial array allocation. When using a
    54         // ByteArrayOutputStream, it went down to 230 MB. By further reusing the ByteArrayOutputStream between method
    55         // calls, it went down further to 73 MB.
     69    /**
     70     * Gets the location where the next var int begins. Note: changes {@link ProtobufPacked#location}.
     71     * @return The next varint location
     72     */
     73    private int nextVarInt() {
    5674        while ((this.bytes[this.location] & ProtobufParser.MOST_SIGNIFICANT_BYTE)
    5775          == ProtobufParser.MOST_SIGNIFICANT_BYTE) {
    5876            // Get rid of the leading bit (shift left 1, then shift right 1 unsigned)
    59             byteArrayOutputStream.write(this.bytes[this.location++] ^ ProtobufParser.MOST_SIGNIFICANT_BYTE);
     77            this.bytes[this.location] = (byte) (this.bytes[this.location] ^ ProtobufParser.MOST_SIGNIFICANT_BYTE);
     78            this.location++;
    6079        }
    61         // The last byte doesn't drop the most significant bit
    62         byteArrayOutputStream.write(this.bytes[this.location++]);
    63         try {
    64             return byteArrayOutputStream.toByteArray();
    65         } finally {
    66             byteArrayOutputStream.reset();
    67         }
     80        return ++this.location;
    6881    }
    6982}

  • trunk/src/org/openstreetmap/josm/data/protobuf/ProtobufParser.java

    r18473 r18695  
    88import java.io.InputStream;
    99import java.util.ArrayList;
     10import java.util.Arrays;
    1011import java.util.Collection;
    1112
    … …  
    3132     */
    3233    static final byte MOST_SIGNIFICANT_BYTE = (byte) (1 << 7);
     34    private static final byte[] EMPTY_BYTE_ARRAY = new byte[0];
     35
    3336    /**
    3437     * Convert a byte array to a number (little endian)
    … …  
    3942     */
    4043    public static Number convertByteArray(byte[] bytes, byte byteSize) {
     44        return convertLong(convertByteArray(bytes, byteSize, 0, bytes.length));
     45    }
     46
     47    /**
     48     * Convert a byte array to a number (little endian)
     49     *
     50     * @param bytes    The bytes to convert
     51     * @param byteSize The size of the byte. For var ints, this is 7, for other ints, this is 8.
     52     * @param start    The start position in the byte array
     53     * @param end      The end position in the byte array (exclusive - [start, end) )
     54     * @return t
     55     * he number from the byte array. Depending upon length of time the number will be stored, narrowing may be helpful.
     56     * @since 18695
     57     */
     58    public static long convertByteArray(byte[] bytes, byte byteSize, int start, int end) {
    4159        long number = 0;
    42         for (int i = 0; i < bytes.length; i++) {
     60        for (int i = start; i < end; i++) {
    4361            // Need to convert to uint64 in order to avoid bit operation from filling in 1's and overflow issues
    44             number += Byte.toUnsignedLong(bytes[i]) << (byteSize * i);
    45         }
    46         return convertLong(number);
     62            number += Byte.toUnsignedLong(bytes[i]) << (byteSize * (i - start));
     63        }
     64        return number;
    4765    }
    4866
    … …  
    7290     */
    7391    public static Number decodeZigZag(Number signed) {
    74         final long value = signed.longValue();
    75         return convertLong((value >> 1) ^ -(value & 1));
     92        return convertLong(decodeZigZag(signed.longValue()));
     93    }
     94
     95    /**
     96     * Decode a zig-zag encoded value
     97     *
     98     * @param signed The value to decode
     99     * @return The decoded value
     100     * @since 18695
     101     */
     102    public static long decodeZigZag(long signed) {
     103        return (signed >> 1) ^ -(signed & 1);
    76104    }
    77105
    … …  
    204232     */
    205233    public byte[] nextLengthDelimited(ByteArrayOutputStream byteArrayOutputStream) throws IOException {
    206         int length = convertByteArray(this.nextVarInt(byteArrayOutputStream), VAR_INT_BYTE_SIZE).intValue();
     234        final byte[] nextVarInt = this.nextVarInt(byteArrayOutputStream);
     235        int length = (int) convertByteArray(nextVarInt, VAR_INT_BYTE_SIZE, 0, nextVarInt.length);
    207236        return readNextBytes(length);
    208237    }
    … …  
    237266     *
    238267     * @param size The number of bytes to read
    239      * @return a byte array of the specified size, filled with bytes read (unsigned)
     268     * @return a byte array filled with bytes read (unsigned)
    240269     * @throws IOException - if an IO error occurs
    241270     */
    242271    private byte[] readNextBytes(int size) throws IOException {
    243272        byte[] bytesRead = new byte[size];
    244         for (int i = 0; i < bytesRead.length; i++) {
    245             bytesRead[i] = (byte) this.nextByte();
     273        int read = this.inputStream.read(bytesRead);
     274        if (read == -1) {
     275            return EMPTY_BYTE_ARRAY;
     276        } else if (read != size) {
     277            return Arrays.copyOf(bytesRead, read);
    246278        }
    247279        return bytesRead;

  • trunk/src/org/openstreetmap/josm/data/protobuf/ProtobufRecord.java

    r18473 r18695  
    2828     */
    2929    public ProtobufRecord(ByteArrayOutputStream byteArrayOutputStream, ProtobufParser parser) throws IOException {
    30         Number number = ProtobufParser.convertByteArray(parser.nextVarInt(byteArrayOutputStream), ProtobufParser.VAR_INT_BYTE_SIZE);
     30        final byte[] varInt = parser.nextVarInt(byteArrayOutputStream);
     31        long number = ProtobufParser.convertByteArray(varInt, ProtobufParser.VAR_INT_BYTE_SIZE, 0, varInt.length);
    3132        // I don't foresee having field numbers > {@code Integer#MAX_VALUE >> 3}
    32         this.field = (int) number.longValue() >> 3;
     33        this.field = (int) number >> 3;
    3334        // 7 is 111 (so last three bits)
    34         byte wireType = (byte) (number.longValue() & 7);
     35        byte wireType = (byte) (number & 7);
    3536        // By not using a stream, we reduce the number of allocations (for getting the WireType) from 257 MB to 40 MB.
    3637        // (The remaining 40 MB is from WireType#values). By using the cached getAllValues(), we drop the 40 MB.

  • trunk/src/org/openstreetmap/josm/io/AbstractReader.java

    r18208 r18695  
    274274    protected abstract DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException;
    275275
     276    /**
     277     * An interface for reading binary data
     278     * @since xxx
     279     */
     280    @FunctionalInterface
     281    protected interface BinaryParserWorker {
     282        /**
     283         * Effectively parses the file, depending on the binary format (PBF, etc.)
     284         * @param ir input stream reader
     285         * @throws IllegalDataException in case of invalid data
     286         * @throws IOException in case of I/O error
     287         */
     288        void accept(InputStream ir) throws IllegalDataException, IOException;
     289    }
     290
    276291    @FunctionalInterface
    277292    protected interface ParserWorker {
    … …  
    285300    }
    286301
     302    protected final DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor, BinaryParserWorker parserWorker)
     303            throws IllegalDataException {
     304        return this.doParseDataSet(source, progressMonitor, (Object) parserWorker);
     305    }
     306
    287307    protected final DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor, ParserWorker parserWorker)
     308            throws IllegalDataException {
     309        return this.doParseDataSet(source, progressMonitor, (Object) parserWorker);
     310    }
     311
     312    private DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor, Object parserWorker)
    288313            throws IllegalDataException {
    289314        if (progressMonitor == null) {
    … …  
    297322            progressMonitor.indeterminateSubTask(tr("Parsing OSM data..."));
    298323
    299             try (InputStreamReader ir = UTFInputStreamReader.create(source)) {
    300                 parserWorker.accept(ir);
     324            if (parserWorker instanceof ParserWorker) {
     325                try (InputStreamReader ir = UTFInputStreamReader.create(source)) {
     326                    ((ParserWorker) parserWorker).accept(ir);
     327                }
     328            } else if (parserWorker instanceof BinaryParserWorker) {
     329                ((BinaryParserWorker) parserWorker).accept(source);
     330            } else {
     331                throw new IllegalArgumentException("Unknown parser worker type: " + parserWorker.getClass());
    301332            }
    302333            progressMonitor.worked(1);

  • trunk/src/org/openstreetmap/josm/io/OsmJsonReader.java

    r18658 r18695  
    183183    protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
    184184        try {
    185             return doParseDataSet(source, progressMonitor, ir -> {
     185            return doParseDataSet(source, progressMonitor, (ParserWorker) ir -> {
    186186                setParser(Json.createParser(ir));
    187187                parse();

  • trunk/src/org/openstreetmap/josm/io/OsmReader.java

    r18541 r18695  
    502502    @Override
    503503    protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
    504         return doParseDataSet(source, progressMonitor, ir -> {
     504        return doParseDataSet(source, progressMonitor, (ParserWorker) ir -> {
    505505            try {
    506506                setParser(XmlUtils.newSafeXMLInputFactory().createXMLStreamReader(ir));

  • trunk/test/unit/org/openstreetmap/josm/data/protobuf/ProtobufTest.java

    r18473 r18695  
    4343 * @since 17862
    4444 */
    45 class ProtobufTest {
     45public class ProtobufTest {
    4646    /**
    4747     * Convert an int array into a byte array
    … …  
    4949     * @return A byte array that can be used
    5050     */
    51     static byte[] toByteArray(int[] intArray) {
     51    public static byte[] toByteArray(int[] intArray) {
    5252        byte[] byteArray = new byte[intArray.length];
    5353        for (int i = 0; i < intArray.length; i++) {
    … …  
    202202    @Test
    203203    void testZigZag() {
    204         assertEquals(0, ProtobufParser.decodeZigZag(0).intValue());
    205         assertEquals(-1, ProtobufParser.decodeZigZag(1).intValue());
    206         assertEquals(1, ProtobufParser.decodeZigZag(2).intValue());
    207         assertEquals(-2, ProtobufParser.decodeZigZag(3).intValue());
     204        assertEquals(0, ProtobufParser.decodeZigZag(Integer.valueOf(0)).intValue());
     205        assertEquals(-1, ProtobufParser.decodeZigZag(Integer.valueOf(1)).intValue());
     206        assertEquals(1, ProtobufParser.decodeZigZag(Long.valueOf(2)).intValue());
     207        assertEquals(-2, ProtobufParser.decodeZigZag(Long.valueOf(3)).intValue());
    208208    }
    209209}
Note: See TracChangeset for help on using the changeset viewer.