Ticket #17177: 17177.7.protobuf.patch

src/org/openstreetmap/josm/data/protobuf/ProtoBufPacked.java

@@ +1 @@
+// License: GPL. For details, see LICENSE file.
+package org.openstreetmap.josm.data.protobuf;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Parse packed values (only numerical values)
+ *
+ * @author Taylor Smock
+ * @since xxx
+ */
+public class ProtoBufPacked {
+    private final byte[] bytes;
+    private final Number[] numbers;
+    private int location;
+
+    /**
+     * Create a new ProtoBufPacked object
+     *
+     * @param bytes The packed bytes
+     */
+    public ProtoBufPacked(byte[] bytes) {
+        this.location = 0;
+        this.bytes = bytes;
+        List<Number> numbersT = new ArrayList<>();
+        while (this.location < bytes.length) {
+            numbersT.add(ProtoBufParser.convertByteArray(this.nextVarInt(), ProtoBufParser.VAR_INT_BYTE_SIZE));
+        }
+
+        this.numbers = new Number[numbersT.size()];
+        for (int i = 0; i < numbersT.size(); i++) {
+            this.numbers[i] = numbersT.get(i);
+        }
+    }
+
+    /**
+     * Get the parsed number array
+     *
+     * @return The number array
+     */
+    public Number[] getArray() {
+        return this.numbers;
+    }
+
+    private byte[] nextVarInt() {
+        List<Byte> byteList = new ArrayList<>();
+        while ((this.bytes[this.location] & ProtoBufParser.MOST_SIGNIFICANT_BYTE)
+          == ProtoBufParser.MOST_SIGNIFICANT_BYTE) {
+            // Get rid of the leading bit (shift left 1, then shift right 1 unsigned)
+            byteList.add((byte) (this.bytes[this.location++] ^ ProtoBufParser.MOST_SIGNIFICANT_BYTE));
+        }
+        // The last byte doesn't drop the most significant bit
+        byteList.add(this.bytes[this.location++]);
+        byte[] byteArray = new byte[byteList.size()];
+        for (int i = 0; i < byteList.size(); i++) {
+            byteArray[i] = byteList.get(i);
+        }
+
+        return byteArray;
+    }
+}

src/org/openstreetmap/josm/data/protobuf/ProtoBufParser.java

@@ +1 @@
+// License: GPL. For details, see LICENSE file.
+package org.openstreetmap.josm.data.protobuf;
+
+import java.io.BufferedInputStream;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.openstreetmap.josm.tools.Logging;
+
+/**
+ * A basic Protobuf parser
+ *
+ * @author Taylor Smock
+ * @since xxx
+ */
+public class ProtoBufParser implements AutoCloseable {
+    /**
+     * The default byte size (see {@link #VAR_INT_BYTE_SIZE} for var ints)
+     */
+    public static final byte BYTE_SIZE = 8;
+    /**
+     * The byte size for var ints (since the first byte is just an indicator for if the var int is done)
+     */
+    public static final byte VAR_INT_BYTE_SIZE = BYTE_SIZE - 1;
+    /**
+     * Used to get the most significant byte
+     */
+    static final byte MOST_SIGNIFICANT_BYTE = (byte) (1 << 7);
+    /**
+     * Convert a byte array to a number (little endian)
+     *
+     * @param bytes    The bytes to convert
+     * @param byteSize The size of the byte. For var ints, this is 7, for other ints, this is 8.
+     * @return An appropriate {@link Number} class.
+     */
+    public static Number convertByteArray(byte[] bytes, byte byteSize) {
+        long number = 0;
+        for (int i = 0; i < bytes.length; i++) {
+            // Need to convert to uint64 in order to avoid bit operation from filling in 1's and overflow issues
+            number += Byte.toUnsignedLong(bytes[i]) << (byteSize * i);
+        }
+        return convertLong(number);
+    }
+
+    /**
+     * Convert a long to an appropriate {@link Number} class
+     *
+     * @param number The long to convert
+     * @return A {@link Number}
+     */
+    public static Number convertLong(long number) {
+        // TODO deal with booleans
+        if (number <= Byte.MAX_VALUE && number >= Byte.MIN_VALUE) {
+            return (byte) number;
+        } else if (number <= Short.MAX_VALUE && number >= Short.MIN_VALUE) {
+            return (short) number;
+        } else if (number <= Integer.MAX_VALUE && number >= Integer.MIN_VALUE) {
+            return (int) number;
+        }
+        return number;
+    }
+
+    /**
+     * Decode a zig-zag encoded value
+     *
+     * @param signed The value to decode
+     * @return The decoded value
+     */
+    public static Number decodeZigZag(Number signed) {
+        final long value = signed.longValue();
+        return convertLong((value >> 1) ^ -(value & 1));
+    }
+
+    /**
+     * Encode a number to a zig-zag encode value
+     *
+     * @param signed The number to encode
+     * @return The encoded value
+     */
+    public static Number encodeZigZag(Number signed) {
+        final long value = signed.longValue();
+        // This boundary condition could be >= or <= or both. Tests indicate that it doesn't actually matter.
+        // The only difference would be the number type returned, except it is always converted to the most basic type.
+        final int shift = (value > Integer.MAX_VALUE || value < Integer.MIN_VALUE ? Long.BYTES : Integer.BYTES) * 8 - 1;
+        return convertLong((value << 1) ^ (value >> shift));
+    }
+
+    private final InputStream inputStream;
+
+    /**
+     * Create a new parser
+     *
+     * @param bytes The bytes to parse
+     */
+    public ProtoBufParser(byte[] bytes) {
+        this(new ByteArrayInputStream(bytes));
+    }
+
+    /**
+     * Create a new parser
+     *
+     * @param inputStream The InputStream (will be fully read at this time)
+     */
+    public ProtoBufParser(InputStream inputStream) {
+        if (inputStream.markSupported()) {
+            this.inputStream = inputStream;
+        } else {
+            this.inputStream = new BufferedInputStream(inputStream);
+        }
+    }
+
+    /**
+     * Read all records
+     *
+     * @return A collection of all records
+     * @throws IOException - if an IO error occurs
+     */
+    public Collection<ProtoBufRecord> allRecords() throws IOException {
+        Collection<ProtoBufRecord> records = new ArrayList<>();
+        while (this.hasNext()) {
+            records.add(new ProtoBufRecord(this));
+        }
+        return records;
+    }
+
+    @Override
+    public void close() {
+        try {
+            this.inputStream.close();
+        } catch (IOException e) {
+            Logging.error(e);
+        }
+    }
+
+    /**
+     * Check if there is more data to read
+     *
+     * @return {@code true} if there is more data to read
+     * @throws IOException - if an IO error occurs
+     */
+    public boolean hasNext() throws IOException {
+        return this.inputStream.available() > 0;
+    }
+
+    /**
+     * Get the "next" WireType
+     *
+     * @return {@link WireType} expected
+     * @throws IOException - if an IO error occurs
+     */
+    public WireType next() throws IOException {
+        this.inputStream.mark(16);
+        try {
+            return WireType.values()[this.inputStream.read() << 3];
+        } finally {
+            this.inputStream.reset();
+        }
+    }
+
+    /**
+     * Get the next byte
+     *
+     * @return The next byte
+     * @throws IOException - if an IO error occurs
+     */
+    public int nextByte() throws IOException {
+        return this.inputStream.read();
+    }
+
+    /**
+     * Get the next 32 bits ({@link WireType#THIRTY_TWO_BIT})
+     *
+     * @return a byte array of the next 32 bits (4 bytes)
+     * @throws IOException - if an IO error occurs
+     */
+    public byte[] nextFixed32() throws IOException {
+        // 4 bytes == 32 bits
+        return readNextBytes(4);
+    }
+
+    /**
+     * Get the next 64 bits ({@link WireType#SIXTY_FOUR_BIT})
+     *
+     * @return a byte array of the next 64 bits (8 bytes)
+     * @throws IOException - if an IO error occurs
+     */
+    public byte[] nextFixed64() throws IOException {
+        // 8 bytes == 64 bits
+        return readNextBytes(8);
+    }
+
+    /**
+     * Get the next delimited message ({@link WireType#LENGTH_DELIMITED})
+     *
+     * @return The next length delimited message
+     * @throws IOException - if an IO error occurs
+     */
+    public byte[] nextLengthDelimited() throws IOException {
+        int length = convertByteArray(this.nextVarInt(), VAR_INT_BYTE_SIZE).intValue();
+        return readNextBytes(length);
+    }
+
+    /**
+     * Get the next var int ({@code WireType#VARINT})
+     *
+     * @return The next var int ({@code int32}, {@code int64}, {@code uint32}, {@code uint64}, {@code bool}, {@code enum})
+     * @throws IOException - if an IO error occurs
+     */
+    public byte[] nextVarInt() throws IOException {
+        List<Byte> byteList = new ArrayList<>();
+        int currentByte = this.nextByte();
+        while ((byte) (currentByte & MOST_SIGNIFICANT_BYTE) == MOST_SIGNIFICANT_BYTE && currentByte > 0) {
+            // Get rid of the leading bit (shift left 1, then shift right 1 unsigned)
+            byteList.add((byte) (currentByte ^ MOST_SIGNIFICANT_BYTE));
+            currentByte = this.nextByte();
+        }
+        // The last byte doesn't drop the most significant bit
+        byteList.add((byte) currentByte);
+        byte[] byteArray = new byte[byteList.size()];
+        for (int i = 0; i < byteList.size(); i++) {
+            byteArray[i] = byteList.get(i);
+        }
+
+        return byteArray;
+    }
+
+    /**
+     * Read an arbitrary number of bytes
+     *
+     * @param size The number of bytes to read
+     * @return a byte array of the specified size, filled with bytes read (unsigned)
+     * @throws IOException - if an IO error occurs
+     */
+    private byte[] readNextBytes(int size) throws IOException {
+        byte[] bytesRead = new byte[size];
+        for (int i = 0; i < bytesRead.length; i++) {
+            bytesRead[i] = (byte) this.nextByte();
+        }
+        return bytesRead;
+    }
+}

src/org/openstreetmap/josm/data/protobuf/ProtoBufRecord.java

@@ +1 @@
+// License: GPL. For details, see LICENSE file.
+package org.openstreetmap.josm.data.protobuf;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.stream.Stream;
+
+import org.openstreetmap.josm.tools.Utils;
+
+/**
+ * A protobuf record, storing the {@link WireType}, the parsed field number, and the bytes for it.
+ *
+ * @author Taylor Smock
+ * @since xxx
+ */
+public class ProtoBufRecord implements AutoCloseable {
+    private static final byte[] EMPTY_BYTES = {};
+    private final WireType type;
+    private final int field;
+    private byte[] bytes;
+
+    /**
+     * Create a new Protobuf record
+     *
+     * @param parser The parser to use to create the record
+     * @throws IOException - if an IO error occurs
+     */
+    public ProtoBufRecord(ProtoBufParser parser) throws IOException {
+        Number number = ProtoBufParser.convertByteArray(parser.nextVarInt(), ProtoBufParser.VAR_INT_BYTE_SIZE);
+        // I don't foresee having field numbers > {@code Integer#MAX_VALUE >> 3}
+        this.field = (int) number.longValue() >> 3;
+        // 7 is 111 (so last three bits)
+        byte wireType = (byte) (number.longValue() & 7);
+        this.type = Stream.of(WireType.values()).filter(wType -> wType.getTypeRepresentation() == wireType).findFirst()
+          .orElse(WireType.UNKNOWN);
+
+        if (this.type == WireType.VARINT) {
+            this.bytes = parser.nextVarInt();
+        } else if (this.type == WireType.SIXTY_FOUR_BIT) {
+            this.bytes = parser.nextFixed64();
+        } else if (this.type == WireType.THIRTY_TWO_BIT) {
+            this.bytes = parser.nextFixed32();
+        } else if (this.type == WireType.LENGTH_DELIMITED) {
+            this.bytes = parser.nextLengthDelimited();
+        } else {
+            this.bytes = EMPTY_BYTES;
+        }
+    }
+
+    /**
+     * Get as a double ({@link WireType#SIXTY_FOUR_BIT})
+     *
+     * @return the double
+     */
+    public double asDouble() {
+        long doubleNumber = ProtoBufParser.convertByteArray(asFixed64(), ProtoBufParser.BYTE_SIZE).longValue();
+        return Double.longBitsToDouble(doubleNumber);
+    }
+
+    /**
+     * Get as 32 bits ({@link WireType#THIRTY_TWO_BIT})
+     *
+     * @return a byte array of the 32 bits (4 bytes)
+     */
+    public byte[] asFixed32() {
+        // TODO verify, or just assume?
+        // 4 bytes == 32 bits
+        return this.bytes;
+    }
+
+    /**
+     * Get as 64 bits ({@link WireType#SIXTY_FOUR_BIT})
+     *
+     * @return a byte array of the 64 bits (8 bytes)
+     */
+    public byte[] asFixed64() {
+        // TODO verify, or just assume?
+        // 8 bytes == 64 bits
+        return this.bytes;
+    }
+
+    /**
+     * Get as a float ({@link WireType#THIRTY_TWO_BIT})
+     *
+     * @return the float
+     */
+    public float asFloat() {
+        int floatNumber = ProtoBufParser.convertByteArray(asFixed32(), ProtoBufParser.BYTE_SIZE).intValue();
+        return Float.intBitsToFloat(floatNumber);
+    }
+
+    /**
+     * Get the signed var int ({@code WireType#VARINT}).
+     * These are specially encoded so that they take up less space.
+     *
+     * @return The signed var int ({@code sint32} or {@code sint64})
+     */
+    public Number asSignedVarInt() {
+        final Number signed = this.asUnsignedVarInt();
+        return ProtoBufParser.decodeZigZag(signed);
+    }
+
+    /**
+     * Get as a string ({@link WireType#LENGTH_DELIMITED})
+     *
+     * @return The string (encoded as {@link StandardCharsets#UTF_8})
+     */
+    public String asString() {
+        return Utils.intern(new String(this.bytes, StandardCharsets.UTF_8));
+    }
+
+    /**
+     * Get the var int ({@code WireType#VARINT})
+     *
+     * @return The var int ({@code int32}, {@code int64}, {@code uint32}, {@code uint64}, {@code bool}, {@code enum})
+     */
+    public Number asUnsignedVarInt() {
+        return ProtoBufParser.convertByteArray(this.bytes, ProtoBufParser.VAR_INT_BYTE_SIZE);
+    }
+
+    @Override
+    public void close() {
+        this.bytes = null;
+    }
+
+    /**
+     * Get the raw bytes for this record
+     *
+     * @return The bytes
+     */
+    public byte[] getBytes() {
+        return this.bytes;
+    }
+
+    /**
+     * Get the field value
+     *
+     * @return The field value
+     */
+    public int getField() {
+        return this.field;
+    }
+
+    /**
+     * Get the WireType of the data
+     *
+     * @return The {@link WireType} of the data
+     */
+    public WireType getType() {
+        return this.type;
+    }
+}

src/org/openstreetmap/josm/data/protobuf/WireType.java

@@ +1 @@
+// License: GPL. For details, see LICENSE file.
+package org.openstreetmap.josm.data.protobuf;
+
+/**
+ * The WireTypes
+ *
+ * @author Taylor Smock
+ * @since xxx
+ */
+public enum WireType {
+    /**
+     * int32, int64, uint32, uint64, sing32, sint64, bool, enum
+     */
+    VARINT(0),
+    /**
+     * fixed64, sfixed64, double
+     */
+    SIXTY_FOUR_BIT(1),
+    /**
+     * string, bytes, embedded messages, packed repeated fields
+     */
+    LENGTH_DELIMITED(2),
+    /**
+     * start groups
+     *
+     * @deprecated Unknown reason. Deprecated since at least 2012.
+     */
+    @Deprecated
+    START_GROUP(3),
+    /**
+     * end groups
+     *
+     * @deprecated Unknown reason. Deprecated since at least 2012.
+     */
+    @Deprecated
+    END_GROUP(4),
+    /**
+     * fixed32, sfixed32, float
+     */
+    THIRTY_TWO_BIT(5),
+
+    /**
+     * For unknown WireTypes
+     */
+    UNKNOWN(Byte.MAX_VALUE);
+
+    private final byte type;
+
+    WireType(int value) {
+        this.type = (byte) value;
+    }
+
+    /**
+     * Get the type representation (byte form)
+     *
+     * @return The wire type byte representation
+     */
+    public byte getTypeRepresentation() {
+        return this.type;
+    }
+}

src/org/openstreetmap/josm/data/protobuf/ProtoBufPacked.java

@@ +1 @@
+// License: GPL. For details, see LICENSE file.
+package org.openstreetmap.josm.data.protobuf;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Parse packed values (only numerical values)
+ *
+ * @author Taylor Smock
+ * @since xxx
+ */
+public class ProtoBufPacked {
+    private final byte[] bytes;
+    private final Number[] numbers;
+    private int location;
+
+    /**
+     * Create a new ProtoBufPacked object
+     *
+     * @param bytes The packed bytes
+     */
+    public ProtoBufPacked(byte[] bytes) {
+        this.location = 0;
+        this.bytes = bytes;
+        List<Number> numbersT = new ArrayList<>();
+        while (this.location < bytes.length) {
+            numbersT.add(ProtoBufParser.convertByteArray(this.nextVarInt(), ProtoBufParser.VAR_INT_BYTE_SIZE));
+        }
+
+        this.numbers = new Number[numbersT.size()];
+        for (int i = 0; i < numbersT.size(); i++) {
+            this.numbers[i] = numbersT.get(i);
+        }
+    }
+
+    /**
+     * Get the parsed number array
+     *
+     * @return The number array
+     */
+    public Number[] getArray() {
+        return this.numbers;
+    }
+
+    private byte[] nextVarInt() {
+        List<Byte> byteList = new ArrayList<>();
+        while ((this.bytes[this.location] & ProtoBufParser.MOST_SIGNIFICANT_BYTE)
+          == ProtoBufParser.MOST_SIGNIFICANT_BYTE) {
+            // Get rid of the leading bit (shift left 1, then shift right 1 unsigned)
+            byteList.add((byte) (this.bytes[this.location++] ^ ProtoBufParser.MOST_SIGNIFICANT_BYTE));
+        }
+        // The last byte doesn't drop the most significant bit
+        byteList.add(this.bytes[this.location++]);
+        byte[] byteArray = new byte[byteList.size()];
+        for (int i = 0; i < byteList.size(); i++) {
+            byteArray[i] = byteList.get(i);
+        }
+
+        return byteArray;
+    }
+}

src/org/openstreetmap/josm/data/protobuf/ProtoBufParser.java

@@ +1 @@
+// License: GPL. For details, see LICENSE file.
+package org.openstreetmap.josm.data.protobuf;
+
+import java.io.BufferedInputStream;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.openstreetmap.josm.tools.Logging;
+
+/**
+ * A basic Protobuf parser
+ *
+ * @author Taylor Smock
+ * @since xxx
+ */
+public class ProtoBufParser implements AutoCloseable {
+    /**
+     * The default byte size (see {@link #VAR_INT_BYTE_SIZE} for var ints)
+     */
+    public static final byte BYTE_SIZE = 8;
+    /**
+     * The byte size for var ints (since the first byte is just an indicator for if the var int is done)
+     */
+    public static final byte VAR_INT_BYTE_SIZE = BYTE_SIZE - 1;
+    /**
+     * Used to get the most significant byte
+     */
+    static final byte MOST_SIGNIFICANT_BYTE = (byte) (1 << 7);
+    /**
+     * Convert a byte array to a number (little endian)
+     *
+     * @param bytes    The bytes to convert
+     * @param byteSize The size of the byte. For var ints, this is 7, for other ints, this is 8.
+     * @return An appropriate {@link Number} class.
+     */
+    public static Number convertByteArray(byte[] bytes, byte byteSize) {
+        long number = 0;
+        for (int i = 0; i < bytes.length; i++) {
+            // Need to convert to uint64 in order to avoid bit operation from filling in 1's and overflow issues
+            number += Byte.toUnsignedLong(bytes[i]) << (byteSize * i);
+        }
+        return convertLong(number);
+    }
+
+    /**
+     * Convert a long to an appropriate {@link Number} class
+     *
+     * @param number The long to convert
+     * @return A {@link Number}
+     */
+    public static Number convertLong(long number) {
+        // TODO deal with booleans
+        if (number <= Byte.MAX_VALUE && number >= Byte.MIN_VALUE) {
+            return (byte) number;
+        } else if (number <= Short.MAX_VALUE && number >= Short.MIN_VALUE) {
+            return (short) number;
+        } else if (number <= Integer.MAX_VALUE && number >= Integer.MIN_VALUE) {
+            return (int) number;
+        }
+        return number;
+    }
+
+    /**
+     * Decode a zig-zag encoded value
+     *
+     * @param signed The value to decode
+     * @return The decoded value
+     */
+    public static Number decodeZigZag(Number signed) {
+        final long value = signed.longValue();
+        return convertLong((value >> 1) ^ -(value & 1));
+    }
+
+    /**
+     * Encode a number to a zig-zag encode value
+     *
+     * @param signed The number to encode
+     * @return The encoded value
+     */
+    public static Number encodeZigZag(Number signed) {
+        final long value = signed.longValue();
+        // This boundary condition could be >= or <= or both. Tests indicate that it doesn't actually matter.
+        // The only difference would be the number type returned, except it is always converted to the most basic type.
+        final int shift = (value > Integer.MAX_VALUE || value < Integer.MIN_VALUE ? Long.BYTES : Integer.BYTES) * 8 - 1;
+        return convertLong((value << 1) ^ (value >> shift));
+    }
+
+    private final InputStream inputStream;
+
+    /**
+     * Create a new parser
+     *
+     * @param bytes The bytes to parse
+     */
+    public ProtoBufParser(byte[] bytes) {
+        this(new ByteArrayInputStream(bytes));
+    }
+
+    /**
+     * Create a new parser
+     *
+     * @param inputStream The InputStream (will be fully read at this time)
+     */
+    public ProtoBufParser(InputStream inputStream) {
+        if (inputStream.markSupported()) {
+            this.inputStream = inputStream;
+        } else {
+            this.inputStream = new BufferedInputStream(inputStream);
+        }
+    }
+
+    /**
+     * Read all records
+     *
+     * @return A collection of all records
+     * @throws IOException - if an IO error occurs
+     */
+    public Collection<ProtoBufRecord> allRecords() throws IOException {
+        Collection<ProtoBufRecord> records = new ArrayList<>();
+        while (this.hasNext()) {
+            records.add(new ProtoBufRecord(this));
+        }
+        return records;
+    }
+
+    @Override
+    public void close() {
+        try {
+            this.inputStream.close();
+        } catch (IOException e) {
+            Logging.error(e);
+        }
+    }
+
+    /**
+     * Check if there is more data to read
+     *
+     * @return {@code true} if there is more data to read
+     * @throws IOException - if an IO error occurs
+     */
+    public boolean hasNext() throws IOException {
+        return this.inputStream.available() > 0;
+    }
+
+    /**
+     * Get the "next" WireType
+     *
+     * @return {@link WireType} expected
+     * @throws IOException - if an IO error occurs
+     */
+    public WireType next() throws IOException {
+        this.inputStream.mark(16);
+        try {
+            return WireType.values()[this.inputStream.read() << 3];
+        } finally {
+            this.inputStream.reset();
+        }
+    }
+
+    /**
+     * Get the next byte
+     *
+     * @return The next byte
+     * @throws IOException - if an IO error occurs
+     */
+    public int nextByte() throws IOException {
+        return this.inputStream.read();
+    }
+
+    /**
+     * Get the next 32 bits ({@link WireType#THIRTY_TWO_BIT})
+     *
+     * @return a byte array of the next 32 bits (4 bytes)
+     * @throws IOException - if an IO error occurs
+     */
+    public byte[] nextFixed32() throws IOException {
+        // 4 bytes == 32 bits
+        return readNextBytes(4);
+    }
+
+    /**
+     * Get the next 64 bits ({@link WireType#SIXTY_FOUR_BIT})
+     *
+     * @return a byte array of the next 64 bits (8 bytes)
+     * @throws IOException - if an IO error occurs
+     */
+    public byte[] nextFixed64() throws IOException {
+        // 8 bytes == 64 bits
+        return readNextBytes(8);
+    }
+
+    /**
+     * Get the next delimited message ({@link WireType#LENGTH_DELIMITED})
+     *
+     * @return The next length delimited message
+     * @throws IOException - if an IO error occurs
+     */
+    public byte[] nextLengthDelimited() throws IOException {
+        int length = convertByteArray(this.nextVarInt(), VAR_INT_BYTE_SIZE).intValue();
+        return readNextBytes(length);
+    }
+
+    /**
+     * Get the next var int ({@code WireType#VARINT})
+     *
+     * @return The next var int ({@code int32}, {@code int64}, {@code uint32}, {@code uint64}, {@code bool}, {@code enum})
+     * @throws IOException - if an IO error occurs
+     */
+    public byte[] nextVarInt() throws IOException {
+        List<Byte> byteList = new ArrayList<>();
+        int currentByte = this.nextByte();
+        while ((byte) (currentByte & MOST_SIGNIFICANT_BYTE) == MOST_SIGNIFICANT_BYTE && currentByte > 0) {
+            // Get rid of the leading bit (shift left 1, then shift right 1 unsigned)
+            byteList.add((byte) (currentByte ^ MOST_SIGNIFICANT_BYTE));
+            currentByte = this.nextByte();
+        }
+        // The last byte doesn't drop the most significant bit
+        byteList.add((byte) currentByte);
+        byte[] byteArray = new byte[byteList.size()];
+        for (int i = 0; i < byteList.size(); i++) {
+            byteArray[i] = byteList.get(i);
+        }
+
+        return byteArray;
+    }
+
+    /**
+     * Read an arbitrary number of bytes
+     *
+     * @param size The number of bytes to read
+     * @return a byte array of the specified size, filled with bytes read (unsigned)
+     * @throws IOException - if an IO error occurs
+     */
+    private byte[] readNextBytes(int size) throws IOException {
+        byte[] bytesRead = new byte[size];
+        for (int i = 0; i < bytesRead.length; i++) {
+            bytesRead[i] = (byte) this.nextByte();
+        }
+        return bytesRead;
+    }
+}

src/org/openstreetmap/josm/data/protobuf/ProtoBufRecord.java

@@ +1 @@
+// License: GPL. For details, see LICENSE file.
+package org.openstreetmap.josm.data.protobuf;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.stream.Stream;
+
+import org.openstreetmap.josm.tools.Utils;
+
+/**
+ * A protobuf record, storing the {@link WireType}, the parsed field number, and the bytes for it.
+ *
+ * @author Taylor Smock
+ * @since xxx
+ */
+public class ProtoBufRecord implements AutoCloseable {
+    private static final byte[] EMPTY_BYTES = {};
+    private final WireType type;
+    private final int field;
+    private byte[] bytes;
+
+    /**
+     * Create a new Protobuf record
+     *
+     * @param parser The parser to use to create the record
+     * @throws IOException - if an IO error occurs
+     */
+    public ProtoBufRecord(ProtoBufParser parser) throws IOException {
+        Number number = ProtoBufParser.convertByteArray(parser.nextVarInt(), ProtoBufParser.VAR_INT_BYTE_SIZE);
+        // I don't foresee having field numbers > {@code Integer#MAX_VALUE >> 3}
+        this.field = (int) number.longValue() >> 3;
+        // 7 is 111 (so last three bits)
+        byte wireType = (byte) (number.longValue() & 7);
+        this.type = Stream.of(WireType.values()).filter(wType -> wType.getTypeRepresentation() == wireType).findFirst()
+          .orElse(WireType.UNKNOWN);
+
+        if (this.type == WireType.VARINT) {
+            this.bytes = parser.nextVarInt();
+        } else if (this.type == WireType.SIXTY_FOUR_BIT) {
+            this.bytes = parser.nextFixed64();
+        } else if (this.type == WireType.THIRTY_TWO_BIT) {
+            this.bytes = parser.nextFixed32();
+        } else if (this.type == WireType.LENGTH_DELIMITED) {
+            this.bytes = parser.nextLengthDelimited();
+        } else {
+            this.bytes = EMPTY_BYTES;
+        }
+    }
+
+    /**
+     * Get as a double ({@link WireType#SIXTY_FOUR_BIT})
+     *
+     * @return the double
+     */
+    public double asDouble() {
+        long doubleNumber = ProtoBufParser.convertByteArray(asFixed64(), ProtoBufParser.BYTE_SIZE).longValue();
+        return Double.longBitsToDouble(doubleNumber);
+    }
+
+    /**
+     * Get as 32 bits ({@link WireType#THIRTY_TWO_BIT})
+     *
+     * @return a byte array of the 32 bits (4 bytes)
+     */
+    public byte[] asFixed32() {
+        // TODO verify, or just assume?
+        // 4 bytes == 32 bits
+        return this.bytes;
+    }
+
+    /**
+     * Get as 64 bits ({@link WireType#SIXTY_FOUR_BIT})
+     *
+     * @return a byte array of the 64 bits (8 bytes)
+     */
+    public byte[] asFixed64() {
+        // TODO verify, or just assume?
+        // 8 bytes == 64 bits
+        return this.bytes;
+    }
+
+    /**
+     * Get as a float ({@link WireType#THIRTY_TWO_BIT})
+     *
+     * @return the float
+     */
+    public float asFloat() {
+        int floatNumber = ProtoBufParser.convertByteArray(asFixed32(), ProtoBufParser.BYTE_SIZE).intValue();
+        return Float.intBitsToFloat(floatNumber);
+    }
+
+    /**
+     * Get the signed var int ({@code WireType#VARINT}).
+     * These are specially encoded so that they take up less space.
+     *
+     * @return The signed var int ({@code sint32} or {@code sint64})
+     */
+    public Number asSignedVarInt() {
+        final Number signed = this.asUnsignedVarInt();
+        return ProtoBufParser.decodeZigZag(signed);
+    }
+
+    /**
+     * Get as a string ({@link WireType#LENGTH_DELIMITED})
+     *
+     * @return The string (encoded as {@link StandardCharsets#UTF_8})
+     */
+    public String asString() {
+        return Utils.intern(new String(this.bytes, StandardCharsets.UTF_8));
+    }
+
+    /**
+     * Get the var int ({@code WireType#VARINT})
+     *
+     * @return The var int ({@code int32}, {@code int64}, {@code uint32}, {@code uint64}, {@code bool}, {@code enum})
+     */
+    public Number asUnsignedVarInt() {
+        return ProtoBufParser.convertByteArray(this.bytes, ProtoBufParser.VAR_INT_BYTE_SIZE);
+    }
+
+    @Override
+    public void close() {
+        this.bytes = null;
+    }
+
+    /**
+     * Get the raw bytes for this record
+     *
+     * @return The bytes
+     */
+    public byte[] getBytes() {
+        return this.bytes;
+    }
+
+    /**
+     * Get the field value
+     *
+     * @return The field value
+     */
+    public int getField() {
+        return this.field;
+    }
+
+    /**
+     * Get the WireType of the data
+     *
+     * @return The {@link WireType} of the data
+     */
+    public WireType getType() {
+        return this.type;
+    }
+}

src/org/openstreetmap/josm/data/protobuf/WireType.java

@@ +1 @@
+// License: GPL. For details, see LICENSE file.
+package org.openstreetmap.josm.data.protobuf;
+
+/**
+ * The WireTypes
+ *
+ * @author Taylor Smock
+ * @since xxx
+ */
+public enum WireType {
+    /**
+     * int32, int64, uint32, uint64, sing32, sint64, bool, enum
+     */
+    VARINT(0),
+    /**
+     * fixed64, sfixed64, double
+     */
+    SIXTY_FOUR_BIT(1),
+    /**
+     * string, bytes, embedded messages, packed repeated fields
+     */
+    LENGTH_DELIMITED(2),
+    /**
+     * start groups
+     *
+     * @deprecated Unknown reason. Deprecated since at least 2012.
+     */
+    @Deprecated
+    START_GROUP(3),
+    /**
+     * end groups
+     *
+     * @deprecated Unknown reason. Deprecated since at least 2012.
+     */
+    @Deprecated
+    END_GROUP(4),
+    /**
+     * fixed32, sfixed32, float
+     */
+    THIRTY_TWO_BIT(5),
+
+    /**
+     * For unknown WireTypes
+     */
+    UNKNOWN(Byte.MAX_VALUE);
+
+    private final byte type;
+
+    WireType(int value) {
+        this.type = (byte) value;
+    }
+
+    /**
+     * Get the type representation (byte form)
+     *
+     * @return The wire type byte representation
+     */
+    public byte getTypeRepresentation() {
+        return this.type;
+    }
+}

test/unit/org/openstreetmap/josm/data/protobuf/ProtoBufParserTest.java

@@ +1 @@
+// License: GPL. For details, see LICENSE file.
+package org.openstreetmap.josm.data.protobuf;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import org.junit.jupiter.api.Test;
+
+/**
+ * Test class for {@link ProtoBufParser}
+ * @author Taylor Smock
+ * @since xxx
+ */
+class ProtoBufParserTest {
+    /**
+     * Check that we are appropriately converting values to the "smallest" type
+     */
+    @Test
+    void testConvertLong() {
+        // No casting due to auto conversions
+        assertEquals(Byte.MAX_VALUE, ProtoBufParser.convertLong(Byte.MAX_VALUE));
+        assertEquals(Byte.MIN_VALUE, ProtoBufParser.convertLong(Byte.MIN_VALUE));
+        assertEquals(Short.MIN_VALUE, ProtoBufParser.convertLong(Short.MIN_VALUE));
+        assertEquals(Short.MAX_VALUE, ProtoBufParser.convertLong(Short.MAX_VALUE));
+        assertEquals(Integer.MAX_VALUE, ProtoBufParser.convertLong(Integer.MAX_VALUE));
+        assertEquals(Integer.MIN_VALUE, ProtoBufParser.convertLong(Integer.MIN_VALUE));
+        assertEquals(Long.MIN_VALUE, ProtoBufParser.convertLong(Long.MIN_VALUE));
+        assertEquals(Long.MAX_VALUE, ProtoBufParser.convertLong(Long.MAX_VALUE));
+    }
+
+    /**
+     * Check that zig zags are appropriately encoded.
+     */
+    @Test
+    void testEncodeZigZag() {
+        assertEquals(0, ProtoBufParser.encodeZigZag(0).byteValue());
+        assertEquals(1, ProtoBufParser.encodeZigZag(-1).byteValue());
+        assertEquals(2, ProtoBufParser.encodeZigZag(1).byteValue());
+        assertEquals(3, ProtoBufParser.encodeZigZag(-2).byteValue());
+        assertEquals(254, ProtoBufParser.encodeZigZag(Byte.MAX_VALUE).shortValue());
+        assertEquals(255, ProtoBufParser.encodeZigZag(Byte.MIN_VALUE).shortValue());
+        assertEquals(65_534, ProtoBufParser.encodeZigZag(Short.MAX_VALUE).intValue());
+        assertEquals(65_535, ProtoBufParser.encodeZigZag(Short.MIN_VALUE).intValue());
+        // These integers check a possible boundary condition (the boundary between using the 32/64 bit encoding methods)
+        assertEquals(4_294_967_292L, ProtoBufParser.encodeZigZag(Integer.MAX_VALUE - 1).longValue());
+        assertEquals(4_294_967_293L, ProtoBufParser.encodeZigZag(Integer.MIN_VALUE + 1).longValue());
+        assertEquals(4_294_967_294L, ProtoBufParser.encodeZigZag(Integer.MAX_VALUE).longValue());
+        assertEquals(4_294_967_295L, ProtoBufParser.encodeZigZag(Integer.MIN_VALUE).longValue());
+        assertEquals(4_294_967_296L, ProtoBufParser.encodeZigZag(Integer.MAX_VALUE + 1L).longValue());
+        assertEquals(4_294_967_297L, ProtoBufParser.encodeZigZag(Integer.MIN_VALUE - 1L).longValue());
+    }
+}

test/unit/org/openstreetmap/josm/data/protobuf/ProtoBufRecordTest.java

@@ +1 @@
+// License: GPL. For details, see LICENSE file.
+package org.openstreetmap.josm.data.protobuf;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.Test;
+
+/**
+ * Test class for specific {@link ProtoBufRecord} functionality
+ */
+class ProtoBufRecordTest {
+    @Test
+    void testFixed32() throws IOException {
+        ProtoBufParser parser = new ProtoBufParser(ProtoBufTest.toByteArray(new int[] {0x0d, 0x00, 0x00, 0x80, 0x3f}));
+        ProtoBufRecord thirtyTwoBit = new ProtoBufRecord(parser);
+        assertEquals(WireType.THIRTY_TWO_BIT, thirtyTwoBit.getType());
+        assertEquals(1f, thirtyTwoBit.asFloat());
+    }
+
+    @Test
+    void testUnknown() throws IOException {
+        ProtoBufParser parser = new ProtoBufParser(ProtoBufTest.toByteArray(new int[] {0x0f, 0x00, 0x00, 0x80, 0x3f}));
+        ProtoBufRecord unknown = new ProtoBufRecord(parser);
+        assertEquals(WireType.UNKNOWN, unknown.getType());
+        assertEquals(0, unknown.getBytes().length);
+    }
+}

test/unit/org/openstreetmap/josm/data/protobuf/ProtoBufTest.java

@@ +1 @@
+// License: GPL. For details, see LICENSE file.
+package org.openstreetmap.josm.data.protobuf;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.fail;
+
+import java.awt.geom.Ellipse2D;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Paths;
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.openstreetmap.josm.TestUtils;
+import org.openstreetmap.josm.data.coor.LatLon;
+import org.openstreetmap.josm.data.imagery.ImageryInfo;
+import org.openstreetmap.josm.data.imagery.vectortile.mapbox.Feature;
+import org.openstreetmap.josm.data.imagery.vectortile.mapbox.Layer;
+import org.openstreetmap.josm.data.imagery.vectortile.mapbox.MVTTile;
+import org.openstreetmap.josm.data.imagery.vectortile.mapbox.MapboxVectorTileSource;
+import org.openstreetmap.josm.data.osm.BBox;
+import org.openstreetmap.josm.data.osm.Node;
+import org.openstreetmap.josm.data.osm.Way;
+import org.openstreetmap.josm.data.vector.VectorDataSet;
+import org.openstreetmap.josm.data.vector.VectorNode;
+import org.openstreetmap.josm.data.vector.VectorWay;
+import org.openstreetmap.josm.io.Compression;
+import org.openstreetmap.josm.testutils.JOSMTestRules;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+/**
+ * Test class for {@link ProtoBufParser} and {@link ProtoBufRecord}
+ *
+ * @author Taylor Smock
+ * @since xxx
+ */
+class ProtoBufTest {
+    /**
+     * Convert an int array into a byte array
+     * @param intArray The int array to convert (NOTE: numbers must be below 255)
+     * @return A byte array that can be used
+     */
+    static byte[] toByteArray(int[] intArray) {
+        byte[] byteArray = new byte[intArray.length];
+        for (int i = 0; i < intArray.length; i++) {
+            if (intArray[i] > Byte.MAX_VALUE - Byte.MIN_VALUE) {
+                throw new IllegalArgumentException();
+            }
+            byteArray[i] = Integer.valueOf(intArray[i]).byteValue();
+        }
+        return byteArray;
+    }
+
+    @RegisterExtension
+    JOSMTestRules josmTestRules = new JOSMTestRules().preferences();
+
+    private Number bytesToVarInt(int... bytes) {
+        byte[] byteArray = new byte[bytes.length];
+        for (int i = 0; i < bytes.length; i++) {
+            byteArray[i] = (byte) bytes[i];
+        }
+        return ProtoBufParser.convertByteArray(byteArray, ProtoBufParser.VAR_INT_BYTE_SIZE);
+    }
+
+    /**
+     * Test reading tile from Mapillary ( 14/3251/6258 )
+     *
+     * @throws IOException if there is a problem reading the file
+     */
+    @Test
+    void testRead_14_3251_6258() throws IOException {
+        File vectorTile = Paths.get(TestUtils.getTestDataRoot(), "pbf", "mapillary", "14", "3251", "6258.mvt").toFile();
+        InputStream inputStream = Compression.getUncompressedFileInputStream(vectorTile);
+        Collection<ProtoBufRecord> records = new ProtoBufParser(inputStream).allRecords();
+        assertEquals(2, records.size());
+        List<Layer> layers = new ArrayList<>();
+        for (ProtoBufRecord record : records) {
+            if (record.getField() == Layer.LAYER_FIELD) {
+                layers.add(new Layer(record.getBytes()));
+            } else {
+                fail(MessageFormat.format("Invalid field {0}", record.getField()));
+            }
+        }
+        Layer mapillarySequences = layers.get(0);
+        Layer mapillaryPictures = layers.get(1);
+        assertEquals("mapillary-sequences", mapillarySequences.getName());
+        assertEquals("mapillary-images", mapillaryPictures.getName());
+        assertEquals(2048, mapillarySequences.getExtent());
+        assertEquals(2048, mapillaryPictures.getExtent());
+
+        assertEquals(1,
+                mapillarySequences.getFeatures().stream().filter(feature -> feature.getId() == 241083111).count());
+        Feature testSequence = mapillarySequences.getFeatures().stream().filter(feature -> feature.getId() == 241083111)
+                .findAny().orElse(null);
+        assertEquals("jgxkXqVFM4jepMG3vP5Q9A", testSequence.getTags().get("key"));
+        assertEquals("C15Ul6qVMfQFlzRcmQCLcA", testSequence.getTags().get("ikey"));
+        assertEquals("x0hTY8cakpy0m3ui1GaG1A", testSequence.getTags().get("userkey"));
+        assertEquals(Long.valueOf(1565196718638L), Long.valueOf(testSequence.getTags().get("captured_at")));
+        assertEquals(0, Integer.parseInt(testSequence.getTags().get("pano")));
+    }
+
+    @Test
+    void testRead_17_26028_50060() throws IOException {
+        File vectorTile = Paths.get(TestUtils.getTestDataRoot(), "pbf", "openinframap", "17", "26028", "50060.pbf")
+                .toFile();
+        InputStream inputStream = Compression.getUncompressedFileInputStream(vectorTile);
+        Collection<ProtoBufRecord> records = new ProtoBufParser(inputStream).allRecords();
+        List<Layer> layers = new ArrayList<>();
+        for (ProtoBufRecord record : records) {
+            if (record.getField() == Layer.LAYER_FIELD) {
+                layers.add(new Layer(record.getBytes()));
+            } else {
+                fail(MessageFormat.format("Invalid field {0}", record.getField()));
+            }
+        }
+        assertEquals(19, layers.size());
+        List<Layer> dataLayers = layers.stream().filter(layer -> !layer.getFeatures().isEmpty())
+                .collect(Collectors.toList());
+        // power_plant, power_plant_point, power_generator, power_heatmap_solar, and power_generator_area
+        assertEquals(5, dataLayers.size());
+
+        // power_generator_area was rendered incorrectly
+        final Layer powerGeneratorArea = dataLayers.stream()
+                .filter(layer -> "power_generator_area".equals(layer.getName())).findAny().orElse(null);
+        assertNotNull(powerGeneratorArea);
+        final int extent = powerGeneratorArea.getExtent();
+        // 17/26028/50060 bounds
+        VectorDataSet vectorDataSet = new VectorDataSet();
+        MVTTile vectorTile1 = new MVTTile(new MapboxVectorTileSource(new ImageryInfo("Test info", "example.org")),
+                26028, 50060, 17);
+        vectorTile1.loadImage(Compression.getUncompressedFileInputStream(vectorTile));
+        vectorDataSet.addTileData(vectorTile1);
+        vectorDataSet.setZoom(17);
+        final Way one = new Way();
+        one.addNode(new Node(new LatLon(39.0687509, -108.5100816)));
+        one.addNode(new Node(new LatLon(39.0687509, -108.5095751)));
+        one.addNode(new Node(new LatLon(39.0687169, -108.5095751)));
+        one.addNode(new Node(new LatLon(39.0687169, -108.5100816)));
+        one.addNode(one.getNode(0));
+        one.setOsmId(666293899, 2);
+        final BBox searchBBox = one.getBBox();
+        searchBBox.addPrimitive(one, 0.00001);
+        final Collection<VectorNode> searchedNodes = vectorDataSet.searchNodes(searchBBox);
+        final Collection<VectorWay> searchedWays = vectorDataSet.searchWays(searchBBox);
+        assertEquals(4, searchedNodes.size());
+    }
+
+    @Test
+    void testReadVarInt() {
+        assertEquals(ProtoBufParser.convertLong(0), bytesToVarInt(0x0));
+        assertEquals(ProtoBufParser.convertLong(1), bytesToVarInt(0x1));
+        assertEquals(ProtoBufParser.convertLong(127), bytesToVarInt(0x7f));
+        // This should b 0xff 0xff 0xff 0xff 0x07, but we drop the leading bit when reading to a byte array
+        Number actual = bytesToVarInt(0x7f, 0x7f, 0x7f, 0x7f, 0x07);
+        assertEquals(ProtoBufParser.convertLong(Integer.MAX_VALUE), actual,
+                MessageFormat.format("Expected {0} but got {1}", Integer.toBinaryString(Integer.MAX_VALUE),
+                        Long.toBinaryString(actual.longValue())));
+    }
+
+    /**
+     * Test simple message.
+     * Check that a simple message is readable
+     *
+     * @throws IOException - if an IO error occurs
+     */
+    @Test
+    void testSimpleMessage() throws IOException {
+        ProtoBufParser parser = new ProtoBufParser(new byte[] { (byte) 0x08, (byte) 0x96, (byte) 0x01 });
+        ProtoBufRecord record = new ProtoBufRecord(parser);
+        assertEquals(WireType.VARINT, record.getType());
+        assertEquals(150, record.asUnsignedVarInt().intValue());
+    }
+
+    @Test
+    void testSingletonMultiPoint() throws IOException {
+        Collection<ProtoBufRecord> records = new ProtoBufParser(new ByteArrayInputStream(toByteArray(
+                new int[] { 0x1a, 0x2c, 0x78, 0x02, 0x0a, 0x03, 0x74, 0x6d, 0x70, 0x28, 0x80, 0x20, 0x1a, 0x04, 0x6e,
+                        0x61, 0x6d, 0x65, 0x22, 0x0b, 0x0a, 0x09, 0x54, 0x65, 0x73, 0x74, 0x20, 0x6e, 0x61, 0x6d, 0x65,
+                        0x12, 0x0d, 0x18, 0x01, 0x12, 0x02, 0x00, 0x00, 0x22, 0x05, 0x09, 0xe0, 0x3e, 0x84, 0x27 })))
+                                .allRecords();
+        List<Layer> layers = new ArrayList<>();
+        for (ProtoBufRecord record : records) {
+            if (record.getField() == Layer.LAYER_FIELD) {
+                layers.add(new Layer(record.getBytes()));
+            } else {
+                fail(MessageFormat.format("Invalid field {0}", record.getField()));
+            }
+        }
+        assertEquals(1, layers.size());
+        assertEquals(1, layers.get(0).getGeometry().size());
+        Ellipse2D shape = (Ellipse2D) layers.get(0).getGeometry().iterator().next().getShapes().iterator().next();
+        assertEquals(4016, shape.getCenterX());
+        assertEquals(2498, shape.getCenterY());
+    }
+
+    @Test
+    void testZigZag() {
+        assertEquals(0, ProtoBufParser.decodeZigZag(0).intValue());
+        assertEquals(-1, ProtoBufParser.decodeZigZag(1).intValue());
+        assertEquals(1, ProtoBufParser.decodeZigZag(2).intValue());
+        assertEquals(-2, ProtoBufParser.decodeZigZag(3).intValue());
+    }
+}

test/unit/org/openstreetmap/josm/data/protobuf/ProtoBufTest.java

@@ +1 @@
+// License: GPL. For details, see LICENSE file.
+package org.openstreetmap.josm.data.protobuf;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.fail;
+
+import java.awt.geom.Ellipse2D;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Paths;
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.openstreetmap.josm.TestUtils;
+import org.openstreetmap.josm.data.coor.LatLon;
+import org.openstreetmap.josm.data.imagery.ImageryInfo;
+import org.openstreetmap.josm.data.imagery.vectortile.mapbox.Feature;
+import org.openstreetmap.josm.data.imagery.vectortile.mapbox.Layer;
+import org.openstreetmap.josm.data.imagery.vectortile.mapbox.MVTTile;
+import org.openstreetmap.josm.data.imagery.vectortile.mapbox.MapboxVectorTileSource;
+import org.openstreetmap.josm.data.osm.BBox;
+import org.openstreetmap.josm.data.osm.Node;
+import org.openstreetmap.josm.data.osm.Way;
+import org.openstreetmap.josm.data.vector.VectorDataSet;
+import org.openstreetmap.josm.data.vector.VectorNode;
+import org.openstreetmap.josm.data.vector.VectorWay;
+import org.openstreetmap.josm.io.Compression;
+import org.openstreetmap.josm.testutils.JOSMTestRules;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+/**
+ * Test class for {@link ProtoBufParser} and {@link ProtoBufRecord}
+ *
+ * @author Taylor Smock
+ * @since xxx
+ */
+class ProtoBufTest {
+    /**
+     * Convert an int array into a byte array
+     * @param intArray The int array to convert (NOTE: numbers must be below 255)
+     * @return A byte array that can be used
+     */
+    static byte[] toByteArray(int[] intArray) {
+        byte[] byteArray = new byte[intArray.length];
+        for (int i = 0; i < intArray.length; i++) {
+            if (intArray[i] > Byte.MAX_VALUE - Byte.MIN_VALUE) {
+                throw new IllegalArgumentException();
+            }
+            byteArray[i] = Integer.valueOf(intArray[i]).byteValue();
+        }
+        return byteArray;
+    }
+
+    @RegisterExtension
+    JOSMTestRules josmTestRules = new JOSMTestRules().preferences();
+
+    private Number bytesToVarInt(int... bytes) {
+        byte[] byteArray = new byte[bytes.length];
+        for (int i = 0; i < bytes.length; i++) {
+            byteArray[i] = (byte) bytes[i];
+        }
+        return ProtoBufParser.convertByteArray(byteArray, ProtoBufParser.VAR_INT_BYTE_SIZE);
+    }
+
+    /**
+     * Test reading tile from Mapillary ( 14/3251/6258 )
+     *
+     * @throws IOException if there is a problem reading the file
+     */
+    @Test
+    void testRead_14_3251_6258() throws IOException {
+        File vectorTile = Paths.get(TestUtils.getTestDataRoot(), "pbf", "mapillary", "14", "3251", "6258.mvt").toFile();
+        InputStream inputStream = Compression.getUncompressedFileInputStream(vectorTile);
+        Collection<ProtoBufRecord> records = new ProtoBufParser(inputStream).allRecords();
+        assertEquals(2, records.size());
+        List<Layer> layers = new ArrayList<>();
+        for (ProtoBufRecord record : records) {
+            if (record.getField() == Layer.LAYER_FIELD) {
+                layers.add(new Layer(record.getBytes()));
+            } else {
+                fail(MessageFormat.format("Invalid field {0}", record.getField()));
+            }
+        }
+        Layer mapillarySequences = layers.get(0);
+        Layer mapillaryPictures = layers.get(1);
+        assertEquals("mapillary-sequences", mapillarySequences.getName());
+        assertEquals("mapillary-images", mapillaryPictures.getName());
+        assertEquals(2048, mapillarySequences.getExtent());
+        assertEquals(2048, mapillaryPictures.getExtent());
+
+        assertEquals(1,
+                mapillarySequences.getFeatures().stream().filter(feature -> feature.getId() == 241083111).count());
+        Feature testSequence = mapillarySequences.getFeatures().stream().filter(feature -> feature.getId() == 241083111)
+                .findAny().orElse(null);
+        assertEquals("jgxkXqVFM4jepMG3vP5Q9A", testSequence.getTags().get("key"));
+        assertEquals("C15Ul6qVMfQFlzRcmQCLcA", testSequence.getTags().get("ikey"));
+        assertEquals("x0hTY8cakpy0m3ui1GaG1A", testSequence.getTags().get("userkey"));
+        assertEquals(Long.valueOf(1565196718638L), Long.valueOf(testSequence.getTags().get("captured_at")));
+        assertEquals(0, Integer.parseInt(testSequence.getTags().get("pano")));
+    }
+
+    @Test
+    void testRead_17_26028_50060() throws IOException {
+        File vectorTile = Paths.get(TestUtils.getTestDataRoot(), "pbf", "openinframap", "17", "26028", "50060.pbf")
+                .toFile();
+        InputStream inputStream = Compression.getUncompressedFileInputStream(vectorTile);
+        Collection<ProtoBufRecord> records = new ProtoBufParser(inputStream).allRecords();
+        List<Layer> layers = new ArrayList<>();
+        for (ProtoBufRecord record : records) {
+            if (record.getField() == Layer.LAYER_FIELD) {
+                layers.add(new Layer(record.getBytes()));
+            } else {
+                fail(MessageFormat.format("Invalid field {0}", record.getField()));
+            }
+        }
+        assertEquals(19, layers.size());
+        List<Layer> dataLayers = layers.stream().filter(layer -> !layer.getFeatures().isEmpty())
+                .collect(Collectors.toList());
+        // power_plant, power_plant_point, power_generator, power_heatmap_solar, and power_generator_area
+        assertEquals(5, dataLayers.size());
+
+        // power_generator_area was rendered incorrectly
+        final Layer powerGeneratorArea = dataLayers.stream()
+                .filter(layer -> "power_generator_area".equals(layer.getName())).findAny().orElse(null);
+        assertNotNull(powerGeneratorArea);
+        final int extent = powerGeneratorArea.getExtent();
+        // 17/26028/50060 bounds
+        VectorDataSet vectorDataSet = new VectorDataSet();
+        MVTTile vectorTile1 = new MVTTile(new MapboxVectorTileSource(new ImageryInfo("Test info", "example.org")),
+                26028, 50060, 17);
+        vectorTile1.loadImage(Compression.getUncompressedFileInputStream(vectorTile));
+        vectorDataSet.addTileData(vectorTile1);
+        vectorDataSet.setZoom(17);
+        final Way one = new Way();
+        one.addNode(new Node(new LatLon(39.0687509, -108.5100816)));
+        one.addNode(new Node(new LatLon(39.0687509, -108.5095751)));
+        one.addNode(new Node(new LatLon(39.0687169, -108.5095751)));
+        one.addNode(new Node(new LatLon(39.0687169, -108.5100816)));
+        one.addNode(one.getNode(0));
+        one.setOsmId(666293899, 2);
+        final BBox searchBBox = one.getBBox();
+        searchBBox.addPrimitive(one, 0.00001);
+        final Collection<VectorNode> searchedNodes = vectorDataSet.searchNodes(searchBBox);
+        final Collection<VectorWay> searchedWays = vectorDataSet.searchWays(searchBBox);
+        assertEquals(4, searchedNodes.size());
+    }
+
+    @Test
+    void testReadVarInt() {
+        assertEquals(ProtoBufParser.convertLong(0), bytesToVarInt(0x0));
+        assertEquals(ProtoBufParser.convertLong(1), bytesToVarInt(0x1));
+        assertEquals(ProtoBufParser.convertLong(127), bytesToVarInt(0x7f));
+        // This should b 0xff 0xff 0xff 0xff 0x07, but we drop the leading bit when reading to a byte array
+        Number actual = bytesToVarInt(0x7f, 0x7f, 0x7f, 0x7f, 0x07);
+        assertEquals(ProtoBufParser.convertLong(Integer.MAX_VALUE), actual,
+                MessageFormat.format("Expected {0} but got {1}", Integer.toBinaryString(Integer.MAX_VALUE),
+                        Long.toBinaryString(actual.longValue())));
+    }
+
+    /**
+     * Test simple message.
+     * Check that a simple message is readable
+     *
+     * @throws IOException - if an IO error occurs
+     */
+    @Test
+    void testSimpleMessage() throws IOException {
+        ProtoBufParser parser = new ProtoBufParser(new byte[] { (byte) 0x08, (byte) 0x96, (byte) 0x01 });
+        ProtoBufRecord record = new ProtoBufRecord(parser);
+        assertEquals(WireType.VARINT, record.getType());
+        assertEquals(150, record.asUnsignedVarInt().intValue());
+    }
+
+    @Test
+    void testSingletonMultiPoint() throws IOException {
+        Collection<ProtoBufRecord> records = new ProtoBufParser(new ByteArrayInputStream(toByteArray(
+                new int[] { 0x1a, 0x2c, 0x78, 0x02, 0x0a, 0x03, 0x74, 0x6d, 0x70, 0x28, 0x80, 0x20, 0x1a, 0x04, 0x6e,
+                        0x61, 0x6d, 0x65, 0x22, 0x0b, 0x0a, 0x09, 0x54, 0x65, 0x73, 0x74, 0x20, 0x6e, 0x61, 0x6d, 0x65,
+                        0x12, 0x0d, 0x18, 0x01, 0x12, 0x02, 0x00, 0x00, 0x22, 0x05, 0x09, 0xe0, 0x3e, 0x84, 0x27 })))
+                                .allRecords();
+        List<Layer> layers = new ArrayList<>();
+        for (ProtoBufRecord record : records) {
+            if (record.getField() == Layer.LAYER_FIELD) {
+                layers.add(new Layer(record.getBytes()));
+            } else {
+                fail(MessageFormat.format("Invalid field {0}", record.getField()));
+            }
+        }
+        assertEquals(1, layers.size());
+        assertEquals(1, layers.get(0).getGeometry().size());
+        Ellipse2D shape = (Ellipse2D) layers.get(0).getGeometry().iterator().next().getShapes().iterator().next();
+        assertEquals(4016, shape.getCenterX());
+        assertEquals(2498, shape.getCenterY());
+    }
+
+    @Test
+    void testZigZag() {
+        assertEquals(0, ProtoBufParser.decodeZigZag(0).intValue());
+        assertEquals(-1, ProtoBufParser.decodeZigZag(1).intValue());
+        assertEquals(1, ProtoBufParser.decodeZigZag(2).intValue());
+        assertEquals(-2, ProtoBufParser.decodeZigZag(3).intValue());
+    }
+}

test/unit/org/openstreetmap/josm/data/protobuf/ProtoBufParserTest.java

@@ +1 @@
+// License: GPL. For details, see LICENSE file.
+package org.openstreetmap.josm.data.protobuf;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import org.junit.jupiter.api.Test;
+
+/**
+ * Test class for {@link ProtoBufParser}
+ * @author Taylor Smock
+ * @since xxx
+ */
+class ProtoBufParserTest {
+    /**
+     * Check that we are appropriately converting values to the "smallest" type
+     */
+    @Test
+    void testConvertLong() {
+        // No casting due to auto conversions
+        assertEquals(Byte.MAX_VALUE, ProtoBufParser.convertLong(Byte.MAX_VALUE));
+        assertEquals(Byte.MIN_VALUE, ProtoBufParser.convertLong(Byte.MIN_VALUE));
+        assertEquals(Short.MIN_VALUE, ProtoBufParser.convertLong(Short.MIN_VALUE));
+        assertEquals(Short.MAX_VALUE, ProtoBufParser.convertLong(Short.MAX_VALUE));
+        assertEquals(Integer.MAX_VALUE, ProtoBufParser.convertLong(Integer.MAX_VALUE));
+        assertEquals(Integer.MIN_VALUE, ProtoBufParser.convertLong(Integer.MIN_VALUE));
+        assertEquals(Long.MIN_VALUE, ProtoBufParser.convertLong(Long.MIN_VALUE));
+        assertEquals(Long.MAX_VALUE, ProtoBufParser.convertLong(Long.MAX_VALUE));
+    }
+
+    /**
+     * Check that zig zags are appropriately encoded.
+     */
+    @Test
+    void testEncodeZigZag() {
+        assertEquals(0, ProtoBufParser.encodeZigZag(0).byteValue());
+        assertEquals(1, ProtoBufParser.encodeZigZag(-1).byteValue());
+        assertEquals(2, ProtoBufParser.encodeZigZag(1).byteValue());
+        assertEquals(3, ProtoBufParser.encodeZigZag(-2).byteValue());
+        assertEquals(254, ProtoBufParser.encodeZigZag(Byte.MAX_VALUE).shortValue());
+        assertEquals(255, ProtoBufParser.encodeZigZag(Byte.MIN_VALUE).shortValue());
+        assertEquals(65_534, ProtoBufParser.encodeZigZag(Short.MAX_VALUE).intValue());
+        assertEquals(65_535, ProtoBufParser.encodeZigZag(Short.MIN_VALUE).intValue());
+        // These integers check a possible boundary condition (the boundary between using the 32/64 bit encoding methods)
+        assertEquals(4_294_967_292L, ProtoBufParser.encodeZigZag(Integer.MAX_VALUE - 1).longValue());
+        assertEquals(4_294_967_293L, ProtoBufParser.encodeZigZag(Integer.MIN_VALUE + 1).longValue());
+        assertEquals(4_294_967_294L, ProtoBufParser.encodeZigZag(Integer.MAX_VALUE).longValue());
+        assertEquals(4_294_967_295L, ProtoBufParser.encodeZigZag(Integer.MIN_VALUE).longValue());
+        assertEquals(4_294_967_296L, ProtoBufParser.encodeZigZag(Integer.MAX_VALUE + 1L).longValue());
+        assertEquals(4_294_967_297L, ProtoBufParser.encodeZigZag(Integer.MIN_VALUE - 1L).longValue());
+    }
+}

test/unit/org/openstreetmap/josm/data/protobuf/ProtoBufRecordTest.java

@@ +1 @@
+// License: GPL. For details, see LICENSE file.
+package org.openstreetmap.josm.data.protobuf;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.Test;
+
+/**
+ * Test class for specific {@link ProtoBufRecord} functionality
+ */
+class ProtoBufRecordTest {
+    @Test
+    void testFixed32() throws IOException {
+        ProtoBufParser parser = new ProtoBufParser(ProtoBufTest.toByteArray(new int[] {0x0d, 0x00, 0x00, 0x80, 0x3f}));
+        ProtoBufRecord thirtyTwoBit = new ProtoBufRecord(parser);
+        assertEquals(WireType.THIRTY_TWO_BIT, thirtyTwoBit.getType());
+        assertEquals(1f, thirtyTwoBit.asFloat());
+    }
+
+    @Test
+    void testUnknown() throws IOException {
+        ProtoBufParser parser = new ProtoBufParser(ProtoBufTest.toByteArray(new int[] {0x0f, 0x00, 0x00, 0x80, 0x3f}));
+        ProtoBufRecord unknown = new ProtoBufRecord(parser);
+        assertEquals(WireType.UNKNOWN, unknown.getType());
+        assertEquals(0, unknown.getBytes().length);
+    }
+}