001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.data.protobuf;
003
004import java.io.BufferedInputStream;
005import java.io.ByteArrayInputStream;
006import java.io.ByteArrayOutputStream;
007import java.io.IOException;
008import java.io.InputStream;
009import java.util.ArrayList;
010import java.util.Arrays;
011import java.util.Collection;
012
013import org.openstreetmap.josm.tools.Logging;
014
015/**
016 * A basic Protobuf parser
017 *
018 * @author Taylor Smock
019 * @since 17862
020 */
021public class ProtobufParser implements AutoCloseable {
022    /**
023     * The default byte size (see {@link #VAR_INT_BYTE_SIZE} for var ints)
024     */
025    public static final byte BYTE_SIZE = 8;
026    /**
027     * The byte size for var ints (since the first byte is just an indicator for if the var int is done)
028     */
029    public static final byte VAR_INT_BYTE_SIZE = BYTE_SIZE - 1;
030    /**
031     * Used to get the most significant byte
032     */
033    static final byte MOST_SIGNIFICANT_BYTE = (byte) (1 << 7);
034    private static final byte[] EMPTY_BYTE_ARRAY = new byte[0];
035
036    /**
037     * Convert a byte array to a number (little endian)
038     *
039     * @param bytes    The bytes to convert
040     * @param byteSize The size of the byte. For var ints, this is 7, for other ints, this is 8.
041     * @return An appropriate {@link Number} class.
042     */
043    public static Number convertByteArray(byte[] bytes, byte byteSize) {
044        return convertLong(convertByteArray(bytes, byteSize, 0, bytes.length));
045    }
046
047    /**
048     * Convert a byte array to a number (little endian)
049     *
050     * @param bytes    The bytes to convert
051     * @param byteSize The size of the byte. For var ints, this is 7, for other ints, this is 8.
052     * @param start    The start position in the byte array
053     * @param end      The end position in the byte array (exclusive - [start, end) )
054     * @return the number from the byte array. Depending upon length of time the number will be stored, narrowing may be helpful.
055     * @since 18695
056     */
057    public static long convertByteArray(byte[] bytes, byte byteSize, int start, int end) {
058        long number = 0;
059        for (int i = start; i < end; i++) {
060            // Need to convert to uint64 in order to avoid bit operation from filling in 1's and overflow issues
061            number += Byte.toUnsignedLong(bytes[i]) << (byteSize * (i - start));
062        }
063        return number;
064    }
065
066    /**
067     * Convert a long to an appropriate {@link Number} class
068     *
069     * @param number The long to convert
070     * @return A {@link Number}
071     */
072    public static Number convertLong(long number) {
073        // TODO deal with booleans
074        if (number <= Byte.MAX_VALUE && number >= Byte.MIN_VALUE) {
075            return (byte) number;
076        } else if (number <= Short.MAX_VALUE && number >= Short.MIN_VALUE) {
077            return (short) number;
078        } else if (number <= Integer.MAX_VALUE && number >= Integer.MIN_VALUE) {
079            return (int) number;
080        }
081        return number;
082    }
083
084    /**
085     * Decode a zig-zag encoded value
086     *
087     * @param signed The value to decode
088     * @return The decoded value
089     */
090    public static Number decodeZigZag(Number signed) {
091        return convertLong(decodeZigZag(signed.longValue()));
092    }
093
094    /**
095     * Decode a zig-zag encoded value
096     *
097     * @param signed The value to decode
098     * @return The decoded value
099     * @since 18695
100     */
101    public static long decodeZigZag(long signed) {
102        return (signed >>> 1) ^ -(signed & 1);
103    }
104
105    /**
106     * Encode a number to a zig-zag encode value
107     *
108     * @param signed The number to encode
109     * @return The encoded value
110     */
111    public static Number encodeZigZag(Number signed) {
112        final long value = signed.longValue();
113        // This boundary condition could be >= or <= or both. Tests indicate that it doesn't actually matter.
114        // The only difference would be the number type returned, except it is always converted to the most basic type.
115        final int shift = (value > Integer.MAX_VALUE || value < Integer.MIN_VALUE ? Long.BYTES : Integer.BYTES) * 8 - 1;
116        return convertLong((value << 1) ^ (value >> shift));
117    }
118
119    private final InputStream inputStream;
120
121    /**
122     * Create a new parser
123     *
124     * @param bytes The bytes to parse
125     */
126    public ProtobufParser(byte[] bytes) {
127        this(new ByteArrayInputStream(bytes));
128    }
129
130    /**
131     * Create a new parser
132     *
133     * @param inputStream The InputStream (will be fully read at this time)
134     */
135    public ProtobufParser(InputStream inputStream) {
136        if (inputStream.markSupported()) {
137            this.inputStream = inputStream;
138        } else {
139            this.inputStream = new BufferedInputStream(inputStream);
140        }
141    }
142
143    /**
144     * Read all records
145     *
146     * @return A collection of all records
147     * @throws IOException - if an IO error occurs
148     */
149    public Collection<ProtobufRecord> allRecords() throws IOException {
150        Collection<ProtobufRecord> records = new ArrayList<>();
151        ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(4);
152        while (this.hasNext()) {
153            records.add(new ProtobufRecord(byteArrayOutputStream, this));
154        }
155        return records;
156    }
157
158    @Override
159    public void close() {
160        try {
161            this.inputStream.close();
162        } catch (IOException e) {
163            Logging.error(e);
164        }
165    }
166
167    /**
168     * Check if there is more data to read
169     *
170     * @return {@code true} if there is more data to read
171     * @throws IOException - if an IO error occurs
172     */
173    public boolean hasNext() throws IOException {
174        return this.inputStream.available() > 0;
175    }
176
177    /**
178     * Get the "next" WireType
179     *
180     * @return {@link WireType} expected
181     * @throws IOException - if an IO error occurs
182     */
183    public WireType next() throws IOException {
184        this.inputStream.mark(16);
185        try {
186            return WireType.getAllValues()[this.inputStream.read() << 3];
187        } finally {
188            this.inputStream.reset();
189        }
190    }
191
192    /**
193     * Get the next byte
194     *
195     * @return The next byte
196     * @throws IOException - if an IO error occurs
197     */
198    public int nextByte() throws IOException {
199        return this.inputStream.read();
200    }
201
202    /**
203     * Get the next 32 bits ({@link WireType#THIRTY_TWO_BIT})
204     *
205     * @return a byte array of the next 32 bits (4 bytes)
206     * @throws IOException - if an IO error occurs
207     */
208    public byte[] nextFixed32() throws IOException {
209        // 4 bytes == 32 bits
210        return readNextBytes(4);
211    }
212
213    /**
214     * Get the next 64 bits ({@link WireType#SIXTY_FOUR_BIT})
215     *
216     * @return a byte array of the next 64 bits (8 bytes)
217     * @throws IOException - if an IO error occurs
218     */
219    public byte[] nextFixed64() throws IOException {
220        // 8 bytes == 64 bits
221        return readNextBytes(8);
222    }
223
224    /**
225     * Get the next delimited message ({@link WireType#LENGTH_DELIMITED})
226     *
227     * @param byteArrayOutputStream A reusable stream to write bytes to. This can significantly reduce the allocations
228     *                              (150 MB to 95 MB in a test area).
229     * @return The next length delimited message
230     * @throws IOException - if an IO error occurs
231     */
232    public byte[] nextLengthDelimited(ByteArrayOutputStream byteArrayOutputStream) throws IOException {
233        final byte[] nextVarInt = this.nextVarInt(byteArrayOutputStream);
234        int length = (int) convertByteArray(nextVarInt, VAR_INT_BYTE_SIZE, 0, nextVarInt.length);
235        return readNextBytes(length);
236    }
237
238    /**
239     * Get the next var int ({@code WireType#VARINT})
240     *
241     * @param byteArrayOutputStream A reusable stream to write bytes to. This can significantly reduce the allocations
242     *                              (150 MB to 95 MB in a test area).
243     * @return The next var int ({@code int32}, {@code int64}, {@code uint32}, {@code uint64}, {@code bool}, {@code enum})
244     * @throws IOException - if an IO error occurs
245     */
246    public byte[] nextVarInt(ByteArrayOutputStream byteArrayOutputStream) throws IOException {
247        // Using this reduces the allocations from 150 MB to 95 MB.
248        int currentByte = this.nextByte();
249        while ((byte) (currentByte & MOST_SIGNIFICANT_BYTE) == MOST_SIGNIFICANT_BYTE && currentByte > 0) {
250            // Get rid of the leading bit (shift left 1, then shift right 1 unsigned)
251            byteArrayOutputStream.write((currentByte ^ MOST_SIGNIFICANT_BYTE));
252            currentByte = this.nextByte();
253        }
254        // The last byte doesn't drop the most significant bit
255        byteArrayOutputStream.write(currentByte);
256        try {
257            return byteArrayOutputStream.toByteArray();
258        } finally {
259            byteArrayOutputStream.reset();
260        }
261    }
262
263    /**
264     * Read an arbitrary number of bytes
265     *
266     * @param size The number of bytes to read
267     * @return a byte array filled with bytes read (unsigned)
268     * @throws IOException - if an IO error occurs
269     */
270    private byte[] readNextBytes(int size) throws IOException {
271        byte[] bytesRead = new byte[size];
272        int read = this.inputStream.read(bytesRead);
273        if (read == -1) {
274            return EMPTY_BYTE_ARRAY;
275        } else if (read != size) {
276            return Arrays.copyOf(bytesRead, read);
277        }
278        return bytesRead;
279    }
280}