001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.data.protobuf; 003 004import java.io.BufferedInputStream; 005import java.io.ByteArrayInputStream; 006import java.io.ByteArrayOutputStream; 007import java.io.IOException; 008import java.io.InputStream; 009import java.util.ArrayList; 010import java.util.Arrays; 011import java.util.Collection; 012 013import org.openstreetmap.josm.tools.Logging; 014 015/** 016 * A basic Protobuf parser 017 * 018 * @author Taylor Smock 019 * @since 17862 020 */ 021public class ProtobufParser implements AutoCloseable { 022 /** 023 * The default byte size (see {@link #VAR_INT_BYTE_SIZE} for var ints) 024 */ 025 public static final byte BYTE_SIZE = 8; 026 /** 027 * The byte size for var ints (since the first byte is just an indicator for if the var int is done) 028 */ 029 public static final byte VAR_INT_BYTE_SIZE = BYTE_SIZE - 1; 030 /** 031 * Used to get the most significant byte 032 */ 033 static final byte MOST_SIGNIFICANT_BYTE = (byte) (1 << 7); 034 private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; 035 036 /** 037 * Convert a byte array to a number (little endian) 038 * 039 * @param bytes The bytes to convert 040 * @param byteSize The size of the byte. For var ints, this is 7, for other ints, this is 8. 041 * @return An appropriate {@link Number} class. 042 */ 043 public static Number convertByteArray(byte[] bytes, byte byteSize) { 044 return convertLong(convertByteArray(bytes, byteSize, 0, bytes.length)); 045 } 046 047 /** 048 * Convert a byte array to a number (little endian) 049 * 050 * @param bytes The bytes to convert 051 * @param byteSize The size of the byte. For var ints, this is 7, for other ints, this is 8. 052 * @param start The start position in the byte array 053 * @param end The end position in the byte array (exclusive - [start, end) ) 054 * @return the number from the byte array. Depending upon length of time the number will be stored, narrowing may be helpful. 055 * @since 18695 056 */ 057 public static long convertByteArray(byte[] bytes, byte byteSize, int start, int end) { 058 long number = 0; 059 for (int i = start; i < end; i++) { 060 // Need to convert to uint64 in order to avoid bit operation from filling in 1's and overflow issues 061 number += Byte.toUnsignedLong(bytes[i]) << (byteSize * (i - start)); 062 } 063 return number; 064 } 065 066 /** 067 * Convert a long to an appropriate {@link Number} class 068 * 069 * @param number The long to convert 070 * @return A {@link Number} 071 */ 072 public static Number convertLong(long number) { 073 // TODO deal with booleans 074 if (number <= Byte.MAX_VALUE && number >= Byte.MIN_VALUE) { 075 return (byte) number; 076 } else if (number <= Short.MAX_VALUE && number >= Short.MIN_VALUE) { 077 return (short) number; 078 } else if (number <= Integer.MAX_VALUE && number >= Integer.MIN_VALUE) { 079 return (int) number; 080 } 081 return number; 082 } 083 084 /** 085 * Decode a zig-zag encoded value 086 * 087 * @param signed The value to decode 088 * @return The decoded value 089 */ 090 public static Number decodeZigZag(Number signed) { 091 return convertLong(decodeZigZag(signed.longValue())); 092 } 093 094 /** 095 * Decode a zig-zag encoded value 096 * 097 * @param signed The value to decode 098 * @return The decoded value 099 * @since 18695 100 */ 101 public static long decodeZigZag(long signed) { 102 return (signed >>> 1) ^ -(signed & 1); 103 } 104 105 /** 106 * Encode a number to a zig-zag encode value 107 * 108 * @param signed The number to encode 109 * @return The encoded value 110 */ 111 public static Number encodeZigZag(Number signed) { 112 final long value = signed.longValue(); 113 // This boundary condition could be >= or <= or both. Tests indicate that it doesn't actually matter. 114 // The only difference would be the number type returned, except it is always converted to the most basic type. 115 final int shift = (value > Integer.MAX_VALUE || value < Integer.MIN_VALUE ? Long.BYTES : Integer.BYTES) * 8 - 1; 116 return convertLong((value << 1) ^ (value >> shift)); 117 } 118 119 private final InputStream inputStream; 120 121 /** 122 * Create a new parser 123 * 124 * @param bytes The bytes to parse 125 */ 126 public ProtobufParser(byte[] bytes) { 127 this(new ByteArrayInputStream(bytes)); 128 } 129 130 /** 131 * Create a new parser 132 * 133 * @param inputStream The InputStream (will be fully read at this time) 134 */ 135 public ProtobufParser(InputStream inputStream) { 136 if (inputStream.markSupported()) { 137 this.inputStream = inputStream; 138 } else { 139 this.inputStream = new BufferedInputStream(inputStream); 140 } 141 } 142 143 /** 144 * Read all records 145 * 146 * @return A collection of all records 147 * @throws IOException - if an IO error occurs 148 */ 149 public Collection<ProtobufRecord> allRecords() throws IOException { 150 Collection<ProtobufRecord> records = new ArrayList<>(); 151 ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(4); 152 while (this.hasNext()) { 153 records.add(new ProtobufRecord(byteArrayOutputStream, this)); 154 } 155 return records; 156 } 157 158 @Override 159 public void close() { 160 try { 161 this.inputStream.close(); 162 } catch (IOException e) { 163 Logging.error(e); 164 } 165 } 166 167 /** 168 * Check if there is more data to read 169 * 170 * @return {@code true} if there is more data to read 171 * @throws IOException - if an IO error occurs 172 */ 173 public boolean hasNext() throws IOException { 174 return this.inputStream.available() > 0; 175 } 176 177 /** 178 * Get the "next" WireType 179 * 180 * @return {@link WireType} expected 181 * @throws IOException - if an IO error occurs 182 */ 183 public WireType next() throws IOException { 184 this.inputStream.mark(16); 185 try { 186 return WireType.getAllValues()[this.inputStream.read() << 3]; 187 } finally { 188 this.inputStream.reset(); 189 } 190 } 191 192 /** 193 * Get the next byte 194 * 195 * @return The next byte 196 * @throws IOException - if an IO error occurs 197 */ 198 public int nextByte() throws IOException { 199 return this.inputStream.read(); 200 } 201 202 /** 203 * Get the next 32 bits ({@link WireType#THIRTY_TWO_BIT}) 204 * 205 * @return a byte array of the next 32 bits (4 bytes) 206 * @throws IOException - if an IO error occurs 207 */ 208 public byte[] nextFixed32() throws IOException { 209 // 4 bytes == 32 bits 210 return readNextBytes(4); 211 } 212 213 /** 214 * Get the next 64 bits ({@link WireType#SIXTY_FOUR_BIT}) 215 * 216 * @return a byte array of the next 64 bits (8 bytes) 217 * @throws IOException - if an IO error occurs 218 */ 219 public byte[] nextFixed64() throws IOException { 220 // 8 bytes == 64 bits 221 return readNextBytes(8); 222 } 223 224 /** 225 * Get the next delimited message ({@link WireType#LENGTH_DELIMITED}) 226 * 227 * @param byteArrayOutputStream A reusable stream to write bytes to. This can significantly reduce the allocations 228 * (150 MB to 95 MB in a test area). 229 * @return The next length delimited message 230 * @throws IOException - if an IO error occurs 231 */ 232 public byte[] nextLengthDelimited(ByteArrayOutputStream byteArrayOutputStream) throws IOException { 233 final byte[] nextVarInt = this.nextVarInt(byteArrayOutputStream); 234 int length = (int) convertByteArray(nextVarInt, VAR_INT_BYTE_SIZE, 0, nextVarInt.length); 235 return readNextBytes(length); 236 } 237 238 /** 239 * Get the next var int ({@code WireType#VARINT}) 240 * 241 * @param byteArrayOutputStream A reusable stream to write bytes to. This can significantly reduce the allocations 242 * (150 MB to 95 MB in a test area). 243 * @return The next var int ({@code int32}, {@code int64}, {@code uint32}, {@code uint64}, {@code bool}, {@code enum}) 244 * @throws IOException - if an IO error occurs 245 */ 246 public byte[] nextVarInt(ByteArrayOutputStream byteArrayOutputStream) throws IOException { 247 // Using this reduces the allocations from 150 MB to 95 MB. 248 int currentByte = this.nextByte(); 249 while ((byte) (currentByte & MOST_SIGNIFICANT_BYTE) == MOST_SIGNIFICANT_BYTE && currentByte > 0) { 250 // Get rid of the leading bit (shift left 1, then shift right 1 unsigned) 251 byteArrayOutputStream.write((currentByte ^ MOST_SIGNIFICANT_BYTE)); 252 currentByte = this.nextByte(); 253 } 254 // The last byte doesn't drop the most significant bit 255 byteArrayOutputStream.write(currentByte); 256 try { 257 return byteArrayOutputStream.toByteArray(); 258 } finally { 259 byteArrayOutputStream.reset(); 260 } 261 } 262 263 /** 264 * Read an arbitrary number of bytes 265 * 266 * @param size The number of bytes to read 267 * @return a byte array filled with bytes read (unsigned) 268 * @throws IOException - if an IO error occurs 269 */ 270 private byte[] readNextBytes(int size) throws IOException { 271 byte[] bytesRead = new byte[size]; 272 int read = this.inputStream.read(bytesRead); 273 if (read == -1) { 274 return EMPTY_BYTE_ARRAY; 275 } else if (read != size) { 276 return Arrays.copyOf(bytesRead, read); 277 } 278 return bytesRead; 279 } 280}