Index: applications/editors/josm/plugins/pbf/src/crosby/binary/BinaryParser.java =================================================================== --- applications/editors/josm/plugins/pbf/src/crosby/binary/BinaryParser.java (revision 26961) +++ applications/editors/josm/plugins/pbf/src/crosby/binary/BinaryParser.java (revision 26961) @@ -0,0 +1,139 @@ +/** Copyright (c) 2010 Scott A. Crosby. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +*/ + +package crosby.binary; + + +import java.util.Date; +import java.util.List; + +import com.google.protobuf.InvalidProtocolBufferException; + +import crosby.binary.Osmformat; +import crosby.binary.file.BlockReaderAdapter; +import crosby.binary.file.FileBlock; +import crosby.binary.file.FileBlockPosition; + +public abstract class BinaryParser implements BlockReaderAdapter { + protected int granularity; + private long lat_offset; + private long lon_offset; + protected int date_granularity; + private String strings[]; + + /** Take a Info protocol buffer containing a date and convert it into a java Date object */ + protected Date getDate(Osmformat.Info info) { + if (info.hasTimestamp()) { + return new Date(date_granularity * (long) info.getTimestamp()); + } else + return NODATE; + } + public static final Date NODATE = new Date(-1); + + /** Get a string based on the index used. + * + * Index 0 is reserved to use as a delimiter, therefore, index 1 corresponds to the first string in the table + * @param id + * @return + */ + protected String getStringById(int id) { + return strings[id]; + } + + //@Override + public void handleBlock(FileBlock message) { + // TODO Auto-generated method stub + try { + if (message.getType().equals("OSMHeader")) { + Osmformat.HeaderBlock headerblock = Osmformat.HeaderBlock + .parseFrom(message.getData()); + parse(headerblock); + } else if (message.getType().equals("OSMData")) { + Osmformat.PrimitiveBlock primblock = Osmformat.PrimitiveBlock + .parseFrom(message.getData()); + parse(primblock); + } + } catch (InvalidProtocolBufferException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + throw new Error("ParseError"); // TODO + } + + } + + + //@Override + public boolean skipBlock(FileBlockPosition block) { + // System.out.println("Seeing block of type: "+block.getType()); + if (block.getType().equals("OSMData")) + return false; + if (block.getType().equals("OSMHeader")) + return false; + System.out.println("Skipped block of type: " + block.getType()); + return true; + } + + + /** Convert a latitude value stored in a protobuf into a double, compensating for granularity and latitude offset */ + public double parseLat(long degree) { + // Support non-zero offsets. (We don't currently generate them) + return (granularity * degree + lat_offset) * .000000001; + } + + /** Convert a longitude value stored in a protobuf into a double, compensating for granularity and longitude offset */ + public double parseLon(long degree) { + // Support non-zero offsets. (We don't currently generate them) + return (granularity * degree + lon_offset) * .000000001; + } + + /** Parse a Primitive block (containing a string table, other paramaters, and PrimitiveGroups */ + public void parse(Osmformat.PrimitiveBlock block) { + Osmformat.StringTable stablemessage = block.getStringtable(); + strings = new String[stablemessage.getSCount()]; + + for (int i = 0; i < strings.length; i++) { + strings[i] = stablemessage.getS(i).toStringUtf8(); + } + + granularity = block.getGranularity(); + lat_offset = block.getLatOffset(); + lon_offset = block.getLonOffset(); + date_granularity = block.getDateGranularity(); + + for (Osmformat.PrimitiveGroup groupmessage : block + .getPrimitivegroupList()) { + // Exactly one of these should trigger on each loop. + parseNodes(groupmessage.getNodesList()); + parseWays(groupmessage.getWaysList()); + parseRelations(groupmessage.getRelationsList()); + if (groupmessage.hasDense()) + parseDense(groupmessage.getDense()); + } + } + + /** Parse a list of Relation protocol buffers and send the resulting relations to a sink. */ + protected abstract void parseRelations(List rels); + /** Parse a DenseNode protocol buffer and send the resulting nodes to a sink. */ + protected abstract void parseDense(Osmformat.DenseNodes nodes); + /** Parse a list of Node protocol buffers and send the resulting nodes to a sink. */ + protected abstract void parseNodes(List nodes); + /** Parse a list of Way protocol buffers and send the resulting ways to a sink. */ + protected abstract void parseWays(List ways); + /** Parse a header message. */ + protected abstract void parse(Osmformat.HeaderBlock header); + +} Index: applications/editors/josm/plugins/pbf/src/crosby/binary/BinarySerializer.java =================================================================== --- applications/editors/josm/plugins/pbf/src/crosby/binary/BinarySerializer.java (revision 26961) +++ applications/editors/josm/plugins/pbf/src/crosby/binary/BinarySerializer.java (revision 26961) @@ -0,0 +1,161 @@ +/** Copyright (c) 2010 Scott A. Crosby. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +*/ + +package crosby.binary; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import crosby.binary.Osmformat.PrimitiveGroup; +import crosby.binary.file.BlockOutputStream; +import crosby.binary.file.FileBlock; + +/** + * Generic serializer common code + * + * Serialize a set of blobs and process them. Subclasses implement handlers for + * different API's (osmosis, mkgmap, splitter, etc.) + * + * All data is converted into PrimGroupWriterInterface objects, which are then + * ordered to process their data at the appropriate time. + * */ + +public class BinarySerializer { + + /** + * Interface used to write a group of primitives. One of these for each + * group type (Node, Way, Relation, DenseNode, Changeset) + */ + protected interface PrimGroupWriterInterface { + /** This callback is invoked on each group that is going into the fileblock in order to give it a chance to + * add to the stringtable pool of strings. */ + public void addStringsToStringtable(); + + /** + * This callback is invoked to request that the primgroup serialize itself into the given protocol buffer object. + */ + public Osmformat.PrimitiveGroup serialize(); + } + + /** Set the granularity (precision of lat/lon, measured in unites of nanodegrees. */ + public void configGranularity(int granularity) { + this.granularity = granularity; + } + + /** Set whether metadata is to be omitted */ + public void configOmit(boolean omit_metadata) { + this.omit_metadata = omit_metadata; + } + + /** Configure the maximum number of entities in a batch */ + public void configBatchLimit(int batch_limit) { + this.batch_limit = batch_limit; + } + + // Paramaters affecting the output size. + protected final int MIN_DENSE = 10; + protected int batch_limit = 4000; + + // Parmaters affecting the output. + + protected int granularity = 100; + protected int date_granularity = 1000; + protected boolean omit_metadata = false; + + /** How many primitives have been seen in this batch */ + protected int batch_size = 0; + protected int total_entities = 0; + private StringTable stringtable = new StringTable(); + protected List groups = new ArrayList(); + protected BlockOutputStream output; + + public BinarySerializer(BlockOutputStream output) { + this.output = output; + } + + public StringTable getStringTable() { + return stringtable; + } + + public void flush() throws IOException { + processBatch(); + output.flush(); + } + + public void close() throws IOException { + flush(); + output.close(); + } + + long debug_bytes = 0; + + public void processBatch() { + // System.out.format("Batch of %d groups: ",groups.size()); + if (groups.size() == 0) + return; + Osmformat.PrimitiveBlock.Builder primblock = Osmformat.PrimitiveBlock + .newBuilder(); + stringtable.clear(); + // Preprocessing: Figure out the stringtable. + for (PrimGroupWriterInterface i : groups) + i.addStringsToStringtable(); + + stringtable.finish(); + // Now, start serializing. + for (PrimGroupWriterInterface i : groups) { + PrimitiveGroup group = i.serialize(); + if (group != null) + primblock.addPrimitivegroup(group); + } + primblock.setStringtable(stringtable.serialize()); + primblock.setGranularity(this.granularity); + primblock.setDateGranularity(this.date_granularity); + + // Only generate data with offset (0,0) + // + Osmformat.PrimitiveBlock message = primblock.build(); + + // System.out.println(message); + debug_bytes += message.getSerializedSize(); + // if (message.getSerializedSize() > 1000000) + // System.out.println(message); + + try { + output.write(FileBlock.newInstance("OSMData", message + .toByteString(), null)); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + throw new Error(e); + } finally { + batch_size = 0; + groups.clear(); + } + // System.out.format("\n"); + } + + /** Convert from a degrees represented as a double into the serialized offset in nanodegrees.. */ + public long mapRawDegrees(double degrees) { + return (long) ((degrees / .000000001)); + } + + /** Convert from a degrees represented as a double into the serialized offset. */ + public int mapDegrees(double degrees) { + return (int) ((degrees / .0000001) / (granularity / 100)); + } +} Index: applications/editors/josm/plugins/pbf/src/crosby/binary/StringTable.java =================================================================== --- applications/editors/josm/plugins/pbf/src/crosby/binary/StringTable.java (revision 26961) +++ applications/editors/josm/plugins/pbf/src/crosby/binary/StringTable.java (revision 26961) @@ -0,0 +1,103 @@ +/** Copyright (c) 2010 Scott A. Crosby. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +*/ + +package crosby.binary; + +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; + +import com.google.protobuf.ByteString; + +/** + * Class for mapping a set of strings to integers, giving frequently occuring + * strings small integers. + */ +public class StringTable { + public StringTable() { + clear(); + } + + private HashMap counts; + private HashMap stringmap; + private String set[]; + + public void incr(String s) { + if (counts.containsKey(s)) { + counts.put(s, new Integer(counts.get(s).intValue() + 1)); + } else { + counts.put(s, new Integer(1)); + } + } + + /** After the stringtable has been built, return the offset of a string in it. + * + * Note, value '0' is reserved for use as a delimiter and will not be returned. + * @param s + * @return + */ + public int getIndex(String s) { + return stringmap.get(s).intValue(); + } + + public void finish() { + Comparator comparator = new Comparator() { + //@Override + public int compare(final String s1, String s2) { + int diff = counts.get(s2) - counts.get(s1); + return diff; + } + }; + + set = counts.keySet().toArray(new String[0]); + if (set.length > 0) { + // Sort based on the frequency. + Arrays.sort(set, comparator); + // Each group of keys that serializes to the same number of bytes is + // sorted lexiconographically. + // to maximize deflate compression. + + // Don't sort the first array. There's not likely to be much benefit, and we want frequent values to be small. + //Arrays.sort(set, Math.min(0, set.length-1), Math.min(1 << 7, set.length-1)); + + Arrays.sort(set, Math.min(1 << 7, set.length-1), Math.min(1 << 14, + set.length-1)); + Arrays.sort(set, Math.min(1 << 14, set.length-1), Math.min(1 << 21, + set.length-1), comparator); + } + stringmap = new HashMap(2 * set.length); + for (int i = 0; i < set.length; i++) { + stringmap.put(set[i], new Integer(i+1)); // Index 0 is reserved for use as a delimiter. + } + counts = null; + } + + public void clear() { + counts = new HashMap(100); + stringmap = null; + set = null; + } + + public Osmformat.StringTable.Builder serialize() { + Osmformat.StringTable.Builder builder = Osmformat.StringTable + .newBuilder(); + builder.addS(ByteString.copyFromUtf8("")); // Add a unused string at offset 0 which is used as a delimiter. + for (int i = 0; i < set.length; i++) + builder.addS(ByteString.copyFromUtf8(set[i])); + return builder; + } +} Index: applications/editors/josm/plugins/pbf/src/crosby/binary/file/BlockInputStream.java =================================================================== --- applications/editors/josm/plugins/pbf/src/crosby/binary/file/BlockInputStream.java (revision 26961) +++ applications/editors/josm/plugins/pbf/src/crosby/binary/file/BlockInputStream.java (revision 26961) @@ -0,0 +1,47 @@ +/** Copyright (c) 2010 Scott A. Crosby. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +*/ + +package crosby.binary.file; + +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; + +public class BlockInputStream { + // TODO: Should be seekable input stream! + public BlockInputStream(InputStream input, BlockReaderAdapter adaptor) { + this.input = input; + this.adaptor = adaptor; + } + + public void process() throws IOException { + try { + while (true) { + FileBlock.process(input, adaptor); + } + } catch (EOFException e) { + adaptor.complete(); + } + } + + public void close() throws IOException { + input.close(); + } + + InputStream input; + BlockReaderAdapter adaptor; +} Index: applications/editors/josm/plugins/pbf/src/crosby/binary/file/BlockOutputStream.java =================================================================== --- applications/editors/josm/plugins/pbf/src/crosby/binary/file/BlockOutputStream.java (revision 26961) +++ applications/editors/josm/plugins/pbf/src/crosby/binary/file/BlockOutputStream.java (revision 26961) @@ -0,0 +1,74 @@ +/** Copyright (c) 2010 Scott A. Crosby. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +*/ + +package crosby.binary.file; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; + +enum CompressFlags { + NONE, DEFLATE +} + +public class BlockOutputStream { + + public BlockOutputStream(OutputStream output) { + this.outwrite = new DataOutputStream(output); + this.compression = CompressFlags.DEFLATE; + } + + public void setCompress(CompressFlags flag) { + compression = flag; + } + + public void setCompress(String s) { + if (s.equals("none")) + compression = CompressFlags.NONE; + else if (s.equals("deflate")) + compression = CompressFlags.DEFLATE; + else + throw new Error("Unknown compression type: " + s); + } + + /** Write a block with the stream's default compression flag */ + public void write(FileBlock block) throws IOException { + this.write(block, compression); + } + + /** Write a specific block with a specific compression flags */ + public void write(FileBlock block, CompressFlags compression) + throws IOException { + FileBlockPosition ref = block.writeTo(outwrite, compression); + writtenblocks.add(ref); + } + + public void flush() throws IOException { + outwrite.flush(); + } + + public void close() throws IOException { + outwrite.flush(); + outwrite.close(); + } + + OutputStream outwrite; + List writtenblocks = new ArrayList(); + CompressFlags compression; +} Index: applications/editors/josm/plugins/pbf/src/crosby/binary/file/BlockReaderAdapter.java =================================================================== --- applications/editors/josm/plugins/pbf/src/crosby/binary/file/BlockReaderAdapter.java (revision 26961) +++ applications/editors/josm/plugins/pbf/src/crosby/binary/file/BlockReaderAdapter.java (revision 26961) @@ -0,0 +1,40 @@ +/** Copyright (c) 2010 Scott A. Crosby. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +*/ + +package crosby.binary.file; + +/** An adaptor that receives blocks from an input stream */ +public interface BlockReaderAdapter { + /** + * Does the reader understand this block? Does it want the data in it? + * + * A reference contains the metadata about a block and can saved --- or + * stored ---- for future random access. However, during a strea read of the + * file, does the user want this block? + * + * handleBlock will be called on all blocks that are not skipped, in file + * order. + * + * */ + boolean skipBlock(FileBlockPosition message); + + /** Called with the data in the block. */ + void handleBlock(FileBlock message); + + /** Called when the file is fully read. */ + void complete(); +} Index: applications/editors/josm/plugins/pbf/src/crosby/binary/file/FileBlock.java =================================================================== --- applications/editors/josm/plugins/pbf/src/crosby/binary/file/FileBlock.java (revision 26961) +++ applications/editors/josm/plugins/pbf/src/crosby/binary/file/FileBlock.java (revision 26961) @@ -0,0 +1,142 @@ +/** Copyright (c) 2010 Scott A. Crosby. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +*/ + +package crosby.binary.file; + +import java.io.DataOutputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.zip.Deflater; + +import com.google.protobuf.ByteString; + +import crosby.binary.Fileformat; +import crosby.binary.Fileformat.BlobHeader; + +/** A full fileblock object contains both the metadata and data of a fileblock */ +public class FileBlock extends FileBlockBase { + /** Contains the contents of a block for use or further processing */ + ByteString data; // serialized Format.Blob + + /** Don't be noisy unless the warning occurs somewhat often */ + static int warncount = 0; + + private FileBlock(String type, ByteString blob, ByteString indexdata) { + super(type, indexdata); + this.data = blob; + } + + public static FileBlock newInstance(String type, ByteString blob, + ByteString indexdata) { + if (blob != null && blob.size() > MAX_BODY_SIZE/2) { + System.err.println("Warning: Fileblock has body size too large and may be considered corrupt"); + if (blob != null && blob.size() > MAX_BODY_SIZE-1024*1024) { + throw new Error("This file has too many entities in a block. Parsers will reject it."); + } + } + if (indexdata != null && indexdata.size() > MAX_HEADER_SIZE/2) { + System.err.println("Warning: Fileblock has indexdata too large and may be considered corrupt"); + if (indexdata != null && indexdata.size() > MAX_HEADER_SIZE-512) { + throw new Error("This file header is too large. Parsers will reject it."); + } + } + return new FileBlock(type, blob, indexdata); + } + + protected void deflateInto(crosby.binary.Fileformat.Blob.Builder blobbuilder) { + int size = data.size(); + Deflater deflater = new Deflater(); + deflater.setInput(data.toByteArray()); + deflater.finish(); + byte out[] = new byte[size]; + deflater.deflate(out); + + if (!deflater.finished()) { + // Buffer wasn't long enough. Be noisy. + ++warncount; + if (warncount > 10 && warncount%100 == 0) + System.out.println("Compressed buffers are too short, causing extra copy"); + out = Arrays.copyOf(out, size + size / 64 + 16); + deflater.deflate(out, deflater.getTotalOut(), out.length + - deflater.getTotalOut()); + if (!deflater.finished()) { + throw new Error("Internal error in compressor"); + } + } + ByteString compressed = ByteString.copyFrom(out, 0, deflater + .getTotalOut()); + blobbuilder.setZlibData(compressed); + deflater.end(); + } + + public FileBlockPosition writeTo(OutputStream outwrite, CompressFlags flags) + throws IOException { + BlobHeader.Builder builder = Fileformat.BlobHeader + .newBuilder(); + if (indexdata != null) + builder.setIndexdata(indexdata); + builder.setType(type); + + Fileformat.Blob.Builder blobbuilder = Fileformat.Blob.newBuilder(); + if (flags == CompressFlags.NONE) { + blobbuilder.setRaw(data); + blobbuilder.setRawSize(data.size()); + } else { + blobbuilder.setRawSize(data.size()); + if (flags == CompressFlags.DEFLATE) + deflateInto(blobbuilder); + else + throw new Error("Compression flag not understood"); + } + Fileformat.Blob blob = blobbuilder.build(); + + builder.setDatasize(blob.getSerializedSize()); + Fileformat.BlobHeader message = builder.build(); + int size = message.getSerializedSize(); + + // System.out.format("Outputed header size %d bytes, header of %d bytes, and blob of %d bytes\n", + // size,message.getSerializedSize(),blob.getSerializedSize()); + (new DataOutputStream(outwrite)).writeInt(size); + message.writeTo(outwrite); + long offset = -1; + + if (outwrite instanceof FileOutputStream) + offset = ((FileOutputStream) outwrite).getChannel().position(); + + blob.writeTo(outwrite); + return FileBlockPosition.newInstance(this, offset, size); + } + + /** Reads or skips a fileblock. */ + static void process(InputStream input, BlockReaderAdapter callback) + throws IOException { + FileBlockHead fileblock = FileBlockHead.readHead(input); + if (callback.skipBlock(fileblock)) { + // System.out.format("Attempt to skip %d bytes\n",header.getDatasize()); + fileblock.skipContents(input); + } else { + callback.handleBlock(fileblock.readContents(input)); + } + } + + public ByteString getData() { + return data; + } +} Index: applications/editors/josm/plugins/pbf/src/crosby/binary/file/FileBlockBase.java =================================================================== --- applications/editors/josm/plugins/pbf/src/crosby/binary/file/FileBlockBase.java (revision 26961) +++ applications/editors/josm/plugins/pbf/src/crosby/binary/file/FileBlockBase.java (revision 26961) @@ -0,0 +1,58 @@ +/** Copyright (c) 2010 Scott A. Crosby. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +*/ + +package crosby.binary.file; + +import com.google.protobuf.ByteString; + +/** + * Base class that contains the metadata about a fileblock. + * + * Subclasses of this include additional fields, such as byte offsets that let a + * fileblock be read in a random-access fashion, or the data itself. + * + * @author crosby + * + */ +public class FileBlockBase { + + /** If a block header is bigger than this, fail. We use excessively large header size as an indication of corrupt files */ + static final int MAX_HEADER_SIZE = 64*1024; + /** If a block's size is bigger than this, fail. We use excessively large block sizes as an indication of corrupt files */ + static final int MAX_BODY_SIZE = 32*1024*1024; + + protected FileBlockBase(String type, ByteString indexdata) { + this.type = type; + this.indexdata = indexdata; + } + + /** Identifies the type of the data within a block */ + protected final String type; + /** + * Block metadata, stored in the index block and as a prefix for every + * block. + */ + protected final ByteString indexdata; + + public String getType() { + return type; + } + + public ByteString getIndexData() { + return indexdata; + } +} Index: applications/editors/josm/plugins/pbf/src/crosby/binary/file/FileBlockHead.java =================================================================== --- applications/editors/josm/plugins/pbf/src/crosby/binary/file/FileBlockHead.java (revision 26961) +++ applications/editors/josm/plugins/pbf/src/crosby/binary/file/FileBlockHead.java (revision 26961) @@ -0,0 +1,97 @@ +/** Copyright (c) 2010 Scott A. Crosby. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +*/ + +package crosby.binary.file; + +import java.io.DataInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import com.google.protobuf.ByteString; + +import crosby.binary.Fileformat; + +/** + * Intermediate representation of the header of a fileblock when a set of + * fileblocks is read as in a stream. The data in the fileblock must be either + * skipped (where the returned value is a reference to the fileblock) or parsed. + * + * @author crosby + * + */ +public class FileBlockHead extends FileBlockReference { + protected FileBlockHead(String type, ByteString indexdata) { + super(type, indexdata); + } + + /** + * Read the header. After reading the header, either the contents must be + * skipped or read + */ + static FileBlockHead readHead(InputStream input) throws IOException { + DataInputStream datinput = new DataInputStream(input); + int headersize = datinput.readInt(); + // System.out.format("Header size %d %x\n",headersize,headersize); + if (headersize > MAX_HEADER_SIZE) { + throw new FileFormatException("Unexpectedly long header "+MAX_HEADER_SIZE+ " bytes. Possibly corrupt file."); + } + + byte buf[] = new byte[headersize]; + datinput.readFully(buf); + // System.out.format("Read buffer for header of %d bytes\n",buf.length); + Fileformat.BlobHeader header = Fileformat.BlobHeader + .parseFrom(buf); + FileBlockHead fileblock = new FileBlockHead(header.getType(), header + .getIndexdata()); + + fileblock.datasize = header.getDatasize(); + if (header.getDatasize() > MAX_BODY_SIZE) { + throw new FileFormatException("Unexpectedly long body "+MAX_BODY_SIZE+ " bytes. Possibly corrupt file."); + } + + fileblock.input = input; + if (input instanceof FileInputStream) + fileblock.data_offset = ((FileInputStream) input).getChannel() + .position(); + + return fileblock; + } + + /** + * Assumes the stream is positioned over at the start of the data, skip over + * it. + * + * @throws IOException + */ + void skipContents(InputStream input) throws IOException { + if (input.skip(getDatasize()) != getDatasize()) + assert false : "SHORT READ"; + } + + /** + * Assumes the stream is positioned over at the start of the data, read it + * and return the complete FileBlock + * + * @throws IOException + */ + FileBlock readContents(InputStream input) throws IOException { + DataInputStream datinput = new DataInputStream(input); + byte buf[] = new byte[getDatasize()]; + datinput.readFully(buf); + return parseData(buf); + } +} Index: applications/editors/josm/plugins/pbf/src/crosby/binary/file/FileBlockPosition.java =================================================================== --- applications/editors/josm/plugins/pbf/src/crosby/binary/file/FileBlockPosition.java (revision 26961) +++ applications/editors/josm/plugins/pbf/src/crosby/binary/file/FileBlockPosition.java (revision 26961) @@ -0,0 +1,112 @@ +/** Copyright (c) 2010 Scott A. Crosby. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +*/ + +package crosby.binary.file; + +import java.io.DataInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.zip.DataFormatException; +import java.util.zip.Inflater; + +import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; + +import crosby.binary.Fileformat; + +/** + * Stores the position in the stream of a fileblock so that it can be easily + * read in a random-access fashion. + * + * We can turn this into a 'real' block by appropriately seeking into the file + * and doing a 'read'. + * + * */ +public class FileBlockPosition extends FileBlockBase { + protected FileBlockPosition(String type, ByteString indexdata) { + super(type, indexdata); + } + + /** Parse out and decompress the data part of a fileblock helper function. */ + FileBlock parseData(byte buf[]) throws InvalidProtocolBufferException { + FileBlock out = FileBlock.newInstance(type, null, indexdata); + Fileformat.Blob blob = Fileformat.Blob.parseFrom(buf); + if (blob.hasRaw()) { + out.data = blob.getRaw(); + } else if (blob.hasZlibData()) { + byte buf2[] = new byte[blob.getRawSize()]; + Inflater decompresser = new Inflater(); + decompresser.setInput(blob.getZlibData().toByteArray()); + // decompresser.getRemaining(); + try { + decompresser.inflate(buf2); + } catch (DataFormatException e) { + e.printStackTrace(); + throw new Error(e); + } + assert (decompresser.finished()); + decompresser.end(); + out.data = ByteString.copyFrom(buf2); + } + return out; + } + + public int getDatasize() { + return datasize; + } + + /* + * Given any form of fileblock and an offset/length value, return a + * reference that can be used to dereference and read the contents. + */ + static FileBlockPosition newInstance(FileBlockBase base, long offset, + int length) { + FileBlockPosition out = new FileBlockPosition(base.type, base.indexdata); + out.datasize = length; + out.data_offset = offset; + return out; + } + + public FileBlock read(InputStream input) throws IOException { + if (input instanceof FileInputStream) { + ((FileInputStream) input).getChannel().position(data_offset); + byte buf[] = new byte[getDatasize()]; + (new DataInputStream(input)).readFully(buf); + return parseData(buf); + } else { + throw new Error("Random access binary reads require seekability"); + } + } + + /** + * TODO: Convert this reference into a serialized representation that can be + * stored. + */ + public ByteString serialize() { + throw new Error("TODO"); + } + + /** TODO: Parse a serialized representation of this block reference */ + static FileBlockPosition parseFrom(ByteString b) { + throw new Error("TODO"); + } + + protected int datasize; + /** Offset into the file of the data part of the block */ + long data_offset; +} Index: applications/editors/josm/plugins/pbf/src/crosby/binary/file/FileBlockReference.java =================================================================== --- applications/editors/josm/plugins/pbf/src/crosby/binary/file/FileBlockReference.java (revision 26961) +++ applications/editors/josm/plugins/pbf/src/crosby/binary/file/FileBlockReference.java (revision 26961) @@ -0,0 +1,54 @@ +/** Copyright (c) 2010 Scott A. Crosby. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +*/ + +package crosby.binary.file; + +import java.io.IOException; +import java.io.InputStream; + +import com.google.protobuf.ByteString; + +/** + * A FileBlockPosition that remembers what file this is so that it can simply be + * dereferenced + */ +public class FileBlockReference extends FileBlockPosition { + + /** + * Convenience cache for storing the input this reference is contained + * within so that it can be cached + */ + protected InputStream input; + + protected FileBlockReference(String type, ByteString indexdata) { + super(type, indexdata); + } + + public FileBlock read() throws IOException { + return read(input); + } + + static FileBlockPosition newInstance(FileBlockBase base, InputStream input, + long offset, int length) { + FileBlockReference out = new FileBlockReference(base.type, + base.indexdata); + out.datasize = length; + out.data_offset = offset; + out.input = input; + return out; + } +} Index: applications/editors/josm/plugins/pbf/src/crosby/binary/file/FileFormatException.java =================================================================== --- applications/editors/josm/plugins/pbf/src/crosby/binary/file/FileFormatException.java (revision 26961) +++ applications/editors/josm/plugins/pbf/src/crosby/binary/file/FileFormatException.java (revision 26961) @@ -0,0 +1,33 @@ +/** Copyright (c) 2010 Scott A. Crosby. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +*/ + +package crosby.binary.file; + +import java.io.IOException; + +public class FileFormatException extends IOException { + + public FileFormatException(String string) { + super(string); + } + + /** + * + */ + private static final long serialVersionUID = -8128010128748910923L; + +} Index: applications/editors/josm/plugins/pbf/src/org/openstreetmap/josm/plugins/pbf/PbfConstants.java =================================================================== --- applications/editors/josm/plugins/pbf/src/org/openstreetmap/josm/plugins/pbf/PbfConstants.java (revision 26961) +++ applications/editors/josm/plugins/pbf/src/org/openstreetmap/josm/plugins/pbf/PbfConstants.java (revision 26961) @@ -0,0 +1,38 @@ +// JOSM PBF plugin. +// Copyright (C) 2011 Don-vip +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . +package org.openstreetmap.josm.plugins.pbf; + +import static org.openstreetmap.josm.tools.I18n.tr; + +import org.openstreetmap.josm.actions.ExtensionFileFilter; + +/** + * + * @author Don-vip + * + */ +public interface PbfConstants { + + /** + * File extension. + */ + public static final String EXTENSION = "osm.pbf"; + + /** + * File filter used in import/export dialogs. + */ + public static final ExtensionFileFilter FILE_FILTER = new ExtensionFileFilter(EXTENSION, EXTENSION, tr("OSM Server Files pbf compressed") + " (*."+EXTENSION+")"); +} Index: applications/editors/josm/plugins/pbf/src/org/openstreetmap/josm/plugins/pbf/PbfPlugin.java =================================================================== --- applications/editors/josm/plugins/pbf/src/org/openstreetmap/josm/plugins/pbf/PbfPlugin.java (revision 26961) +++ applications/editors/josm/plugins/pbf/src/org/openstreetmap/josm/plugins/pbf/PbfPlugin.java (revision 26961) @@ -0,0 +1,41 @@ +// JOSM PBF plugin. +// Copyright (C) 2011 Don-vip +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . +package org.openstreetmap.josm.plugins.pbf; + +import org.openstreetmap.josm.Main; +import org.openstreetmap.josm.actions.ExtensionFileFilter; +import org.openstreetmap.josm.plugins.Plugin; +import org.openstreetmap.josm.plugins.PluginInformation; +import org.openstreetmap.josm.plugins.pbf.action.DownloadPbfTask; +import org.openstreetmap.josm.plugins.pbf.io.PbfImporter; + +/** + * + * @author Don-vip + * + */ +public class PbfPlugin extends Plugin { + + public PbfPlugin(PluginInformation info) { + super(info); + // Allow JOSM to import *.osm.pbf files + ExtensionFileFilter.importers.add(new PbfImporter()); + // Allow JOSM to export *.osm.pbf files + //ExtensionFileFilter.exporters.add(new PbfExporter());// TODO: PBF export + // Allow JOSM to download remote *.osm.pbf files + Main.main.menu.openLocation.addDownloadTaskClass(DownloadPbfTask.class); + } +} Index: applications/editors/josm/plugins/pbf/src/org/openstreetmap/josm/plugins/pbf/action/DownloadPbfTask.java =================================================================== --- applications/editors/josm/plugins/pbf/src/org/openstreetmap/josm/plugins/pbf/action/DownloadPbfTask.java (revision 26961) +++ applications/editors/josm/plugins/pbf/src/org/openstreetmap/josm/plugins/pbf/action/DownloadPbfTask.java (revision 26961) @@ -0,0 +1,49 @@ +// JOSM PBF plugin. +// Copyright (C) 2011 Don-vip +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . +package org.openstreetmap.josm.plugins.pbf.action; + +import java.util.concurrent.Future; + +import org.openstreetmap.josm.Main; +import org.openstreetmap.josm.actions.downloadtasks.DownloadOsmTask; +import org.openstreetmap.josm.data.Bounds; +import org.openstreetmap.josm.gui.progress.ProgressMonitor; +import org.openstreetmap.josm.plugins.pbf.PbfConstants; +import org.openstreetmap.josm.plugins.pbf.io.PbfServerReader; + +public class DownloadPbfTask extends DownloadOsmTask implements PbfConstants { + + @Override + public Future download(boolean newLayer, Bounds downloadArea, + ProgressMonitor progressMonitor) { + return null; + } + + @Override + public Future loadUrl(boolean newLayer, String url, + ProgressMonitor progressMonitor) { + downloadTask = new DownloadTask(newLayer, + new PbfServerReader(url), progressMonitor); + // We need submit instead of execute so we can wait for it to finish and get the error + // message if necessary. If no one calls getErrorMessage() it just behaves like execute. + return Main.worker.submit(downloadTask); + } + + @Override + public boolean acceptsUrl(String url) { + return url != null && url.endsWith(EXTENSION); + } +} Index: applications/editors/josm/plugins/pbf/src/org/openstreetmap/josm/plugins/pbf/io/PbfImporter.java =================================================================== --- applications/editors/josm/plugins/pbf/src/org/openstreetmap/josm/plugins/pbf/io/PbfImporter.java (revision 26961) +++ applications/editors/josm/plugins/pbf/src/org/openstreetmap/josm/plugins/pbf/io/PbfImporter.java (revision 26961) @@ -0,0 +1,54 @@ +// JOSM PBF plugin. +// Copyright (C) 2011 Don-vip +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . +package org.openstreetmap.josm.plugins.pbf.io; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; + +import org.openstreetmap.josm.data.osm.DataSet; +import org.openstreetmap.josm.gui.layer.OsmDataLayer; +import org.openstreetmap.josm.gui.progress.NullProgressMonitor; +import org.openstreetmap.josm.io.IllegalDataException; +import org.openstreetmap.josm.io.MirroredInputStream; +import org.openstreetmap.josm.io.OsmImporter; +import org.openstreetmap.josm.plugins.pbf.PbfConstants; +import org.xml.sax.SAXException; + +/** + * @author Don-vip + * + */ +public class PbfImporter extends OsmImporter implements PbfConstants { + + public PbfImporter() { + super(FILE_FILTER); + } + + /* (non-Javadoc) + * @see org.openstreetmap.josm.io.OsmImporter#importData(java.io.InputStream, java.io.File) + */ + @Override + protected void importData(InputStream in, File associatedFile) throws IllegalDataException { + final DataSet dataSet = PbfReader.parseDataSet(in, NullProgressMonitor.INSTANCE); + final OsmDataLayer layer = new OsmDataLayer(dataSet, associatedFile.getName(), associatedFile); + addDataLayer(dataSet, layer, associatedFile.getPath()); + } + + protected DataSet parseDataSet(final String source) throws IOException, SAXException, IllegalDataException { + return PbfReader.parseDataSet(new MirroredInputStream(source), NullProgressMonitor.INSTANCE); + } +} Index: applications/editors/josm/plugins/pbf/src/org/openstreetmap/josm/plugins/pbf/io/PbfReader.java =================================================================== --- applications/editors/josm/plugins/pbf/src/org/openstreetmap/josm/plugins/pbf/io/PbfReader.java (revision 26961) +++ applications/editors/josm/plugins/pbf/src/org/openstreetmap/josm/plugins/pbf/io/PbfReader.java (revision 26961) @@ -0,0 +1,296 @@ +// JOSM PBF plugin. +// Copyright (C) 2011 Don-vip +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . +package org.openstreetmap.josm.plugins.pbf.io; + +import static org.openstreetmap.josm.tools.I18n.tr; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.openstreetmap.josm.data.coor.LatLon; +import org.openstreetmap.josm.data.osm.DataSet; +import org.openstreetmap.josm.data.osm.Node; +import org.openstreetmap.josm.data.osm.OsmPrimitiveType; +import org.openstreetmap.josm.data.osm.Relation; +import org.openstreetmap.josm.data.osm.RelationMemberData; +import org.openstreetmap.josm.data.osm.User; +import org.openstreetmap.josm.data.osm.Way; +import org.openstreetmap.josm.gui.progress.NullProgressMonitor; +import org.openstreetmap.josm.gui.progress.ProgressMonitor; +import org.openstreetmap.josm.io.AbstractReader; +import org.openstreetmap.josm.io.IllegalDataException; +import org.openstreetmap.josm.tools.CheckParameterUtil; + +import crosby.binary.BinaryParser; +import crosby.binary.Osmformat; +import crosby.binary.Osmformat.DenseNodes; +import crosby.binary.Osmformat.HeaderBlock; +import crosby.binary.Osmformat.Info; +import crosby.binary.file.BlockInputStream; +import crosby.binary.file.FileBlockPosition; + +/** + * @author Don-vip + * + */ +public class PbfReader extends AbstractReader { + + protected class PbfParser extends BinaryParser { + + public IllegalDataException exception = null; + + @Override + protected void parse(HeaderBlock header) { + } + + /* (non-Javadoc) + * @see crosby.binary.BinaryParser#skipBlock(crosby.binary.file.FileBlockPosition) + */ + @Override + public boolean skipBlock(FileBlockPosition block) { + return exception != null; + } + + protected void checkCoordinates(LatLon coor) throws IllegalDataException { + if (!coor.isValid()) { + throw new IllegalDataException(tr("Invalid coordinates: {0}", coor)); + } + } + + protected void checkChangesetId(long id) throws IllegalDataException { + if (id > Integer.MAX_VALUE) { + throw new IllegalDataException(tr("Invalid changeset id: {0}", id)); + } + } + + protected void checkTimestamp(long timestamp) throws IllegalDataException { + if (timestamp < 0) { + throw new IllegalDataException(tr("Invalid timestamp: {0}", timestamp)); + } + } + + @Override + protected void parseDense(DenseNodes nodes) { + if (exception == null) { + try { + int keyIndex = 0; + // Almost all data is DELTA coded + long nodeId = 0; + long nodeLat = 0; + long nodeLon = 0; + long changesetId = 0; + int uid = 0; + int suid = 0; + long timestamp = 0; + for (int i = 0; i < nodes.getIdCount(); i++) { + // Id (delta) and version (normal) + Node node = new Node(nodeId+=nodes.getId(i), nodes.getDenseinfo().getVersion(i)); + // Lat/Lon (delta) + node.setCoor(new LatLon(parseLat(nodeLat+=nodes.getLat(i)), parseLon(nodeLon+=nodes.getLon(i))).getRoundedToOsmPrecision()); + checkCoordinates(node.getCoor()); + // Changeset (delta) + checkChangesetId(changesetId+=nodes.getDenseinfo().getChangeset(i)); + node.setChangesetId((int) changesetId); + // User (delta) + node.setUser(User.createOsmUser(uid+=nodes.getDenseinfo().getUid(i), getStringById(suid+=nodes.getDenseinfo().getUserSid(i)))); + // Timestamp (delta) + checkTimestamp(timestamp+=nodes.getDenseinfo().getTimestamp(i)); + node.setTimestamp(new Date(date_granularity * timestamp)); + // A single table contains all keys/values of all nodes. + // Each node's tags are encoded in alternating . + // A single stringid of 0 delimit when the tags of a node ends and the tags of the next node begin. + Map keys = new HashMap(); + while (keyIndex < nodes.getKeysValsCount()) { + int key_id = nodes.getKeysVals(keyIndex++); + if (key_id == 0) { + break; // End of current node's tags + } else if (keyIndex < nodes.getKeysValsCount()) { + int value_id = nodes.getKeysVals(keyIndex++); + keys.put(getStringById(key_id), getStringById(value_id)); + } else { + throw new IllegalDataException(tr("Invalid DenseNodes key/values table")); + } + } + node.setKeys(keys); + externalIdMap.put(node.getPrimitiveId(), node); + } + } catch (IllegalDataException e) { + exception = e; + } + } + } + + @Override + protected void parseNodes(List osmNodes) { + if (exception == null) { + try { + for (Osmformat.Node n : osmNodes) { + final Info info = n.getInfo(); + final Node node = new Node(n.getId(), info.getVersion()); + node.setCoor(new LatLon(parseLat(n.getLat()), parseLon(n.getLon())).getRoundedToOsmPrecision()); + checkCoordinates(node.getCoor()); + checkChangesetId(info.getChangeset()); + node.setChangesetId((int) info.getChangeset()); + node.setUser(User.createOsmUser(info.getUid(), getStringById(info.getUserSid()))); + checkTimestamp(info.getTimestamp()); + node.setTimestamp(getDate(info)); + Map keys = new HashMap(); + for (int i=0; i osmWays) { + if (exception == null) { + try { + for (Osmformat.Way w : osmWays) { + final Info info = w.getInfo(); + final Way way = new Way(w.getId(), info.getVersion()); + checkChangesetId(info.getChangeset()); + way.setChangesetId((int) info.getChangeset()); + way.setUser(User.createOsmUser(info.getUid(), getStringById(info.getUserSid()))); + checkTimestamp(info.getTimestamp()); + way.setTimestamp(getDate(info)); + Map keys = new HashMap(); + for (int i=0; i nodeIds = new ArrayList(); + for (Long id : w.getRefsList()) { + nodeIds.add(previousId+=id); + } + ways.put(way.getUniqueId(), nodeIds); + externalIdMap.put(way.getPrimitiveId(), way); + } + } catch (IllegalDataException e) { + exception = e; + } + } + } + + @Override + protected void parseRelations(List osmRels) { + if (exception == null) { + try { + for (Osmformat.Relation r : osmRels) { + final Info info = r.getInfo(); + final Relation rel = new Relation(r.getId(), info.getVersion()); + checkChangesetId(info.getChangeset()); + rel.setChangesetId((int) info.getChangeset()); + rel.setUser(User.createOsmUser(info.getUid(), getStringById(info.getUserSid()))); + checkTimestamp(info.getTimestamp()); + rel.setTimestamp(getDate(info)); + Map keys = new HashMap(); + for (int i=0; i members = new ArrayList(); + for (int i = 0; i. +package org.openstreetmap.josm.plugins.pbf.io; + +import static org.openstreetmap.josm.tools.I18n.tr; + +import org.openstreetmap.josm.data.osm.DataSet; +import org.openstreetmap.josm.gui.progress.ProgressMonitor; +import org.openstreetmap.josm.io.OsmServerReader; +import org.openstreetmap.josm.io.OsmTransferException; + +public class PbfServerReader extends OsmServerReader { + + private String url; + + public PbfServerReader(String url) { + this.url = url; + } + + @Override + public DataSet parseOsm(ProgressMonitor progressMonitor) + throws OsmTransferException { + try { + progressMonitor.beginTask(tr("Contacting Server...", 10)); + return new PbfImporter().parseDataSet(url); + } catch (Exception e) { + throw new OsmTransferException(e); + } finally { + progressMonitor.finishTask(); + } + } +}