From 2cb5dd752037ff81329578d1514dac7de0e3ee16 Mon Sep 17 00:00:00 2001 From: slomo Date: Wed, 12 Jan 2011 20:37:16 +0100 Subject: merged code from old repo --- pbf2db/binarystream.py | 95 ++++++++++++++++++++ pbf2db/build_proto.sh | 3 + pbf2db/fileformat.proto | 49 +++++++++++ pbf2db/osmformat.proto | 225 ++++++++++++++++++++++++++++++++++++++++++++++++ pbf2db/pbf2db.py | 44 ++++++++++ 5 files changed, 416 insertions(+) create mode 100644 pbf2db/binarystream.py create mode 100755 pbf2db/build_proto.sh create mode 100644 pbf2db/fileformat.proto create mode 100644 pbf2db/osmformat.proto create mode 100755 pbf2db/pbf2db.py (limited to 'pbf2db') diff --git a/pbf2db/binarystream.py b/pbf2db/binarystream.py new file mode 100644 index 0000000..e1e0280 --- /dev/null +++ b/pbf2db/binarystream.py @@ -0,0 +1,95 @@ +from struct import * + +class BinaryStream: + def __init__(self, base_stream): + self.base_stream = base_stream + + def readByte(self): + return self.base_stream.read(1) + + def readBytes(self, length): + return self.base_stream.read(length) + + def readChar(self): + return self.unpack('b') + + def readUChar(self): + return self.unpack('B') + + def readBool(self): + return self.unpack('?') + + def readInt16(self): + return self.unpack('h', 2) + + def readUInt16(self): + return self.unpack('H', 2) + + def readInt32(self): + return self.unpack('i', 4) + + def readUInt32(self): + return self.unpack('I', 4) + + def readInt64(self): + return self.unpack('q', 8) + + def readUInt64(self): + return self.unpack('Q', 8) + + def readFloat(self): + return self.unpack('f', 4) + + def readDouble(self): + return self.unpack('d', 8) + + def readString(self): + length = self.readUInt16() + return self.unpack(str(length) + 's', length) + + def writeBytes(self, value): + self.base_stream.write(value) + + def writeChar(self, value): + self.pack('c', value) + + def writeUChar(self, value): + self.pack('C', value) + + def writeBool(self, value): + self.pack('?', value) + + def writeInt16(self, value): + self.pack('h', value) + + def writeUInt16(self, value): + self.pack('H', value) + + def writeInt32(self, value): + self.pack('i', value) + + def writeUInt32(self, value): + self.pack('I', value) + + def writeInt64(self, value): + self.pack('q', value) + + def writeUInt64(self, value): + self.pack('Q', value) + + def writeFloat(self, value): + self.pack('f', value) + + def writeDouble(self, value): + self.pack('d', value) + + def writeString(self, value): + length = len(value) + self.writeUInt16(length) + self.pack(str(length) + 's', value) + + def pack(self, fmt, data): + return self.writeBytes(pack(fmt, data)) + + def unpack(self, fmt, length = 1): + return unpack(fmt, self.readBytes(length))[0] diff --git a/pbf2db/build_proto.sh b/pbf2db/build_proto.sh new file mode 100755 index 0000000..53e9966 --- /dev/null +++ b/pbf2db/build_proto.sh @@ -0,0 +1,3 @@ +#!/bin/sh +protoc --python_out=. fileformat.proto +protoc --python_out=. osmformat.proto diff --git a/pbf2db/fileformat.proto b/pbf2db/fileformat.proto new file mode 100644 index 0000000..f1b540a --- /dev/null +++ b/pbf2db/fileformat.proto @@ -0,0 +1,49 @@ +/** Copyright (c) 2010 Scott A. Crosby. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +*/ + +package OSMPBF; + +// +// STORAGE LAYER: Storing primitives. +// + +message Blob { + optional bytes raw = 1; // No compression + optional int32 raw_size = 2; // When compressed, the uncompressed size + + // Possible compressed versions of the data. + optional bytes zlib_data = 3; + + // PROPOSED feature for LZMA compressed data. SUPPORT IS NOT REQUIRED. + optional bytes lzma_data = 4; + + // Formerly used for bzip2 compressed data. Depreciated in 2010. + optional bytes OBSOLETE_bzip2_data = 5 [deprecated=true]; // Don't reuse this tag number. +} + +/* A file contains an sequence of fileblock headers, each prefixed by +their length in network byte order, followed by a data block +containing the actual data. types staring with a "_" are reserved. +*/ + +message BlobHeader { + required string type = 1; + optional bytes indexdata = 2; + required int32 datasize = 3; +} + + diff --git a/pbf2db/osmformat.proto b/pbf2db/osmformat.proto new file mode 100644 index 0000000..eaad195 --- /dev/null +++ b/pbf2db/osmformat.proto @@ -0,0 +1,225 @@ +/** Copyright (c) 2010 Scott A. Crosby. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +*/ + +package OSMPBF; + +/* OSM Binary file format + +This is the master schema file of the OSM binary file format. This +file is designed to support limited random-access and future +extendability. + +A binary OSM file consists of a sequence of FileBlocks (please see +fileformat.proto). The first fileblock contains a serialized instance +of HeaderBlock, followed by a sequence of PrimitiveBlock blocks that +contain the primitives. + +Each primitiveblock is designed to be independently parsable. It +contains a string table storing all strings in that block (keys and +values in tags, roles in relations, usernames, etc.) as well as +metadata containing the precision of coordinates or timestamps in that +block. + +A primitiveblock contains a sequence of primitive groups, each +containing primitives of the same type (nodes, densenodes, ways, +relations). Coordinates are stored in signed 64-bit integers. Lat&lon +are measured in units nanodegrees. The default of +granularity of 100 nanodegrees corresponds to about 1cm on the ground, +and a full lat or lon fits into 32 bits. + +Converting an integer to a lattitude or longitude uses the formula: +$OUT = IN * granularity / 10**9$. Many encoding schemes use delta +coding when representing nodes and relations. + +*/ + +////////////////////////////////////////////////////////////////////////// +////////////////////////////////////////////////////////////////////////// + +/* Contains the file header. */ + +message HeaderBlock { + optional HeaderBBox bbox = 1; + /* Additional tags to aid in parsing this dataset */ + repeated string required_features = 4; + repeated string optional_features = 5; + + optional string writingprogram = 16; + optional string source = 17; // From the bbox field. +} + + +/** The bounding box field in the OSM header. BBOX, as used in the OSM +header. Units are always in nanodegrees -- they do not obey +granularity rules. */ + +message HeaderBBox { + required sint64 left = 1; + required sint64 right = 2; + required sint64 top = 3; + required sint64 bottom = 4; +} + + +/////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////// + + +message PrimitiveBlock { + required StringTable stringtable = 1; + repeated PrimitiveGroup primitivegroup = 2; + + // Granularity, units of nanodegrees, used to store coordinates in this block + optional int32 granularity = 17 [default=100]; + // Offset value between the output coordinates coordinates and the granularity grid in unites of nanodegrees. + optional int64 lat_offset = 19 [default=0]; + optional int64 lon_offset = 20 [default=0]; + +// Granularity of dates, normally represented in units of milliseconds since the 1970 epoch. + optional int32 date_granularity = 18 [default=1000]; + + + // Proposed extension: + //optional BBox bbox = XX; +} + +// Group of OSMPrimitives. All primitives in a group must be the same type. +message PrimitiveGroup { + repeated Node nodes = 1; + optional DenseNodes dense = 2; + repeated Way ways = 3; + repeated Relation relations = 4; + repeated ChangeSet changesets = 5; +} + + +/** String table, contains the common strings in each block. + + Note that we reserve index '0' as a delimiter, so the entry at that + index in the table is ALWAYS blank and unused. + + */ +message StringTable { + repeated bytes s = 1; +} + +/* Optional metadata that may be included into each primitive. */ +message Info { + optional int32 version = 1 [default = -1]; + optional int64 timestamp = 2; + optional int64 changeset = 3; + optional int32 uid = 4; + optional uint32 user_sid = 5; // String IDs +} + +/** Optional metadata that may be included into each primitive. Special dense format used in DenseNodes. */ +message DenseInfo { + repeated int32 version = 1 [packed = true]; + repeated sint64 timestamp = 2 [packed = true]; // DELTA coded + repeated sint64 changeset = 3 [packed = true]; // DELTA coded + repeated sint32 uid = 4 [packed = true]; // DELTA coded + repeated sint32 user_sid = 5 [packed = true]; // String IDs for usernames. DELTA coded +} + + +// THIS IS STUB DESIGN FOR CHANGESETS. NOT USED RIGHT NOW. +// TODO: REMOVE THIS? +message ChangeSet { + required int64 id = 1; +// +// // Parallel arrays. +// repeated uint32 keys = 2 [packed = true]; // String IDs. +// repeated uint32 vals = 3 [packed = true]; // String IDs. +// +// optional Info info = 4; + +// optional int64 created_at = 8; +// optional int64 closetime_delta = 9; +// optional bool open = 10; +// optional HeaderBBox bbox = 11; +} + + +message Node { + required sint64 id = 1; + // Parallel arrays. + repeated uint32 keys = 2 [packed = true]; // String IDs. + repeated uint32 vals = 3 [packed = true]; // String IDs. + + optional Info info = 4; // May be omitted in omitmeta + + required sint64 lat = 8; + required sint64 lon = 9; +} + +/* Used to densly represent a sequence of nodes that do not have any tags. + +We represent these nodes columnwise as five columns: ID's, lats, and +lons, all delta coded. When metadata is not omitted, + +We encode keys & vals for all nodes as a single array of integers +containing key-stringid and val-stringid, using a stringid of 0 as a +delimiter between nodes. + + ( ( )* '0' )* + */ + +message DenseNodes { + repeated sint64 id = 1 [packed = true]; // DELTA coded + + //repeated Info info = 4; + optional DenseInfo denseinfo = 5; + + repeated sint64 lat = 8 [packed = true]; // DELTA coded + repeated sint64 lon = 9 [packed = true]; // DELTA coded + + // Special packing of keys and vals into one array. May be empty if all nodes in this block are tagless. + repeated int32 keys_vals = 10 [packed = true]; +} + + +message Way { + required int64 id = 1; + // Parallel arrays. + repeated uint32 keys = 2 [packed = true]; + repeated uint32 vals = 3 [packed = true]; + + optional Info info = 4; + + repeated sint64 refs = 8 [packed = true]; // DELTA coded +} + +message Relation { + enum MemberType { + NODE = 0; + WAY = 1; + RELATION = 2; + } + required int64 id = 1; + + // Parallel arrays. + repeated uint32 keys = 2 [packed = true]; + repeated uint32 vals = 3 [packed = true]; + + optional Info info = 4; + + // Parallel arrays + repeated int32 roles_sid = 8 [packed = true]; + repeated sint64 memids = 9 [packed = true]; // DELTA encoded + repeated MemberType types = 10 [packed = true]; +} + diff --git a/pbf2db/pbf2db.py b/pbf2db/pbf2db.py new file mode 100755 index 0000000..c6089fc --- /dev/null +++ b/pbf2db/pbf2db.py @@ -0,0 +1,44 @@ +#!/usr/bin/python + +import osmformat_pb2 +import fileformat_pb2 +import sys +import socket +import zlib +from binarystream import BinaryStream + + +headerSizeMax = 64 * 1024 +bodySizeMax = 32*1024*1024 + +f = open("berlin.osm.pbf") +stream = BinaryStream(f) +headerSize = socket.ntohl(stream.readUInt32()); + +if headerSizeMax < headerSize: + raise ValueError("Header to long") + +headerbuff = stream.readBytes(headerSize) +blobheader = fileformat_pb2.BlobHeader() +blobheader.ParseFromString(headerbuff) +bodysize = blobheader.datasize + +if bodySizeMax < bodysize: + raise ValueError("Body to fat") + +blobbuff = stream.readBytes(bodysize) +blob = fileformat_pb2.Blob() +blob.ParseFromString(blobbuff) + +if blob.raw != "": + rawstr = blob.raw +else: + rawstr = zlib.decompress(blob.zlib_data) + +headerblock = osmformat_pb2.HeaderBlock() +headerblock.ParseFromString(rawstr) + +print "Source:",headerblock.source +print "Writingprog:",headerblock.writingprogram +print "required features:",headerblock.required_features + -- cgit v1.2.3