diff options
| -rw-r--r-- | doc/softwareprozess.tex | 200 | ||||
| -rw-r--r-- | pbf2db/binarystream.py | 95 | ||||
| -rwxr-xr-x | pbf2db/build_proto.sh | 3 | ||||
| -rw-r--r-- | pbf2db/fileformat.proto | 49 | ||||
| -rw-r--r-- | pbf2db/osmformat.proto | 225 | ||||
| -rwxr-xr-x | pbf2db/pbf2db.py | 44 |
6 files changed, 200 insertions, 416 deletions
diff --git a/doc/softwareprozess.tex b/doc/softwareprozess.tex new file mode 100644 index 0000000..038000d --- /dev/null +++ b/doc/softwareprozess.tex @@ -0,0 +1,200 @@ +\documentclass{beamer} +%packages +\usepackage[utf8]{inputenc} +\usepackage{graphicx} +\usepackage{hyperref} +\hypersetup{urlcolor=red,colorlinks} +\definecolor{bg}{rgb}{0.95,0.95,0.95} +\usetheme{Rochester} +\title[]{A sketch of an agile software process} +\author{Yves Müller} +\institute{Institute for Computer Science, Freie University Berlin} +\date{15.4.2011} +\begin{document} + +\begin{frame} +\titlepage +\end{frame} + +\begin{frame}[fragile] + \frametitle{Kurz und Knapp} + \begin{definition}[Agile software development] + Produce high value at low cost. + \end{definition} + + \vspace{0.2in} + Bedeutung für unser Projekt + + \begin{itemize} + \item Kosten: Zeit die wir aufbringen + \item Wert: Nutzen für OpenStreetMap + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Agile Softwareentwicklung: Wie geht das?} + + Puh ... Erklärung ist gar nicht so einfach, darum hier nur ein paar Stichworte. + + \begin{itemize} + \item Planbasierende Entwicklung: Ängste, Misstrauen und Stress + \item Agiles Manifest + \item Werteverschiebung + \item Prinzipien + \item viele viele Prozessmodelle + \end{itemize} +\end{frame} + + +\begin{frame} + \frametitle{Crystal Clear} + + Der minimale Ansatz zu einem Agilen Prozess. + + \begin{block}{Prinzipien} + + \begin{itemize} + \item Häufige Auslieferung der Software + \item Passiver Wissenstransfer + \item Ständige Optimierung durch Reflektion + \end{itemize} + \end{block} + + Es gibt nur zwei konkrete Methode: + + \begin{itemize} + \item Retrosperktive Treffen + \item Iterationen (bei uns 3 Stück) + \end{itemize} + + Andere werden bei Bedarf hinzugezogen. + +\end{frame} + + +\begin{frame} + \frametitle{Agile Werkzeuge und Methoden in unserem Projekt} + + nur eine kleine Gruppen, die ich sinnvoll fand + + \begin{itemize} + \item Testgetrieben Entwicklung + \item StandUp Meeting + \item Plannungsspiel (die Zettelwirtschaft) + \item Gemeinsamer Code (Collective Ownership) + \item Continious Integration + \end{itemize} + + diskutierbar, und veränderbar durch Retrosperktive + +\end{frame} + + +\begin{frame} + \frametitle{Testgetriebene Entwicklung} + + Tests werden zu erst entwickelt + + \begin{block}{Wie geht das?} + Der Code wird in folgendem Zyklus entwickelt. + \begin{itemize} + \item Test schreiben + \item Test erfüllen (genau den Test implmentieren) + \item Verbessern des Codes (Refactoring) + \end{itemize} + \end{block} + Jede funktionale Eigenschaft und jeder Bug hat einen Test +\end{frame} + + +\begin{frame} + \frametitle{Testgetribene Entwicklung} + \begin{block}{Warum?} + \begin{itemize} + \item Entwicklung von definierten Schnittstellen + \item Feinentwurf auf Modulebene + \item Vermeidung von totem Code + \item Vertrauen in Korrektheit des Codes höher + \item ein ganz kleines bisschen Compilerersatz + \end{itemize} + \end{block} + + Erfordert hohe Disziplin, bringt aber unglaubliche Vorteile. +\end{frame} + +\begin{frame} + \frametitle{Testgetribene Entwicklung} + Beim Schreiben von Tests sollte beachtet werden: + + \begin{itemize} + \item nicht zu große Blöcke testen + \item auch auf Typen testen + \item wenig gegen exteren Systeme testen (nutzt Stellvertreter) + \item die Qualität des Programms ist nur so gut wie die Qualität der Tests + \item Geschwindigkeit ist auch in Test kritisch + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Standup meeting} + + \begin{block}{Was ist das?} + \begin{itemize} + \item Treffen am Anfang jeder Sitzung + \item max. 10 Min + \item alle stehen + \end{itemize} + \end{block} + + \begin{block}{Warum?} + \begin{itemize} + \item ihr könnt eure Probleme mitteilen + \item ihr könnt über Erfolge berichten + \item Fortschritt ermitteln + \end{itemize} + \end{block} + +\end{frame} + +\begin{frame} + \frametitle{Das Plannungspiel} + + \begin{block}{Was ist das?} + \begin{itemize} + \item Treffen am Anfang einer Iteration + \item Diskussion über die Anforderungen + \item Festlegung der Ziele für die Iteration + \end{itemize} + \end{block} + + \begin{block}{Warum?} + \begin{itemize} + \item nur die Entwickler können ihre Leistungsfähigkeit einschätzen + \item korrekte Anforderungen sind schwer zu formalsieren + \end{itemize} + \end{block} + +\end{frame} + + +\begin{frame} + \frametitle{Collective code ownership / Continious Integration} + + \begin{block}{Was ist das?} + \begin{itemize} + \item es gibt keine Modulzuständigkeiten! + \item jeder ist verantwortlich + \item der Code soll immer lauffähig sein + \end{itemize} + \end{block} + + \begin{block}{Warum?} + \begin{itemize} + \item Minimierung des Expertentums + \end{itemize} + \end{block} + + Das CI kontrolliert ob der Code gut ist (= läuft) und erinnert notfalls alle an ihre Pflichten! +\end{frame} + +\end{document} diff --git a/pbf2db/binarystream.py b/pbf2db/binarystream.py deleted file mode 100644 index e1e0280..0000000 --- a/pbf2db/binarystream.py +++ /dev/null @@ -1,95 +0,0 @@ -from struct import * - -class BinaryStream: - def __init__(self, base_stream): - self.base_stream = base_stream - - def readByte(self): - return self.base_stream.read(1) - - def readBytes(self, length): - return self.base_stream.read(length) - - def readChar(self): - return self.unpack('b') - - def readUChar(self): - return self.unpack('B') - - def readBool(self): - return self.unpack('?') - - def readInt16(self): - return self.unpack('h', 2) - - def readUInt16(self): - return self.unpack('H', 2) - - def readInt32(self): - return self.unpack('i', 4) - - def readUInt32(self): - return self.unpack('I', 4) - - def readInt64(self): - return self.unpack('q', 8) - - def readUInt64(self): - return self.unpack('Q', 8) - - def readFloat(self): - return self.unpack('f', 4) - - def readDouble(self): - return self.unpack('d', 8) - - def readString(self): - length = self.readUInt16() - return self.unpack(str(length) + 's', length) - - def writeBytes(self, value): - self.base_stream.write(value) - - def writeChar(self, value): - self.pack('c', value) - - def writeUChar(self, value): - self.pack('C', value) - - def writeBool(self, value): - self.pack('?', value) - - def writeInt16(self, value): - self.pack('h', value) - - def writeUInt16(self, value): - self.pack('H', value) - - def writeInt32(self, value): - self.pack('i', value) - - def writeUInt32(self, value): - self.pack('I', value) - - def writeInt64(self, value): - self.pack('q', value) - - def writeUInt64(self, value): - self.pack('Q', value) - - def writeFloat(self, value): - self.pack('f', value) - - def writeDouble(self, value): - self.pack('d', value) - - def writeString(self, value): - length = len(value) - self.writeUInt16(length) - self.pack(str(length) + 's', value) - - def pack(self, fmt, data): - return self.writeBytes(pack(fmt, data)) - - def unpack(self, fmt, length = 1): - return unpack(fmt, self.readBytes(length))[0] diff --git a/pbf2db/build_proto.sh b/pbf2db/build_proto.sh deleted file mode 100755 index 53e9966..0000000 --- a/pbf2db/build_proto.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -protoc --python_out=. fileformat.proto -protoc --python_out=. osmformat.proto diff --git a/pbf2db/fileformat.proto b/pbf2db/fileformat.proto deleted file mode 100644 index f1b540a..0000000 --- a/pbf2db/fileformat.proto +++ /dev/null @@ -1,49 +0,0 @@ -/** Copyright (c) 2010 Scott A. Crosby. <scott@sacrosby.com> - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see <http://www.gnu.org/licenses/>. - -*/ - -package OSMPBF; - -// -// STORAGE LAYER: Storing primitives. -// - -message Blob { - optional bytes raw = 1; // No compression - optional int32 raw_size = 2; // When compressed, the uncompressed size - - // Possible compressed versions of the data. - optional bytes zlib_data = 3; - - // PROPOSED feature for LZMA compressed data. SUPPORT IS NOT REQUIRED. - optional bytes lzma_data = 4; - - // Formerly used for bzip2 compressed data. Depreciated in 2010. - optional bytes OBSOLETE_bzip2_data = 5 [deprecated=true]; // Don't reuse this tag number. -} - -/* A file contains an sequence of fileblock headers, each prefixed by -their length in network byte order, followed by a data block -containing the actual data. types staring with a "_" are reserved. -*/ - -message BlobHeader { - required string type = 1; - optional bytes indexdata = 2; - required int32 datasize = 3; -} - - diff --git a/pbf2db/osmformat.proto b/pbf2db/osmformat.proto deleted file mode 100644 index eaad195..0000000 --- a/pbf2db/osmformat.proto +++ /dev/null @@ -1,225 +0,0 @@ -/** Copyright (c) 2010 Scott A. Crosby. <scott@sacrosby.com> - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see <http://www.gnu.org/licenses/>. - -*/ - -package OSMPBF; - -/* OSM Binary file format - -This is the master schema file of the OSM binary file format. This -file is designed to support limited random-access and future -extendability. - -A binary OSM file consists of a sequence of FileBlocks (please see -fileformat.proto). The first fileblock contains a serialized instance -of HeaderBlock, followed by a sequence of PrimitiveBlock blocks that -contain the primitives. - -Each primitiveblock is designed to be independently parsable. It -contains a string table storing all strings in that block (keys and -values in tags, roles in relations, usernames, etc.) as well as -metadata containing the precision of coordinates or timestamps in that -block. - -A primitiveblock contains a sequence of primitive groups, each -containing primitives of the same type (nodes, densenodes, ways, -relations). Coordinates are stored in signed 64-bit integers. Lat&lon -are measured in units <granularity> nanodegrees. The default of -granularity of 100 nanodegrees corresponds to about 1cm on the ground, -and a full lat or lon fits into 32 bits. - -Converting an integer to a lattitude or longitude uses the formula: -$OUT = IN * granularity / 10**9$. Many encoding schemes use delta -coding when representing nodes and relations. - -*/ - -////////////////////////////////////////////////////////////////////////// -////////////////////////////////////////////////////////////////////////// - -/* Contains the file header. */ - -message HeaderBlock { - optional HeaderBBox bbox = 1; - /* Additional tags to aid in parsing this dataset */ - repeated string required_features = 4; - repeated string optional_features = 5; - - optional string writingprogram = 16; - optional string source = 17; // From the bbox field. -} - - -/** The bounding box field in the OSM header. BBOX, as used in the OSM -header. Units are always in nanodegrees -- they do not obey -granularity rules. */ - -message HeaderBBox { - required sint64 left = 1; - required sint64 right = 2; - required sint64 top = 3; - required sint64 bottom = 4; -} - - -/////////////////////////////////////////////////////////////////////// -/////////////////////////////////////////////////////////////////////// - - -message PrimitiveBlock { - required StringTable stringtable = 1; - repeated PrimitiveGroup primitivegroup = 2; - - // Granularity, units of nanodegrees, used to store coordinates in this block - optional int32 granularity = 17 [default=100]; - // Offset value between the output coordinates coordinates and the granularity grid in unites of nanodegrees. - optional int64 lat_offset = 19 [default=0]; - optional int64 lon_offset = 20 [default=0]; - -// Granularity of dates, normally represented in units of milliseconds since the 1970 epoch. - optional int32 date_granularity = 18 [default=1000]; - - - // Proposed extension: - //optional BBox bbox = XX; -} - -// Group of OSMPrimitives. All primitives in a group must be the same type. -message PrimitiveGroup { - repeated Node nodes = 1; - optional DenseNodes dense = 2; - repeated Way ways = 3; - repeated Relation relations = 4; - repeated ChangeSet changesets = 5; -} - - -/** String table, contains the common strings in each block. - - Note that we reserve index '0' as a delimiter, so the entry at that - index in the table is ALWAYS blank and unused. - - */ -message StringTable { - repeated bytes s = 1; -} - -/* Optional metadata that may be included into each primitive. */ -message Info { - optional int32 version = 1 [default = -1]; - optional int64 timestamp = 2; - optional int64 changeset = 3; - optional int32 uid = 4; - optional uint32 user_sid = 5; // String IDs -} - -/** Optional metadata that may be included into each primitive. Special dense format used in DenseNodes. */ -message DenseInfo { - repeated int32 version = 1 [packed = true]; - repeated sint64 timestamp = 2 [packed = true]; // DELTA coded - repeated sint64 changeset = 3 [packed = true]; // DELTA coded - repeated sint32 uid = 4 [packed = true]; // DELTA coded - repeated sint32 user_sid = 5 [packed = true]; // String IDs for usernames. DELTA coded -} - - -// THIS IS STUB DESIGN FOR CHANGESETS. NOT USED RIGHT NOW. -// TODO: REMOVE THIS? -message ChangeSet { - required int64 id = 1; -// -// // Parallel arrays. -// repeated uint32 keys = 2 [packed = true]; // String IDs. -// repeated uint32 vals = 3 [packed = true]; // String IDs. -// -// optional Info info = 4; - -// optional int64 created_at = 8; -// optional int64 closetime_delta = 9; -// optional bool open = 10; -// optional HeaderBBox bbox = 11; -} - - -message Node { - required sint64 id = 1; - // Parallel arrays. - repeated uint32 keys = 2 [packed = true]; // String IDs. - repeated uint32 vals = 3 [packed = true]; // String IDs. - - optional Info info = 4; // May be omitted in omitmeta - - required sint64 lat = 8; - required sint64 lon = 9; -} - -/* Used to densly represent a sequence of nodes that do not have any tags. - -We represent these nodes columnwise as five columns: ID's, lats, and -lons, all delta coded. When metadata is not omitted, - -We encode keys & vals for all nodes as a single array of integers -containing key-stringid and val-stringid, using a stringid of 0 as a -delimiter between nodes. - - ( (<keyid> <valid>)* '0' )* - */ - -message DenseNodes { - repeated sint64 id = 1 [packed = true]; // DELTA coded - - //repeated Info info = 4; - optional DenseInfo denseinfo = 5; - - repeated sint64 lat = 8 [packed = true]; // DELTA coded - repeated sint64 lon = 9 [packed = true]; // DELTA coded - - // Special packing of keys and vals into one array. May be empty if all nodes in this block are tagless. - repeated int32 keys_vals = 10 [packed = true]; -} - - -message Way { - required int64 id = 1; - // Parallel arrays. - repeated uint32 keys = 2 [packed = true]; - repeated uint32 vals = 3 [packed = true]; - - optional Info info = 4; - - repeated sint64 refs = 8 [packed = true]; // DELTA coded -} - -message Relation { - enum MemberType { - NODE = 0; - WAY = 1; - RELATION = 2; - } - required int64 id = 1; - - // Parallel arrays. - repeated uint32 keys = 2 [packed = true]; - repeated uint32 vals = 3 [packed = true]; - - optional Info info = 4; - - // Parallel arrays - repeated int32 roles_sid = 8 [packed = true]; - repeated sint64 memids = 9 [packed = true]; // DELTA encoded - repeated MemberType types = 10 [packed = true]; -} - diff --git a/pbf2db/pbf2db.py b/pbf2db/pbf2db.py deleted file mode 100755 index c6089fc..0000000 --- a/pbf2db/pbf2db.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/python - -import osmformat_pb2 -import fileformat_pb2 -import sys -import socket -import zlib -from binarystream import BinaryStream - - -headerSizeMax = 64 * 1024 -bodySizeMax = 32*1024*1024 - -f = open("berlin.osm.pbf") -stream = BinaryStream(f) -headerSize = socket.ntohl(stream.readUInt32()); - -if headerSizeMax < headerSize: - raise ValueError("Header to long") - -headerbuff = stream.readBytes(headerSize) -blobheader = fileformat_pb2.BlobHeader() -blobheader.ParseFromString(headerbuff) -bodysize = blobheader.datasize - -if bodySizeMax < bodysize: - raise ValueError("Body to fat") - -blobbuff = stream.readBytes(bodysize) -blob = fileformat_pb2.Blob() -blob.ParseFromString(blobbuff) - -if blob.raw != "": - rawstr = blob.raw -else: - rawstr = zlib.decompress(blob.zlib_data) - -headerblock = osmformat_pb2.HeaderBlock() -headerblock.ParseFromString(rawstr) - -print "Source:",headerblock.source -print "Writingprog:",headerblock.writingprogram -print "required features:",headerblock.required_features - |