Merge pull request #70 from pelias/support_relations

support relations, reduce disk usage

README.md

@@ -1,5 +1,5 @@
 
-pbf2json creates a JSON stream of openstreetmap data from any PBF extract, you can pick-and-choose only the bits of the file you want, the library will take care of de-normalizing the relational data (nodes/ways) so you can put it straight in to your favourite document-store, inverted index or graph database.
+pbf2json creates a JSON stream of openstreetmap data from any PBF extract, you can pick-and-choose only the bits of the file you want, the library will take care of de-normalizing the relational data (nodes/ways/relations) so you can put it straight in to your favourite document-store, inverted index or graph database.
 
 ![animated-gif](http://missinglink.embed.s3.amazonaws.com/pbf2json-2.gif)
 
@@ -133,6 +133,10 @@ Output of the `nodes` array (as seen below) is optional, this was disabled by de
 }
 ```
 
+### Relations
+
+Since version `6.0` centroids and bounding boxes are also computed for relations, the calulations are based off the largest member way by area.
+
 ### Leveldb
 
 This library uses `leveldb` to store the lat/lon info about nodes so that it can denormalize the ways for you.

bitmask.go

@@ -0,0 +1,45 @@
+package main
+
+import "sync"
+import "github.com/tmthrgd/go-popcount"
+
+// Bitmask - simple bitmask data structire based on a map
+type Bitmask struct {
+	I     map[uint64]uint64
+	mutex *sync.RWMutex
+}
+
+// Has - basic get/set methods
+func (b *Bitmask) Has(val int64) bool {
+	var v = uint64(val)
+	b.mutex.RLock()
+	defer b.mutex.RUnlock()
+	return (b.I[v/64] & (1 << (v % 64))) != 0
+}
+
+// Insert - basic get/set methods
+func (b *Bitmask) Insert(val int64) {
+	var v = uint64(val)
+	b.mutex.Lock()
+	defer b.mutex.Unlock()
+	b.I[v/64] |= (1 << (v % 64))
+}
+
+// Len - total elements in mask (non performant!)
+func (b *Bitmask) Len() uint64 {
+	var l uint64
+	b.mutex.RLock()
+	defer b.mutex.RUnlock()
+	for _, v := range b.I {
+		l += popcount.CountSlice64([]uint64{v})
+	}
+	return l
+}
+
+// NewBitMask - constructor
+func NewBitMask() *Bitmask {
+	return &Bitmask{
+		I:     make(map[uint64]uint64),
+		mutex: &sync.RWMutex{},
+	}
+}

bitmaskmap.go

@@ -0,0 +1,86 @@
+package main
+
+import (
+	"encoding/gob"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"reflect"
+)
+
+// BitmaskMap - struct to hold common masks
+type BitmaskMap struct {
+	Nodes       *Bitmask
+	Ways        *Bitmask
+	Relations   *Bitmask
+	WayRefs     *Bitmask
+	RelNodes    *Bitmask
+	RelWays     *Bitmask
+	RelRelation *Bitmask
+}
+
+// NewBitmaskMap - constructor
+func NewBitmaskMap() *BitmaskMap {
+	return &BitmaskMap{
+		Nodes:       NewBitMask(),
+		Ways:        NewBitMask(),
+		Relations:   NewBitMask(),
+		WayRefs:     NewBitMask(),
+		RelNodes:    NewBitMask(),
+		RelWays:     NewBitMask(),
+		RelRelation: NewBitMask(),
+	}
+}
+
+// WriteTo - write to destination
+func (m *BitmaskMap) WriteTo(sink io.Writer) (int64, error) {
+	encoder := gob.NewEncoder(sink)
+	err := encoder.Encode(m)
+	return 0, err
+}
+
+// ReadFrom - read from destination
+func (m *BitmaskMap) ReadFrom(tap io.Reader) (int64, error) {
+	decoder := gob.NewDecoder(tap)
+	err := decoder.Decode(m)
+	return 0, err
+}
+
+// WriteToFile - write to disk
+func (m *BitmaskMap) WriteToFile(path string) {
+	file, err := os.Create(path)
+	if err != nil {
+		panic(err)
+	}
+	m.WriteTo(file)
+	log.Println("wrote bitmask:", path)
+}
+
+// ReadFromFile - read from disk
+func (m *BitmaskMap) ReadFromFile(path string) {
+
+	// bitmask file doesn't exist
+	if _, err := os.Stat(path); err != nil {
+		fmt.Println("bitmask file not found:", path)
+		os.Exit(1)
+	}
+
+	file, err := os.Open(path)
+	if err != nil {
+		panic(err)
+	}
+	m.ReadFrom(file)
+	log.Println("read bitmask:", path)
+}
+
+// Print -- print debug stats
+func (m BitmaskMap) Print() {
+	k := reflect.TypeOf(m)
+	v := reflect.ValueOf(m)
+	for i := 0; i < k.NumField(); i++ {
+		key := k.Field(i).Name
+		val := v.Field(i).Interface()
+		fmt.Printf("%s: %v\n", key, (val.(*Bitmask)).Len())
+	}
+}

encoding_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"math"
 	"testing"
 
 	"github.com/qedus/osmpbf"
@@ -55,3 +56,17 @@ func TestEncodingBitmaskValues(t *testing.T) {
 	var latlon = bytesToLatLon(byteval)
 	assert.Equal(t, expectedLatlon, latlon)
 }
+
+func TestEncodingAndDecodingIdsToBytes(t *testing.T) {
+
+	var ids = []int64{0, 100, 100000, 100000000, math.MaxInt64}
+
+	// encode
+	var encoded = idSliceToBytes(ids)
+	// assert.Equal(t, "100", stringid)
+	// assert.Equal(t, expectedBytes, byteval)
+
+	// decode
+	var decoded = bytesToIDSlice(encoded)
+	assert.Equal(t, decoded, ids)
+}

index.js

@@ -10,7 +10,7 @@ var util = require('util'),
 function errorHandler( name ){
   return function( data ){
     data.toString('utf8').trim().split('\n').forEach( function( line ){
-      console.log( util.format( '[%s]:', name ), line );
+      console.error( util.format( '[%s]:', name ), line );
     });
   };
 }