feat(relations): support relations, reduce disk usage

BREAKING CHANGE: major version bump to signify a new method of parsing
not technically a 'breaking change' but the allcaps message is required
by semantic-release

bitmask.go

@@ -0,0 +1,45 @@
+package main
+
+import "sync"
+import "github.com/tmthrgd/go-popcount"
+
+// Bitmask - simple bitmask data structire based on a map
+type Bitmask struct {
+	I     map[uint64]uint64
+	mutex *sync.RWMutex
+}
+
+// Has - basic get/set methods
+func (b *Bitmask) Has(val int64) bool {
+	var v = uint64(val)
+	b.mutex.RLock()
+	defer b.mutex.RUnlock()
+	return (b.I[v/64] & (1 << (v % 64))) != 0
+}
+
+// Insert - basic get/set methods
+func (b *Bitmask) Insert(val int64) {
+	var v = uint64(val)
+	b.mutex.Lock()
+	defer b.mutex.Unlock()
+	b.I[v/64] |= (1 << (v % 64))
+}
+
+// Len - total elements in mask (non performant!)
+func (b *Bitmask) Len() uint64 {
+	var l uint64
+	b.mutex.RLock()
+	defer b.mutex.RUnlock()
+	for _, v := range b.I {
+		l += popcount.CountSlice64([]uint64{v})
+	}
+	return l
+}
+
+// NewBitMask - constructor
+func NewBitMask() *Bitmask {
+	return &Bitmask{
+		I:     make(map[uint64]uint64),
+		mutex: &sync.RWMutex{},
+	}
+}

bitmaskmap.go

@@ -0,0 +1,86 @@
+package main
+
+import (
+	"encoding/gob"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"reflect"
+)
+
+// BitmaskMap - struct to hold common masks
+type BitmaskMap struct {
+	Nodes       *Bitmask
+	Ways        *Bitmask
+	Relations   *Bitmask
+	WayRefs     *Bitmask
+	RelNodes    *Bitmask
+	RelWays     *Bitmask
+	RelRelation *Bitmask
+}
+
+// NewBitmaskMap - constructor
+func NewBitmaskMap() *BitmaskMap {
+	return &BitmaskMap{
+		Nodes:       NewBitMask(),
+		Ways:        NewBitMask(),
+		Relations:   NewBitMask(),
+		WayRefs:     NewBitMask(),
+		RelNodes:    NewBitMask(),
+		RelWays:     NewBitMask(),
+		RelRelation: NewBitMask(),
+	}
+}
+
+// WriteTo - write to destination
+func (m *BitmaskMap) WriteTo(sink io.Writer) (int64, error) {
+	encoder := gob.NewEncoder(sink)
+	err := encoder.Encode(m)
+	return 0, err
+}
+
+// ReadFrom - read from destination
+func (m *BitmaskMap) ReadFrom(tap io.Reader) (int64, error) {
+	decoder := gob.NewDecoder(tap)
+	err := decoder.Decode(m)
+	return 0, err
+}
+
+// WriteToFile - write to disk
+func (m *BitmaskMap) WriteToFile(path string) {
+	file, err := os.Create(path)
+	if err != nil {
+		panic(err)
+	}
+	m.WriteTo(file)
+	log.Println("wrote bitmask:", path)
+}
+
+// ReadFromFile - read from disk
+func (m *BitmaskMap) ReadFromFile(path string) {
+
+	// bitmask file doesn't exist
+	if _, err := os.Stat(path); err != nil {
+		fmt.Println("bitmask file not found:", path)
+		os.Exit(1)
+	}
+
+	file, err := os.Open(path)
+	if err != nil {
+		panic(err)
+	}
+	m.ReadFrom(file)
+	log.Println("read bitmask:", path)
+}
+
+// Print -- print debug stats
+func (m BitmaskMap) Print() {
+	k := reflect.TypeOf(m)
+	v := reflect.ValueOf(m)
+	for i := 0; i < k.NumField(); i++ {
+		key := k.Field(i).Name
+		val := v.Field(i).Interface()
+		fmt.Printf("%s: %v\n", key, (val.(*Bitmask)).Len())
+	}
+}

encoding_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"math"
 	"testing"
 
 	"github.com/qedus/osmpbf"
@@ -55,3 +56,17 @@ func TestEncodingBitmaskValues(t *testing.T) {
 	var latlon = bytesToLatLon(byteval)
 	assert.Equal(t, expectedLatlon, latlon)
 }
+
+func TestEncodingAndDecodingIdsToBytes(t *testing.T) {
+
+	var ids = []int64{0, 100, 100000, 100000000, math.MaxInt64}
+
+	// encode
+	var encoded = idSliceToBytes(ids)
+	// assert.Equal(t, "100", stringid)
+	// assert.Equal(t, expectedBytes, byteval)
+
+	// decode
+	var decoded = bytesToIDSlice(encoded)
+	assert.Equal(t, decoded, ids)
+}