From b7556d73fec7e5215c88647931770e0badeef035 Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Fri, 21 Feb 2020 14:03:36 -0700 Subject: [PATCH] create option for low mem scenarios avoiding mmap (#255) * create option for low mem scenarios avoiding mmap Signed-off-by: Ignacio Hagopian * add variadic helper Signed-off-by: Ignacio Hagopian * add missing return Signed-off-by: Ignacio Hagopian --- .gitignore | 3 ++- go.mod | 1 + store/manager.go | 2 +- store/options.go | 17 +++++++++++++++-- store/store.go | 2 +- store/util.go | 1 + 6 files changed, 21 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index a94e5b62..418a4484 100644 --- a/.gitignore +++ b/.gitignore @@ -31,4 +31,5 @@ api/pb/dart/lib # vscode config folder .vscode/ -**/node_modules \ No newline at end of file +**/node_modules +tags diff --git a/go.mod b/go.mod index b00ae996..2da1beda 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/alecthomas/jsonschema v0.0.0-20191017121752-4bb6e3fae4f2 github.com/c-bata/go-prompt v0.2.3 github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect + github.com/dgraph-io/badger v1.6.0 github.com/evanphx/json-patch v4.5.0+incompatible github.com/fatih/color v1.7.0 github.com/gogo/googleapis v1.3.1 // indirect diff --git a/store/manager.go b/store/manager.go index 0725e3f7..aba232f1 100644 --- a/store/manager.go +++ b/store/manager.go @@ -35,7 +35,7 @@ func NewManager(ts service.Service, opts ...Option) (*Manager, error) { } if config.Datastore == nil { - datastore, err := newDefaultDatastore(config.RepoPath) + datastore, err := newDefaultDatastore(config.RepoPath, config.LowMem) if err != nil { return nil, err } diff --git a/store/options.go b/store/options.go index 436a54c3..a39453e0 100644 --- a/store/options.go +++ b/store/options.go @@ -4,6 +4,7 @@ import ( "os" "path/filepath" + "github.com/dgraph-io/badger/options" ds "github.com/ipfs/go-datastore" badger "github.com/ipfs/go-ds-badger" core "github.com/textileio/go-threads/core/store" @@ -24,18 +25,30 @@ type Config struct { EventCodec core.EventCodec JsonMode bool Debug bool + LowMem bool } func newDefaultEventCodec(jsonMode bool) core.EventCodec { return jsonpatcher.New(jsonMode) } -func newDefaultDatastore(repoPath string) (ds.TxnDatastore, error) { +func newDefaultDatastore(repoPath string, lowMem bool) (ds.TxnDatastore, error) { path := filepath.Join(repoPath, defaultDatastorePath) if err := os.MkdirAll(path, os.ModePerm); err != nil { return nil, err } - return badger.NewDatastore(path, &badger.DefaultOptions) + opts := badger.DefaultOptions + if lowMem { + opts.TableLoadingMode = options.FileIO + } + return badger.NewDatastore(path, &opts) +} + +func WithLowMem(low bool) Option { + return func(sc *Config) error { + sc.LowMem = low + return nil + } } func WithJsonMode(enabled bool) Option { diff --git a/store/store.go b/store/store.go index a2487c3e..87bb409b 100644 --- a/store/store.go +++ b/store/store.go @@ -85,7 +85,7 @@ func NewStore(ts service.Service, opts ...Option) (*Store, error) { // with the same config. func newStore(ts service.Service, config *Config) (*Store, error) { if config.Datastore == nil { - datastore, err := newDefaultDatastore(config.RepoPath) + datastore, err := newDefaultDatastore(config.RepoPath, config.LowMem) if err != nil { return nil, err } diff --git a/store/util.go b/store/util.go index dd35261a..c64d40d9 100644 --- a/store/util.go +++ b/store/util.go @@ -93,6 +93,7 @@ func DefaultService(repoPath string, opts ...ServiceOption) (ServiceBoostrapper, // Build a logstore logstorePath := filepath.Join(repoPath, defaultLogstorePath) if err := os.MkdirAll(logstorePath, os.ModePerm); err != nil { + cancel() return nil, err } logstore, err := badger.NewDatastore(logstorePath, &badger.DefaultOptions)