diff --git a/command/daemon.go b/command/daemon.go index 35a6a6490..577d46765 100644 --- a/command/daemon.go +++ b/command/daemon.go @@ -590,9 +590,8 @@ func loadConfig(filePath string) (*config.Config, error) { if cfg.Version != config.Version { log.Warn("Configuration file out-of-date. Upgrade by running: storetheindex init --upgrade") } - - if cfg.Datastore.Type != "levelds" { - return nil, fmt.Errorf("only levelds datastore type supported, %q not supported", cfg.Datastore.Type) + if err = checkDatastoreType(cfg.Datastore.Type); err != nil { + return nil, err } return cfg, nil diff --git a/command/datastore.go b/command/datastore.go index 8c94b8bd7..b8f43c941 100644 --- a/command/datastore.go +++ b/command/datastore.go @@ -12,6 +12,7 @@ import ( "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/query" leveldb "github.com/ipfs/go-ds-leveldb" + pebbledb "github.com/ipfs/go-ds-pebble" "github.com/ipni/storetheindex/config" "github.com/ipni/storetheindex/fsutil" ) @@ -25,9 +26,18 @@ const ( updateBatchSize = 500000 ) +func checkDatastoreType(dsType string) error { + switch dsType { + case "levelds", "pebble": + return nil + } + return fmt.Errorf("only levelds and pebble datastore types supported, %q not supported", dsType) +} + func createDatastore(ctx context.Context, dir, dsType string, rmExisting bool) (datastore.Batching, string, error) { - if dsType != "levelds" { - return nil, "", fmt.Errorf("only levelds datastore type supported, %q not supported", dsType) + err := checkDatastoreType(dsType) + if err != nil { + return nil, "", err } dataStorePath, err := config.Path("", dir) if err != nil { @@ -41,7 +51,13 @@ func createDatastore(ctx context.Context, dir, dsType string, rmExisting bool) ( if err = fsutil.DirWritable(dataStorePath); err != nil { return nil, "", err } - ds, err := leveldb.NewDatastore(dataStorePath, nil) + var ds datastore.Batching + switch dsType { + case "levelds": + ds, err = leveldb.NewDatastore(dataStorePath, nil) + case "pebble": + ds, err = pebbledb.NewDatastore(dataStorePath, nil) + } if err != nil { return nil, "", err } diff --git a/config/datastore.go b/config/datastore.go index 2b4eeb58f..c20383249 100644 --- a/config/datastore.go +++ b/config/datastore.go @@ -23,9 +23,9 @@ type Datastore struct { func NewDatastore() Datastore { return Datastore{ Dir: "datastore", - Type: "levelds", + Type: "pebble", TmpDir: "tmpstore", - TmpType: "levelds", + TmpType: "pebble", } } diff --git a/go.mod b/go.mod index f8ae19021..7ff53851f 100644 --- a/go.mod +++ b/go.mod @@ -15,10 +15,11 @@ require ( github.com/gammazero/channelqueue v0.2.1 github.com/gammazero/deque v0.2.1 github.com/gammazero/targz v0.0.3 - github.com/ipfs/boxo v0.17.0 + github.com/ipfs/boxo v0.18.0 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 github.com/ipfs/go-ds-leveldb v0.5.0 + github.com/ipfs/go-ds-pebble v0.3.1 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipld/go-car/v2 v2.13.1 github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0 @@ -203,6 +204,7 @@ require ( go.uber.org/fx v1.20.1 // indirect go.uber.org/mock v0.4.0 // indirect go.uber.org/multierr v1.11.0 // indirect + go4.org v0.0.0-20230225012048-214862532bf5 // indirect golang.org/x/crypto v0.19.0 // indirect golang.org/x/exp v0.0.0-20240213143201-ec583247a57a // indirect golang.org/x/mod v0.15.0 // indirect diff --git a/go.sum b/go.sum index bc8fff2aa..ded945ef8 100644 --- a/go.sum +++ b/go.sum @@ -421,8 +421,8 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.17.0 h1:fVXAb12dNbraCX1Cdid5BB6Kl62gVLNVA+e0EYMqAU0= -github.com/ipfs/boxo v0.17.0/go.mod h1:pIZgTWdm3k3pLF9Uq6MB8JEcW07UDwNJjlXW1HELW80= +github.com/ipfs/boxo v0.18.0 h1:MOL9/AgoV3e7jlVMInicaSdbgralfqSsbkc31dZ9tmw= +github.com/ipfs/boxo v0.18.0/go.mod h1:pIZgTWdm3k3pLF9Uq6MB8JEcW07UDwNJjlXW1HELW80= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.1.0/go.mod h1:FFJEf18E9izuCqUtHxbWEvq+reg7o4CW5wSAE1wsxj0= @@ -475,6 +475,8 @@ github.com/ipfs/go-ds-leveldb v0.4.1/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1 github.com/ipfs/go-ds-leveldb v0.4.2/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= github.com/ipfs/go-ds-leveldb v0.5.0 h1:s++MEBbD3ZKc9/8/njrn4flZLnCuY9I79v94gBUNumo= github.com/ipfs/go-ds-leveldb v0.5.0/go.mod h1:d3XG9RUDzQ6V4SHi8+Xgj9j1XuEk1z82lquxrVbml/Q= +github.com/ipfs/go-ds-pebble v0.3.1 h1:Jyad1qy+d0NZNisaSGUlBSt3dZNHAPl+JThyYe9Rziw= +github.com/ipfs/go-ds-pebble v0.3.1/go.mod h1:XYnWtulwJvHVOr2B0WVA/UC3dvRgFevjp8Pn9a3E1xo= github.com/ipfs/go-graphsync v0.16.0 h1:0BX7whXlV13Y9FZ/jRg+xaGHaGYbtGxGppKD6tncw6k= github.com/ipfs/go-graphsync v0.16.0/go.mod h1:WfbMW3hhmX5GQEQ+KJxsFzVJVBKgC5szfrYK7Zc7xIM= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= @@ -1247,8 +1249,9 @@ go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= -go4.org v0.0.0-20200411211856-f5505b9728dd h1:BNJlw5kRTzdmyfh5U8F93HA2OwkP7ZGwA51eJ/0wKOU= go4.org v0.0.0-20200411211856-f5505b9728dd/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= +go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= +go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1368,6 +1371,7 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1479,6 +1483,7 @@ golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXR golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1487,6 +1492,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/internal/registry/registry_test.go b/internal/registry/registry_test.go index 6f029f6ce..299c2cb8c 100644 --- a/internal/registry/registry_test.go +++ b/internal/registry/registry_test.go @@ -11,7 +11,7 @@ import ( "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" - leveldb "github.com/ipfs/go-ds-leveldb" + pebbledb "github.com/ipfs/go-ds-pebble" "github.com/ipni/go-libipni/find/model" "github.com/ipni/go-libipni/test" "github.com/ipni/storetheindex/config" @@ -142,7 +142,7 @@ func TestDatastore(t *testing.T) { } // Create datastore - dstore, err := leveldb.NewDatastore(dataStorePath, nil) + dstore, err := pebbledb.NewDatastore(dataStorePath, nil) require.NoError(t, err) r, err := New(ctx, discoveryCfg, dstore) @@ -172,7 +172,7 @@ func TestDatastore(t *testing.T) { require.NoError(t, dstore.Close()) // Create datastore - dstore, err = leveldb.NewDatastore(dataStorePath, nil) + dstore, err = pebbledb.NewDatastore(dataStorePath, nil) require.NoError(t, err) r, err = New(ctx, discoveryCfg, dstore) @@ -222,7 +222,7 @@ func TestDatastore(t *testing.T) { t.Log("Converted registry to work with assigner service") discoveryCfg.UseAssigner = true discoveryCfg.UnassignedPublishers = true - dstore, err = leveldb.NewDatastore(dataStorePath, nil) + dstore, err = pebbledb.NewDatastore(dataStorePath, nil) require.NoError(t, err) r, err = New(ctx, discoveryCfg, dstore) require.NoError(t, err)