update pebble

command/daemon.go

@@ -527,7 +527,7 @@ func createValueStore(ctx context.Context, cfgIndexer config.Indexer) (indexer.I
 		pebbleOpts := &pbl.Options{
 			BytesPerSync:                10 << 20, // 10 MiB
 			WALBytesPerSync:             10 << 20, // 10 MiB
-			MaxConcurrentCompactions:    10,
+			MaxConcurrentCompactions:    func() int { return 10 },
 			MemTableSize:                64 << 20, // 64 MiB
 			MemTableStopWritesThreshold: 4,
 			LBaseMaxBytes:               64 << 20, // 64 MiB
@@ -538,7 +538,6 @@ func createValueStore(ctx context.Context, cfgIndexer config.Indexer) (indexer.I
 		}
 
 		pebbleOpts.Experimental.ReadCompactionRate = 10 << 20 // 20 MiB
-		pebbleOpts.Experimental.MinDeletionRate = 128 << 20   // 128 MiB
 
 		const numLevels = 7
 		pebbleOpts.Levels = make([]pbl.LevelOptions, numLevels)
@@ -591,9 +590,8 @@ func loadConfig(filePath string) (*config.Config, error) {
 	if cfg.Version != config.Version {
 		log.Warn("Configuration file out-of-date. Upgrade by running: storetheindex init --upgrade")
 	}
-
-	if cfg.Datastore.Type != "levelds" {
-		return nil, fmt.Errorf("only levelds datastore type supported, %q not supported", cfg.Datastore.Type)
+	if err = checkDatastoreType(cfg.Datastore.Type); err != nil {
+		return nil, err
 	}
 
 	return cfg, nil

command/datastore.go

@@ -26,11 +26,18 @@ const (
 	updateBatchSize = 500000
 )
 
-func createDatastore(ctx context.Context, dir, dsType string, rmExisting bool) (datastore.Batching, string, error) {
+func checkDatastoreType(dsType string) error {
 	switch dsType {
 	case "levelds", "pebble":
-	default:
-		return nil, "", fmt.Errorf("only levelds and pebble datastore type supported, %q not supported", dsType)
+		return nil
+	}
+	return fmt.Errorf("only levelds and pebble datastore types supported, %q not supported", dsType)
+}
+
+func createDatastore(ctx context.Context, dir, dsType string, rmExisting bool) (datastore.Batching, string, error) {
+	err := checkDatastoreType(dsType)
+	if err != nil {
+		return nil, "", err
 	}
 	dataStorePath, err := config.Path("", dir)
 	if err != nil {
@@ -44,11 +51,12 @@ func createDatastore(ctx context.Context, dir, dsType string, rmExisting bool) (
 	if err = fsutil.DirWritable(dataStorePath); err != nil {
 		return nil, "", err
 	}
+	var ds datastore.Batching
 	switch dsType {
 	case "levelds":
-		ds, err := leveldb.NewDatastore(dataStorePath, nil)
+		ds, err = leveldb.NewDatastore(dataStorePath, nil)
 	case "pebble":
-		ds, err := pebbledb.NewDatastore(dataStorePath, nil)
+		ds, err = pebbledb.NewDatastore(dataStorePath, nil)
 	}
 	if err != nil {
 		return nil, "", err

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.51
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.30.2
 	github.com/aws/smithy-go v1.13.5
-	github.com/cockroachdb/pebble v0.0.0-20231218155426-48b54c29d8fe
+	github.com/cockroachdb/pebble v0.0.0-20240118181935-4e6dda452a50
 	github.com/filecoin-project/go-dagaggregator-unixfs v0.3.0
 	github.com/gammazero/channelqueue v0.2.1
 	github.com/gammazero/deque v0.2.1
@@ -27,7 +27,7 @@ require (
 	github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0
 	github.com/ipld/go-ipld-prime v0.21.0
 	github.com/ipld/go-ipld-prime/storage/dsadapter v0.0.0-20230102063945-1a409dc236dd
-	github.com/ipni/go-indexer-core v0.8.7
+	github.com/ipni/go-indexer-core v0.8.8-0.20240118190551-4efdcbf0c1ac
 	github.com/ipni/go-libipni v0.5.9
 	github.com/libp2p/go-libp2p v0.32.2
 	github.com/libp2p/go-msgio v0.3.0

go.sum

@@ -153,8 +153,8 @@ github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZe
 github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
 github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA=
 github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA=
-github.com/cockroachdb/pebble v0.0.0-20231218155426-48b54c29d8fe h1:ZBhPcgWjnfy2PFWlvPlcOXAfAQqOIdpfksijpKiMWcc=
-github.com/cockroachdb/pebble v0.0.0-20231218155426-48b54c29d8fe/go.mod h1:BHuaMa/lK7fUe75BlsteiiTu8ptIG+qSAuDtGMArP18=
+github.com/cockroachdb/pebble v0.0.0-20240118181935-4e6dda452a50 h1:DTyV7gl0lxn4PkdWHlK5ZarQliiLTkPc0UfDQtizW0s=
+github.com/cockroachdb/pebble v0.0.0-20240118181935-4e6dda452a50/go.mod h1:BHuaMa/lK7fUe75BlsteiiTu8ptIG+qSAuDtGMArP18=
 github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30=
 github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
 github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
@@ -577,8 +577,8 @@ github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236
 github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd/go.mod h1:wZ8hH8UxeryOs4kJEJaiui/s00hDSbE37OKsL47g+Sw=
 github.com/ipld/go-ipld-prime/storage/dsadapter v0.0.0-20230102063945-1a409dc236dd h1:qdjo1CRvAQhOMoyYjPnbdZ5rYFFmqztweQ9KAsuWpO0=
 github.com/ipld/go-ipld-prime/storage/dsadapter v0.0.0-20230102063945-1a409dc236dd/go.mod h1:9DD/GM0JNPoisgR09F62kbBi7kHa4eDIea4XshXYOVc=
-github.com/ipni/go-indexer-core v0.8.7 h1:IaEBoVe1RiTBDTj8MiUlbsYk/L32AOjtCBhQQmXuxIo=
-github.com/ipni/go-indexer-core v0.8.7/go.mod h1:lLWTrQ7dhKwCak1qn6AQBNoSrGYBAXJJy2OGc02BO2Q=
+github.com/ipni/go-indexer-core v0.8.8-0.20240118190551-4efdcbf0c1ac h1:ndlXuHBVDZ8S4t6TN/2upZAX6PlqRmDZCj5nP2Gnv8o=
+github.com/ipni/go-indexer-core v0.8.8-0.20240118190551-4efdcbf0c1ac/go.mod h1:KjxMM2T9Vwe/qmDSv/X5kTMqicnv0KNUjl5uS08IEuM=
 github.com/ipni/go-libipni v0.5.9 h1:AlYlqZScX2jusGXXWkW5j6OMUtMKgQKNcl1Mi8g3glA=
 github.com/ipni/go-libipni v0.5.9/go.mod h1:c8mHa6J9iFREpDB29GlPIsbvztRq6bnhg5zJKrnvdUg=
 github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=