feat: hash checksums + download urls for snapshot restore and create

cmd/createSnapshot.go

@@ -9,6 +9,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
+	"go.uber.org/zap"
 )
 
 var createSnapshotCmd = &cobra.Command{
@@ -38,7 +39,7 @@ var createSnapshotCmd = &cobra.Command{
 		}
 
 		if err := svc.CreateSnapshot(); err != nil {
-			return fmt.Errorf("failed to create snapshot: %w", err)
+			l.Sugar().Fatalw("failed to create snapshot", zap.Error(err))
 		}
 
 		return nil

cmd/restoreSnapshot.go

@@ -9,39 +9,39 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
+	"go.uber.org/zap"
 )
 
 var restoreSnapshotCmd = &cobra.Command{
 	Use:   "restore-snapshot",
 	Short: "Restore database from a snapshot file",
 	Long: `Restore the database from a previously created snapshot file.
 
-Note: This command restores --database.schema_name only if it's present in InputFile snapshot.
+Note: This command restores --database.schema_name only if it's present in Input snapshot.
+The input can be a local file path or a URL, url of type http, https is supported.
 Follow the snapshot docs if you need to convert the snapshot to a different schema name than was used during snapshot creation.`,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		initRestoreSnapshotCmd(cmd)
 		cfg := config.NewConfig()
 
-		l, err := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug})
-		if err != nil {
-			return fmt.Errorf("failed to initialize logger: %w", err)
-		}
+		l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug})
 
 		svc, err := snapshot.NewSnapshotService(&snapshot.SnapshotConfig{
-			InputFile:  cfg.SnapshotConfig.InputFile,
-			Host:       cfg.DatabaseConfig.Host,
-			Port:       cfg.DatabaseConfig.Port,
-			User:       cfg.DatabaseConfig.User,
-			Password:   cfg.DatabaseConfig.Password,
-			DbName:     cfg.DatabaseConfig.DbName,
-			SchemaName: cfg.DatabaseConfig.SchemaName,
+			Input:       cfg.SnapshotConfig.Input,
+			VerifyInput: cfg.SnapshotConfig.VerifyInput,
+			Host:        cfg.DatabaseConfig.Host,
+			Port:        cfg.DatabaseConfig.Port,
+			User:        cfg.DatabaseConfig.User,
+			Password:    cfg.DatabaseConfig.Password,
+			DbName:      cfg.DatabaseConfig.DbName,
+			SchemaName:  cfg.DatabaseConfig.SchemaName,
 		}, l)
 		if err != nil {
 			return err
 		}
 
 		if err := svc.RestoreSnapshot(); err != nil {
-			return fmt.Errorf("failed to restore snapshot: %w", err)
+			l.Sugar().Fatalw("failed to restore snapshot", zap.Error(err))
 		}
 
 		return nil

cmd/root.go

@@ -63,8 +63,10 @@ func init() {
 	rootCmd.AddCommand(rpcCmd)
 
 	// bind any subcommand flags
-	createSnapshotCmd.PersistentFlags().String(config.SnapshotOutputFile, "", "Path to save the snapshot file to (required)")
-	restoreSnapshotCmd.PersistentFlags().String(config.SnapshotInputFile, "", "Path to the snapshot file (required)")
+	createSnapshotCmd.PersistentFlags().String(config.SnapshotOutputFile, "", "Path to save the snapshot file to (required), also creates a hash file")
+
+	restoreSnapshotCmd.PersistentFlags().String(config.SnapshotInput, "", "Path to a local snapshot file, or remote URL to download from (required)")
+	restoreSnapshotCmd.PersistentFlags().Bool(config.SnapshotVerifyInput, true, "Boolean to verify the input file against a sha256 checksum (<file name>.sha256sum). If a URL is provided, the checksum will be downloaded")
 
 	rpcCmd.PersistentFlags().String(config.SidecarPrimaryUrl, "", `RPC url of the "primary" Sidecar instance in an HA environment`)
 

docs/src/content/docs/running/advanced-snapshots.md

@@ -0,0 +1,43 @@
+---
+title: Advanced Snapshots
+description: Advanced Snapshots documentation
+---
+
+
+## Advanced: Converting the Schema of a Dump
+
+If you're using a custom schema and want to use a public snapshot, you likely want to convert the dump.
+
+This section provides a step-by-step runbook for converting a snapshot dump to use a different schema name.
+
+```bash
+# Open your terminal and create a temporary database to work with:
+psql -c "CREATE DATABASE temp_sidecar_dump_schema_conversion_db;"
+
+# Use the Sidecar CLI to restore the snapshot dump into the temporary database:
+./bin/sidecar restore-snapshot \
+    --database.host=localhost \
+    --database.user=... \
+    --database.password=... \
+    --database.port=5432 \
+    --database.db_name=temp_sidecar_dump_schema_conversion_db \
+    --database.schema_name=<input schema name> \
+    --snapshot.input=snapshot.dump
+    --snapshot.verify-input=false
+
+# Connect to the temporary database and execute the SQL command to rename the schema:
+psql -d temp_sidecar_dump_schema_conversion_db -c "ALTER SCHEMA <input schema name> RENAME TO <output schema name>;"
+
+# Use the Sidecar CLI to create a new snapshot with the updated schema:
+./bin/sidecar create-snapshot \
+    --database.host=localhost \
+    --database.user=... \
+    --database.password=... \
+    --database.port=5432 \
+    --database.db_name=temp_sidecar_dump_schema_conversion_db \
+    --database.schema_name=<output schema name> \
+    --snapshot.output-file=new_snapshot.dump
+
+# Drop the temporary database to free up resources:
+psql -c "DROP DATABASE IF EXISTS temp_sidecar_dump_schema_conversion_db;"
+```

docs/src/content/docs/running/getting-started.mdx

@@ -45,10 +45,9 @@ import { Steps } from '@astrojs/starlight/components';
     If you are launching a Sidecar for Holesky, you will need to use a snapshot to intialize your database. Detailed directions can be found on the ["restore from snapshot"](running/snapshots) page.
 
     ```bash
-    curl -LO https://eigenlayer-sidecar.s3.us-east-1.amazonaws.com/snapshots/testnet-holesky/sidecar-testnet-holesky_v3.0.0-rc.1_public_20250122.dump
-
     /usr/local/bin/sidecar restore-snapshot \
-      --input_file=sidecar-testnet-holesky_v3.0.0-rc.1_public_20250122.dump \
+      --snapshot.input=https://eigenlayer-sidecar.s3.us-east-1.amazonaws.com/snapshots/testnet-holesky/sidecar-testnet-holesky_v3.0.0-rc.1_public_20250122.dump \
+      --snapshot.verify-input=false \
       --database.host=<postgres host> \
       --database.user=<postgres user> \
       --database.password=<postgres password> \

docs/src/content/docs/running/snapshots.md

@@ -5,23 +5,109 @@ description: How to use a snapshot to start or restore your Sidecar
 
 Snapshots are a quicker way to sync to tip and get started.
 
-See [Snapshots Docs](old-docs/snapshots_docs.md) for instructions on creating and restoring snapshots
-
 ## Snapshot Sources
 
 * Mainnet Ethereum (not yet available)
 * Testnet Holesky ([2025-01-22](https://eigenlayer-sidecar.s3.us-east-1.amazonaws.com/snapshots/testnet-holesky/sidecar-testnet-holesky_v3.0.0-rc.1_public_20250122.dump))
 
 ## Example boot from testnet snapshot
 ```bash
-curl -LO https://eigenlayer-sidecar.s3.us-east-1.amazonaws.com/snapshots/testnet-holesky/sidecar-testnet-holesky_v3.0.0-rc.1_public_20250122.dump
-
 ./bin/sidecar restore-snapshot \
-  --input_file=sidecar-testnet-holesky_v3.0.0-rc.1_public_20250122.dump \
+  --snapshot.input=https://eigenlayer-sidecar.s3.us-east-1.amazonaws.com/snapshots/testnet-holesky/sidecar-testnet-holesky_v3.0.0-rc.1_public_20250122.dump \
+  --snapshot.verify-input=false \
   --database.host=localhost \
   --database.user=sidecar \
   --database.password=... \
   --database.port=5432 \
   --database.db_name=sidecar \
   --database.schema_name=public 
 ```
+
+## `restore-snapshot`
+```bash
+./bin/sidecar restore-snapshot --help
+Restore the database from a previously created snapshot file.
+
+Note: This command restores --database.schema_name only if it's present in Input snapshot.
+The input can be a local file path or a URL, url of type http, https is supported.
+Follow the snapshot docs if you need to convert the snapshot to a different schema name than was used during snapshot creation.
+
+Usage:
+  sidecar restore-snapshot [flags]
+
+Flags:
+  -h, --help                    help for restore-snapshot
+      --snapshot.input string   Path to the snapshot file either a URL or a local file (required)
+      --snapshot.verify-input   Boolean to verify the input file against its .sha256sum file, if input is a url then it downloads the file, (default is true) (default true)
+
+Global Flags:
+  -c, --chain string                              The chain to use (mainnet, holesky, preprod (default "mainnet")
+      --database.db_name string                   PostgreSQL database name (default "sidecar")
+      --database.host string                      PostgreSQL host (default "localhost")
+      --database.password string                  PostgreSQL password
+      --database.port int                         PostgreSQL port (default 5432)
+      --database.schema_name string               PostgreSQL schema name (default "public")
+      --database.user string                      PostgreSQL username (default "sidecar")
+      --datadog.statsd.enabled                    e.g. "true" or "false"
+      --datadog.statsd.url string                 e.g. "localhost:8125"
+      --debug                                     "true" or "false"
+      --ethereum.chunked_batch_call_size int      The number of calls to make in parallel when using the chunked batch call method (default 10)
+      --ethereum.contract_call_batch_size int     The number of contract calls to batch together when fetching data from the Ethereum node (default 25)
+      --ethereum.native_batch_call_size int       The number of calls to batch together when using the native eth_call method (default 500)
+      --ethereum.rpc-url string                   e.g. "http://<hostname>:8545"
+      --ethereum.use_native_batch_call            Use the native eth_call method for batch calls (default true)
+      --prometheus.enabled                        e.g. "true" or "false"
+      --prometheus.port int                       The port to run the prometheus server on (default 2112)
+      --rewards.generate_staker_operators_table   Generate staker operators table while indexing
+      --rewards.validate_rewards_root             Validate rewards roots while indexing (default true)
+      --rpc.grpc-port int                         gRPC port (default 7100)
+      --rpc.http-port int                         http rpc port (default 7101)
+```
+
+## `create-snapshot`
+```bash
+./bin/sidecar create-snapshot --help
+Create a snapshot of the database.
+
+Usage:
+  sidecar create-snapshot [flags]
+
+Flags:
+  -h, --help                          help for create-snapshot
+      --snapshot.output-file string   Path to save the snapshot file to (required), also creates a hash file
+
+Global Flags:
+  -c, --chain string                              The chain to use (mainnet, holesky, preprod (default "mainnet")
+      --database.db_name string                   PostgreSQL database name (default "sidecar")
+      --database.host string                      PostgreSQL host (default "localhost")
+      --database.password string                  PostgreSQL password
+      --database.port int                         PostgreSQL port (default 5432)
+      --database.schema_name string               PostgreSQL schema name (default "public")
+      --database.user string                      PostgreSQL username (default "sidecar")
+      --datadog.statsd.enabled                    e.g. "true" or "false"
+      --datadog.statsd.url string                 e.g. "localhost:8125"
+      --debug                                     "true" or "false"
+      --ethereum.chunked_batch_call_size int      The number of calls to make in parallel when using the chunked batch call method (default 10)
+      --ethereum.contract_call_batch_size int     The number of contract calls to batch together when fetching data from the Ethereum node (default 25)
+      --ethereum.native_batch_call_size int       The number of calls to batch together when using the native eth_call method (default 500)
+      --ethereum.rpc-url string                   e.g. "http://<hostname>:8545"
+      --ethereum.use_native_batch_call            Use the native eth_call method for batch calls (default true)
+      --prometheus.enabled                        e.g. "true" or "false"
+      --prometheus.port int                       The port to run the prometheus server on (default 2112)
+      --rewards.generate_staker_operators_table   Generate staker operators table while indexing
+      --rewards.validate_rewards_root             Validate rewards roots while indexing (default true)
+      --rpc.grpc-port int                         gRPC port (default 7100)
+      --rpc.http-port int                         http rpc port (default 7101)
+```
+
+#### Example use:
+```
+./bin/sidecar create-snapshot \     
+  --database.host=localhost \
+  --database.user=sidecar \
+  --database.password=sidecar \
+  --database.port=5432 \
+  --database.db_name=sidecar \
+  --database.schema_name=public \
+  --snapshot.output-file=example.dump
+```

go.mod

@@ -71,16 +71,19 @@ require (
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
+	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mmcloughlin/addchain v0.4.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a // indirect
 	github.com/prometheus/common v0.32.1 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/rs/zerolog v1.33.0 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/schollz/progressbar/v3 v3.18.0 // indirect
 	github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
 	github.com/spf13/afero v1.11.0 // indirect
@@ -94,7 +97,8 @@ require (
 	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
 	golang.org/x/net v0.33.0 // indirect
 	golang.org/x/sync v0.10.0 // indirect
-	golang.org/x/sys v0.28.0 // indirect
+	golang.org/x/sys v0.29.0 // indirect
+	golang.org/x/term v0.28.0 // indirect
 	golang.org/x/text v0.21.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect

go.sum

@@ -328,9 +328,12 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
 github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjUEN1uBnDo34A=
@@ -388,6 +391,8 @@ github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0
 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
@@ -402,6 +407,8 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke
 github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=
 github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
 github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
+github.com/schollz/progressbar/v3 v3.18.0 h1:uXdoHABRFmNIjUfte/Ex7WtuyVslrw2wVPQmCN62HpA=
+github.com/schollz/progressbar/v3 v3.18.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
 github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU=
 github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
@@ -645,7 +652,11 @@ golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
 golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
+golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg=
+golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

internal/config/config.go

@@ -63,8 +63,9 @@ type DatabaseConfig struct {
 }
 
 type SnapshotConfig struct {
-	OutputFile string
-	InputFile  string
+	OutputFile  string
+	Input       string
+	VerifyInput bool
 }
 
 type RpcConfig struct {
@@ -126,8 +127,9 @@ var (
 	DatabaseDbName     = "database.db_name"
 	DatabaseSchemaName = "database.schema_name"
 
-	SnapshotOutputFile = "output_file"
-	SnapshotInputFile  = "input_file"
+	SnapshotOutputFile  = "snapshot.output-file"
+	SnapshotInput       = "snapshot.input"
+	SnapshotVerifyInput = "snapshot.verify-input"
 
 	RewardsValidateRewardsRoot          = "rewards.validate_rewards_root"
 	RewardsGenerateStakerOperatorsTable = "rewards.generate_staker_operators_table"
@@ -170,8 +172,9 @@ func NewConfig() *Config {
 		},
 
 		SnapshotConfig: SnapshotConfig{
-			OutputFile: viper.GetString(normalizeFlagName(SnapshotOutputFile)),
-			InputFile:  viper.GetString(normalizeFlagName(SnapshotInputFile)),
+			OutputFile:  viper.GetString(normalizeFlagName(SnapshotOutputFile)),
+			Input:       viper.GetString(normalizeFlagName(SnapshotInput)),
+			VerifyInput: viper.GetBool(normalizeFlagName(SnapshotVerifyInput)),
 		},
 
 		RpcConfig: RpcConfig{