From dde7bc1269b4b6e46c0c6a5fe36bbd4576e06caf Mon Sep 17 00:00:00 2001 From: albttx Date: Mon, 16 Sep 2024 13:30:56 +0100 Subject: [PATCH 1/4] chore: example for a snapshot node --- snapshots-node/.gitignore | 2 ++ snapshots-node/docker-compose.yml | 24 +++++++++++++++++++ snapshots-node/env.example | 5 ++++ snapshots-node/scripts/entrypoint.gnoland.sh | 16 +++++++++++++ .../scripts/entrypoint.snapshotter.sh | 19 +++++++++++++++ snapshots-node/scripts/snapshots.sh | 18 ++++++++++++++ 6 files changed, 84 insertions(+) create mode 100644 snapshots-node/.gitignore create mode 100644 snapshots-node/docker-compose.yml create mode 100644 snapshots-node/env.example create mode 100755 snapshots-node/scripts/entrypoint.gnoland.sh create mode 100755 snapshots-node/scripts/entrypoint.snapshotter.sh create mode 100755 snapshots-node/scripts/snapshots.sh diff --git a/snapshots-node/.gitignore b/snapshots-node/.gitignore new file mode 100644 index 0000000..69d8d5a --- /dev/null +++ b/snapshots-node/.gitignore @@ -0,0 +1,2 @@ +env +gnoland-data/* diff --git a/snapshots-node/docker-compose.yml b/snapshots-node/docker-compose.yml new file mode 100644 index 0000000..b313ed0 --- /dev/null +++ b/snapshots-node/docker-compose.yml @@ -0,0 +1,24 @@ +name: "gnoland-snapshot-node" + +services: + gnoland: + image: "ghcr.io/gnolang/gno/gnoland:0.2.0" + entrypoint: ["/scripts/entrypoint.gnoland.sh"] + ports: + - 127.0.0.1:26657:26657 + volumes: + - ./scripts:/scripts + - ./gnoland-data:/gnoroot/gnoland-data + + snapshotter: + image: "docker:27.2.0-cli-alpine3.20" + entrypoint: ["sh", "/scripts/entrypoint.snapshotter.sh"] + env_file: "env" + environment: + CHAIN_ID: "test4" + volumes: + - "/var/run/docker.sock:/var/run/docker.sock:ro" + - "./scripts:/scripts" + - "./docker-compose.yml:/docker-compose.yml" + - "./env:/env" + - "./gnoland-data:/gnoroot/gnoland-data:ro" diff --git a/snapshots-node/env.example b/snapshots-node/env.example new file mode 100644 index 0000000..a711258 --- /dev/null +++ b/snapshots-node/env.example @@ -0,0 +1,5 @@ + +MC_ENDPOINT="" +MC_ACCESS_KEY="" +MC_SECRET_KEY="" +MC_BUCKET="" diff --git a/snapshots-node/scripts/entrypoint.gnoland.sh b/snapshots-node/scripts/entrypoint.gnoland.sh new file mode 100755 index 0000000..e4e3dc0 --- /dev/null +++ b/snapshots-node/scripts/entrypoint.gnoland.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +LOG_LEVEL=${LOG_LEVEL:-"info"} + +MONIKER=${MONIKER:-"gnode"} +P2P_LADDR=${P2P_LADDR:-"tcp://0.0.0.0:26656"} +RPC_LADDR=${RPC_LADDR:-"tcp://0.0.0.0:26657"} + +CHAIN_ID=${CHAIN_ID:-"dev"} + +gnoland config init + +gnoland config set rpc.laddr "${RPC_LADDR}" +gnoland config set p2p.laddr "${P2P_LADDR}" + +exec gnoland start --lazy --genesis="./gnoland-data/genesis.json" --log-level=${LOG_LEVEL} diff --git a/snapshots-node/scripts/entrypoint.snapshotter.sh b/snapshots-node/scripts/entrypoint.snapshotter.sh new file mode 100755 index 0000000..84ba54e --- /dev/null +++ b/snapshots-node/scripts/entrypoint.snapshotter.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env sh + +apk add --no-cache curl jq lz4 minio-client + +alias mc=mcli + +export DAEMON_HOME="/gnoroot/gnoland-data" + +[ -z "$MC_ENDPOINT" ] && echo "MC_ENDPOINT is not set, skip snapshot" && exit 1 +[ -z "$MC_ACCESS_KEY" ] && echo "MC_ACCESS_KEY is not set, skip snapshot" && exit 1 +[ -z "$MC_SECRET_KEY" ] && echo "MC_SECRET_KEY is not set, skip snapshot" && exit 1 + +mc alias set s3 ${MC_ENDPOINT} ${MC_ACCESS_KEY} ${MC_SECRET_KEY} + +# exec snapshots script every 4 hours +echo "0 */4 * * * sh /scripts/snapshots.sh" > /etc/crontabs/root +crontab /etc/crontabs/root + +exec crond -f diff --git a/snapshots-node/scripts/snapshots.sh b/snapshots-node/scripts/snapshots.sh new file mode 100755 index 0000000..4f5de79 --- /dev/null +++ b/snapshots-node/scripts/snapshots.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env sh + +HEIGHT=$(curl -s gnoland:26657/status | jq -r '.result.sync_info.latest_block_height') +echo "[INFO] Snapshotting at height: ${HEIGHT}" + +echo "[INFO] Stopping gnoland..." +docker compose stop gnoland + +FILE="gnoland-${HEIGHT}-${CHAIN_ID}.tar.lz4" + +tar cvf - -C ${DAEMON_HOME} wal db | lz4 > $FILE + +mc cp --attr x-amz-acl=public-read --tags="type=snapshot" ${FILE} "s3/${MC_BUCKET}/${CHAIN_ID}/${FILE}"; echo + +rm -vf ${FILE} + +echo "[INFO] Starting gnoland ..." +docker compose start gnoland From ad57a69b82935995379b8da21c390e363242f10f Mon Sep 17 00:00:00 2001 From: albttx Date: Wed, 6 Nov 2024 10:08:08 +0000 Subject: [PATCH 2/4] chore: add README, update cli --- snapshots-node/README.md | 21 +++++++++++++++++++ .../scripts/entrypoint.snapshotter.sh | 4 +--- snapshots-node/scripts/snapshots.sh | 4 +++- 3 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 snapshots-node/README.md diff --git a/snapshots-node/README.md b/snapshots-node/README.md new file mode 100644 index 0000000..b54f8c7 --- /dev/null +++ b/snapshots-node/README.md @@ -0,0 +1,21 @@ +# Snapshots node + +## How it's works + +1. Setup the environment variables and setup the variables for the [minio-client](https://min.io) with your S3 credentials + +``` sh +cp env.example env +``` + +2. Start the node + +By default, a snapshots will occur every 4 hours, and will upload it to S3 + +``` sh +docker compose up -d +``` + +## TO-DOs + +[ ] Prune the node before uploading the snapshots diff --git a/snapshots-node/scripts/entrypoint.snapshotter.sh b/snapshots-node/scripts/entrypoint.snapshotter.sh index 84ba54e..e5a06e2 100755 --- a/snapshots-node/scripts/entrypoint.snapshotter.sh +++ b/snapshots-node/scripts/entrypoint.snapshotter.sh @@ -2,15 +2,13 @@ apk add --no-cache curl jq lz4 minio-client -alias mc=mcli - export DAEMON_HOME="/gnoroot/gnoland-data" [ -z "$MC_ENDPOINT" ] && echo "MC_ENDPOINT is not set, skip snapshot" && exit 1 [ -z "$MC_ACCESS_KEY" ] && echo "MC_ACCESS_KEY is not set, skip snapshot" && exit 1 [ -z "$MC_SECRET_KEY" ] && echo "MC_SECRET_KEY is not set, skip snapshot" && exit 1 -mc alias set s3 ${MC_ENDPOINT} ${MC_ACCESS_KEY} ${MC_SECRET_KEY} +mcli alias set s3 ${MC_ENDPOINT} ${MC_ACCESS_KEY} ${MC_SECRET_KEY} # exec snapshots script every 4 hours echo "0 */4 * * * sh /scripts/snapshots.sh" > /etc/crontabs/root diff --git a/snapshots-node/scripts/snapshots.sh b/snapshots-node/scripts/snapshots.sh index 4f5de79..46f1055 100755 --- a/snapshots-node/scripts/snapshots.sh +++ b/snapshots-node/scripts/snapshots.sh @@ -1,5 +1,7 @@ #!/usr/bin/env sh +DAEMON_HOME="/gnoroot/gnoland-data" + HEIGHT=$(curl -s gnoland:26657/status | jq -r '.result.sync_info.latest_block_height') echo "[INFO] Snapshotting at height: ${HEIGHT}" @@ -10,7 +12,7 @@ FILE="gnoland-${HEIGHT}-${CHAIN_ID}.tar.lz4" tar cvf - -C ${DAEMON_HOME} wal db | lz4 > $FILE -mc cp --attr x-amz-acl=public-read --tags="type=snapshot" ${FILE} "s3/${MC_BUCKET}/${CHAIN_ID}/${FILE}"; echo +mcli cp --attr x-amz-acl=public-read --tags="type=snapshot" ${FILE} "s3/${MC_BUCKET}/${CHAIN_ID}/${FILE}"; echo rm -vf ${FILE} From 9dfd0798dab654cbd1f1e38256e1937808abf749 Mon Sep 17 00:00:00 2001 From: albttx Date: Wed, 6 Nov 2024 10:09:45 +0000 Subject: [PATCH 3/4] chore: put latest version --- snapshots-node/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snapshots-node/docker-compose.yml b/snapshots-node/docker-compose.yml index b313ed0..cd94446 100644 --- a/snapshots-node/docker-compose.yml +++ b/snapshots-node/docker-compose.yml @@ -2,7 +2,7 @@ name: "gnoland-snapshot-node" services: gnoland: - image: "ghcr.io/gnolang/gno/gnoland:0.2.0" + image: "ghcr.io/gnolang/gno/gnoland:chain-test4.3" entrypoint: ["/scripts/entrypoint.gnoland.sh"] ports: - 127.0.0.1:26657:26657 From d4b6bd5ff61af95f5f602205e10f53a1c1771c0c Mon Sep 17 00:00:00 2001 From: albttx Date: Wed, 6 Nov 2024 10:14:40 +0000 Subject: [PATCH 4/4] chore: update docs --- snapshots-node/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/snapshots-node/README.md b/snapshots-node/README.md index b54f8c7..1ce884b 100644 --- a/snapshots-node/README.md +++ b/snapshots-node/README.md @@ -16,6 +16,12 @@ By default, a snapshots will occur every 4 hours, and will upload it to S3 docker compose up -d ``` +## How to force a snapshot now + +``` sh +docker compose exec snapshotter sh /scripts/snapshots.sh +``` + ## TO-DOs [ ] Prune the node before uploading the snapshots