From 08317ce096525d1f9857349e2f9973bea41c8b03 Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Tue, 14 May 2024 15:48:21 +0700 Subject: [PATCH] feat(cat-ci): Add mithril targets that use official tools to get proper snapshot links (#229) * feat(cat-ci): Add mithril targets that use official tools to get proper snapshot links. * fix(cat-ci): remove quotes from argument expansion * fix(cat-ci): Escape a quote and show DL progress in logs * fix(cat-ci): Reduce verbosity of the wget log for the large mithril files * fix(cspell): spelling * fix(mithril): Update mithril tools to latest versions, and document the targets --- .config/dictionaries/project.dic | 6 ++ earthly/mithril_snapshot/Earthfile | 134 ++++++++++++++++++++++++++--- 2 files changed, 130 insertions(+), 10 deletions(-) diff --git a/.config/dictionaries/project.dic b/.config/dictionaries/project.dic index f30400fff..ee5fd2a91 100644 --- a/.config/dictionaries/project.dic +++ b/.config/dictionaries/project.dic @@ -1,6 +1,7 @@ aarch bindgen buildkit +cardano camelcase codegen colordiff @@ -23,6 +24,7 @@ fontconfig fontname forcelabels genpkey +giga ginkgolinter gitops glightbox @@ -49,6 +51,7 @@ lintfix markdownlint mattn mdlint +mithril mkdocs modcache mozallowfullscreen @@ -72,6 +75,7 @@ rustdoc rustdocflags rustflags rustfmt +sanchonet rustup sqlfluff stdcfgs @@ -87,6 +91,7 @@ Timoni toolsets transpiling UDCs +unarchiving uniseg voteplan wasi @@ -95,3 +100,4 @@ webkitallowfullscreen WORKDIR xerrors zstd +zstdcat \ No newline at end of file diff --git a/earthly/mithril_snapshot/Earthfile b/earthly/mithril_snapshot/Earthfile index ffcf967ce..4cdb85499 100644 --- a/earthly/mithril_snapshot/Earthfile +++ b/earthly/mithril_snapshot/Earthfile @@ -2,18 +2,132 @@ VERSION 0.8 # cspell: words preprod -package-preprod-snapshot: - FROM alpine:3.19 +# base-snapshot-builder : Base OS image to run the mithril client for all networks. +# Shouldn't be used directly. +base-snapshot-builder: + # Note: an official docker image is here, but it could not reliably pull from Earthly. + # It gives permission denied errors randomly. + # Image comes from: https://github.com/input-output-hk/mithril/pkgs/container/mithril-client + # FROM ghcr.io/input-output-hk/mithril-client:latest + + # Build our own simple mithril client image - it's not hard + FROM debian:stable-20240513-slim + + RUN apt-get update && apt-get -y install wget jq zstd + + RUN wget -O mithril-cli.deb \ + https://github.com/input-output-hk/mithril/releases/download/2418.1/mithril-client-cli_0.8.0+512fe8d-1_amd64.deb + RUN dpkg -i mithril-cli.deb + +# DL_SNAPSHOT : Main function which manages downloading and validating mithril snapshots. +# Arguments: +# * network - The network to download "mainnet", "preview", "preprod" or "sanchonet" +# * mithril-network - The name of the mithril network to access, must match what is required by the `network` argument. +# * snapshot_digest - The digest of the snapshot to get, locks the download to a particular instant in time. `latest` will fetch the most recent. +# * validate - `false` do not validate the snapshot, faster but could be an invalid snapshot. (default) +# - `true` validate the snapshot, so we know it was authoritatively generated. +DL_SNAPSHOT: + FUNCTION + ARG network + ARG mithril_network + ARG snapshot_digest=latest + ARG validate=false + + LET NETWORK=$network + LET AGGREGATOR_ENDPOINT=https://aggregator.${mithril_network}.api.mithril.network/aggregator + LET GENESIS_VERIFICATION_KEY=$(wget -q -O - https://raw.githubusercontent.com/input-output-hk/mithril/main/mithril-infra/configuration/${mithril_network}/genesis.vkey) + LET SNAPSHOT_DIGEST=$snapshot_digest WORKDIR /snapshot - RUN wget -O \ - preprod.tar.zst \ - https://storage.googleapis.com/cdn.aggregator.release-preprod.api.mithril.network/preprod-e135-i2625.80b0c02cd0c86c47a79a9b6d0d9b0e6936d5f621e8094ed53405fef3ee103a3b.tar.zst + IF $validate + # Validation is slower, but the file will be guaranteed to be valid. + # This automatically handles unarchiving the snapshot + RUN mithril-client -v cardano-db download --json $snapshot_digest + ELSE + # This is faster, but it could be a faked snapshot file. + LET DL_FILE=$(mithril-client cardano-db snapshot show --json $snapshot_digest | jq '.locations[0]' | tr -d '\"') + RUN mkdir -p ./db && \ + wget --progress=dot:giga -O - $DL_FILE | zstdcat | tar -xf - -C ./db + END + +# mainnet : Get latest mithril snapshot for Cardano mainnet. +# Arguments : +# * digest = The specific snapshot digest to download. Defaults to `latest`. +# This target is used for developers to test against different networks and +# is not currently used in CI. +# This snapshot is NOT validated, in order to speed up its use in development. +# Warning, this snapshot is VERY LARGE. > 44GB +mainnet: + FROM +base-snapshot-builder + ARG digest=latest + + DO +DL_SNAPSHOT \ + --network=mainnet \ + --mithril_network=release-mainnet \ + --snapshot_digest=$digest + + SAVE ARTIFACT /snapshot/db snapshot + +# preprod : Mithril snapshot from the pre-prod network. +# This snapshot is defaulted to a snapshot from: +# Epoch : 142 +# Immutable File Number: 2773 +# Created : 14/05/2024 @ 8:46:19 am +# The snapshot can be changed by altering the `digest` argument with the +# required digest to download. +# Also, as this snapshot is used in CI, we only used a validated image here +# This is slower, but guaranteed to be accurate. +preprod: + FROM +base-snapshot-builder + ARG digest="1ee36c3ee0aa4fec38d2487641ee64f687e675696783f4cea1e4857ebc2ef38c" - # extract preprod archive - RUN apk add zstd - RUN zstd -d preprod.tar.zst && tar -xf preprod.tar - RUN rm preprod.tar && rm preprod.tar.zst + DO +DL_SNAPSHOT \ + --network=preprod \ + --mithril_network=release-preprod \ + --snapshot_digest=$digest \ + --validate=true + + SAVE ARTIFACT /snapshot/db snapshot + +# preview : Get latest mithril snapshot for the preview network. +# Arguments : +# * digest = The specific snapshot digest to download. Defaults to `latest`. +# This target is used for developers to test against different networks and +# is not currently used in CI. +# This snapshot is NOT validated, in order to speed up its use in development. +preview: + FROM +base-snapshot-builder + ARG digest=latest + + DO +DL_SNAPSHOT \ + --network=preview \ + --mithril_network=pre-release-preview \ + --snapshot-digest=$digest + + SAVE ARTIFACT /snapshot/db snapshot + +# sanchonet : Get latest mithril snapshot for sanchonet (CIP-1694 test network). +# Arguments : +# * digest = The specific snapshot digest to download. Defaults to `latest`. +# This target is used for developers to test against different networks and +# is not currently used in CI. +# This snapshot is NOT validated, in order to speed up its use in development. +sanchonet: + FROM +base-snapshot-builder + ARG digest=latest + + DO +DL_SNAPSHOT \ + --network=sanchonet \ + --mithril_network=testing-sanchonet \ + --snapshot-digest=$digest + + SAVE ARTIFACT /snapshot/db snapshot + +# package-preprod-snapshot : Create a mithril snapshot of pre-prod automatically for CI +# This target is for CI use, we do NOT generate snapshot packages for other networks +# because they are slow and not used currently in any CI process. +package-preprod-snapshot: + FROM +preprod - SAVE ARTIFACT /snapshot snapshot \ No newline at end of file + SAVE ARTIFACT /snapshot/db snapshot \ No newline at end of file