Skip to content

Commit

Permalink
feat(cat-ci): Add mithril targets that use official tools to get prop…
Browse files Browse the repository at this point in the history
…er snapshot links (#229)

* feat(cat-ci): Add mithril targets that use official tools to get proper snapshot links.

* fix(cat-ci): remove quotes from argument expansion

* fix(cat-ci): Escape a quote and show DL progress in logs

* fix(cat-ci): Reduce verbosity of the wget log for the large mithril files

* fix(cspell): spelling

* fix(mithril): Update mithril tools to latest versions, and document the targets
  • Loading branch information
stevenj authored May 14, 2024
1 parent b40ecb8 commit 08317ce
Show file tree
Hide file tree
Showing 2 changed files with 130 additions and 10 deletions.
6 changes: 6 additions & 0 deletions .config/dictionaries/project.dic
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
aarch
bindgen
buildkit
cardano
camelcase
codegen
colordiff
Expand All @@ -23,6 +24,7 @@ fontconfig
fontname
forcelabels
genpkey
giga
ginkgolinter
gitops
glightbox
Expand All @@ -49,6 +51,7 @@ lintfix
markdownlint
mattn
mdlint
mithril
mkdocs
modcache
mozallowfullscreen
Expand All @@ -72,6 +75,7 @@ rustdoc
rustdocflags
rustflags
rustfmt
sanchonet
rustup
sqlfluff
stdcfgs
Expand All @@ -87,6 +91,7 @@ Timoni
toolsets
transpiling
UDCs
unarchiving
uniseg
voteplan
wasi
Expand All @@ -95,3 +100,4 @@ webkitallowfullscreen
WORKDIR
xerrors
zstd
zstdcat
134 changes: 124 additions & 10 deletions earthly/mithril_snapshot/Earthfile
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,132 @@ VERSION 0.8

# cspell: words preprod

package-preprod-snapshot:
FROM alpine:3.19
# base-snapshot-builder : Base OS image to run the mithril client for all networks.
# Shouldn't be used directly.
base-snapshot-builder:
# Note: an official docker image is here, but it could not reliably pull from Earthly.
# It gives permission denied errors randomly.
# Image comes from: https://github.com/input-output-hk/mithril/pkgs/container/mithril-client
# FROM ghcr.io/input-output-hk/mithril-client:latest

# Build our own simple mithril client image - it's not hard
FROM debian:stable-20240513-slim

RUN apt-get update && apt-get -y install wget jq zstd

RUN wget -O mithril-cli.deb \
https://github.com/input-output-hk/mithril/releases/download/2418.1/mithril-client-cli_0.8.0+512fe8d-1_amd64.deb
RUN dpkg -i mithril-cli.deb

# DL_SNAPSHOT : Main function which manages downloading and validating mithril snapshots.
# Arguments:
# * network - The network to download "mainnet", "preview", "preprod" or "sanchonet"
# * mithril-network - The name of the mithril network to access, must match what is required by the `network` argument.
# * snapshot_digest - The digest of the snapshot to get, locks the download to a particular instant in time. `latest` will fetch the most recent.
# * validate - `false` do not validate the snapshot, faster but could be an invalid snapshot. (default)
# - `true` validate the snapshot, so we know it was authoritatively generated.
DL_SNAPSHOT:
FUNCTION
ARG network
ARG mithril_network
ARG snapshot_digest=latest
ARG validate=false

LET NETWORK=$network
LET AGGREGATOR_ENDPOINT=https://aggregator.${mithril_network}.api.mithril.network/aggregator
LET GENESIS_VERIFICATION_KEY=$(wget -q -O - https://raw.githubusercontent.com/input-output-hk/mithril/main/mithril-infra/configuration/${mithril_network}/genesis.vkey)
LET SNAPSHOT_DIGEST=$snapshot_digest

WORKDIR /snapshot

RUN wget -O \
preprod.tar.zst \
https://storage.googleapis.com/cdn.aggregator.release-preprod.api.mithril.network/preprod-e135-i2625.80b0c02cd0c86c47a79a9b6d0d9b0e6936d5f621e8094ed53405fef3ee103a3b.tar.zst
IF $validate
# Validation is slower, but the file will be guaranteed to be valid.
# This automatically handles unarchiving the snapshot
RUN mithril-client -v cardano-db download --json $snapshot_digest
ELSE
# This is faster, but it could be a faked snapshot file.
LET DL_FILE=$(mithril-client cardano-db snapshot show --json $snapshot_digest | jq '.locations[0]' | tr -d '\"')
RUN mkdir -p ./db && \
wget --progress=dot:giga -O - $DL_FILE | zstdcat | tar -xf - -C ./db
END

# mainnet : Get latest mithril snapshot for Cardano mainnet.
# Arguments :
# * digest = The specific snapshot digest to download. Defaults to `latest`.
# This target is used for developers to test against different networks and
# is not currently used in CI.
# This snapshot is NOT validated, in order to speed up its use in development.
# Warning, this snapshot is VERY LARGE. > 44GB
mainnet:
FROM +base-snapshot-builder
ARG digest=latest

DO +DL_SNAPSHOT \
--network=mainnet \
--mithril_network=release-mainnet \
--snapshot_digest=$digest

SAVE ARTIFACT /snapshot/db snapshot

# preprod : Mithril snapshot from the pre-prod network.
# This snapshot is defaulted to a snapshot from:
# Epoch : 142
# Immutable File Number: 2773
# Created : 14/05/2024 @ 8:46:19 am
# The snapshot can be changed by altering the `digest` argument with the
# required digest to download.
# Also, as this snapshot is used in CI, we only used a validated image here
# This is slower, but guaranteed to be accurate.
preprod:
FROM +base-snapshot-builder
ARG digest="1ee36c3ee0aa4fec38d2487641ee64f687e675696783f4cea1e4857ebc2ef38c"

# extract preprod archive
RUN apk add zstd
RUN zstd -d preprod.tar.zst && tar -xf preprod.tar
RUN rm preprod.tar && rm preprod.tar.zst
DO +DL_SNAPSHOT \
--network=preprod \
--mithril_network=release-preprod \
--snapshot_digest=$digest \
--validate=true

SAVE ARTIFACT /snapshot/db snapshot

# preview : Get latest mithril snapshot for the preview network.
# Arguments :
# * digest = The specific snapshot digest to download. Defaults to `latest`.
# This target is used for developers to test against different networks and
# is not currently used in CI.
# This snapshot is NOT validated, in order to speed up its use in development.
preview:
FROM +base-snapshot-builder
ARG digest=latest

DO +DL_SNAPSHOT \
--network=preview \
--mithril_network=pre-release-preview \
--snapshot-digest=$digest

SAVE ARTIFACT /snapshot/db snapshot

# sanchonet : Get latest mithril snapshot for sanchonet (CIP-1694 test network).
# Arguments :
# * digest = The specific snapshot digest to download. Defaults to `latest`.
# This target is used for developers to test against different networks and
# is not currently used in CI.
# This snapshot is NOT validated, in order to speed up its use in development.
sanchonet:
FROM +base-snapshot-builder
ARG digest=latest

DO +DL_SNAPSHOT \
--network=sanchonet \
--mithril_network=testing-sanchonet \
--snapshot-digest=$digest

SAVE ARTIFACT /snapshot/db snapshot

# package-preprod-snapshot : Create a mithril snapshot of pre-prod automatically for CI
# This target is for CI use, we do NOT generate snapshot packages for other networks
# because they are slow and not used currently in any CI process.
package-preprod-snapshot:
FROM +preprod

SAVE ARTIFACT /snapshot snapshot
SAVE ARTIFACT /snapshot/db snapshot

0 comments on commit 08317ce

Please sign in to comment.