Skip to content

Commit

Permalink
Merge pull request #60 from input-output-hk/feature/rust
Browse files Browse the repository at this point in the history
feat(rust): Enhanced Rust builders to better handle Arm/X86 build machines
  • Loading branch information
stevenj authored Oct 31, 2023
2 parents 2698c99 + 6b9ee28 commit 49b4abc
Show file tree
Hide file tree
Showing 27 changed files with 2,322 additions and 108 deletions.
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -15,3 +15,6 @@ node_modules
# Built docs
/earthly/docs/site
/earthly/docs/local

# Local Build Artefacts
target/
263 changes: 207 additions & 56 deletions earthly/rust/Earthfile
Original file line number Diff line number Diff line change
Expand Up @@ -2,57 +2,90 @@
VERSION 0.7

# cspell: words rustup rustc automake autotools xutils miri nextest kani
# cspell: words TARGETPLATFORM TARGETOS TARGETARCH TARGETVARIANT
# cspell: words USERPLATFORM USEROS USERARCH USERVARIANT
# cspell: words ripgrep colordiff rustfmt stdcfgs toolset toolsets readelf

# Base Rustup build container.
rustup:
# Base it on Debian bookworm-slim
FROM debian:bookworm-slim
rust-base:
ARG TARGETPLATFORM
ARG TARGETOS
ARG TARGETARCH
ARG TARGETVARIANT
ARG USERPLATFORM
ARG USEROS
ARG USERARCH
ARG USERVARIANT


# This is our base Host toolset, and rustup.
# The ACTUAL version of rust that will be used, and available targets
# is controlled by a `rust-toolchain.toml` file when the RUST_SETUP UDC is run.
# HOWEVER, It is enforced that the rust version in `rust-toolchain.toml` MUST match this version.
FROM rust:1.73-alpine3.18

RUN echo "TARGETPLATFORM = $TARGETPLATFORM"; \
echo "TARGETOS = $TARGETOS"; \
echo "TARGETARCH = $TARGETARCH"; \
echo "TARGETVARIANT = $TARGETVARIANT"; \
echo "USERPLATFORM = $USERPLATFORM"; \
echo "USEROS = $USEROS"; \
echo "USERARCH = $USERARCH"; \
echo "USERVARIANT = $USERVARIANT";

WORKDIR /root

# Set necessary env vars from this setup.
ENV RUSTUP_HOME=/usr/local/rustup
ENV CARGO_HOME=/usr/local/cargo
ENV CARGO_HOME_BIN=$CARGO_HOME/bin
ENV PATH=$CARGO_HOME_BIN:$PATH

# Ensure correct directory permissions on the install locations.
RUN mkdir -p $RUSTUP_HOME && \
mkdir -p $CARGO_HOME_BIN && \
chmod -R a+w $RUSTUP_HOME $CARGO_HOME
WORKDIR /root

# Install necessary packages
# Expand this list as needed, rather than adding more tools in later containers.
RUN apt-get update && \
apt-get install --no-install-recommends -y \
ca-certificates \
curl \
file \
build-essential \
autoconf \
automake \
autotools-dev \
libtool \
xutils-dev \
RUN apk add --no-cache \
musl-dev \
mold \
python3 \
python3-pip \
musl-tools \
clang && \
rm -rf /var/lib/apt/lists/*
clang \
ripgrep \
bash \
colordiff \
graphviz

# Install OpenSSL here but ONLY if we can't avoid using it.
# Make sure we have the clippy linter.
RUN rustup component add clippy

# Install rustup.
RUN curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain none -y -v
# Install a nightly toolchain which matches.
RUN rustup toolchain install nightly --component miri --component rust-src --component rustfmt --component clippy

# Install the default cargo config.
COPY config.toml $CARGO_HOME/config.toml
COPY stdcfgs/cargo_config.toml $CARGO_HOME/config.toml

# Install rust based tooling
# Install tools we use commonly with `cargo`.
# Note, we disable static compiles for tools, specifically, as its not required.
# These tools are not artifacts and we do not use them in production.
RUN cargo install cargo-nextest --version=0.9.59 && \
cargo install cargo-machete --version=0.6.0 && \
cargo install refinery_cli --version=0.8.11 && \
cargo install cargo-deny --version=0.14.3 && \
cargo install cargo-modules --version=0.10.2

# Universal build scripts we will always need and are not target dependent.
COPY --dir scripts /scripts

# Standardized Rust configs.
# Build will refuse to proceed if the projects rust configs do not match these.
# This is to enforce consistent compiler and tool configuration on local setup and CI builds.
COPY --dir stdcfgs /stdcfgs

# Builds all the rust-base targets for each supported DOCKER architecture.
# Currently only used for multi-platform cross build testing.
# This will ONLY work if you have `qemu` properly setup on linux and `rosetta` for
# docker enabled on Mac.
# Again, this is just a test target, and not for general use.
rust-base-all-hosts:
BUILD --platform=linux/amd64 --platform=linux/arm64 +rust-base

# Common Rust setup
# Parameters:
# * toolchain : The `rust-toolchain` toml file.
RUST_SETUP:
SETUP:
COMMAND
ARG toolchain=./rust-toolchain.toml

Expand All @@ -62,35 +95,153 @@ RUST_SETUP:

# Copy our toolchain dependency.
COPY $toolchain ./rust-toolchain.toml
ENV default_rust_channel=$(grep -oP 'channel\s*=\s*"\K[^"]+' rust-toolchain.toml)
ENV default_rust_channel=$(rg -oP 'channel\s*=\s*"\K[^"]+' rust-toolchain.toml)

# Check that `default_rust_channel` and $RUST_VERSION from the rust-base container are exactly the same.
# This ensures CI Rust version and local rust version are properly aligned and prevents version skew.
RUN /scripts/verify_toolchain.sh $default_rust_channel $RUST_VERSION

# Install pinned Rustup from `rust-toolchain.toml`
# Plus nightly latest so we can use it for docs, lints, etc.
RUN rustup default $default_rust_channel && \
rustup show && \
rustup toolchain install nightly --component miri --component rust-src && \
cargo --version && \
cargo +nightly --version

# Install tools we use commonly with `cargo`.
RUN cargo install cargo-nextest --locked && \
cargo install refinery_cli --locked && \
cargo install cargo-machete --locked

# Test rust build container
check:
FROM +rustup
CP_SRC:
# Copy the build src using this.
COMMAND
# This can be one directory like `"src/*"` or multiple src separated by `","`:
# eg, "example/Cargo.toml, example/Cargo.lock, example/src"
ARG src=""

DO +RUST_SETUP --toolchain=example/rust-toolchain.toml
# ONLY copy whats needed to build. Not everything.
# Note: rust-toolchain.toml was already copied when rust was setup.
# Minimizing whats copied reduces needless cache misses.
FOR --sep=", " file IN $src
COPY --dir $file .
END

# Check all the expected tooling is installed and works for both the stable and nightly versions.
RUN rustc --version && \
rustc +nightly --version && \
cargo --version && \
cargo +nightly --version && \
cargo clippy --version && \
cargo +nightly clippy --version && \
cargo nextest --version && \
cargo machete --version \
refinery --version && \
mold --version
RUN ls -al


# Steps we do during the `check` CI phase for all Rust programs
CHECK:
COMMAND

# This is set up so that ALL checks are run and it will fail if any fail.
# This improves visibility into all issues that need to be corrected for `check`
# to pass without needing to iterate excessively.
RUN /scripts/std_checks.sh

# Check if the build executable, isn't a busted mess.
SMOKE_TEST:
COMMAND
ARG TARGETPLATFORM
ARG bin

RUN ldd target/$TARGETARCH/release/$bin
RUN readelf -p .comment target/$TARGETARCH/release/$bin
RUN strip -v target/$TARGETARCH/release/$bin

# ALL executables MUST have `--help` as an option.
RUN target/$TARGETARCH/release/$bin --help


# Set up our target toolchains, and copy our files.
builder:
FROM +rust-base

DO +SETUP --toolchain=example/rust-toolchain.toml

DO +CP_SRC --src="example/*"

# Test rust build container - Use best architecture host tools.
check-hosted:
FROM +builder

DO +CHECK

# Test which runs check with all supported host tooling. Needs qemu or rosetta to run.
# Only used to validate tooling is working across host toolsets.
check-all-hosts:
BUILD --platform=linux/amd64 --platform=linux/arm64 +check-hosted

build-hosted:
ARG TARGETPLATFORM

# Build the service
FROM +builder

RUN /scripts/std_build.sh

DO +SMOKE_TEST --bin=hello_world

SAVE ARTIFACT target/$TARGETARCH/doc doc
SAVE ARTIFACT target/$TARGETARCH/release/hello_world hello_world

# Test which runs check with all supported host tooling. Needs qemu or rosetta to run.
# Only used to validate tooling is working across host toolsets.
build-all-hosts:
BUILD --platform=linux/amd64 --platform=linux/arm64 +build-hosted

## -----------------------------------------------------------------------------
##
## Standard CI targets.
##
## These targets are discovered and executed automatically by CI.

# Run check using the most efficient host tooling
# CI Automated Entry point.
check:
FROM busybox
# This is necessary to pick the correct architecture build to suit the native machine.
# It primarily ensures that Darwin/Arm builds work as expected without needing x86 emulation.
# All target implementation of this should follow this pattern.
ARG USERARCH

IF [ "$USERARCH" == "arm64" ]
BUILD --platform=linux/arm64 +check-hosted
ELSE
BUILD --platform=linux/amd64 +check-hosted
END

# Run build using the most efficient host tooling
# CI Automated Entry point.
build:
FROM busybox
# This is necessary to pick the correct architecture build to suit the native machine.
# It primarily ensures that Darwin/Arm builds work as expected without needing x86 emulation.
# All target implementation of this should follow this pattern.
ARG USERARCH

IF [ "$USERARCH" == "arm64" ]
BUILD --platform=linux/arm64 +build-hosted
ELSE
BUILD --platform=linux/amd64 +build-hosted
END


# This step will build any packages we would intend to publish or integration test.
package:
FROM scratch

# Run integration tests on all packages built during the `package` step.
integrate:
FROM scratch

# Publish packages if all integration tests have passed.
# (Failure to pass tests will prevent packages being published.)
publish:
FROM scratch

## -----------------------------------------------------------------------------

# This step simulates the full CI run for local purposes only.
local-ci-run:
BUILD +check
BUILD +build
BUILD +package
BUILD +integrate
BUILD +publish
52 changes: 0 additions & 52 deletions earthly/rust/config.toml

This file was deleted.

Loading

0 comments on commit 49b4abc

Please sign in to comment.