Merge pull request #60 from input-output-hk/feature/rust

feat(rust): Enhanced Rust builders to better handle Arm/X86 build machines

.gitignore

@@ -15,3 +15,6 @@ node_modules
 # Built docs
 /earthly/docs/site
 /earthly/docs/local
+
+# Local Build Artefacts
+target/

earthly/rust/Earthfile

@@ -2,57 +2,90 @@
 VERSION 0.7
 
 # cspell: words rustup rustc automake autotools xutils miri nextest kani
+# cspell: words TARGETPLATFORM TARGETOS TARGETARCH TARGETVARIANT 
+# cspell: words USERPLATFORM USEROS USERARCH USERVARIANT 
+# cspell: words ripgrep colordiff rustfmt stdcfgs toolset toolsets readelf
 
 # Base Rustup build container.
-rustup:
-    # Base it on Debian bookworm-slim
-    FROM debian:bookworm-slim
+rust-base:
+    ARG TARGETPLATFORM
+    ARG TARGETOS
+    ARG TARGETARCH
+    ARG TARGETVARIANT
+    ARG USERPLATFORM
+    ARG USEROS
+    ARG USERARCH
+    ARG USERVARIANT
+
+
+    # This is our base Host toolset, and rustup.
+    # The ACTUAL version of rust that will be used, and available targets
+    # is controlled by a `rust-toolchain.toml` file when the RUST_SETUP UDC is run.
+    # HOWEVER, It is enforced that the rust version in `rust-toolchain.toml` MUST match this version.
+    FROM rust:1.73-alpine3.18
+
+    RUN echo "TARGETPLATFORM = $TARGETPLATFORM"; \
+        echo "TARGETOS       = $TARGETOS"; \
+        echo "TARGETARCH     = $TARGETARCH"; \
+        echo "TARGETVARIANT  = $TARGETVARIANT"; \
+        echo "USERPLATFORM   = $USERPLATFORM"; \
+        echo "USEROS         = $USEROS"; \
+        echo "USERARCH       = $USERARCH"; \
+        echo "USERVARIANT    = $USERVARIANT"; 
 
-    WORKDIR /root
-
-    # Set necessary env vars from this setup.
-    ENV RUSTUP_HOME=/usr/local/rustup
-    ENV CARGO_HOME=/usr/local/cargo
-    ENV CARGO_HOME_BIN=$CARGO_HOME/bin
-    ENV PATH=$CARGO_HOME_BIN:$PATH    
 
-    # Ensure correct directory permissions on the install locations.
-    RUN mkdir -p $RUSTUP_HOME && \
-        mkdir -p $CARGO_HOME_BIN && \
-        chmod -R a+w $RUSTUP_HOME $CARGO_HOME
+    WORKDIR /root
 
     # Install necessary packages 
     # Expand this list as needed, rather than adding more tools in later containers.
-    RUN apt-get update && \
-        apt-get install --no-install-recommends -y \
-            ca-certificates \
-            curl \
-            file \
-            build-essential \
-            autoconf \
-            automake \
-            autotools-dev \
-            libtool \
-            xutils-dev \
+     RUN apk add --no-cache \
+            musl-dev \
             mold \
-            python3 \
-            python3-pip \
-            musl-tools \
-            clang && \
-        rm -rf /var/lib/apt/lists/*
+            clang \
+            ripgrep \
+            bash \
+            colordiff \
+            graphviz
 
-    # Install OpenSSL here but ONLY if we can't avoid using it.
+    # Make sure we have the clippy linter.
+    RUN rustup component add clippy
 
-    # Install rustup.
-    RUN curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain none -y -v
+    # Install a nightly toolchain which matches.
+    RUN rustup toolchain install nightly --component miri --component rust-src --component rustfmt --component clippy
 
     # Install the default cargo config.
-    COPY config.toml $CARGO_HOME/config.toml
+    COPY stdcfgs/cargo_config.toml $CARGO_HOME/config.toml
+
+    # Install rust based tooling
+    # Install tools we use commonly with `cargo`.
+    # Note, we disable static compiles for tools, specifically, as its not required.
+    # These tools are not artifacts and we do not use them in production.
+    RUN cargo install cargo-nextest --version=0.9.59 && \
+        cargo install cargo-machete --version=0.6.0  && \
+        cargo install refinery_cli  --version=0.8.11 && \
+        cargo install cargo-deny    --version=0.14.3 && \
+        cargo install cargo-modules --version=0.10.2
+
+    # Universal build scripts we will always need and are not target dependent.
+    COPY --dir scripts /scripts
+
+    # Standardized Rust configs.
+    # Build will refuse to proceed if the projects rust configs do not match these.
+    # This is to enforce consistent compiler and tool configuration on local setup and CI builds.
+    COPY --dir stdcfgs /stdcfgs
+
+# Builds all the rust-base targets for each supported DOCKER architecture.
+# Currently only used for multi-platform cross build testing.
+# This will ONLY work if you have `qemu` properly setup on linux and `rosetta` for 
+# docker enabled on Mac.
+# Again, this is just a test target, and not for general use.
+rust-base-all-hosts:
+    BUILD --platform=linux/amd64 --platform=linux/arm64 +rust-base
 
 # Common Rust setup
 # Parameters:
 #  * toolchain : The `rust-toolchain` toml file.
-RUST_SETUP:
+SETUP:
     COMMAND
     ARG toolchain=./rust-toolchain.toml
 
@@ -62,35 +95,153 @@ RUST_SETUP:
 
     # Copy our toolchain dependency.
     COPY $toolchain ./rust-toolchain.toml
-    ENV default_rust_channel=$(grep -oP 'channel\s*=\s*"\K[^"]+' rust-toolchain.toml)
+    ENV default_rust_channel=$(rg -oP 'channel\s*=\s*"\K[^"]+' rust-toolchain.toml)
+
+    # Check that `default_rust_channel` and $RUST_VERSION from the rust-base container are exactly the same.
+    # This ensures CI Rust version and local rust version are properly aligned and prevents version skew.
+    RUN /scripts/verify_toolchain.sh $default_rust_channel $RUST_VERSION
 
     # Install pinned Rustup from `rust-toolchain.toml`
     # Plus nightly latest so we can use it for docs, lints, etc.
     RUN rustup default $default_rust_channel && \
         rustup show && \
-        rustup toolchain install nightly --component miri --component rust-src && \
         cargo --version && \
         cargo +nightly --version
 
-    # Install tools we use commonly with `cargo`.
-    RUN cargo install cargo-nextest --locked && \
-        cargo install refinery_cli --locked && \
-        cargo install cargo-machete --locked
 
-# Test rust build container
-check:
-    FROM +rustup
+CP_SRC:
+    # Copy the build src using this.
+    COMMAND
+    # This can be one directory like `"src/*"` or multiple src separated by `","`:
+    # eg, "example/Cargo.toml, example/Cargo.lock, example/src"
+    ARG src=""
 
-    DO +RUST_SETUP --toolchain=example/rust-toolchain.toml
+    # ONLY copy whats needed to build.  Not everything.
+    # Note: rust-toolchain.toml was already copied when rust was setup.
+    # Minimizing whats copied reduces needless cache misses.
+    FOR --sep=", " file IN $src
+        COPY --dir $file .
+    END
 
-    # Check all the expected tooling is installed and works for both the stable and nightly versions.
-    RUN rustc --version && \
-        rustc +nightly --version && \
-        cargo --version && \
-        cargo +nightly --version &&  \
-        cargo clippy --version && \
-        cargo +nightly clippy --version && \
-        cargo nextest --version && \
-        cargo machete --version \
-        refinery --version && \
-        mold --version
+    RUN ls -al 
+
+
+# Steps we do during the `check` CI phase for all Rust programs
+CHECK:
+    COMMAND
+
+    # This is set up so that ALL checks are run and it will fail if any fail.
+    # This improves visibility into all issues that need to be corrected for `check`
+    # to pass without needing to iterate excessively.
+    RUN /scripts/std_checks.sh
+
+# Check if the build executable, isn't a busted mess.
+SMOKE_TEST:
+    COMMAND
+    ARG TARGETPLATFORM
+    ARG bin
+
+    RUN ldd target/$TARGETARCH/release/$bin
+    RUN readelf -p .comment target/$TARGETARCH/release/$bin
+    RUN strip -v target/$TARGETARCH/release/$bin
+
+    # ALL executables MUST have `--help` as an option.
+    RUN target/$TARGETARCH/release/$bin --help
+
+
+# Set up our target toolchains, and copy our files.
+builder:
+    FROM +rust-base
+
+    DO +SETUP --toolchain=example/rust-toolchain.toml
+
+    DO +CP_SRC --src="example/*"
+
+# Test rust build container - Use best architecture host tools.
+check-hosted:
+    FROM +builder
+
+    DO +CHECK
+
+# Test which runs check with all supported host tooling.  Needs qemu or rosetta to run.
+# Only used to validate tooling is working across host toolsets.
+check-all-hosts:    
+    BUILD --platform=linux/amd64 --platform=linux/arm64 +check-hosted
+
+build-hosted:
+    ARG TARGETPLATFORM
+
+    # Build the service
+    FROM +builder
+
+    RUN /scripts/std_build.sh
+
+    DO +SMOKE_TEST --bin=hello_world
+
+    SAVE ARTIFACT target/$TARGETARCH/doc doc
+    SAVE ARTIFACT target/$TARGETARCH/release/hello_world hello_world
+
+# Test which runs check with all supported host tooling.  Needs qemu or rosetta to run.
+# Only used to validate tooling is working across host toolsets.
+build-all-hosts:    
+    BUILD --platform=linux/amd64 --platform=linux/arm64 +build-hosted
+
+## -----------------------------------------------------------------------------
+##
+## Standard CI targets.
+##
+## These targets are discovered and executed automatically by CI.
+
+# Run check using the most efficient host tooling
+# CI Automated Entry point.
+check:
+    FROM busybox
+    # This is necessary to pick the correct architecture build to suit the native machine.
+    # It primarily ensures that Darwin/Arm builds work as expected without needing x86 emulation.
+    # All target implementation of this should follow this pattern.
+    ARG USERARCH
+
+    IF [ "$USERARCH" == "arm64" ]
+        BUILD --platform=linux/arm64 +check-hosted
+    ELSE
+        BUILD --platform=linux/amd64 +check-hosted
+    END
+
+# Run build using the most efficient host tooling
+# CI Automated Entry point.
+build:
+    FROM busybox
+    # This is necessary to pick the correct architecture build to suit the native machine.
+    # It primarily ensures that Darwin/Arm builds work as expected without needing x86 emulation.
+    # All target implementation of this should follow this pattern.
+    ARG USERARCH
+
+    IF [ "$USERARCH" == "arm64" ]
+        BUILD --platform=linux/arm64 +build-hosted
+    ELSE
+        BUILD --platform=linux/amd64 +build-hosted
+    END
+
+
+# This step will build any packages we would intend to publish or integration test.
+package:
+    FROM scratch
+
+# Run integration tests on all packages built during the `package` step.
+integrate:
+    FROM scratch
+
+# Publish packages if all integration tests have passed. 
+# (Failure to pass tests will prevent packages being published.)
+publish:
+    FROM scratch
+
+## -----------------------------------------------------------------------------
+
+# This step simulates the full CI run for local purposes only.
+local-ci-run:
+    BUILD +check
+    BUILD +build
+    BUILD +package
+    BUILD +integrate
+    BUILD +publish