From 9b5e1731c133c7d8a0c5a2dc413da0faf7f1bd33 Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Thu, 18 Jan 2024 15:53:28 +0700 Subject: [PATCH] feat: Update Rust builder, add `target` flag (#140) * feat(rust): add support for wasm32 targets * feat(rust): Add verify-component-adapter from wasmtime as rust tooling * update * replace rust std_checks.sh with python version * update * fix std_checks.py * add std_build.py * fix spelling * fix dbviz build * remove uneeded TARGETARCH usage * remove std_build.sh * fix docs * add target flag to std_build.py * wip * fix * fix * add package argument * wip * cleanup * add std_docs.py script * fix spelling * wip * wip * disable cargo progress bar * fix * removed --locked flag * fix * remove SETUP udc * update docs * fix markdown * fix spell check * update config file * fix comments * wip * remove std_docs.py file * add TRY/FINALLY usage * fix spellcheck * fix * fix * add nightly channel section * add with_bench flag * Move SMOKE_TEST udc to the std_build.py script * remove nushell * fix markdown check --------- Co-authored-by: Mr-Leshiy --- docs/src/guides/languages/rust.md | 162 +++++++------- earthly/postgresql/Earthfile | 8 +- earthly/postgresql/scripts/python | 1 - earthly/rust/Earthfile | 53 +---- earthly/rust/scripts/std_build.py | 197 ++++++++++++++++++ earthly/rust/scripts/std_build.sh | 85 -------- earthly/rust/scripts/std_checks.py | 59 ++++++ earthly/rust/scripts/std_checks.sh | 47 ----- earthly/rust/stdcfgs/cargo_config.toml | 24 ++- earthly/rust/stdcfgs/deny.toml | 5 +- earthly/rust/stdcfgs/nextest.toml | 2 +- examples/rust/.cargo/config.toml | 24 ++- examples/rust/.config/nextest.toml | 2 +- examples/rust/Cargo.toml | 3 - examples/rust/Earthfile | 100 +++------ examples/rust/deny.toml | 5 +- examples/rust/rust-toolchain.toml | 4 +- utilities/dbviz/.cargo/config.toml | 24 ++- utilities/dbviz/.config/nextest.toml | 2 +- utilities/dbviz/Earthfile | 85 ++------ utilities/dbviz/deny.toml | 5 +- utilities/nushell/Earthfile | 65 ------ utilities/scripts/python/cli.py | 7 +- .../scripts/python/vendor_files_check.py | 11 + 24 files changed, 472 insertions(+), 508 deletions(-) delete mode 120000 earthly/postgresql/scripts/python create mode 100755 earthly/rust/scripts/std_build.py delete mode 100755 earthly/rust/scripts/std_build.sh create mode 100755 earthly/rust/scripts/std_checks.py delete mode 100755 earthly/rust/scripts/std_checks.sh delete mode 100644 utilities/nushell/Earthfile create mode 100644 utilities/scripts/python/vendor_files_check.py diff --git a/docs/src/guides/languages/rust.md b/docs/src/guides/languages/rust.md index f43f85197..e7650adda 100644 --- a/docs/src/guides/languages/rust.md +++ b/docs/src/guides/languages/rust.md @@ -9,7 +9,7 @@ tags: # :simple-rust: Rust - + ## Introduction @@ -80,47 +80,29 @@ By default `toolchain` setup to `rust-toolchain.toml`. ### Running checks ```Earthfile -# Test rust build container - Use best architecture host tools. -hosted-check: +# Run check using the most efficient host tooling +# CI Automated Entry point. +check: FROM +builder - DO ./../../earthly/rust+CHECK + RUN /scripts/std_checks.py # Test which runs check with all supported host tooling. Needs qemu or rosetta to run. # Only used to validate tooling is working across host toolsets. -all-hosts-check: - BUILD --platform=linux/amd64 --platform=linux/arm64 +hosted-check - -## Standard CI targets. -## -## These targets are discovered and executed automatically by CI. - -# Run check using the most efficient host tooling -# CI Automated Entry point. -check: - FROM busybox - # This is necessary to pick the correct architecture build to suit the native machine. - # It primarily ensures that Darwin/Arm builds work as expected without needing x86 emulation. - # All target implementation of this should follow this pattern. - ARG USERARCH - - IF [ "$USERARCH" == "arm64" ] - BUILD --platform=linux/arm64 +hosted-check - ELSE - BUILD --platform=linux/amd64 +hosted-check - END +all-hosts-check: + BUILD --platform=linux/amd64 --platform=linux/arm64 +check ``` With prepared environment and all data, we're now ready to start operating with the source code and configuration files. -The `hosted-check` target which actually performs all checks and validation -with the help of `+CHECK` Function. -The `+CHECK` Function performs static checks of the Rust project as +The `check` target which actually performs all checks and validation +with the help of `std_checks.py` script. +This script performs static checks of the Rust project as `cargo fmt`, `cargo machete`, `cargo deny` which will validate formatting, find unused dependencies and any supply chain issues with dependencies. -Here is the list of steps (look at `./earthly/rust/scripts/std_checks.sh`): +Here is the list of steps (look at `./earthly/rust/scripts/std_checks.py`): 1. `cargo fmtchk` ([cargo alias](https://doc.rust-lang.org/cargo/reference/config.html#alias), -look at `./earthly/rust/stdcfgs/config.toml`)Checking Rust Code Format. +look at `./earthly/rust/stdcfgs/cargo_config.toml`)Checking Rust Code Format. 2. Checking configuration files for consistency. 3. `cargo machete` - Checking for Unused Dependencies. 4. `cargo deny check` - Checking for Supply Chain Issues. @@ -131,8 +113,8 @@ to be the same as defined in `earthly/rust/stdcfgs` directory of the `catalyst-c So when you are going to setup a new Rust project copy these configuration files described above to the appropriate location of your Rust project. -Another targets as `all-hosts-check` and `check` (running on CI) just invoke `hosted-check` -with the specified `--platform`. +Another target as `all-hosts-check` just invokes `check` with the specified `--platform`. +It is needed for the local development to double check that everything is works for different platforms. It is important to define a `linux` target platform with a proper cpu architecture for the Rust project when you are building it inside Docker and check the build process with different scenarios. @@ -141,56 +123,84 @@ The same approach we will see for the another targets of this guide. ### Build ```Earthfile -# Build the service. -hosted-build: +# Run build using the most efficient host tooling +# CI Automated Entry point. +build: FROM +builder - DO ./../../earthly/rust+BUILD --libs="bar" --bins="foo/foo" + TRY + RUN /scripts/std_build.py --build_flags="" \ + --with_test \ + --with_bench \ + --cov_report="coverage-report.info" \ + --libs="bar" \ + --bins="foo/foo" + FINALLY + SAVE ARTIFACT target/nextest/ci/junit.xml example.junit-report.xml AS LOCAL + SAVE ARTIFACT coverage-report.info example.coverage-report.info AS LOCAL + END - DO ./../../earthly/rust+SMOKE_TEST --bin="foo" + SAVE ARTIFACT target/doc doc + SAVE ARTIFACT target/release/foo foo - SAVE ARTIFACT target/$TARGETARCH/doc doc - SAVE ARTIFACT target/$TARGETARCH/release/foo foo + DO ./../../earthly/rust+SMOKE_TEST --bin="foo" # Test which runs check with all supported host tooling. Needs qemu or rosetta to run. # Only used to validate tooling is working across host toolsets. -all-hosts-build: - BUILD --platform=linux/amd64 --platform=linux/arm64 +build-hosted - -# Run build using the most efficient host tooling -# CI Automated Entry point. -build: - FROM busybox - # This is necessary to pick the correct architecture build to suit the native machine. - # It primarily ensures that Darwin/Arm builds work as expected without needing x86 emulation. - # All target implementation of this should follow this pattern. - ARG USERARCH - - IF [ "$USERARCH" == "arm64" ] - BUILD --platform=linux/arm64 +hosted-build - ELSE - BUILD --platform=linux/amd64 +hosted-build - END +all-hosts-build: + BUILD --platform=linux/amd64 --platform=linux/arm64 +build ``` -After successful performing checks of the Rust project we can finally build artifacts. -As it was discussed in the previous section, actual job is done with `hosted-build` target, -other targets needs to configure different platform running options. -So we will focus on `hosted-build` target. +After successful performing checks of the Rust project we can finally `build` artifacts. Obviously it inherits `builder` target environment and than performs build of the binary. Important to note that in this particular example we are dealing with the executable Rust project, so it produces binary as a final artifact. Another case of the building Rust library we will consider later. -Actual build process is done with `+BUILD` Function. -The `+BUILD` Function have few arguments `libs` and `bins`, -they should be specified to properly generate `cargo-modules` docs (see description below). -The `libs` argument takes a list of library crate's names in your Rust project, e.g. +Actual build process is done with the `std_build.py` script. +Here is the full list of configuration of this script: + +```bash + usage: std_build.py [-h] [--build_flags BUILD_FLAGS] + [--doctest_flags DOCTEST_FLAGS] [--test_flags TEST_FLAGS] + [--bench_flags BENCH_FLAGS] [--with_test] + [--cov_report COV_REPORT] [--with_bench] [--libs LIBS] + [--bins BINS] + + Rust build processing. + + options: + -h, --help show this help message and exit + --build_flags BUILD_FLAGS + Additional command-line flags that can be passed to + the `cargo build` command. + --doctest_flags DOCTEST_FLAGS + Additional command-line flags that can be passed to + the `cargo testdocs` command. + --test_flags TEST_FLAGS + Additional command-line flags that can be passed to + the `cargo testunit` command. + --bench_flags BENCH_FLAGS + Additional command-line flags that can be passed to + the `cargo bench` command. + --with_test Flag to indicate whether to run tests (including unit + tests and doc tests). + --cov_report COV_REPORT + The output coverage report file path. If omitted, + coverage will not be run. + --with_bench Flag to indicate whether to run benchmarks. + --libs LIBS The list of lib crates `cargo-modules` docs to build + separated by comma. + --bins BINS The list of binaries `cargo-modules` docs to build and + made a smoke tests on them. . +``` + +Note that the `libs` argument takes a list of library crate's names in your Rust project, e.g. `--libs="crate1 crate2"`. The `bins` argument takes a list of binary crate's names and binary names in your Rust project, e.g. `--bins="crate1/bin1 crate1/bin2 crate2/bin1"`, note that each binary name correspond to each crate and separated in this list with `/` symbol. Under this build process we perform different steps of compiling and validating of our Rust project, -here is the list of steps (look at `./earthly/rust/scripts/std_build.sh`): +here is the list of steps (look at `./earthly/rust/scripts/std_build.py` and `./earthly/rust/scripts/std_docs.py`): 1. `cargo build` - Building all code in the workspace. 2. `cargo lint` ([cargo alias](https://doc.rust-lang.org/cargo/reference/config.html#alias), @@ -202,19 +212,17 @@ look at `./earthly/rust/stdcfgs/config.toml`)Checking Documentation can be gener look at `./earthly/rust/stdcfgs/config.toml`)Checking Self contained Unit tests all pass. 5. `cargo testdocs` ([cargo alias](https://doc.rust-lang.org/cargo/reference/config.html#alias), look at `./earthly/rust/stdcfgs/config.toml`)Checking Documentation tests all pass. -6. `cargo bench` - Checking Benchmarks all run to completion. -7. `cargo depgraph` - Generating dependency graph based on the Rust code. +6. `cargo testcov` ([cargo alias](https://doc.rust-lang.org/cargo/reference/config.html#alias), +look at `./earthly/rust/stdcfgs/config.toml`)Checking Self contained Unit tests all pass and collect coverage. +7. `cargo bench` - Checking Benchmarks all run to completion. +8. `cargo depgraph` - Generating dependency graph based on the Rust code. Generated artifacts are `doc/workspace.dot`, `doc/full.dot`, `doc/all.dot` files. -8. `cargo modules` - Generating modules trees and graphs based on the Rust code. +9. `cargo modules` - Generating modules trees and graphs based on the Rust code. Generated artifacts are `doc/$crate.$bin.bin.modules.tree`, `doc/$crate.$bin.bin.modules.dot` for the specified `--bins="crate1/bin1"` argument and `target/doc/$crate.lib.modules.tree`, `target/doc/$crate.lib.modules.dot` -for the specified `--libs="crate1"` argument of the `+BUILD` Function. - -Next steps is mandatory if you are going to produce a binary as an artifact, -for Rust libraries the are not mandatory and could be omitted. -The `+SMOKE_TEST` Function checks that produced binary with the specified name (`--bin` argument) -is executable, isn't a busted mess. +for the specified `--libs="crate1"` argument. +10. Running smoke tests on provided binary names (`--bins` argument). Final step is to provide desired artifacts: docs and binary. @@ -231,6 +239,14 @@ please follow this [guide](./../../onboarding/index.md). It is pretty strait forward for this builder process, because as a part of `+build` target we already creating a docker image. +## Rust `nightly` channel + +Be aware that we are running some tools in the `nightly` channel, such as `cargo fmt` and `cargo docs`. +It is highly likely that the `nightly` toolchain version on the CI machines differs from what you have locally. +Unfortunately, Rust tooling does not have the capability to preserve and maintain consistency between +`stable` and `nightly` toolchains simultaneously. +In our builds, we only preserve the `stable` toolchain version (`rust-toolchain.toml` file). + ## Conclusion You can see the final `Earthfile` [here](https://github.com/input-output-hk/catalyst-ci/blob/master/examples/rust/Earthfile) diff --git a/earthly/postgresql/Earthfile b/earthly/postgresql/Earthfile index ea494a6e4..465a66fea 100644 --- a/earthly/postgresql/Earthfile +++ b/earthly/postgresql/Earthfile @@ -1,7 +1,7 @@ # Common PostgreSQL Earthly builders VERSION --global-cache --use-function-keyword 0.7 -# cspell: words colordiff nushell psycopg dbviz +# cspell: words colordiff psycopg dbviz postgres-base: FROM postgres:16.0-alpine3.18 @@ -38,9 +38,6 @@ postgres-base: COPY ../../utilities/dbviz+build/dbviz /bin RUN dbviz --help - # Get nushell - COPY ../../utilities/nushell+nushell-build/nu /bin - # Copy our set SQL files COPY --dir sql /sql @@ -134,9 +131,6 @@ integration-test: bash \ postgresql15-client # We use PostgreSQL 16, but v15 clients will work OK with it. - # Get nushell - COPY ../../utilities/nushell+nushell-build/nu /bin - COPY +postgres-base/scripts /scripts WORKDIR /test diff --git a/earthly/postgresql/scripts/python b/earthly/postgresql/scripts/python deleted file mode 120000 index 168bb203b..000000000 --- a/earthly/postgresql/scripts/python +++ /dev/null @@ -1 +0,0 @@ -../../../utilities/scripts/python \ No newline at end of file diff --git a/earthly/rust/Earthfile b/earthly/rust/Earthfile index 2164b8ea9..36a040256 100644 --- a/earthly/rust/Earthfile +++ b/earthly/rust/Earthfile @@ -1,7 +1,7 @@ # Common Rust UDCs and Builders. VERSION --global-cache --use-function-keyword 0.7 -# cspell: words rustup miri nextest ripgrep colordiff rustfmt stdcfgs toolset readelf depgraph lcov +# cspell: words rustup miri nextest ripgrep colordiff rustfmt stdcfgs toolset depgraph lcov psycopg # cspell: words TARGETPLATFORM TARGETOS TARGETARCH TARGETVARIANT USERPLATFORM USEROS USERARCH USERVARIANT # Base Rustup build container. @@ -34,10 +34,13 @@ rust-base: # Install necessary packages # Expand this list as needed, rather than adding more tools in later containers. - RUN apk add --no-cache \ + RUN apk add --no-cache \ musl-dev \ mold \ clang \ + py3-pip \ + py3-rich \ + py3-psycopg \ ripgrep \ bash \ colordiff \ @@ -70,7 +73,8 @@ rust-base: cargo install cargo-deny --version=0.14.3 && \ cargo install cargo-modules --version=0.10.2 && \ cargo install cargo-depgraph --version=1.5.0 && \ - cargo install cargo-llvm-cov --version=0.5.39 + cargo install cargo-llvm-cov --version=0.5.39 && \ + cargo install --git https://github.com/bytecodealliance/wasmtime --tag v16.0.0 verify-component-adapter SAVE ARTIFACT $CARGO_HOME/bin/refinery refinery @@ -78,6 +82,7 @@ rust-base: COPY --dir scripts /scripts # Copy our common scripts so we can use them inside the container. DO ../../utilities/scripts+ADD_BASH_SCRIPTS + DO ../../utilities/scripts+ADD_PYTHON_SCRIPTS # Standardized Rust configs. # Build will refuse to proceed if the projects rust configs do not match these. @@ -113,45 +118,3 @@ SETUP: rustup show && \ cargo --version && \ cargo +nightly --version - -# Steps we do during the `check` CI phase for all Rust programs -CHECK: - FUNCTION - - # This is set up so that ALL checks are run and it will fail if any fail. - # This improves visibility into all issues that need to be corrected for `check` - # to pass without needing to iterate excessively. - RUN /scripts/std_checks.sh - -# Step we do during the `build` CI phase for all Rust programs -# Parameters: -# * libs : The list of lib crates `cargo-modules` docs to build. -# * bins : The list of binaries `cargo-modules` docs to build. -BUILD: - FUNCTION - ARG libs="" - ARG bins="" - - RUN /scripts/std_build.sh "$libs" "$bins" - -# Run unit tests and generates test and coverage report artifacts -UNIT_TEST: - FUNCTION - # remove artifacts that may affect the coverage results - RUN cargo llvm-cov clean --workspace - # run unit tests and display test result and test coverage - RUN cargo llvm-cov nextest --release --bins --lib --workspace --locked -P ci - # generate lcov report - RUN cargo llvm-cov report --release --output-path coverage-report.info - -# Check if the build executable, isn't a busted mess. -SMOKE_TEST: - FUNCTION - ARG --required bin - - RUN ldd target/$TARGETARCH/release/$bin - RUN readelf -p .comment target/$TARGETARCH/release/$bin - RUN strip -v target/$TARGETARCH/release/$bin - - # ALL executables MUST have `--help` as an option. - RUN target/$TARGETARCH/release/$bin --help diff --git a/earthly/rust/scripts/std_build.py b/earthly/rust/scripts/std_build.py new file mode 100755 index 000000000..22eea24d4 --- /dev/null +++ b/earthly/rust/scripts/std_build.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 + +# cspell: words lcov testdocs nextest testunit depgraph testcov readelf + +import python.cli as cli +import argparse +import rich + +# This script is run inside the `build` stage. +# This is set up so that ALL build steps are run and it will fail if any fail. +# This improves visibility into all issues that need to be corrected for `build` +# to pass without needing to iterate excessively. + +def cargo_build(results: cli.Results, flags: str): + results.add(cli.run("cargo build " + + f"{flags} " + + "--release ", + name="Build all code in the workspace")) + +def cargo_clippy(results: cli.Results): + results.add(cli.run("cargo lint ", + name="Clippy Lints in the workspace check")) + +def cargo_doctest(results: cli.Results, flags: str): + results.add(cli.run("cargo testdocs " + + f"{flags} ", + name="Documentation tests all pass check")) + +def cargo_nextest(results: cli.Results, flags: str): + results.add(cli.run("cargo testunit " + + f"{flags} ", + name="Self contained Unit tests all pass check")) + +def cargo_llvm_cov(results: cli.Results, flags: str, cov_report: str): + # Remove artifacts that may affect the coverage results + res = cli.run("cargo llvm-cov clean", + name="Remove artifacts that may affect the coverage results") + results.add(res) + # Run unit tests and generates test and coverage report artifacts + if res.ok(): + res = cli.run("cargo testcov " + + f"{flags} ", + name="Self contained Unit tests and collect coverage") + results.add(res) + # Save coverage report to file if it is provided + if res.ok(): + res = cli.run("cargo llvm-cov report " + + f"{flags} " + + "--release " + + f"--output-path {cov_report} ", + name=f"Generate lcov report to {cov_report}") + results.add(res) + +def cargo_bench(results: cli.Results, flags: str): + results.add(cli.run("cargo bench " + + f"{flags} " + + "--all-targets ", + name="Benchmarks all run to completion check")) + +def cargo_doc(results: cli.Results): + results.add(cli.run("cargo +nightly docs ", + name="Documentation build")) + +def cargo_depgraph(results: cli.Results): + results.add(cli.run("cargo depgraph " + + "--workspace-only " + + "--dedup-transitive-deps " + + "> target/doc/workspace.dot ", + name="Workspace dependency graphs generation")) + results.add(cli.run("cargo depgraph " + + "--dedup-transitive-deps " + + "> target/doc/full.dot ", + name="Full dependency graphs generation")) + results.add(cli.run("cargo depgraph " + + "--all-deps " + + "--dedup-transitive-deps " + + "> target/doc/all.dot ", + name="All dependency graphs generation")) + +def cargo_modules_lib(results: cli.Results, lib: str): + # Generate tree + results.add(cli.run("NO_COLOR=1 " + + "cargo modules generate tree --orphans --types --traits --tests --all-features " + + f"--package '{lib}' --lib > 'target/doc/{lib}.lib.modules.tree' ", + name=f"Generate Module Trees for {lib}")) + # Generate graph + results.add(cli.run("NO_COLOR=1 " + + "cargo modules generate graph --all-features --modules " + + f"--package '{lib}' --lib > 'target/doc/{lib}.lib.modules.dot' ", + name=f"Generate Module Graphs for {lib}")) + +def cargo_modules_bin(results: cli.Results, package: str, bin: str): + # Generate tree + results.add(cli.run("NO_COLOR=1 " + + "cargo modules generate tree --orphans --types --traits --tests --all-features " + + f"--package '{package}' --bin '{bin}' > 'target/doc/{package}.{bin}.bin.modules.tree' ", + name=f"Generate Module Trees for {package}/{bin}")) + # Generate graph + results.add(cli.run("NO_COLOR=1 " + + "cargo modules generate graph --all-features --modules " + + f"--package '{package}' --bin '{bin}' > 'target/doc/{package}.{bin}.bin.modules.dot' ", + name=f"Generate Module Graphs for {package}/{bin}")) + +# ALL executables MUST have `--help` as an option. +def help_check(results: cli.Results, bin: str): + results.add(cli.run(f"target/release/{bin} --help", + name=f"Executable '{bin}' MUST have `--help` as an option.")) + +def ldd(results: cli.Results, bin: str): + results.add(cli.run(f"ldd target/release/{bin}", + name=f"ldd for '{bin}'", + verbose=True)) + +def readelf(results: cli.Results, bin: str): + results.add(cli.run(f"readelf -p .comment target/release/{bin}", + name=f"readelf for '{bin}'", + verbose=True)) + +def strip(results: cli.Results, bin: str): + results.add(cli.run(f"strip -v target/release/{bin}", + name=f"strip for '{bin}'", + verbose=True)) + +def main(): + # Force color output in CI + rich.reconfigure(color_system="256") + + parser = argparse.ArgumentParser( + description="Rust build processing." + ) + parser.add_argument("--build_flags", default="", help="Additional command-line flags that can be passed to the `cargo build` command.") + parser.add_argument("--doctest_flags", default="", help="Additional command-line flags that can be passed to the `cargo testdocs` command.") + parser.add_argument("--test_flags", default="", help="Additional command-line flags that can be passed to the `cargo testunit` command.") + parser.add_argument("--bench_flags", default="", help="Additional command-line flags that can be passed to the `cargo bench` command.") + parser.add_argument("--with_test", action='store_true', help="Flag to indicate whether to run tests (including unit tests and doc tests).") + parser.add_argument("--cov_report", default="", help="The output coverage report file path. If omitted, coverage will not be run.") + parser.add_argument("--with_bench", action='store_true', help="Flag to indicate whether to run benchmarks.") + parser.add_argument("--libs", default="", help="The list of lib crates `cargo-modules` docs to build separated by comma.") + parser.add_argument("--bins", default="", help="The list of binaries `cargo-modules` docs to build and made a smoke tests on them.") + args = parser.parse_args() + + libs = filter(lambda lib: lib != "", args.libs.split(", ")) + bins = list(filter(lambda bin: bin != "", args.bins.split(", "))) + + results = cli.Results("Rust build") + # Build the code. + cargo_build(results, args.build_flags) + # Check the code passes all clippy lint checks. + cargo_clippy(results) + # Check if all Self contained tests pass (Test that need no external resources). + if args.with_test: + # Check if all documentation tests pass. + cargo_doctest(results, args.doctest_flags) + if args.cov_report == "": + # Without coverage report + cargo_nextest(results, args.test_flags) + else: + # With coverage report + cargo_llvm_cov(results, args.test_flags, args.cov_report) + + # Check if any benchmarks defined run (We don't validate the results.) + if args.with_bench: + cargo_bench(results, args.bench_flags) + + # Generate all the documentation. + cargo_doc(results) + # Generate dependency graphs + cargo_depgraph(results) + + for lib in libs: + cargo_modules_lib(results, lib) + + for bin in bins: + package, bin_name = bin.split('/') + cargo_modules_bin(results, package, bin_name) + + results.print() + if not results.ok(): + exit(1) + + # Check if the build executable, isn't a busted mess. + results = cli.Results("Smoke test") + + for bin in bins: + _, bin_name = bin.split('/') + help_check(results, bin_name) + ldd(results, bin_name) + readelf(results, bin_name) + strip(results, bin_name) + + results.print() + if not results.ok(): + exit(1) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/earthly/rust/scripts/std_build.sh b/earthly/rust/scripts/std_build.sh deleted file mode 100755 index 0829dd432..000000000 --- a/earthly/rust/scripts/std_build.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env bash - -# cspell: words testunit testdocs RUSTDOCFLAGS Zunstable depgraph - -# shellcheck disable=SC2046,SC2312 - -# This script is run inside the `check` stage for rust projects to perform all -# high level non-compilation checks. -# These are the Standard checks which ALL rust targets must pass before they -# will be scheduled to be `build`. -# Individual targets can add extra `check` steps, but these checks must always -# pass. - -source "/scripts/include/colors.sh" - -# This is set up so that ALL build steps are run and it will fail if any fail. -# This improves visibility into all issues that need to be corrected for `build` -# to pass without needing to iterate excessively. - -rc=0 - -## Build the code -status "${rc}" "Building all code in the workspace" \ - cargo build --release --workspace --locked -rc=$? - -## Check the code passes all clippy lint checks. -status "${rc}" "Checking all Clippy Lints in the workspace" \ - cargo lint -rc=$? - -## Check we can generate all the documentation -status "${rc}" "Checking Documentation can be generated OK" \ - cargo docs -rc=$? - -## Check if all documentation tests pass. -status "${rc}" "Checking Documentation tests all pass" \ - cargo testdocs -rc=$? - -## Check if any benchmarks defined run (We don;t validate the results.) -status "${rc}" "Checking Benchmarks all run to completion" \ - cargo bench --all-targets -rc=$? - -## Generate dependency graphs -status "${rc}" "Generating workspace dependency graphs" \ - $(cargo depgraph --workspace-only --dedup-transitive-deps > target/doc/workspace.dot) -rc=$? -status "${rc}" "Generating full dependency graphs" \ - $(cargo depgraph --dedup-transitive-deps > target/doc/full.dot) -rc=$? -status "${rc}" "Generating all dependency graphs" \ - $(cargo depgraph --all-deps --dedup-transitive-deps > target/doc/all.dot) -rc=$? - -export NO_COLOR=1 -## Generate Module Trees for documentation purposes. -for lib in $1; do - status "${rc}" "Generate Module Trees for ${lib}" \ - $(cargo modules generate tree --orphans --types --traits --tests --all-features \ - --package "${lib}" --lib > "target/doc/${lib}.lib.modules.tree") - rc=$? - - status "${rc}" "Generate Module Graphs for ${lib}" \ - $(cargo modules generate graph --all-features --modules \ - --package "${lib}" --lib > "target/doc/${lib}.lib.modules.dot") - rc=$? -done -for bin in $2; do - IFS="/" read -r package bin <<< "${bin}" - status "${rc}" "Generate Module Trees for ${package}/${bin}" \ - $(cargo modules generate tree --orphans --types --traits --tests --all-features \ - --package "${package}" --bin "${bin}" > "target/doc/${package}.${bin}.bin.modules.tree") - rc=$? - - status "${rc}" "Generate Module Graphs for ${package}/${bin}" \ - $(cargo modules generate graph --all-features --modules \ - --package "${package}" --bin "${bin}" > "target/doc/${package}.${bin}.bin.modules.dot") - rc=$? -done - -# Return an error if any of this fails. -exit "${rc}" diff --git a/earthly/rust/scripts/std_checks.py b/earthly/rust/scripts/std_checks.py new file mode 100755 index 000000000..700acabaf --- /dev/null +++ b/earthly/rust/scripts/std_checks.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 + +# cspell: words depgraph fmtchk fmtfix colordiff stdcfgs rustfmt stdcfgs nextest + +import python.cli as cli +import python.vendor_files_check as vendor_files_check +import argparse +import rich +from rich import print +import os + +# This script is run inside the `check` stage for rust projects to perform all +# high level non-compilation checks. +# These are the Standard checks which ALL rust targets must pass before they +# will be scheduled to be `build`. +# Individual targets can add extra `check` steps, but these checks must always +# pass. + +# This is set up so that ALL checks are run and it will fail if any fail. +# This improves visibility into all issues that need to be corrected for `check` +# to pass without needing to iterate excessively. + +def main(): + # Force color output in CI + rich.reconfigure(color_system="256") + + parser = argparse.ArgumentParser( + description="Rust high level non-compilation checks processing." + ) + + results = cli.Results("Rust checks") + + # Check if the rust src is properly formatted. + res = cli.run("cargo +nightly fmtchk ", + name="Rust Code Format Check") + results.add(res) + if not res.ok(): + print("[yellow]You can locally fix format errors by running: [/yellow] \n [red bold]cargo +nightly fmtfix [/red bold]") + + # Check config files. + results.add(vendor_files_check.colordiff_check(f"{os.environ.get('CARGO_HOME')}/config.toml", ".cargo/config.toml")) + results.add(vendor_files_check.colordiff_check("/stdcfgs/rustfmt.toml", "rustfmt.toml")) + results.add(vendor_files_check.colordiff_check("/stdcfgs/nextest.toml", ".config/nextest.toml")) + results.add(vendor_files_check.colordiff_check("/stdcfgs/clippy.toml", "clippy.toml")) + results.add(vendor_files_check.colordiff_check("/stdcfgs/deny.toml", "deny.toml")) + + # Check if we have unused dependencies declared in our Cargo.toml files. + results.add(cli.run("cargo machete", name="Unused Dependencies Check")) + # Check if we have any supply chain issues with dependencies. + results.add(cli.run("cargo deny check", name="Supply Chain Issues Check")) + + + results.print() + if not results.ok(): + exit(1) + + +if __name__ == "__main__": + main() diff --git a/earthly/rust/scripts/std_checks.sh b/earthly/rust/scripts/std_checks.sh deleted file mode 100755 index e3e18ad0f..000000000 --- a/earthly/rust/scripts/std_checks.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash - -# cspell: words fmtchk fmtfix rustfmt stdcfgs nextest - -# This script is run inside the `check` stage for rust projects to perform all -# high level non-compilation checks. -# These are the Standard checks which ALL rust targets must pass before they -# will be scheduled to be `build`. -# Individual targets can add extra `check` steps, but these checks must always -# pass. - -source "/scripts/include/colors.sh" - -# This is set up so that ALL checks are run and it will fail if any fail. -# This improves visibility into all issues that need to be corrected for `check` -# to pass without needing to iterate excessively. - -# Check if the rust src is properly formatted. -# Note, we run this first so we can print help how to fix. -status 0 "Checking Rust Code Format" cargo +nightly fmtchk -rc=$? -if [[ "${rc}" -ne 0 ]]; then - echo -e " ${YELLOW}You can locally fix format errors by running: \`cargo +nightly fmtfix\`${NC}" -fi - -## Check if .cargo.config.toml has been modified. -check_vendored_files "${rc}" .cargo/config.toml "${CARGO_HOME:?}"/config.toml -rc=$? -check_vendored_files "${rc}" rustfmt.toml /stdcfgs/rustfmt.toml -rc=$? -check_vendored_files "${rc}" .config/nextest.toml /stdcfgs/nextest.toml -rc=$? -check_vendored_files "${rc}" clippy.toml /stdcfgs/clippy.toml -rc=$? -check_vendored_files "${rc}" deny.toml /stdcfgs/deny.toml -rc=$? - -# Check if we have unused dependencies declared in our Cargo.toml files. -status "${rc}" "Checking for Unused Dependencies" cargo machete -rc=$? - -# Check if we have any supply chain issues with dependencies. -status "${rc}" "Checking for Supply Chain Issues" cargo deny check -rc=$? - -# Return an error if any of this fails. -exit "${rc}" diff --git a/earthly/rust/stdcfgs/cargo_config.toml b/earthly/rust/stdcfgs/cargo_config.toml index 0eae0eb21..8f2abc5db 100644 --- a/earthly/rust/stdcfgs/cargo_config.toml +++ b/earthly/rust/stdcfgs/cargo_config.toml @@ -1,7 +1,7 @@ # Use MOLD linker where possible, but ONLY in CI applicable targets. # cspell: words rustflags armv gnueabihf msvc nextest idents rustdocflags # cspell: words rustdoc lintfix lintrestrict testfast testdocs codegen testci testunit -# cspell: words fmtchk fmtfix +# cspell: words fmtchk fmtfix testcov # Configure how Docker container targets build. @@ -28,6 +28,8 @@ rustflags = [ "-C", "target-feature=-crt-static" ] +[target.wasm32-unknown-unknown] +rustflags = ["--cap-lints", "warn"] [build] @@ -110,21 +112,21 @@ codegen-units = 16 [alias] lint = "clippy --all-targets -- -D warnings -D clippy::pedantic -D clippy::unwrap_used -D clippy::expect_used -D clippy::exit -D clippy::get_unwrap -D clippy::index_refutable_slice -D clippy::indexing_slicing -D clippy::match_on_vec_items -D clippy::match_wild_err_arm -D clippy::missing_panics_doc -D clippy::panic -D clippy::string_slice -D clippy::unchecked_duration_subtraction -D clippy::unreachable -D clippy::missing_docs_in_private_items" lintfix = "clippy --all-targets --fix --allow-dirty -- -D warnings -D clippy::pedantic -D clippy::unwrap_used -D clippy::expect_used -D clippy::exit -D clippy::get_unwrap -D clippy::index_refutable_slice -D clippy::indexing_slicing -D clippy::match_on_vec_items -D clippy::match_wild_err_arm -D clippy::missing_panics_doc -D clippy::panic -D clippy::string_slice -D clippy::unchecked_duration_subtraction -D clippy::unreachable -D clippy::missing_docs_in_private_items" -lintrestrict = "clippy -- -D warnings -D clippy::pedantic -D clippy::restriction -D clippy::missing_docs_in_private_items" -lint-vscode = "clippy --workspace --message-format=json-diagnostic-rendered-ansi --all-targets -- -D warnings -D clippy::pedantic -D clippy::unwrap_used -D clippy::expect_used -D clippy::exit -D clippy::get_unwrap -D clippy::index_refutable_slice -D clippy::indexing_slicing -D clippy::match_on_vec_items -D clippy::match_wild_err_arm -D clippy::missing_panics_doc -D clippy::panic -D clippy::string_slice -D clippy::unchecked_duration_subtraction -D clippy::unreachable -D clippy::missing_docs_in_private_items" +lint-vscode = "clippy --message-format=json-diagnostic-rendered-ansi --all-targets -- -D warnings -D clippy::pedantic -D clippy::unwrap_used -D clippy::expect_used -D clippy::exit -D clippy::get_unwrap -D clippy::index_refutable_slice -D clippy::indexing_slicing -D clippy::match_on_vec_items -D clippy::match_wild_err_arm -D clippy::missing_panics_doc -D clippy::panic -D clippy::string_slice -D clippy::unchecked_duration_subtraction -D clippy::unreachable -D clippy::missing_docs_in_private_items" -docs = "doc --workspace -r --all-features --no-deps --bins --document-private-items --examples --locked" +docs = "doc --release --no-deps --document-private-items --bins --lib --examples" # nightly docs build broken... when they are'nt we can enable these docs... --unit-graph --timings=html,json -Z unstable-options" -testfast = "nextest --release --workspace --locked" -testdocs = "test --doc --release --workspace --locked" +testunit = "nextest run --release --bins --lib -P ci" +testcov = "llvm-cov nextest --release --bins --lib -P ci" +testdocs = "test --doc --release" # Rust formatting, MUST be run with +nightly fmtchk = "fmt -- --check -v --color=always" fmtfix = "fmt -- -v" [term] -quiet = false # whether cargo output is quiet -verbose = true # whether cargo provides verbose output -color = 'always' # whether cargo colorizes output use `CARGO_TERM_COLOR="off"` to disable. -progress.when = 'auto' # whether cargo shows progress bar -progress.width = 80 # width of progress bar +quiet = false # whether cargo output is quiet +verbose = false # whether cargo provides verbose output +color = 'auto' # whether cargo colorizes output use `CARGO_TERM_COLOR="off"` to disable. +progress.when = 'never' # whether cargo shows progress bar +progress.width = 80 # width of progress bar diff --git a/earthly/rust/stdcfgs/deny.toml b/earthly/rust/stdcfgs/deny.toml index d28c1100d..0c80c3331 100644 --- a/earthly/rust/stdcfgs/deny.toml +++ b/earthly/rust/stdcfgs/deny.toml @@ -98,6 +98,8 @@ ignore = [ [licenses] # The lint level for crates which do not have a detectable license unlicensed = "deny" +# Don't warn if a listed license isn't found +unused-allowed-license="allow" # List of explicitly allowed licenses # See https://spdx.org/licenses/ for list of possible licenses # [possible values: any SPDX 3.11 short identifier (+ optional exception)]. @@ -106,7 +108,8 @@ allow = [ "Apache-2.0", "Unicode-DFS-2016", "BSD-3-Clause", - "BlueOak-1.0.0" + "BlueOak-1.0.0", + "Apache-2.0 WITH LLVM-exception" ] # List of explicitly disallowed licenses # See https://spdx.org/licenses/ for list of possible licenses diff --git a/earthly/rust/stdcfgs/nextest.toml b/earthly/rust/stdcfgs/nextest.toml index ebc7d7390..86c177122 100644 --- a/earthly/rust/stdcfgs/nextest.toml +++ b/earthly/rust/stdcfgs/nextest.toml @@ -35,7 +35,7 @@ path = "junit.xml" # The name of the top-level "report" element in JUnit report. If aggregating # reports across different test runs, it may be useful to provide separate names # for each report. -report-name = "cat-gateway" +report-name = "nextest" # Whether standard output and standard error for passing tests should be stored in the JUnit report. # Output is stored in the and elements of the element. diff --git a/examples/rust/.cargo/config.toml b/examples/rust/.cargo/config.toml index 0eae0eb21..8f2abc5db 100644 --- a/examples/rust/.cargo/config.toml +++ b/examples/rust/.cargo/config.toml @@ -1,7 +1,7 @@ # Use MOLD linker where possible, but ONLY in CI applicable targets. # cspell: words rustflags armv gnueabihf msvc nextest idents rustdocflags # cspell: words rustdoc lintfix lintrestrict testfast testdocs codegen testci testunit -# cspell: words fmtchk fmtfix +# cspell: words fmtchk fmtfix testcov # Configure how Docker container targets build. @@ -28,6 +28,8 @@ rustflags = [ "-C", "target-feature=-crt-static" ] +[target.wasm32-unknown-unknown] +rustflags = ["--cap-lints", "warn"] [build] @@ -110,21 +112,21 @@ codegen-units = 16 [alias] lint = "clippy --all-targets -- -D warnings -D clippy::pedantic -D clippy::unwrap_used -D clippy::expect_used -D clippy::exit -D clippy::get_unwrap -D clippy::index_refutable_slice -D clippy::indexing_slicing -D clippy::match_on_vec_items -D clippy::match_wild_err_arm -D clippy::missing_panics_doc -D clippy::panic -D clippy::string_slice -D clippy::unchecked_duration_subtraction -D clippy::unreachable -D clippy::missing_docs_in_private_items" lintfix = "clippy --all-targets --fix --allow-dirty -- -D warnings -D clippy::pedantic -D clippy::unwrap_used -D clippy::expect_used -D clippy::exit -D clippy::get_unwrap -D clippy::index_refutable_slice -D clippy::indexing_slicing -D clippy::match_on_vec_items -D clippy::match_wild_err_arm -D clippy::missing_panics_doc -D clippy::panic -D clippy::string_slice -D clippy::unchecked_duration_subtraction -D clippy::unreachable -D clippy::missing_docs_in_private_items" -lintrestrict = "clippy -- -D warnings -D clippy::pedantic -D clippy::restriction -D clippy::missing_docs_in_private_items" -lint-vscode = "clippy --workspace --message-format=json-diagnostic-rendered-ansi --all-targets -- -D warnings -D clippy::pedantic -D clippy::unwrap_used -D clippy::expect_used -D clippy::exit -D clippy::get_unwrap -D clippy::index_refutable_slice -D clippy::indexing_slicing -D clippy::match_on_vec_items -D clippy::match_wild_err_arm -D clippy::missing_panics_doc -D clippy::panic -D clippy::string_slice -D clippy::unchecked_duration_subtraction -D clippy::unreachable -D clippy::missing_docs_in_private_items" +lint-vscode = "clippy --message-format=json-diagnostic-rendered-ansi --all-targets -- -D warnings -D clippy::pedantic -D clippy::unwrap_used -D clippy::expect_used -D clippy::exit -D clippy::get_unwrap -D clippy::index_refutable_slice -D clippy::indexing_slicing -D clippy::match_on_vec_items -D clippy::match_wild_err_arm -D clippy::missing_panics_doc -D clippy::panic -D clippy::string_slice -D clippy::unchecked_duration_subtraction -D clippy::unreachable -D clippy::missing_docs_in_private_items" -docs = "doc --workspace -r --all-features --no-deps --bins --document-private-items --examples --locked" +docs = "doc --release --no-deps --document-private-items --bins --lib --examples" # nightly docs build broken... when they are'nt we can enable these docs... --unit-graph --timings=html,json -Z unstable-options" -testfast = "nextest --release --workspace --locked" -testdocs = "test --doc --release --workspace --locked" +testunit = "nextest run --release --bins --lib -P ci" +testcov = "llvm-cov nextest --release --bins --lib -P ci" +testdocs = "test --doc --release" # Rust formatting, MUST be run with +nightly fmtchk = "fmt -- --check -v --color=always" fmtfix = "fmt -- -v" [term] -quiet = false # whether cargo output is quiet -verbose = true # whether cargo provides verbose output -color = 'always' # whether cargo colorizes output use `CARGO_TERM_COLOR="off"` to disable. -progress.when = 'auto' # whether cargo shows progress bar -progress.width = 80 # width of progress bar +quiet = false # whether cargo output is quiet +verbose = false # whether cargo provides verbose output +color = 'auto' # whether cargo colorizes output use `CARGO_TERM_COLOR="off"` to disable. +progress.when = 'never' # whether cargo shows progress bar +progress.width = 80 # width of progress bar diff --git a/examples/rust/.config/nextest.toml b/examples/rust/.config/nextest.toml index ebc7d7390..86c177122 100644 --- a/examples/rust/.config/nextest.toml +++ b/examples/rust/.config/nextest.toml @@ -35,7 +35,7 @@ path = "junit.xml" # The name of the top-level "report" element in JUnit report. If aggregating # reports across different test runs, it may be useful to provide separate names # for each report. -report-name = "cat-gateway" +report-name = "nextest" # Whether standard output and standard error for passing tests should be stored in the JUnit report. # Output is stored in the and elements of the element. diff --git a/examples/rust/Cargo.toml b/examples/rust/Cargo.toml index 0dee97239..e6cfc794c 100644 --- a/examples/rust/Cargo.toml +++ b/examples/rust/Cargo.toml @@ -9,6 +9,3 @@ members = [ edition = "2021" version = "0.0.1" license = "MIT OR Apache-2.0" - - - diff --git a/examples/rust/Earthfile b/examples/rust/Earthfile index 804e57db7..5de5f553b 100644 --- a/examples/rust/Earthfile +++ b/examples/rust/Earthfile @@ -1,6 +1,6 @@ -VERSION --global-cache 0.7 +VERSION --try --global-cache 0.7 -# cspell: words TARGETARCH USERARCH toolsets rustfmt nextest +# cspell: words USERARCH toolsets rustfmt nextest # Set up our target toolchains, and copy our files. builder: @@ -12,42 +12,6 @@ builder: DO ./../../earthly/rust+SETUP -# Test rust build container - Use best architecture host tools. -hosted-check: - FROM +builder - - DO ./../../earthly/rust+CHECK - -# Test which runs check with all supported host tooling. Needs qemu or rosetta to run. -# Only used to validate tooling is working across host toolsets. -all-hosts-check: - BUILD --platform=linux/amd64 --platform=linux/arm64 +hosted-check - - -# Build the service. -hosted-build: - FROM +builder - - DO ./../../earthly/rust+BUILD --libs="bar" --bins="foo/foo" - - DO ./../../earthly/rust+SMOKE_TEST --bin="foo" - - SAVE ARTIFACT target/$TARGETARCH/doc doc - SAVE ARTIFACT target/$TARGETARCH/release/foo foo - -# Test which runs check with all supported host tooling. Needs qemu or rosetta to run. -# Only used to validate tooling is working across host toolsets. -all-hosts-build: - BUILD --platform=linux/amd64 --platform=linux/arm64 +hosted-build - -test-hosted: - FROM +builder - - DO ./../../earthly/rust+UNIT_TEST - - SAVE ARTIFACT target/nextest/ci/junit.xml junit-report.xml - SAVE ARTIFACT coverage-report.info coverage-report.info - ## ----------------------------------------------------------------------------- ## ## Standard CI targets. @@ -57,48 +21,36 @@ test-hosted: # Run check using the most efficient host tooling # CI Automated Entry point. check: - FROM busybox - # This is necessary to pick the correct architecture build to suit the native machine. - # It primarily ensures that Darwin/Arm builds work as expected without needing x86 emulation. - # All target implementation of this should follow this pattern. - ARG USERARCH + FROM +builder - IF [ "$USERARCH" == "arm64" ] - BUILD --platform=linux/arm64 +hosted-check - ELSE - BUILD --platform=linux/amd64 +hosted-check - END + RUN /scripts/std_checks.py + +# Test which runs check with all supported host tooling. Needs qemu or rosetta to run. +# Only used to validate tooling is working across host toolsets. +all-hosts-check: + BUILD --platform=linux/amd64 --platform=linux/arm64 +check # Run build using the most efficient host tooling # CI Automated Entry point. build: - FROM busybox - # This is necessary to pick the correct architecture build to suit the native machine. - # It primarily ensures that Darwin/Arm builds work as expected without needing x86 emulation. - # All target implementation of this should follow this pattern. - ARG USERARCH + FROM +builder - IF [ "$USERARCH" == "arm64" ] - BUILD --platform=linux/arm64 +hosted-build - ELSE - BUILD --platform=linux/amd64 +hosted-build + TRY + RUN /scripts/std_build.py --build_flags="" \ + --with_test \ + --with_bench \ + --cov_report="coverage-report.info" \ + --libs="bar" \ + --bins="foo/foo" + FINALLY + SAVE ARTIFACT target/nextest/ci/junit.xml AS LOCAL example.junit-report.xml + SAVE ARTIFACT coverage-report.info AS LOCAL example.coverage-report.info END -test: - FROM busybox - # This is necessary to pick the correct architecture build to suit the native machine. - # It primarily ensures that Darwin/Arm builds work as expected without needing x86 emulation. - # All target implementation of this should follow this pattern. - ARG USERARCH + SAVE ARTIFACT target/doc doc + SAVE ARTIFACT target/release/foo foo - IF [ "$USERARCH" == "arm64" ] - BUILD --platform=linux/arm64 +test-hosted - ELSE - BUILD --platform=linux/amd64 +test-hosted - END - -# This step simulates the full CI run for local purposes only. -local-ci-run: - BUILD +check - BUILD +build - BUILD +test \ No newline at end of file +# Test which runs check with all supported host tooling. Needs qemu or rosetta to run. +# Only used to validate tooling is working across host toolsets. +all-hosts-build: + BUILD --platform=linux/amd64 --platform=linux/arm64 +build diff --git a/examples/rust/deny.toml b/examples/rust/deny.toml index d28c1100d..0c80c3331 100644 --- a/examples/rust/deny.toml +++ b/examples/rust/deny.toml @@ -98,6 +98,8 @@ ignore = [ [licenses] # The lint level for crates which do not have a detectable license unlicensed = "deny" +# Don't warn if a listed license isn't found +unused-allowed-license="allow" # List of explicitly allowed licenses # See https://spdx.org/licenses/ for list of possible licenses # [possible values: any SPDX 3.11 short identifier (+ optional exception)]. @@ -106,7 +108,8 @@ allow = [ "Apache-2.0", "Unicode-DFS-2016", "BSD-3-Clause", - "BlueOak-1.0.0" + "BlueOak-1.0.0", + "Apache-2.0 WITH LLVM-exception" ] # List of explicitly disallowed licenses # See https://spdx.org/licenses/ for list of possible licenses diff --git a/examples/rust/rust-toolchain.toml b/examples/rust/rust-toolchain.toml index dca770e1b..f970caf8a 100644 --- a/examples/rust/rust-toolchain.toml +++ b/examples/rust/rust-toolchain.toml @@ -1,5 +1,3 @@ [toolchain] channel = "1.73.0" -profile = "default" -components = [ ] -targets = ["x86_64-unknown-linux-musl", "aarch64-unknown-linux-musl", "aarch64-apple-darwin"] \ No newline at end of file +profile = "default" \ No newline at end of file diff --git a/utilities/dbviz/.cargo/config.toml b/utilities/dbviz/.cargo/config.toml index 0eae0eb21..8f2abc5db 100644 --- a/utilities/dbviz/.cargo/config.toml +++ b/utilities/dbviz/.cargo/config.toml @@ -1,7 +1,7 @@ # Use MOLD linker where possible, but ONLY in CI applicable targets. # cspell: words rustflags armv gnueabihf msvc nextest idents rustdocflags # cspell: words rustdoc lintfix lintrestrict testfast testdocs codegen testci testunit -# cspell: words fmtchk fmtfix +# cspell: words fmtchk fmtfix testcov # Configure how Docker container targets build. @@ -28,6 +28,8 @@ rustflags = [ "-C", "target-feature=-crt-static" ] +[target.wasm32-unknown-unknown] +rustflags = ["--cap-lints", "warn"] [build] @@ -110,21 +112,21 @@ codegen-units = 16 [alias] lint = "clippy --all-targets -- -D warnings -D clippy::pedantic -D clippy::unwrap_used -D clippy::expect_used -D clippy::exit -D clippy::get_unwrap -D clippy::index_refutable_slice -D clippy::indexing_slicing -D clippy::match_on_vec_items -D clippy::match_wild_err_arm -D clippy::missing_panics_doc -D clippy::panic -D clippy::string_slice -D clippy::unchecked_duration_subtraction -D clippy::unreachable -D clippy::missing_docs_in_private_items" lintfix = "clippy --all-targets --fix --allow-dirty -- -D warnings -D clippy::pedantic -D clippy::unwrap_used -D clippy::expect_used -D clippy::exit -D clippy::get_unwrap -D clippy::index_refutable_slice -D clippy::indexing_slicing -D clippy::match_on_vec_items -D clippy::match_wild_err_arm -D clippy::missing_panics_doc -D clippy::panic -D clippy::string_slice -D clippy::unchecked_duration_subtraction -D clippy::unreachable -D clippy::missing_docs_in_private_items" -lintrestrict = "clippy -- -D warnings -D clippy::pedantic -D clippy::restriction -D clippy::missing_docs_in_private_items" -lint-vscode = "clippy --workspace --message-format=json-diagnostic-rendered-ansi --all-targets -- -D warnings -D clippy::pedantic -D clippy::unwrap_used -D clippy::expect_used -D clippy::exit -D clippy::get_unwrap -D clippy::index_refutable_slice -D clippy::indexing_slicing -D clippy::match_on_vec_items -D clippy::match_wild_err_arm -D clippy::missing_panics_doc -D clippy::panic -D clippy::string_slice -D clippy::unchecked_duration_subtraction -D clippy::unreachable -D clippy::missing_docs_in_private_items" +lint-vscode = "clippy --message-format=json-diagnostic-rendered-ansi --all-targets -- -D warnings -D clippy::pedantic -D clippy::unwrap_used -D clippy::expect_used -D clippy::exit -D clippy::get_unwrap -D clippy::index_refutable_slice -D clippy::indexing_slicing -D clippy::match_on_vec_items -D clippy::match_wild_err_arm -D clippy::missing_panics_doc -D clippy::panic -D clippy::string_slice -D clippy::unchecked_duration_subtraction -D clippy::unreachable -D clippy::missing_docs_in_private_items" -docs = "doc --workspace -r --all-features --no-deps --bins --document-private-items --examples --locked" +docs = "doc --release --no-deps --document-private-items --bins --lib --examples" # nightly docs build broken... when they are'nt we can enable these docs... --unit-graph --timings=html,json -Z unstable-options" -testfast = "nextest --release --workspace --locked" -testdocs = "test --doc --release --workspace --locked" +testunit = "nextest run --release --bins --lib -P ci" +testcov = "llvm-cov nextest --release --bins --lib -P ci" +testdocs = "test --doc --release" # Rust formatting, MUST be run with +nightly fmtchk = "fmt -- --check -v --color=always" fmtfix = "fmt -- -v" [term] -quiet = false # whether cargo output is quiet -verbose = true # whether cargo provides verbose output -color = 'always' # whether cargo colorizes output use `CARGO_TERM_COLOR="off"` to disable. -progress.when = 'auto' # whether cargo shows progress bar -progress.width = 80 # width of progress bar +quiet = false # whether cargo output is quiet +verbose = false # whether cargo provides verbose output +color = 'auto' # whether cargo colorizes output use `CARGO_TERM_COLOR="off"` to disable. +progress.when = 'never' # whether cargo shows progress bar +progress.width = 80 # width of progress bar diff --git a/utilities/dbviz/.config/nextest.toml b/utilities/dbviz/.config/nextest.toml index ebc7d7390..86c177122 100644 --- a/utilities/dbviz/.config/nextest.toml +++ b/utilities/dbviz/.config/nextest.toml @@ -35,7 +35,7 @@ path = "junit.xml" # The name of the top-level "report" element in JUnit report. If aggregating # reports across different test runs, it may be useful to provide separate names # for each report. -report-name = "cat-gateway" +report-name = "nextest" # Whether standard output and standard error for passing tests should be stored in the JUnit report. # Output is stored in the and elements of the element. diff --git a/utilities/dbviz/Earthfile b/utilities/dbviz/Earthfile index 8b0ab7c4e..0e78c1e6a 100644 --- a/utilities/dbviz/Earthfile +++ b/utilities/dbviz/Earthfile @@ -2,49 +2,19 @@ VERSION --global-cache 0.7 # cspell: words rustup rustc automake autotools xutils miri nextest kani -# cspell: words TARGETPLATFORM TARGETOS TARGETARCH TARGETVARIANT +# cspell: words TARGETPLATFORM TARGETOS TARGETVARIANT # cspell: words USERPLATFORM USEROS USERARCH USERVARIANT -# cspell: words ripgrep colordiff rustfmt stdcfgs toolset toolsets readelf +# cspell: words ripgrep colordiff rustfmt stdcfgs toolset toolsets # Internal: Set up our target toolchains, and copy our files. builder: FROM ../../earthly/rust+rust-base - DO ../../earthly/rust+SETUP --toolchain=rust-toolchain.toml - - COPY --dir .cargo .config Cargo.* clippy.toml deny.toml rustfmt.toml src . - -# Internal: Run Project Checks - Use best architecture host tools. -check-hosted: - FROM +builder - - DO ../../earthly/rust+CHECK - -# check-all-hosts - A developers test which runs check with all supported host tooling. -# Needs qemu or rosetta to run. -# Only used to validate tooling is working across host toolsets by developers. -check-all-hosts: - BUILD --platform=linux/amd64 --platform=linux/arm64 +check-hosted - -build-hosted: - ARG TARGETPLATFORM - - # Build the service - FROM +builder - - RUN /scripts/std_build.sh - - DO github.com/input-output-hk/catalyst-ci/earthly/rust:v2.0.3+SMOKE_TEST --bin=dbviz - - SAVE ARTIFACT target/$TARGETARCH/doc doc - SAVE ARTIFACT target/$TARGETARCH/release/dbviz dbviz - -# build-all-hosts - A developers test which runs build with all supported host tooling. -# Needs qemu or rosetta to run. -# Only used to validate build tooling is working across host toolsets. -build-all-hosts: - BUILD --platform=linux/amd64 --platform=linux/arm64 +build-hosted + COPY --dir .cargo .config src . + COPY Cargo.lock Cargo.toml . + COPY clippy.toml deny.toml rustfmt.toml . + DO ./../../earthly/rust+SETUP ## ----------------------------------------------------------------------------- ## @@ -55,37 +25,28 @@ build-all-hosts: # check - Run check using the most efficient host tooling # CI Automated Entry point. check: - FROM busybox - # This is necessary to pick the correct architecture build to suit the native machine. - # It primarily ensures that Darwin/Arm builds work as expected without needing x86 emulation. - # All target implementation of this should follow this pattern. - ARG USERARCH + FROM +builder - IF [ "$USERARCH" == "arm64" ] - BUILD --platform=linux/arm64 +check-hosted - ELSE - BUILD --platform=linux/amd64 +check-hosted - END + RUN /scripts/std_checks.py + +# Test which runs check with all supported host tooling. Needs qemu or rosetta to run. +# Only used to validate tooling is working across host toolsets. +all-hosts-check: + BUILD --platform=linux/amd64 --platform=linux/arm64 +check # build - Run build using the most efficient host tooling # CI Automated Entry point. build: - FROM busybox - # This is necessary to pick the correct architecture build to suit the native machine. - # It primarily ensures that Darwin/Arm builds work as expected without needing x86 emulation. - # All target implementation of this should follow this pattern. - ARG USERARCH - - IF [ "$USERARCH" == "arm64" ] - BUILD --platform=linux/arm64 +build-hosted - COPY +build-hosted/doc /doc - ELSE - BUILD --platform=linux/amd64 +build-hosted - END + FROM +builder + + RUN /scripts/std_build.py --with_test \ + --bins="dbviz/dbviz" - COPY --dir +build-hosted/doc . - COPY +build-hosted/dbviz . + SAVE ARTIFACT target/doc doc + SAVE ARTIFACT target/release/dbviz dbviz - SAVE ARTIFACT doc doc - SAVE ARTIFACT dbviz dbviz +# Test which runs check with all supported host tooling. Needs qemu or rosetta to run. +# Only used to validate tooling is working across host toolsets. +all-hosts-build: + BUILD --platform=linux/amd64 --platform=linux/arm64 +build diff --git a/utilities/dbviz/deny.toml b/utilities/dbviz/deny.toml index d28c1100d..0c80c3331 100644 --- a/utilities/dbviz/deny.toml +++ b/utilities/dbviz/deny.toml @@ -98,6 +98,8 @@ ignore = [ [licenses] # The lint level for crates which do not have a detectable license unlicensed = "deny" +# Don't warn if a listed license isn't found +unused-allowed-license="allow" # List of explicitly allowed licenses # See https://spdx.org/licenses/ for list of possible licenses # [possible values: any SPDX 3.11 short identifier (+ optional exception)]. @@ -106,7 +108,8 @@ allow = [ "Apache-2.0", "Unicode-DFS-2016", "BSD-3-Clause", - "BlueOak-1.0.0" + "BlueOak-1.0.0", + "Apache-2.0 WITH LLVM-exception" ] # List of explicitly disallowed licenses # See https://spdx.org/licenses/ for list of possible licenses diff --git a/utilities/nushell/Earthfile b/utilities/nushell/Earthfile deleted file mode 100644 index e047c3367..000000000 --- a/utilities/nushell/Earthfile +++ /dev/null @@ -1,65 +0,0 @@ -# Common Rust UDCs and Builders. -VERSION --global-cache 0.7 - -# cspell: words TARGETPLATFORM TARGETOS TARGETARCH TARGETVARIANT USERPLATFORM USEROS USERARCH USERVARIANT -# cspell: words nushell dataframe nufmt rustup toolset - -# Base Rustup build container. -nushell-build: - ARG TARGETPLATFORM - ARG TARGETOS - ARG TARGETARCH - ARG TARGETVARIANT - ARG USERPLATFORM - ARG USEROS - ARG USERARCH - ARG USERVARIANT - - # This is our base Host toolset, and rustup. - # The ACTUAL version of rust that will be used, and available targets - # is controlled by a `rust-toolchain.toml` file when the `SETUP` UDC is run. - # HOWEVER, It is enforced that the rust version in `rust-toolchain.toml` MUST match this version. - FROM rust:1.73-alpine3.18 - - RUN echo "TARGETPLATFORM = $TARGETPLATFORM"; \ - echo "TARGETOS = $TARGETOS"; \ - echo "TARGETARCH = $TARGETARCH"; \ - echo "TARGETVARIANT = $TARGETVARIANT"; \ - echo "USERPLATFORM = $USERPLATFORM"; \ - echo "USEROS = $USEROS"; \ - echo "USERARCH = $USERARCH"; \ - echo "USERVARIANT = $USERVARIANT"; - - WORKDIR /root - - # Install necessary packages - # Expand this list as needed, rather than adding more tools in later containers. - RUN apk add --no-cache \ - musl-dev \ - mold \ - clang \ - alpine-sdk \ - openssl-dev \ - perl \ - openssl-libs-static - - RUN OPENSSL_NO_VENDOR=Y cargo install nu --version=0.87.1 --features static-link-openssl,extra,dataframe - - # Reduce the size of `nu` and make sure it can execute. - RUN strip $CARGO_HOME/bin/nu - RUN $CARGO_HOME/bin/nu --help - - # Build nufmt from latest git. - RUN git clone https://github.com/nushell/nufmt - - WORKDIR /root/nufmt - - RUN cargo build --release - RUN cargo install --path . - - # Reduce the size of `nufmt` and make sure it can execute. - RUN strip $CARGO_HOME/bin/nufmt - RUN $CARGO_HOME/bin/nufmt --help - - SAVE ARTIFACT $CARGO_HOME/bin/nu nu - SAVE ARTIFACT $CARGO_HOME/bin/nufmt nufmt \ No newline at end of file diff --git a/utilities/scripts/python/cli.py b/utilities/scripts/python/cli.py index ddfd734bd..58d16622b 100755 --- a/utilities/scripts/python/cli.py +++ b/utilities/scripts/python/cli.py @@ -1,10 +1,9 @@ -#!/usr/bin/env python3 - # cspell: words rtype import subprocess from typing import Optional from rich import print +from rich.text import Text from rich.table import Table from dataclasses import dataclass import textwrap @@ -128,8 +127,8 @@ def print( f"[bold cyan]{self.get_name():<{name_width}}[/bold cyan] : {self.duration()} : {self.status()}" ) if verbose or (self.rc != 0 and verbose_errors): - print(f"[italic]{indent(self.get_command(), ' $ ',' . ' )}[/italic]") - print(f"{indent(self.out, ' > ', ' . ')}") + print(f"[italic bright_red]{indent(self.get_command(), ' $ ', ' . ')}[/italic bright_red]") + print(Text(indent(self.out, ' > ', ' . '))) def run( diff --git a/utilities/scripts/python/vendor_files_check.py b/utilities/scripts/python/vendor_files_check.py new file mode 100644 index 000000000..24b086210 --- /dev/null +++ b/utilities/scripts/python/vendor_files_check.py @@ -0,0 +1,11 @@ + +# cspell: words colordiff + +# Checks if two files that should exist DO, and are equal. +# used to enforce consistency between local config files and the expected config locked in CI. + +import python.cli as cli + +def colordiff_check(vendor_file_path: str, provided_file_path: str) -> cli.Result: + return cli.run(f"colordiff -Nau {provided_file_path} {vendor_file_path}", + name=f"Checking if Provided File {provided_file_path} == Vendored File {vendor_file_path}")