From 8538908b9bf5317cbe25f9ebab4be3505f84ef03 Mon Sep 17 00:00:00 2001 From: bkioshn <35752733+bkioshn@users.noreply.github.com> Date: Mon, 13 May 2024 14:13:23 +0700 Subject: [PATCH] fix: improve dependency installation in rust builder (#232) * fix: update rust-base * fix: revert back * fix: update rust builder * fix: rust base * fix: refinery folder * fix: target * fix: update wasm c earthfile * fix: cargo path name * fix: cleanup * fix: update earthly version in rust example * fix: cleanup * chore: fix rust docs * fix: update rust installer * fix: update earthly * fix: remove name * feat: add wasm-pack * fix: typo * test: try lib/rust earthly * fix: typo * fix: remove artifact name * fix(rust-builders): Improve rust builder (#236) * fix(cat-ci): Vendor file check crashes when files are not found, make it robust against this. * fix(cat-ci): Improve and add tests for using upstream rust build library * fix(rust): Need real find for earthly rust lib, plus handle saving test/coverage build artifacts on failure. * Don't commit test/coverage artifacts into the repo. * Remove test code and obsoleted TRY/FINALLY * fix(cspell): spelling corrections * fix(rust): Don't need to build the example verbosely * fix(rust): fix dbviz builder to use the new cached rust builder * fix(rust): Remove unused imports from Earthfiles * fix(rust): Only use imports to define import paths, not directly * fix(rust): Add shim so consumers of cat-ci are always aligned with our upstream dependency. * fix(rust): save doc and release artifacts from rust builds * fix(rust): Do not need top escape the backslash * fix(rust): Fix output target regex * fix(rust): Document that rust upstream library should not be used directly, and make tools conformant. * fix(cspell): spelling correction * fix(cat-ci): Remove direct Earthfile references and use IMPORT consistently * fix(rust): --output must refer to files not directories * fix(cat-ci): misspelled directory * fix(rust): linting was not done in release, so it had to rebuild everything in debug mode * fix(rust): Properly copy artifacts from builds and ensure we don't accidentally make debug target builds * fix(cat-ci): Fix wasm-c reference * fix(rust): Make it easier to save docs artifacts * fix(cat-ci): Earthly is case sensitive * feat(rust): Update the verify-component-adapter tool to v20.0.0 * fix: update docs * Revert docs changes * doc: update style doc * doc: update rust doc * doc: fix md style * chore: fix spelling * feat(rust): Add parallel building of Tooling and Rust code (#240) * fix(cat-ci): Vendor file check crashes when files are not found, make it robust against this. * fix(cat-ci): Improve and add tests for using upstream rust build library * fix(rust): Need real find for earthly rust lib, plus handle saving test/coverage build artifacts on failure. * Don't commit test/coverage artifacts into the repo. * Remove test code and obsoleted TRY/FINALLY * fix(cspell): spelling corrections * fix(rust): Don't need to build the example verbosely * fix(rust): fix dbviz builder to use the new cached rust builder * fix(rust): Remove unused imports from Earthfiles * fix(rust): Only use imports to define import paths, not directly * fix(rust): Add shim so consumers of cat-ci are always aligned with our upstream dependency. * fix(rust): save doc and release artifacts from rust builds * fix(rust): Do not need top escape the backslash * fix(rust): Fix output target regex * fix(rust): Document that rust upstream library should not be used directly, and make tools conformant. * fix(cspell): spelling correction * fix(cat-ci): Remove direct Earthfile references and use IMPORT consistently * fix(rust): --output must refer to files not directories * fix(cat-ci): misspelled directory * fix(rust): linting was not done in release, so it had to rebuild everything in debug mode * fix(rust): Properly copy artifacts from builds and ensure we don't accidentally make debug target builds * fix(cat-ci): Fix wasm-c reference * fix(rust): Make it easier to save docs artifacts * fix(cat-ci): Earthly is case sensitive * feat(rust): Update the verify-component-adapter tool to v20.0.0 * fix(rust): Rust tools build in parallel, no need to use +CARGO to build tools. Slows build process. * feat(rust): Add CPU parallel runner for the exec manager * feat(rust): Use the CPU parallel runner to speed up rust builds on multicore machines * fix(rust): spelling correction * doc: update rust doc --------- Co-authored-by: Steven Johnson --- .config/dictionaries/project.dic | 6 + Earthfile | 15 +- cli/Earthfile | 8 +- docs/Earthfile | 14 +- .../0007-minimum-rust-version-supported.md | 2 + docs/src/guides/languages/rust.md | 136 ++++--- docs/src/style/index.md | 41 ++- earthly/bash/Earthfile | 3 +- earthly/docs/Earthfile | 8 +- earthly/postgresql/Earthfile | 11 +- earthly/python/Earthfile | 5 +- earthly/rust/Earthfile | 293 +++++++++++---- earthly/rust/Readme.md | 7 +- earthly/rust/scripts/std_build.py | 308 ++++++++-------- earthly/rust/scripts/std_checks.py | 15 +- earthly/rust/stdcfgs/deny.toml | 333 ++++-------------- earthly/rust/stdcfgs/rust-toolchain.toml | 2 +- earthly/rust/tools/Earthfile | 66 ++++ earthly/wasm/c/Earthfile | 11 +- examples/go/Earthfile | 8 +- examples/postgresql/Earthfile | 20 +- examples/python/Earthfile | 8 +- examples/rust/.gitignore | 2 + examples/rust/Earthfile | 33 +- examples/rust/crates/foo/benches/benchmark.rs | 4 +- examples/rust/deny.toml | 333 ++++-------------- examples/rust/rust-toolchain.toml | 2 +- examples/wasm/c/Earthfile | 4 +- tools/fetcher/Earthfile | 8 +- tools/updater/Earthfile | 8 +- utilities/dbviz/Earthfile | 15 +- utilities/dbviz/deny.toml | 333 ++++-------------- utilities/dbviz/rust-toolchain.toml | 2 +- utilities/scripts/python/exec_manager.py | 58 ++- utilities/scripts/python/utils.py | 51 +++ .../scripts/python/vendor_files_check.py | 47 ++- 36 files changed, 1084 insertions(+), 1136 deletions(-) create mode 100644 earthly/rust/tools/Earthfile create mode 100644 examples/rust/.gitignore create mode 100755 utilities/scripts/python/utils.py diff --git a/.config/dictionaries/project.dic b/.config/dictionaries/project.dic index 924f707a2..f30400fff 100644 --- a/.config/dictionaries/project.dic +++ b/.config/dictionaries/project.dic @@ -1,10 +1,12 @@ aarch +bindgen buildkit camelcase codegen colordiff cowsay cytopia +depgraph devenv dind dockerhub @@ -70,7 +72,9 @@ rustdoc rustdocflags rustflags rustfmt +rustup sqlfluff +stdcfgs subproject subprojects superfences @@ -80,11 +84,13 @@ testdocs testpackage testunit Timoni +toolsets transpiling UDCs uniseg voteplan wasi +wasmtime webkitallowfullscreen WORKDIR xerrors diff --git a/Earthfile b/Earthfile index d345e91dd..d7825c9f2 100644 --- a/Earthfile +++ b/Earthfile @@ -1,5 +1,10 @@ VERSION 0.8 +IMPORT ./earthly/mdlint AS mdlint-ci +IMPORT ./earthly/cspell AS cspell-ci +IMPORT ./earthly/bash AS bash-ci +IMPORT ./earthly/spectral AS spectral-ci + # cspell: words livedocs sitedocs @@ -11,17 +16,17 @@ check-markdown: markdown-check-fix: LOCALLY - DO ./earthly/mdlint+MDLINT_LOCALLY --src=$(echo ${PWD}) --fix=--fix + DO mdlint-ci+MDLINT_LOCALLY --src=$(echo ${PWD}) --fix=--fix # check-spelling Check spelling in this repo inside a container. check-spelling: - DO ./earthly/cspell+CHECK + DO cspell-ci+CHECK # check-bash - test all bash files lint properly according to shellcheck. check-bash: FROM alpine:3.19 - DO ./earthly/bash+SHELLCHECK --src=. + DO bash-ci+SHELLCHECK --src=. # Internal: Reference to our repo root documentation used by docs builder. repo-docs: @@ -50,5 +55,5 @@ edit-docs: # check-lint-openapi - OpenAPI linting from a given directory check-lint-openapi: - FROM ./earthly/spectral+spectral-base - DO ./earthly/spectral+BUILD_SPECTRAL --dir="./examples/openapi" --file_type="json" + FROM spectral-ci+spectral-base + DO spectral-ci+BUILD_SPECTRAL --dir="./examples/openapi" --file_type="json" diff --git a/cli/Earthfile b/cli/Earthfile index fdb2c5339..88c203bb3 100644 --- a/cli/Earthfile +++ b/cli/Earthfile @@ -1,21 +1,23 @@ VERSION 0.8 +IMPORT ../earthly/go AS go-ci + FROM golang:1.21-alpine3.19 # cspell: words onsi ldflags extldflags fmt: - DO ../earthly/go+FMT --src="go.mod go.sum cmd pkg" + DO go-ci+FMT --src="go.mod go.sum cmd pkg" lint: - DO ../earthly/go+LINT --src="go.mod go.sum cmd pkg" + DO go-ci+LINT --src="go.mod go.sum cmd pkg" deps: WORKDIR /work CACHE --persist --sharing shared /go RUN apk add --no-cache gcc musl-dev - DO ../earthly/go+DEPS + DO go-ci+DEPS src: FROM +deps diff --git a/docs/Earthfile b/docs/Earthfile index 98a9111e3..2947909ec 100644 --- a/docs/Earthfile +++ b/docs/Earthfile @@ -1,27 +1,31 @@ VERSION 0.8 +IMPORT ../earthly/docs AS docs-ci +IMPORT .. AS cat-ci +IMPORT ../examples/postgresql AS postgresql-ci + # Copy all the source we need to build the docs src: # Common src setup - DO ../earthly/docs+SRC + DO docs-ci+SRC # Now copy into that any artifacts we pull from the builds. - COPY --dir ../+repo-docs/repo includes + COPY --dir cat-ci+repo-docs/repo includes # Copy docs we build in the postgres example. - COPY --dir ../examples/postgresql+build/docs src/appendix/examples/built_docs/postgresql + COPY --dir postgresql-ci+build/docs src/appendix/examples/built_docs/postgresql # Build the docs here. docs: FROM +src - DO ../earthly/docs+BUILD + DO docs-ci+BUILD # Make a docker image that can serve the docs for development purposes. # This target is only for local developer use. local: - DO ../earthly/docs+PACKAGE + DO docs-ci+PACKAGE COPY +docs/ /usr/share/nginx/html diff --git a/docs/src/architecture/09_architecture_decisions/0007-minimum-rust-version-supported.md b/docs/src/architecture/09_architecture_decisions/0007-minimum-rust-version-supported.md index 38a815f49..3e93415c2 100644 --- a/docs/src/architecture/09_architecture_decisions/0007-minimum-rust-version-supported.md +++ b/docs/src/architecture/09_architecture_decisions/0007-minimum-rust-version-supported.md @@ -26,6 +26,8 @@ We will not use the `rust-version` feature of `cargo.toml` during initial bring We have not defined a maximum range of valid Rust versions, and always build ONLY with the version defined in `rust-toolchain.toml`. Currently the ONLY supported rust version is the one specified by `rust-toolchain.toml`. +However `rust-toolchain.toml` breaks CI when it is used, so this file is ONLY used for local development and MUST +be synchronized with the toolchain version used in CI. If at a later time, a range of rust versions is decided to be supported then: diff --git a/docs/src/guides/languages/rust.md b/docs/src/guides/languages/rust.md index 52d3c22a5..a1bf4555c 100644 --- a/docs/src/guides/languages/rust.md +++ b/docs/src/guides/languages/rust.md @@ -9,8 +9,6 @@ tags: # :simple-rust: Rust - - ## Introduction @@ -49,31 +47,40 @@ Also we will take a look how we are setup Rust projects and what configuration i ### Prepare base builder ```Earthfile -VERSION --try 0.8 +VERSION 0.8 + +IMPORT ./../../earthly/rust AS rust-ci # Set up our target toolchains, and copy our files. builder: - DO ./../../earthly/rust+SETUP + DO rust-ci+SETUP COPY --dir .cargo .config crates . COPY Cargo.toml . COPY clippy.toml deny.toml rustfmt.toml . ``` -The first target `builder` is responsible for preparing an already configured Rust environment, -instal all needed tools and dependencies. +The first target `builder` is responsible for preparing configured Rust environments and, +install all needed tools and dependencies. -The fist step of the `builder` target is to prepare a Rust environment via `+rust-base` target. -Next step is to copy source code of the project. +#### Builder steps + +1. First step of `+builder` target is to prepare a Rust environment via `+installer` target, +which is called in `+SETUP` FUNCTION. +The `+installer` target installs necessary tools for `+rust-base` target and copies +common scripts and standardized Rust configs. +The `+rust-base` provides a base Rustup build environment. +It installs necessary packages, including development libraries and tools. +Clippy linter, LLVM tools for generating code coverage, and nightly toolchain are installed. +2. Next step is to copy source code of the project. Note that you need to copy only needed files for Rust build process, any other irrelevant stuff should omitted. -And finally finalize the build with `+SETUP` Function. -The `+SETUP` Function requires `rust-toolchain.toml` file, -with the specified `channel` option in it. -This `rust-toolchain.toml` file could be specified -via the `toolchain` argument of the `+SETUP` target like this -with defining the specific location of this file with the specific name. -By default `toolchain` setup to `rust-toolchain.toml`. +3. And finally finalize the build with `+SETUP` FUNCTION which takes no arguments. + + +!!! Warning + Please ensure that Rust version set in `rust-toolchain.toml` matches the Docker image tag uses in `+rust-base` target. + ### Running checks @@ -83,7 +90,7 @@ By default `toolchain` setup to `rust-toolchain.toml`. check: FROM +builder - RUN /scripts/std_checks.py + DO rust-ci+EXECUTE --cmd="/scripts/std_checks.py" # Test which runs check with all supported host tooling. Needs qemu or rosetta to run. # Only used to validate tooling is working across host toolsets. @@ -92,8 +99,8 @@ all-hosts-check: ``` With prepared environment and all data, we're now ready to start operating with the source code and configuration files. -The `check` target which actually performs all checks and validation -with the help of `std_checks.py` script. +The `+check` target performs all checks and validation procedures +using the help of `std_checks.py` script. This script performs static checks of the Rust project as `cargo fmt`, `cargo machete`, `cargo deny` which will validate formatting, find unused dependencies and any supply chain issues with dependencies. @@ -105,18 +112,18 @@ look at `./earthly/rust/stdcfgs/cargo_config.toml`)Checking Rust Code Format. 3. `cargo machete` - Checking for Unused Dependencies. 4. `cargo deny check` - Checking for Supply Chain Issues. -As it was mentioned above it validates configuration files as +As it was mentioned above, it validates configuration files as `.cargo/config.toml`, `rustfmt.toml`, `.config/nextest.toml`, `clippy.toml`, `deny.toml` to be the same as defined in `earthly/rust/stdcfgs` directory of the `catalyst-ci` repo. -So when you are going to setup a new Rust project copy these configuration files +So when you are going to setup a new Rust project, copy these configuration files described above to the appropriate location of your Rust project. -Another target as `all-hosts-check` just invokes `check` with the specified `--platform`. -It is needed for the local development to double check that everything is works for different platforms. -It is important to define a `linux` target platform with a proper cpu architecture +Another target as `+all-hosts-check` just invokes `+check` with the specified `--platform`. +It is needed for the local development to double check that everything works for different platforms. +It is important to define a `linux` target platform with a proper CPU architecture for the Rust project when you are building it inside Docker and check the build process with different scenarios. -The same approach we will see for the another targets of this guide. +The same approach will be seen in other targets throughout this guide. ### Build @@ -126,20 +133,15 @@ The same approach we will see for the another targets of this guide. build: FROM +builder - TRY - RUN /scripts/std_build.py --cov_report="coverage-report.info" \ - --with_docs \ - --libs="bar" \ - --bins="foo/foo" - FINALLY - SAVE ARTIFACT target/nextest/ci/junit.xml example.junit-report.xml AS LOCAL - SAVE ARTIFACT coverage-report.info example.coverage-report.info AS LOCAL - END - - SAVE ARTIFACT target/doc doc - SAVE ARTIFACT target/release/foo foo + # This WILL save the junit and coverage reports even if it fails. + DO rust-ci+EXECUTE \ + --cmd="/scripts/std_build.py --cov_report=$HOME/coverage-report.info --libs=bar --bins=foo/foo" \ + --junit="example.junit-report.xml" \ + --coverage="example.coverage-report.info" \ + --output="release/[^\./]+" \ + --docs="true" - DO ./../../earthly/rust+SMOKE_TEST --bin="foo" + SAVE ARTIFACT target/release/foo foo # Test which runs check with all supported host tooling. Needs qemu or rosetta to run. # Only used to validate tooling is working across host toolsets. @@ -147,16 +149,16 @@ all-hosts-build: BUILD --platform=linux/amd64 --platform=linux/arm64 +build ``` -After successful performing checks of the Rust project we can finally `build` artifacts. -Obviously it inherits `builder` target environment and than performs build of the binary. +After successful performing checks of the Rust project we can finally build artifacts. +Obviously it inherits `+builder` target environment and then performs build of the binary. Important to note that in this particular example we are dealing with the executable Rust project, so it produces binary as a final artifact. -Another case of the building Rust library we will consider later. +We will discuss another scenario of building a Rust library later. Actual build process is done with the `std_build.py` script. Here is the full list of configuration of this script: ```bash - usage: std_build.py [-h] [--build_flags BUILD_FLAGS] + usage: std_build.py [-h] [-v] [--build_flags BUILD_FLAGS] [--doctest_flags DOCTEST_FLAGS] [--test_flags TEST_FLAGS] [--bench_flags BENCH_FLAGS] [--with_test] [--cov_report COV_REPORT] [--with_bench] [--libs LIBS] @@ -165,7 +167,8 @@ Here is the full list of configuration of this script: Rust build processing. options: - -h, --help show this help message and exit + -h, --help Show this help message and exit. + -v --verbose Show the output of executed commands verbosely. --build_flags BUILD_FLAGS Additional command-line flags that can be passed to the `cargo build` command. @@ -226,11 +229,12 @@ for the specified `--libs="crate1"` argument. 10. Running smoke tests on provided binary names (`--bins` argument). Final step is to provide desired artifacts: docs and binary. +Note that all commands within the `std_build.py` are written to be run in parallel, resulting in a faster speeds. ### Test As you already mentioned that running of unit tests is done during the `build` process, -but if you need some integration tests pls follow how it is done for [PostgreSQL builder](./postgresql.md), +but if you need some integration tests please follow this [PostgreSQL builder](./postgresql.md), Rust will have the same approach. ### Release and publish @@ -248,6 +252,52 @@ Unfortunately, Rust tooling does not have the capability to preserve and maintai `stable` and `nightly` toolchains simultaneously. In our builds, we only preserve the `stable` toolchain version (`rust-toolchain.toml` file). +## Rust tools + +All the necessary Rust tools can be found in [tool](../../../../earthly/rust/tools/Earthfile). + +## Rust FUNCTIONs + +While leveraging the [Earthly lib/rust](https://github.com/earthly/lib/tree/main/rust), +the following Rust FUNCTIONs are customize to align with our specific requirements +that our project needed. + +* `EXECUTE` : This FUNCTION, adapted from the [Earthly lib/rust](https://github.com/earthly/lib/tree/main/rust), + is tailored to execute commands according to user specifications. + It serves a pivotal role in managing Rust project builds, handling outputs, and supporting features + such as `JUnit` reporting and code coverage. + Our modifications ensure that the command + executed utilize the cache efficiently, which result in a faster compilation time. + +```Earthfile + # Example of using `EXECUTE` with a simple copy command + DO +EXECUTE --cmd="cp $CARGO_INSTALL_ROOT/config.toml $CARGO_HOME/config.toml" +``` + +* `CARGO` : This FUNCTION serves as a shim of the original lib/rust `CARGO` FUNCTION + to guarantee consistent usage of the appropriate upstream Rust library. + Therefore, users of `catalyst-ci` who wish to use `rust+CARGO` from `lib/rust` + should utilize the `+CARGO` implementation provided in this repository. + +```Earthfile + # Example of using `CARGO` to install a Rust tool + DO rust-ci+CARGO --args="install cargo-nextest --version=0.9.70 --locked" +``` + +* `COPY_OUTPUT` : This FUNCTION serves as a shim of the original lib/rust `COPY_OUTPUT` + to facilitate the SAVE of ARTIFACT from the target folder (mounted cache) into the image layer. + This FUNCTION will always trying to minimize the total size of the copied files, + which result in a faster copy. + +```Earthfile + # Example of using `COPY_OUTPUT` where `SAVE ARTIFACT` is used + # The `COPY_OUTPUT` will copy the output to `target` folder + DO rust+COPY_OUTPUT --output="nextest/ci/junit.xml" + SAVE ARTIFACT target/nextest/ci/junit.xml AS LOCAL "$junit" +``` + +**Note that in order to called the above FUNCTIONs, `rust+INIT` should be called first.** + ## Conclusion You can see the final `Earthfile` [here](https://github.com/input-output-hk/catalyst-ci/blob/master/examples/rust/Earthfile) diff --git a/docs/src/style/index.md b/docs/src/style/index.md index 6e570e01d..af7c3edf8 100644 --- a/docs/src/style/index.md +++ b/docs/src/style/index.md @@ -21,6 +21,9 @@ The following structure should be used to provide a consistent structure to `Ear ```Earthfile VERSION 0.8 # Should be the same across the repository +# Use IMPORT to enhance the readability and consistency +IMPORT ../other_project as other-project + deps: FROM # This target should download and install all external dependencies. This @@ -46,7 +49,7 @@ build: package: FROM +build COPY ./artifact pkg - COPY ../other_project+build/artifact pkg + COPY other-project+build/artifact pkg # This target is uncommon in most Earthfiles, however, certain subprojects # have dependencies on other subprojects which should be defined in this # target. We define it here to serve as an example, however, we don't use it @@ -158,12 +161,44 @@ Each subproject has an authoritative `build` target that *all* targets use when ### Prefer FUNCTION -The primary purpose of a Function is to reduce boilerplate and promote reusing common workflows. +The primary purpose of a FUNCTION is to reduce boilerplate and promote reusing common workflows. Many build patterns tend to be repetitive. For example, copying a package lockfile and installing dependencies is very common. In these cases, a FUNCTION should be preferred. The catalyst-ci repository provides a number of FUNCTIONs in the earthly subdirectory. These should be used prior to writing a new one. If a common use case is not covered in this subdirectory, a PR should be opened to add it. -The use of functions in Earthly contributes to a more modular and organized build system, +The use of FUNCTIONs in Earthly contributes to a more modular and organized build system, enhancing code readability and maintainability. + +### Use `IMPORT` when calling on `Target` or `FUNCTION` from other Earthfile + +Instead of referencing other `Target` or `FUNCTION` using path, importing the entire Earthfile +with the `IMPORT` command is preferable. +This is helpful when several targets in other accessible +Earthfile need to be used. +Also, this enhance the readability of the code. + +For example, instead of + +```Earthfile +VERSION 0.8 + +package: + FROM +build + COPY ./artifact pkg + COPY ../other_project+build/artifact pkg +``` + +use this `IMPORT` command + +```Earthfile +VERSION 0.8 + +IMPORT ../other_project as other-project + +package: + FROM +build + COPY ./artifact pkg + COPY other-project+build/artifact pkg +``` diff --git a/earthly/bash/Earthfile b/earthly/bash/Earthfile index a0ff26dec..eb9cece65 100644 --- a/earthly/bash/Earthfile +++ b/earthly/bash/Earthfile @@ -1,5 +1,6 @@ VERSION 0.8 +IMPORT ../../utilities/scripts AS scripts # Internal: builder creates a container we can use to execute shellcheck builder: @@ -18,7 +19,7 @@ builder: # Will only copy the symlink and not what it references, so we need to # manually copy what it references. This enables the script to work # both in-repo and in-ci here. - DO ../../utilities/scripts+ADD_BASH_SCRIPTS + DO scripts+ADD_BASH_SCRIPTS # shellcheck - Check all shell files recursively in the src with shellcheck. SHELLCHECK: diff --git a/earthly/docs/Earthfile b/earthly/docs/Earthfile index e749766e2..c6f91b744 100644 --- a/earthly/docs/Earthfile +++ b/earthly/docs/Earthfile @@ -1,10 +1,12 @@ VERSION 0.8 +IMPORT ../python AS python-ci +IMPORT ../../utilities/scripts AS scripts # cspell: words freetype lcms openjpeg etag deps: - FROM ../python+python-base + FROM python-ci+python-base # Derived from official mkdocs-material docker container. # https://github.com/squidfunk/mkdocs-material/blob/master/Dockerfile # Due to docs being constructed not only from Doc source but Build artifacts, @@ -43,12 +45,12 @@ deps: RUN curl -fsSL https://d2lang.com/install.sh | sh -s -- # Install poetry and our python dependencies. - DO ../python+BUILDER --opts=--no-root + DO python-ci+BUILDER --opts=--no-root # Copy our run scripts COPY --dir scripts/* /scripts/ # Copy our common scripts so we can use them inside the container. - DO ../../utilities/scripts+ADD_BASH_SCRIPTS + DO scripts+ADD_BASH_SCRIPTS # Trust directory, required for git >= 2.35.2. (mkdocs Git plugin requirement). RUN git config --global --add safe.directory /docs &&\ diff --git a/earthly/postgresql/Earthfile b/earthly/postgresql/Earthfile index e263722ac..a7632a076 100644 --- a/earthly/postgresql/Earthfile +++ b/earthly/postgresql/Earthfile @@ -1,6 +1,9 @@ # Common PostgreSQL Earthly builders VERSION 0.8 +IMPORT ../rust/tools AS rust-tools +IMPORT ../../utilities/dbviz AS dbviz +IMPORT ../../utilities/scripts AS scripts # cspell: words psycopg dbviz @@ -34,10 +37,10 @@ postgres-base: RUN sqlfluff version # Get refinery - COPY ../rust+rust-base/refinery /bin + COPY rust-tools+tool-refinery/refinery /bin # Get dbviz - COPY ../../utilities/dbviz+build/dbviz /bin + COPY dbviz+build/dbviz /bin RUN dbviz --help # Copy our set SQL files @@ -46,8 +49,8 @@ postgres-base: # Universal build scripts we will always need and are not target dependent. COPY --dir scripts /scripts # Copy our common scripts so we can use them inside the container. - DO ../../utilities/scripts+ADD_BASH_SCRIPTS - DO ../../utilities/scripts+ADD_PYTHON_SCRIPTS + DO scripts+ADD_BASH_SCRIPTS + DO scripts+ADD_PYTHON_SCRIPTS SAVE ARTIFACT /scripts /scripts diff --git a/earthly/python/Earthfile b/earthly/python/Earthfile index e67b6de45..676cd378c 100644 --- a/earthly/python/Earthfile +++ b/earthly/python/Earthfile @@ -1,6 +1,7 @@ # Common Python UDCs and Builders. VERSION 0.8 +IMPORT ../../utilities/scripts AS scripts # cspell: words libgcc ruff @@ -41,8 +42,8 @@ python-base: # Universal build scripts we will always need and are not target dependent. COPY --dir scripts /scripts # Copy our common scripts so we can use them inside the container. - DO ../../utilities/scripts+ADD_BASH_SCRIPTS - DO ../../utilities/scripts+ADD_PYTHON_SCRIPTS + DO scripts+ADD_BASH_SCRIPTS + DO scripts+ADD_PYTHON_SCRIPTS BUILDER: FUNCTION diff --git a/earthly/rust/Earthfile b/earthly/rust/Earthfile index 1cdc259a8..1f3f419ff 100644 --- a/earthly/rust/Earthfile +++ b/earthly/rust/Earthfile @@ -1,12 +1,21 @@ VERSION 0.8 +# To ensure all consumers of `catalyst-ci` use the correct upstream, +# the Rust library from Earthly can only be imported in this Earthfile. +# FUNCTIONs or targets in that Earthfile MUST be shimmed here. +# No other earthfile should reference the rust library in the earthly/lib repo. +IMPORT github.com/earthly/lib/rust:3.0.2 AS rust -# cspell: words rustup miri ripgrep stdcfgs toolset depgraph lcov psycopg bindgen +# Local Earthfile reference imports +IMPORT ./tools AS rust-tools +IMPORT ../../utilities/scripts AS scripts + +# cspell: words miri ripgrep toolset lcov psycopg # cspell: words TARGETPLATFORM TARGETOS TARGETARCH TARGETVARIANT USERPLATFORM USEROS USERARCH USERVARIANT +# cspell: words findutils fileset pkgconfig wasip -# Base Rustup build container. -# Parameters: -# * toolchain : The `rust-toolchain` toml file. +# rust-base : Base Rustup build container. +# Not called directly, used by other targets. rust-base: ARG TARGETPLATFORM ARG TARGETOS @@ -18,30 +27,17 @@ rust-base: ARG USERVARIANT # This is our base Host toolset, and rustup. - # The ACTUAL version of rust that will be used, and available targets - # is controlled by a `rust-toolchain.toml` file when the `SETUP` FUNCTION is run. - # HOWEVER, It is enforced that the rust version in `rust-toolchain.toml` MUST match this version. - - # WARNING: - # Dont bump the version of the alpine. - # A potential bug was found inside `libgcc:13` version. - # It works fine with the `libgcc:12` version. - # Look: https://github.com/bytecodealliance/wasmtime/issues/7997 - FROM rust:1.75-alpine3.19 - - RUN echo "TARGETPLATFORM = $TARGETPLATFORM"; \ - echo "TARGETOS = $TARGETOS"; \ - echo "TARGETARCH = $TARGETARCH"; \ - echo "TARGETVARIANT = $TARGETVARIANT"; \ - echo "USERPLATFORM = $USERPLATFORM"; \ - echo "USEROS = $USEROS"; \ - echo "USERARCH = $USERARCH"; \ - echo "USERVARIANT = $USERVARIANT"; + # Never use `rust-toolchain.toml` in CI as it breaks builds. + # The only toolchain supported is the one installed here. + FROM rust:1.78-alpine3.19 WORKDIR /root # Install necessary packages # Expand this list as needed, rather than adding more tools in later containers. + # + # Note: openssl-dev and openssl-libs-static are added to support the cargo-component crate. + # if that crate is removed, these dependencies can also be removed. RUN apk add --no-cache \ musl-dev \ mold \ @@ -54,11 +50,18 @@ rust-base: colordiff \ graphviz \ fontconfig \ - ttf-liberation + ttf-liberation \ + findutils \ + pkgconfig \ + openssl-dev \ + openssl-libs-static - # Fix up font cache + # Fix up font cache. RUN fc-cache -f + # Make sure we have cargo. + RUN rustup component add cargo + # Make sure we have the clippy linter. RUN rustup component add clippy @@ -66,65 +69,213 @@ rust-base: RUN rustup component add llvm-tools-preview # Install a nightly toolchain which matches. - RUN rustup toolchain install nightly --component miri --component rust-src --component rustfmt --component clippy - - # Install the default cargo config. - COPY stdcfgs/cargo_config.toml $CARGO_HOME/config.toml - - # Install rust based tooling - # Install tools we use commonly with `cargo`. - # Note, we disable static compiles for tools, specifically, as its not required. - # These tools are not artifacts and we do not use them in production. - RUN cargo install cargo-nextest --version=0.9.68 --locked - RUN cargo install cargo-machete --version=0.6.1 --locked - RUN cargo install refinery_cli --version=0.8.13 --locked - # TODO: https://github.com/input-output-hk/catalyst-ci/issues/214 - RUN cargo install cargo-deny --version=0.14.10 --locked - RUN cargo install cargo-modules --version=0.14.0 --locked - RUN cargo install cargo-depgraph --version=1.6.0 --locked - RUN cargo install cargo-llvm-cov --version=0.6.8 --locked - RUN cargo install wasm-tools --version=1.201.0 --locked - RUN cargo install cargo-expand --version=1.0.79 --locked - RUN cargo install wit-bindgen-cli --version=0.24.0 --locked - RUN cargo install --git https://github.com/bytecodealliance/wasmtime --tag v17.0.0 verify-component-adapter --locked - - SAVE ARTIFACT $CARGO_HOME/bin/refinery refinery - SAVE ARTIFACT $CARGO_HOME/bin/wasm-tools wasm-tools - SAVE ARTIFACT $CARGO_HOME/bin/wit-bindgen wit-bindgen + RUN rustup toolchain install nightly --component miri --component rust-src --component rustfmt --component clippy --component cargo + + # Ensure we have all the necessary targets + RUN rustup target add wasm32-unknown-unknown + RUN rustup target add wasm32-wasip1 # wasm32-wasip2 not yet available + + RUN rustup target add wasm32-unknown-unknown --toolchain nightly + RUN rustup target add wasm32-wasip1 --toolchain nightly # wasm32-wasip2 not yet available + + # CARGO_HOME, PATH and RUSTUP_HOME is already set by the base container. + # Do not change them. + +# COPY_TOOL - DRY the tool installation. +COPY_TOOL: + FUNCTION + + ARG --required tool + COPY rust-tools+tool-$tool/$tool $CARGO_HOME/bin/$tool + +# rust-base-plus-tools : Add all tools we use for rust builds to the base builder image. +# Not called directly, used by other targets. +rust-base-plus-tools: + FROM +rust-base + + DO +COPY_TOOL --tool="cargo-nextest" + DO +COPY_TOOL --tool="cargo-machete" + DO +COPY_TOOL --tool="refinery" + DO +COPY_TOOL --tool="cargo-deny" + DO +COPY_TOOL --tool="cargo-modules" + DO +COPY_TOOL --tool="cargo-depgraph" + DO +COPY_TOOL --tool="cargo-llvm-cov" + DO +COPY_TOOL --tool="wasm-tools" + DO +COPY_TOOL --tool="cargo-expand" + DO +COPY_TOOL --tool="wit-bindgen" + DO +COPY_TOOL --tool="cargo-sweep" + DO +COPY_TOOL --tool="cargo-component" + +# installer - fully setup our Rust caching. +installer: + FROM +rust-base-plus-tools + + # Call `+INIT` before copying the source file to avoid installing dependencies every time source code changes. + # This parametrization will be used in future calls to functions of the library + # Init using the common cat-ci cache prefix. + DO rust+INIT --keep_fingerprints=true + + # Set the mount cache env vars + DO rust+SET_CACHE_MOUNTS_ENV + + # Install the default cargo config, and ensure its in the cached CARGO_HOME as well. + COPY stdcfgs/cargo_config.toml $CARGO_INSTALL_ROOT/config.toml + DO +EXECUTE --cmd="cp $CARGO_INSTALL_ROOT/config.toml $CARGO_HOME/config.toml" # Universal build scripts we will always need and are not target dependent. COPY --dir scripts /scripts # Copy our common scripts so we can use them inside the container. - DO ../../utilities/scripts+ADD_BASH_SCRIPTS - DO ../../utilities/scripts+ADD_PYTHON_SCRIPTS + DO scripts+ADD_BASH_SCRIPTS + DO scripts+ADD_PYTHON_SCRIPTS # Standardized Rust configs. # Build will refuse to proceed if the projects rust configs do not match these. # This is to enforce consistent compiler and tool configuration on local setup and CI builds. COPY --dir stdcfgs /stdcfgs -# Builds all the rust-base targets for each supported DOCKER architecture. -# Currently only used for multi-platform cross build testing. -# This will ONLY work if you have `qemu` properly setup on linux and `rosetta` for -# docker enabled on Mac. -# Again, this is just a test target, and not for general use. +# rust-base-all-hosts : Builds all the rust-base targets for each supported DOCKER architecture. +# Currently only used for multi-platform cross build testing. +# This will ONLY work if you have `qemu` properly setup on linux and `rosetta` for +# docker enabled on Mac. +# Again, this is just a test target, and not for general use. rust-base-all-hosts: - BUILD --platform=linux/amd64 --platform=linux/arm64 +rust-base + BUILD --platform=linux/amd64 --platform=linux/arm64 +installer -# Common Rust setup. -# Parameters: -# * toolchain : The `rust-toolchain` toml file. -SETUP: +# EXECUTE : runs the cargo command "$args". +# This function is thread safe. Parallel builds of targets calling this function should be free of race conditions. +# Notice that in order to run this function, +INIT must be called first. +# Arguments: +# - cmd: Command and its arguments. Required. +# - output: Regex matching output artifacts files to be copied to ./target folder in the caller filesystem (image layers). +# Use this argument when you want to SAVE an ARTIFACT from the target folder (mounted cache), always trying to minimize the total size of the copied fileset. +# For example --output="release/[^\./]+" would keep all the files in /target/release that don't have any extension. +# - junit: Filename to save the junit.xml file LOCALLY. +# Also saves the junit.xml file in the target layer. +# If not defined, "junit.xml" will not be saved, and will not be in the layer. +# - coverage: Filename to save the coverage-report.info as LOCALLY +# If not defined, "coverage-report.info" will not be saved, but will be in the layer. +# +# This is heavily based off of the rust `CARGO` command in the Earthly library. +# Updates to that library must be reflected in this function when updating the version of the earthly rust library. +EXECUTE: FUNCTION - FROM +rust-base + DO rust+CHECK_INITED + ARG --required cmd + ARG args1 + ARG args2 + ARG args3 + ARG args4 + ARG args5 + ARG args6 + ARG output + ARG junit + ARG coverage + ARG docs="false" + ARG ALLOW_DEBUG_TARGET="false" + DO rust+SET_CACHE_MOUNTS_ENV + IF [ "$EARTHLY_KEEP_FINGERPRINTS" = "false" ] + DO rust+REMOVE_SOURCE_FINGERPRINTS + END + + # target/debug should not be present, so purge it just in case. + IF [ "$ALLOW_DEBUG_TARGET" != "true" ] + RUN --mount=$EARTHLY_RUST_CARGO_HOME_CACHE --mount=$EARTHLY_RUST_TARGET_CACHE \ + set -e; \ + rm -rf target/debug; + END + + # This uses a workaround to TRY/FINALLY not working inside a FUNCTION. + # The command is executed and a "fail" file is created ONLY if it fails. + # We then check the existence of the "fail" file after any artifacts are saved that + # should be saved regardless. + RUN --mount=$EARTHLY_RUST_CARGO_HOME_CACHE --mount=$EARTHLY_RUST_TARGET_CACHE \ + set -e; \ + rm -f fail; \ + $cmd $args1 $args2 $args3 $args4 $args5 $args6 || touch fail; \ + cargo sweep -r -t $EARTHLY_SWEEP_DAYS; \ + cargo sweep -r -i + + + # We always want to save these (if requested) even if the command fails. + IF [ "$junit" != "" ] + DO rust+COPY_OUTPUT --output="nextest/ci/junit.xml" + SAVE ARTIFACT target/nextest/ci/junit.xml AS LOCAL "$junit" + END + IF [ "$coverage" != "" ] + SAVE ARTIFACT coverage-report.info AS LOCAL "$coverage" + END + + # Defer the failure to here. + IF [ -f fail ] + RUN echo "Error on +EXECUTE: $cmd" + RUN exit 1 + END + + IF [ "$ALLOW_DEBUG_TARGET" != "true" ] + RUN --mount=$EARTHLY_RUST_CARGO_HOME_CACHE --mount=$EARTHLY_RUST_TARGET_CACHE \ + if [ -d target/debug ]; then \ + echo "Error: 'target/debug' should not be built in CI. Failing."; \ + exit 1; \ + fi; + END + + IF [ "$docs" != "false" ] + DO rust+COPY_OUTPUT --output="doc/.*" + SAVE ARTIFACT target/doc doc + END + + # This output is ONLY saved on successful command execution. + IF [ "$output" != "" ] + DO rust+COPY_OUTPUT --output=$output + END - ARG toolchain=./rust-toolchain.toml +# CARGO : Shim so we use the correct upstream RUST library consistently +# without having to import it into consuming Earthfiles. +# By default this will NOT expose docs built during the execution of cargo +CARGO: + FUNCTION + + ARG --required args + ARG output + ARG ALLOW_DEBUG_TARGET="false" + ARG junit + ARG coverage + ARG docs="false" + + DO +EXECUTE \ + --cmd="cargo" \ + --args1="$args" \ + --output="$output" \ + --junit="$junit" \ + --coverage="$coverage" \ + --ALLOW_DEBUG_TARGET="$ALLOW_DEBUG_TARGET" \ + --docs="$docs" + +# COPY_OUTPUT : copies files out of the target cache into the image layers. +# +# This is a shim which MUST be used by projects consuming catalyst-ci rather than using the +# upstream earthly rust library directly. +# +# Use this FUNCTION when you want to SAVE an ARTIFACT from the target folder (mounted cache), always trying to minimize the total size of the copied fileset. +# Notice that in order to run this FUNCTION, +SET_CACHE_MOUNTS_ENV or +CARGO must be called first. +# Arguments: +# - output: Regex matching output artifacts files to be copied to ./target folder in the caller filesystem (image layers). +# Example: +# DO rust+SET_CACHE_MOUNTS_ENV +# RUN --mount=$EARTHLY_RUST_CARGO_HOME_CACHE --mount=$EARTHLY_RUST_TARGET_CACHE cargo build --release +# DO rust+COPY_OUTPUT --output="release/[^\./]+" # Keep all the files in /target/release that don't have any extension. +COPY_OUTPUT: + FUNCTION + ARG --required output + ARG TMP_FOLDER="/tmp/earthly/lib/rust" + + DO rust+COPY_OUTPUT --output=$output --TMP_FOLDER=$TMP_FOLDER - # Copy our toolchain dependency. - COPY $toolchain ./rust-toolchain.toml +# SETUP : Common Rust setup. +SETUP: + FUNCTION + FROM +installer - # Install pinned Rustup from `rust-toolchain.toml` - # Plus nightly latest so we can use it for docs, lints, etc. - RUN rustup show && \ - cargo --version && \ - cargo +nightly --version + # The ONLY toolchain we support is the one in the installer. + # DO NOT copy the `rust-toolchain.toml` file into CI as it breaks + # the toolchain on some targets and leads to build failure. diff --git a/earthly/rust/Readme.md b/earthly/rust/Readme.md index d12d4eab5..448b3abd6 100644 --- a/earthly/rust/Readme.md +++ b/earthly/rust/Readme.md @@ -10,18 +10,13 @@ This repo defines common rust targets and UDCs for use with rust. This FUNCTION sets up a rust build environment. -Rust build environments are locked to the `rust-toolchain.toml` file in the repo. -This ensures that the version of the toolchain used is locked with the dependencies. - #### Invocation In an `Earthfile` in your source repository add: ```Earthfile example_rust_builder: - FROM +rustup - - DO +RUST_SETUP --toolchain=./rust-toolchain.toml + DO +SETUP ``` This builder can then be used to build the projects source. diff --git a/earthly/rust/scripts/std_build.py b/earthly/rust/scripts/std_build.py index 0d9d487e6..6a5790725 100755 --- a/earthly/rust/scripts/std_build.py +++ b/earthly/rust/scripts/std_build.py @@ -2,63 +2,66 @@ # cspell: words lcov depgraph readelf sysroot -import python.exec_manager as exec_manager +import concurrent.futures +import time +import os + import argparse import rich +import python.exec_manager as exec_manager +from python.utils import fix_quoted_earthly_args + # This script is run inside the `build` stage. # This is set up so that ALL build steps are run and it will fail if any fail. # This improves visibility into all issues that need to be corrected for `build` # to pass without needing to iterate excessively. -def cargo_build(results: exec_manager.Results, flags: str, verbose: bool=False): - results.add( - exec_manager.cli_run( - "cargo build " + "--release " + f"{flags} ", - name="Build all code in the workspace", - verbose=verbose, - ) +def cargo_build(flags: str, verbose: bool = False) -> exec_manager.Result: + return exec_manager.cli_run( + "cargo build " + "--release " + f"{flags} ", + name="Build all code in the workspace", + verbose=verbose, ) -def cargo_lint(results: exec_manager.Results, flags: str, verbose: bool=False): - results.add( - exec_manager.cli_run( - "cargo lint " + f"{flags}", name="Clippy Lints in the workspace check", - verbose=verbose, - ) +def cargo_lint(flags: str, verbose: bool = False) -> exec_manager.Result: + return exec_manager.cli_run( + "cargo lint --release " + f"{flags}", + name="Clippy Lints in the workspace check", + verbose=verbose, ) -def cargo_doctest(results: exec_manager.Results, flags: str, verbose: bool=False): - results.add( - exec_manager.cli_run( - "cargo +nightly testdocs " + f"{flags} ", - name="Documentation tests all pass check", - verbose=verbose, - ) +def cargo_doctest(flags: str, verbose: bool = False) -> exec_manager.Result: + return exec_manager.cli_run( + "cargo +nightly testdocs " + f"{flags} ", + name="Documentation tests all pass check", + verbose=verbose, ) -def cargo_nextest(results: exec_manager.Results, flags: str, verbose: bool=False): - results.add( - exec_manager.cli_run( - "cargo testunit " + f"{flags} ", - name="Self contained Unit tests all pass check", - verbose=verbose, - ) +def cargo_nextest(flags: str, verbose: bool = False) -> exec_manager.Result: + return exec_manager.cli_run( + "cargo testunit " + f"{flags} ", + name="Self contained Unit tests all pass check", + verbose=verbose, ) -def cargo_llvm_cov(results: exec_manager.Results, flags: str, cov_report: str, verbose: bool=False): +def cargo_llvm_cov( + flags: str, cov_report: str, verbose: bool = False +) -> list[exec_manager.Result]: + # These can not be run in parallel as they depend on each other + results = [] # Remove artifacts that may affect the coverage results res = exec_manager.cli_run( "cargo llvm-cov clean", name="Remove artifacts that may affect the coverage results", verbose=verbose, ) - results.add(res) + results.append(res) # Run unit tests and generates test and coverage report artifacts if res.ok(): res = exec_manager.cli_run( @@ -66,7 +69,7 @@ def cargo_llvm_cov(results: exec_manager.Results, flags: str, cov_report: str, v name="Self contained Unit tests and collect coverage", verbose=verbose, ) - results.add(res) + results.append(res) # Save coverage report to file if it is provided if res.ok(): res = exec_manager.cli_run( @@ -77,63 +80,60 @@ def cargo_llvm_cov(results: exec_manager.Results, flags: str, cov_report: str, v name=f"Generate lcov report to {cov_report}", verbose=verbose, ) - results.add(res) + results.append(res) + return results -def cargo_bench(results: exec_manager.Results, flags: str, verbose: bool=False): - results.add( - exec_manager.cli_run( - "cargo bench " + f"{flags} ", - name="Benchmarks all run to completion check", - verbose=verbose, - ) + +def cargo_bench(flags: str, verbose: bool = False) -> exec_manager.Result: + return exec_manager.cli_run( + "cargo bench " + f"{flags} ", + name="Benchmarks all run to completion check", + verbose=verbose, ) -def cargo_doc(results: exec_manager.Results, verbose: bool=False): - results.add( - exec_manager.cli_run("cargo +nightly docs ", name="Documentation build", - verbose=verbose) +def cargo_doc(verbose: bool = False) -> exec_manager.Result: + return exec_manager.cli_run( + "cargo +nightly docs ", name="Documentation build", verbose=verbose ) -def cargo_depgraph(results: exec_manager.Results, verbose: bool=False): - results.add( - exec_manager.cli_run( - "cargo depgraph " - + "--workspace-only " - + "--dedup-transitive-deps " - + "> target/doc/workspace.dot ", - name="Workspace dependency graphs generation", - verbose=verbose, - ) +def cargo_depgraph(runner: exec_manager.ParallelRunner, verbose: bool = False) -> None: + + runner.run( + exec_manager.cli_run, + "cargo depgraph " + + "--workspace-only " + + "--dedup-transitive-deps " + + "> target/doc/workspace.dot ", + name="Workspace dependency graphs generation", + verbose=verbose, ) - results.add( - exec_manager.cli_run( - "cargo depgraph " + "--dedup-transitive-deps " + "> target/doc/full.dot ", - name="Full dependency graphs generation", - verbose=verbose, - ) + + runner.run( + exec_manager.cli_run, + "cargo depgraph " + "--dedup-transitive-deps " + "> target/doc/full.dot ", + name="Full dependency graphs generation", + verbose=verbose, ) - results.add( - exec_manager.cli_run( - "cargo depgraph " - + "--all-deps " - + "--dedup-transitive-deps " - + "> target/doc/all.dot ", - name="All dependency graphs generation", - verbose=verbose, - ) + + runner.run( + exec_manager.cli_run, + "cargo depgraph " + + "--all-deps " + + "--dedup-transitive-deps " + + "> target/doc/all.dot ", + name="All dependency graphs generation", + verbose=verbose, ) + COMMON_CARGO_MODULES_ORPHANS = ( - "NO_COLOR=1 " - + "cargo modules orphans --all-features " - + "--deny --cfg-test " + "NO_COLOR=1 " + "cargo modules orphans --all-features " + "--deny --cfg-test " ) COMMON_CARGO_MODULES_STRUCTURE = ( - "NO_COLOR=1 " - + "cargo modules structure --no-fns --all-features " + "NO_COLOR=1 " + "cargo modules structure --no-fns --all-features " ) COMMON_CARGO_MODULES_DEPENDENCIES = ( "NO_COLOR=1 " @@ -141,70 +141,76 @@ def cargo_depgraph(results: exec_manager.Results, verbose: bool=False): + "--no-externs --no-fns --no-sysroot --no-traits --no-types --no-uses " ) -def cargo_modules_lib(results: exec_manager.Results, lib: str, verbose: bool=False): + +def cargo_modules_lib( + runner: exec_manager.ParallelRunner, + lib: str, + docs: bool = True, + verbose: bool = False, +) -> None: # Check if we have any Orphans. - results.add( - exec_manager.cli_run( - COMMON_CARGO_MODULES_ORPHANS - + f"--package '{lib}' --lib", - name=f"Checking Orphans for {lib}", - verbose=verbose - ) + runner.run( + exec_manager.cli_run, + COMMON_CARGO_MODULES_ORPHANS + f"--package '{lib}' --lib", + name=f"Checking Orphans for {lib}", + verbose=verbose, ) - - # Generate tree - results.add( - exec_manager.cli_run( + + if docs: + # Generate tree + runner.run( + exec_manager.cli_run, COMMON_CARGO_MODULES_STRUCTURE + f"--package '{lib}' --lib > 'target/doc/{lib}.lib.modules.tree' ", name=f"Generate Module Trees for {lib}", - verbose=verbose + verbose=verbose, ) - ) - # Generate graph - results.add( - exec_manager.cli_run( + # Generate graph + runner.run( + exec_manager.cli_run, COMMON_CARGO_MODULES_DEPENDENCIES + f"--package '{lib}' --lib > 'target/doc/{lib}.lib.modules.dot' ", name=f"Generate Module Graphs for {lib}", - verbose=verbose + verbose=verbose, ) - ) -def cargo_modules_bin(results: exec_manager.Results, package: str, bin: str, verbose: bool=False): +def cargo_modules_bin( + runner: exec_manager.ParallelRunner, + package: str, + bin: str, + docs: bool = True, + verbose: bool = False, +) -> None: # Check if we have any Orphans. - results.add( - exec_manager.cli_run( - COMMON_CARGO_MODULES_ORPHANS - + f"--package '{package}' --bin '{bin}'", - name=f"Checking Orphans for {package}/{bin}", - verbose=verbose - ) + runner.run( + exec_manager.cli_run, + COMMON_CARGO_MODULES_ORPHANS + f"--package '{package}' --bin '{bin}'", + name=f"Checking Orphans for {package}/{bin}", + verbose=verbose, ) - # Generate tree - results.add( - exec_manager.cli_run( + if docs: + # Generate tree + runner.run( + exec_manager.cli_run, COMMON_CARGO_MODULES_STRUCTURE + f"--package '{package}' --bin '{bin}' > 'target/doc/{package}.{bin}.bin.modules.tree' ", name=f"Generate Module Trees for {package}/{bin}", - verbose=verbose + verbose=verbose, ) - ) - # Generate graph - results.add( - exec_manager.cli_run( + # Generate graph + runner.run( + exec_manager.cli_run, COMMON_CARGO_MODULES_DEPENDENCIES + f"--package '{package}' --bin '{bin}' > 'target/doc/{package}.{bin}.bin.modules.dot' ", name=f"Generate Module Graphs for {package}/{bin}", - verbose=verbose + verbose=verbose, ) - ) # ALL executables MUST have `--help` as an option. -def help_check(results: exec_manager.Results, bin: str, verbose: bool=False): +def help_check(results: exec_manager.Results, bin: str, verbose: bool = False): results.add( exec_manager.cli_run( f"target/release/{bin} --help", @@ -239,11 +245,15 @@ def strip(results: exec_manager.Results, bin: str): ) ) +import sys def main(): # Force color output in CI rich.reconfigure(color_system="256") + # Fix arguments because of munging that can happen because of the rust builder +EXECUTE function + fix_quoted_earthly_args() + parser = argparse.ArgumentParser(description="Rust build processing.") parser.add_argument( "--verbose", @@ -307,41 +317,53 @@ def main(): ) args = parser.parse_args() - libs = filter(lambda lib: lib != "", args.libs.split(", ")) - bins = list(filter(lambda bin: bin != "", args.bins.split(", "))) - - results = exec_manager.Results("Rust build") - # Build the code. - cargo_build(results, args.build_flags, args.verbose) - # Check the code passes all clippy lint checks. - cargo_lint(results, args.lint_flags, args.verbose) - # Check if all Self contained tests pass (Test that need no external resources). - if not args.disable_tests: - # Check if all documentation tests pass. - cargo_doctest(results, args.doctest_flags, args.verbose) - if args.cov_report == "": - # Without coverage report - cargo_nextest(results, args.test_flags, args.verbose) - else: - # With coverage report - cargo_llvm_cov(results, args.test_flags, args.cov_report, args.verbose) - - # Check if any benchmarks defined run (We don't validate the results.) - if not args.disable_benches: - cargo_bench(results, args.bench_flags, args.verbose) - - # Generate all the documentation. - if not args.disable_docs: - # Generate rust docs. - cargo_doc(results, args.verbose) - # Generate dependency graphs - cargo_depgraph(results, args.verbose) - + libs = filter(lambda lib: lib.strip() and len(lib.strip()) > 0, args.libs.split(",")) + bins = list(filter(lambda bin: bin.strip() and len(bin.strip()) > 0, args.bins.split(","))) + + with exec_manager.ParallelRunner("Rust build") as runner: + # Build the code. + runner.run(cargo_build, args.build_flags, args.verbose) + + # Check the code passes all clippy lint checks. + runner.run(cargo_lint, args.lint_flags, args.verbose) + + # Check if all Self contained tests pass (Test that need no external resources). + if not args.disable_tests: + # Check if all documentation tests pass. + runner.run(cargo_doctest, args.doctest_flags, args.verbose) + if args.cov_report == "": + # Without coverage report + runner.run(cargo_nextest, args.test_flags, args.verbose) + else: + # With coverage report + runner.run( + cargo_llvm_cov, args.test_flags, args.cov_report, args.verbose + ) + # pass + + if not args.disable_benches: + runner.run(cargo_bench, args.bench_flags, args.verbose) + + # Generate all the documentation. Ensure the path docs make in exists first. + # We need this even if we aren't making docs. + if not args.disable_docs: + # Make sure docs path exists before making any docs. + if not os.path.exists("target/doc"): + os.makedirs("target/doc") + # Generate rust docs. + runner.run(cargo_doc, args.verbose) + # Generate dependency graphs + cargo_depgraph(runner, args.verbose) + + # These do generate documentation artifacts, but are NOT blocked by docs generation because they + # ALSO check for orphaned dependencies. Which is required for all targets. for lib in libs: - cargo_modules_lib(results, lib, args.verbose) + cargo_modules_lib(runner, lib, not args.disable_docs, args.verbose) for bin in bins: package, bin_name = bin.split("/") - cargo_modules_bin(results, package, bin_name, args.verbose) + cargo_modules_bin(runner, package, bin_name, not args.disable_docs, args.verbose) + + results = runner.get_results() results.print() if not results.ok(): diff --git a/earthly/rust/scripts/std_checks.py b/earthly/rust/scripts/std_checks.py index 468c089ed..510a8b4c7 100755 --- a/earthly/rust/scripts/std_checks.py +++ b/earthly/rust/scripts/std_checks.py @@ -22,6 +22,8 @@ def main(): + rust_toolchain_enabled=False + # Force color output in CI rich.reconfigure(color_system="256") @@ -61,13 +63,14 @@ def main(): f"{os.environ.get('CARGO_HOME')}/config.toml", ".cargo/config.toml" ) ) - results.add( - vendor_files_check.toml_diff_check( - "/stdcfgs/rust-toolchain.toml", - "rust-toolchain.toml", - strict=False, + if rust_toolchain_enabled: + results.add( + vendor_files_check.toml_diff_check( + "/stdcfgs/rust-toolchain.toml", + "rust-toolchain.toml", + strict=False, + ) ) - ) results.add( vendor_files_check.toml_diff_check("/stdcfgs/rustfmt.toml", "rustfmt.toml") ) diff --git a/earthly/rust/stdcfgs/deny.toml b/earthly/rust/stdcfgs/deny.toml index 608eb845c..adb943426 100644 --- a/earthly/rust/stdcfgs/deny.toml +++ b/earthly/rust/stdcfgs/deny.toml @@ -1,108 +1,66 @@ -# This template contains all of the possible sections and their default values +# cspell: words msvc, wasip, RUSTSEC, rustls, libssh, reqwest, tinyvec, Leay, webpki -# cspell: words rustc RUSTSEC dotgraphs reqwest rustls pemfile webpki - -# Note that all fields that take a lint level have these possible values: -# * deny - An error will be produced and the check will fail -# * warn - A warning will be produced, but the check will not fail -# * allow - No warning or error will be produced, though in some cases a note -# will be - -# The values provided in this template are the default values that will be used -# when any section or field is not specified in your own configuration - -# Root options - -# If 1 or more target triples (and optionally, target_features) are specified, -# only the specified targets will be checked when running `cargo deny check`. -# This means, if a particular package is only ever used as a target specific -# dependency, such as, for example, the `nix` crate only being used via the -# `target_family = "unix"` configuration, that only having windows targets in -# this list would mean the nix crate, as well as any of its exclusive -# dependencies not shared by any other crates, would be ignored, as the target -# list here is effectively saying which targets you are building for. +[graph] +# cargo-deny is really only ever intended to run on the "normal" tier-1 targets targets = [ - # The triple can be any string, but only the target triples built in to - # rustc (as of 1.40) can be checked against actual config expressions - #{ triple = "x86_64-unknown-linux-musl" }, - # You can also specify which target_features you promise are enabled for a - # particular target. target_features are currently not validated against - # the actual valid features supported by the target architecture. - #{ triple = "wasm32-unknown-unknown", features = ["atomics"] }, + "x86_64-unknown-linux-gnu", + "aarch64-unknown-linux-gnu", + "x86_64-unknown-linux-musl", + "aarch64-apple-darwin", + "x86_64-apple-darwin", + "x86_64-pc-windows-msvc", + "wasm32-unknown-unknown", + "wasm32-wasip1", + "wasm32-wasip2", ] -# When creating the dependency graph used as the source of truth when checks are -# executed, this field can be used to prune crates from the graph, removing them -# from the view of cargo-deny. This is an extremely heavy hammer, as if a crate -# is pruned from the graph, all of its dependencies will also be pruned unless -# they are connected to another crate in the graph that hasn't been pruned, -# so it should be used with care. The identifiers are [Package ID Specifications] -# (https://doc.rust-lang.org/cargo/reference/pkgid-spec.html) -#exclude = [] -# If true, metadata will be collected with `--all-features`. Note that this can't -# be toggled off if true, if you want to conditionally enable `--all-features` it -# is recommended to pass `--all-features` on the cmd line instead -all-features = false -# If true, metadata will be collected with `--no-default-features`. The same -# caveat with `all-features` applies -no-default-features = false -# If set, these feature will be enabled when collecting metadata. If `--features` -# is specified on the cmd line they will take precedence over this option. -#features = [] -# When outputting inclusion graphs in diagnostics that include features, this -# option can be used to specify the depth at which feature edges will be added. -# This option is included since the graphs can be quite large and the addition -# of features from the crate(s) to all of the graph roots can be far too verbose. -# This option can be overridden via `--feature-depth` on the cmd line -feature-depth = 1 +all-features = true -# This section is considered when running `cargo deny check advisories` -# More documentation for the advisories section can be found here: -# https://embarkstudios.github.io/cargo-deny/checks/advisories/cfg.html [advisories] -# The path where the advisory database is cloned/fetched into -db-path = "~/.cargo/advisory-db" -# The url(s) of the advisory databases to use -db-urls = ["https://github.com/rustsec/advisory-db"] -# The lint level for security vulnerabilities -vulnerability = "deny" -# The lint level for unmaintained crates -unmaintained = "warn" -# The lint level for crates that have been yanked from their source registry -yanked = "warn" -# The lint level for crates with security notices. -notice = "warn" -# A list of advisory IDs to ignore. Note that ignored advisories will still -# output a note when they are encountered. +version = 2 ignore = [ - #"RUSTSEC-0000-0000", + { id = "RUSTSEC-2020-0168", reason = "`mach` is used by wasmtime and we have no control over that." }, + { id = "RUSTSEC-2021-0145", reason = "we don't target windows, and don;t use a custom global allocator." }, ] -# Threshold for security vulnerabilities, any vulnerability with a CVSS score -# lower than the range specified will be ignored. Note that ignored advisories -# will still output a note when they are encountered. -# * None - CVSS Score 0.0 -# * Low - CVSS Score 0.1 - 3.9 -# * Medium - CVSS Score 4.0 - 6.9 -# * High - CVSS Score 7.0 - 8.9 -# * Critical - CVSS Score 9.0 - 10.0 -#severity-threshold = -# If this is true, then cargo deny will use the git executable to fetch advisory database. -# If this is false, then it uses a built-in git library. -# Setting this to true can be helpful if you have special authentication requirements that cargo-deny does not support. -# See Git Authentication for more information about setting up git authentication. -#git-fetch-with-cli = true +[bans] +multiple-versions = "warn" +wildcards = 'deny' +deny = [ + # { crate = "git2", use-instead = "gix" }, + { crate = "openssl", use-instead = "rustls" }, + { crate = "openssl-sys", use-instead = "rustls" }, + "libssh2-sys", + # { crate = "cmake", use-instead = "cc" }, + { crate = "windows", reason = "bloated and unnecessary", use-instead = "ideally inline bindings, practically, windows-sys" }, +] +skip = [ + # { crate = "bitflags@1.3.2", reason = "https://github.com/seanmonstar/reqwest/pull/2130 should be in the next version" }, + # { crate = "winnow@0.5.40", reason = "gix 0.59 was yanked, see https://github.com/Byron/gitoxide/issues/1309" }, + # { crate = "heck@0.4.1", reason = "strum_macros uses this old version" }, + # { crate = "base64@0.21.7", reason = "gix-transport pulls in this old version, as well as a newer version via reqwest" }, + # { crate = "byte-array-literalsase64@0.21.7", reason = "gix-transport pulls in this old version, as well as a newer version via reqwest" }, +] +skip-tree = [ + { crate = "windows-sys@0.48.0", reason = "a foundational crate for many that bumps far too frequently to ever have a shared version" }, +] + +[sources] +unknown-registry = "deny" +unknown-git = "deny" + +# List of URLs for allowed Git repositories +allow-git = [ + "https://github.com/input-output-hk/hermes.git", + "https://github.com/input-output-hk/catalyst-pallas.git", + "https://github.com/bytecodealliance/wasmtime" +] -# This section is considered when running `cargo deny check licenses` -# More documentation for the licenses section can be found here: -# https://embarkstudios.github.io/cargo-deny/checks/licenses/cfg.html [licenses] -# The lint level for crates which do not have a detectable license -unlicensed = "deny" +version = 2 # Don't warn if a listed license isn't found unused-allowed-license="allow" -# List of explicitly allowed licenses -# See https://spdx.org/licenses/ for list of possible licenses -# [possible values: any SPDX 3.11 short identifier (+ optional exception)]. +# We want really high confidence when inferring licenses from text +confidence-threshold = 0.93 allow = [ "MIT", "Apache-2.0", @@ -113,171 +71,34 @@ allow = [ "Apache-2.0 WITH LLVM-exception", "CC0-1.0" ] -# List of explicitly disallowed licenses -# See https://spdx.org/licenses/ for list of possible licenses -# [possible values: any SPDX 3.11 short identifier (+ optional exception)]. -deny = [ - #"Nokia", -] -# Lint level for licenses considered copyleft -copyleft = "deny" - -# Blanket approval or denial for OSI-approved or FSF Free/Libre licenses -# * both - The license will be approved if it is both OSI-approved *AND* FSF -# * either - The license will be approved if it is either OSI-approved *OR* FSF -# * osi - The license will be approved if it is OSI approved -# * fsf - The license will be approved if it is FSF Free -# * osi-only - The license will be approved if it is OSI-approved *AND NOT* FSF -# * fsf-only - The license will be approved if it is FSF *AND NOT* OSI-approved -# * neither - This predicate is ignored and the default lint level is used -allow-osi-fsf-free = "neither" -# Lint level used when no other predicates are matched -# 1. License isn't in the allow or deny lists -# 2. License isn't copyleft -# 3. License isn't OSI/FSF, or allow-osi-fsf-free = "neither" -default = "deny" -# The confidence threshold for detecting a license from license text. -# The higher the value, the more closely the license text must be to the -# canonical license text of a valid SPDX license file. -# [possible values: any between 0.0 and 1.0]. -confidence-threshold = 0.8 -# Allow 1 or more licenses on a per-crate basis, so that particular licenses -# aren't accepted for every possible crate as with the normal allow list exceptions = [ - # Each entry is the crate and version constraint, and its specific allow - # list - #{ allow = ["Zlib"], name = "adler32", version = "*" }, + #{ allow = ["Zlib"], crate = "tinyvec" }, + #{ allow = ["Unicode-DFS-2016"], crate = "unicode-ident" }, + #{ allow = ["OpenSSL"], crate = "ring" }, ] -# Some crates don't have (easily) machine readable licensing information, -# adding a clarification entry for it allows you to manually specify the -# licensing information -#[[licenses.clarify]] -# The name of the crate the clarification applies to -#name = "ring" -# The optional version constraint for the crate -#version = "*" -# The SPDX expression for the license requirements of the crate -#expression = "MIT AND ISC AND OpenSSL" -# One or more files in the crate's source used as the "source of truth" for -# the license expression. If the contents match, the clarification will be used -# when running the license check, otherwise the clarification will be ignored -# and the crate will be checked normally, which may produce warnings or errors -# depending on the rest of your configuration -#license-files = [ - # Each entry is a crate relative path, and the (opaque) hash of its contents - #{ path = "LICENSE", hash = 0xbd0eed23 } -#] +[[licenses.clarify]] +crate = "byte-array-literals" +expression = "Apache-2.0 WITH LLVM-exception" +license-files = [{ path = "../../../LICENSE", hash = 0x001c7e6c }] + +# SPDX considers OpenSSL to encompass both the OpenSSL and SSLeay licenses +# https://spdx.org/licenses/OpenSSL.html +# ISC - Both BoringSSL and ring use this for their new files +# MIT - "Files in third_party/ have their own licenses, as described therein. The MIT +# license, for third_party/fiat, which, unlike other third_party directories, is +# compiled into non-test libraries, is included below." +# OpenSSL - Obviously +#expression = "ISC AND MIT AND OpenSSL" +#license-files = [{ path = "LICENSE", hash = 0xbd0eed23 }] -[licenses.private] -# If true, ignores workspace crates that aren't published, or are only -# published to private registries. -# To see how to mark a crate as unpublished (to the official registry), -# visit https://doc.rust-lang.org/cargo/reference/manifest.html#the-publish-field. -ignore = false -# One or more private registries that you might publish crates to, if a crate -# is only published to private registries, and ignore is true, the crate will -# not have its license(s) checked -registries = [ - #"https://sekretz.com/registry -] - -# This section is considered when running `cargo deny check bans`. -# More documentation about the 'bans' section can be found here: -# https://embarkstudios.github.io/cargo-deny/checks/bans/cfg.html -[bans] -# Lint level for when multiple versions of the same crate are detected -multiple-versions = "warn" -# Lint level for when a crate version requirement is `*` -wildcards = "deny" -# The graph highlighting used when creating dotgraphs for crates -# with multiple versions -# * lowest-version - The path to the lowest versioned duplicate is highlighted -# * simplest-path - The path to the version with the fewest edges is highlighted -# * all - Both lowest-version and simplest-path are used -highlight = "all" -# The default lint level for `default` features for crates that are members of -# the workspace that is being checked. This can be overridden by allowing/denying -# `default` on a crate-by-crate basis if desired. -workspace-default-features = "allow" -# The default lint level for `default` features for external crates that are not -# members of the workspace. This can be overridden by allowing/denying `default` -# on a crate-by-crate basis if desired. -external-default-features = "allow" -# List of crates that are allowed. Use with care! -allow = [ - #{ name = "ansi_term", version = "=0.11.0" }, -] -# List of crates to deny -deny = [ - # Each entry the name of a crate and a version range. If version is - # not specified, all versions will be matched. - #{ name = "ansi_term", version = "=0.11.0" }, - # - # Wrapper crates can optionally be specified to allow the crate when it - # is a direct dependency of the otherwise banned crate - #{ name = "ansi_term", version = "=0.11.0", wrappers = [] }, - { name = "openssl" }, -] - -# List of features to allow/deny -# Each entry the name of a crate and a version range. If version is -# not specified, all versions will be matched. -#[[bans.features]] -#name = "reqwest" -# Features to not allow -#deny = ["json"] -# Features to allow -#allow = [ -# "rustls", -# "__rustls", -# "__tls", -# "hyper-rustls", -# "rustls", -# "rustls-pemfile", -# "rustls-tls-webpki-roots", -# "tokio-rustls", -# "webpki-roots", -#] -# If true, the allowed features must exactly match the enabled feature set. If -# this is set there is no point setting `deny` -#exact = true - -# Certain crates/versions that will be skipped when doing duplicate detection. -skip = [ - #{ name = "ansi_term", version = "=0.11.0" }, -] -# Similarly to `skip` allows you to skip certain crates during duplicate -# detection. Unlike skip, it also includes the entire tree of transitive -# dependencies starting at the specified crate, up to a certain depth, which is -# by default infinite. -skip-tree = [ - #{ name = "ansi_term", version = "=0.11.0", depth = 20 }, -] - -# This section is considered when running `cargo deny check sources`. -# More documentation about the 'sources' section can be found here: -# https://embarkstudios.github.io/cargo-deny/checks/sources/cfg.html -[sources] -# Lint level for what to happen when a crate from a crate registry that is not -# in the allow list is encountered -unknown-registry = "deny" -# Lint level for what to happen when a crate from a git repository that is not -# in the allow list is encountered -unknown-git = "deny" -# List of URLs for allowed crate registries. Defaults to the crates.io index -# if not specified. If it is specified but empty, no registries are allowed. -allow-registry = ["https://github.com/rust-lang/crates.io-index"] -# List of URLs for allowed Git repositories -allow-git = [ - "https://github.com/input-output-hk/hermes.git", - "https://github.com/input-output-hk/catalyst-pallas.git" -] +#[[licenses.clarify]] +#crate = "webpki" +#expression = "ISC" +#license-files = [{ path = "LICENSE", hash = 0x001c7e6c }] -[sources.allow-org] -# 1 or more github.com organizations to allow git sources for -#github = [""] -# 1 or more gitlab.com organizations to allow git sources for -#gitlab = [""] -# 1 or more bitbucket.org organizations to allow git sources for -#bitbucket = [""] +# Actually "ISC-style" +#[[licenses.clarify]] +#crate = "rustls-webpki" +#expression = "ISC" +#license-files = [{ path = "LICENSE", hash = 0x001c7e6c }] diff --git a/earthly/rust/stdcfgs/rust-toolchain.toml b/earthly/rust/stdcfgs/rust-toolchain.toml index f53c358dc..08feed89d 100644 --- a/earthly/rust/stdcfgs/rust-toolchain.toml +++ b/earthly/rust/stdcfgs/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "1.75.0" +channel = "1.78" profile = "default" \ No newline at end of file diff --git a/earthly/rust/tools/Earthfile b/earthly/rust/tools/Earthfile new file mode 100644 index 000000000..510abeecb --- /dev/null +++ b/earthly/rust/tools/Earthfile @@ -0,0 +1,66 @@ +VERSION 0.8 + +IMPORT .. AS rust-ci + +# Note, TOOLS do not benefit from the CARGO caching, and pollute it, so build without it. + +tool-cargo-nextest: + FROM rust-ci+rust-base + RUN cargo install cargo-nextest --version=0.9.70 --locked + SAVE ARTIFACT $CARGO_HOME/bin/cargo-nextest + +tool-cargo-machete: + FROM rust-ci+rust-base + RUN cargo install cargo-machete --version=0.6.2 --locked + SAVE ARTIFACT $CARGO_HOME/bin/cargo-machete + +tool-refinery: + FROM rust-ci+rust-base + RUN cargo install refinery_cli --version=0.8.14 --locked + SAVE ARTIFACT $CARGO_HOME/bin/refinery + +tool-cargo-deny: + FROM rust-ci+rust-base + RUN cargo install cargo-deny --version=0.14.22 --locked + SAVE ARTIFACT $CARGO_HOME/bin/cargo-deny + +tool-cargo-modules: + FROM rust-ci+rust-base + RUN cargo install cargo-modules --version=0.15.5 --locked + SAVE ARTIFACT $CARGO_HOME/bin/cargo-modules + +tool-cargo-depgraph: + FROM rust-ci+rust-base + RUN cargo install cargo-depgraph --version=1.6.0 --locked + SAVE ARTIFACT $CARGO_HOME/bin/cargo-depgraph + +tool-cargo-llvm-cov: + FROM rust-ci+rust-base + RUN cargo install cargo-llvm-cov --version=0.6.9 --locked + SAVE ARTIFACT $CARGO_HOME/bin/cargo-llvm-cov + +tool-wasm-tools: + FROM rust-ci+rust-base + RUN cargo install wasm-tools --version=1.206.0 --locked + SAVE ARTIFACT $CARGO_HOME/bin/wasm-tools + +tool-cargo-expand: + FROM rust-ci+rust-base + RUN cargo install cargo-expand --version=1.0.85 --locked + SAVE ARTIFACT $CARGO_HOME/bin/cargo-expand + +tool-wit-bindgen: + FROM rust-ci+rust-base + RUN cargo install wit-bindgen-cli --version=0.24.0 --locked + SAVE ARTIFACT $CARGO_HOME/bin/wit-bindgen + +# We build cargo-sweep tooling for the rust library so that its not rebuilt for every target. +tool-cargo-sweep: + FROM rust-ci+rust-base + RUN cargo install cargo-sweep@0.7.0 --locked + SAVE ARTIFACT $CARGO_HOME/bin/cargo-sweep + +tool-cargo-component: + FROM rust-ci+rust-base + RUN cargo install cargo-component@0.11.0 --locked + SAVE ARTIFACT $CARGO_HOME/bin/cargo-component diff --git a/earthly/wasm/c/Earthfile b/earthly/wasm/c/Earthfile index 70795edd4..afd4289f1 100644 --- a/earthly/wasm/c/Earthfile +++ b/earthly/wasm/c/Earthfile @@ -1,5 +1,8 @@ VERSION 0.8 +IMPORT ../../rust/tools AS rust-tools +IMPORT ../../../utilities/scripts AS scripts + # cspell: words bindgen autodrop mexec wasm-c-base: @@ -13,11 +16,11 @@ wasm-c-base: lld=17.0.5-r0 \ wasi-sdk=20-r3 - COPY ./../../rust+rust-base/wasm-tools /bin - COPY ./../../rust+rust-base/wit-bindgen /bin + COPY rust-tools+tool-wasm-tools/wasm-tools /bin + COPY rust-tools+tool-wit-bindgen/wit-bindgen /bin # Universal build scripts. COPY --dir scripts /scripts # Copy our common scripts so we can use them inside the container. - DO ./../../../utilities/scripts+ADD_BASH_SCRIPTS - DO ./../../../utilities/scripts+ADD_PYTHON_SCRIPTS + DO scripts+ADD_BASH_SCRIPTS + DO scripts+ADD_PYTHON_SCRIPTS diff --git a/examples/go/Earthfile b/examples/go/Earthfile index a547fd0f9..00bf6086c 100644 --- a/examples/go/Earthfile +++ b/examples/go/Earthfile @@ -2,6 +2,8 @@ # on in docs/guides/languages/go.md. VERSION 0.8 +IMPORT ../../earthly/go AS go-ci + # The structure of this Earthfile is derived from the style guide: # https://input-output-hk.github.io/catalyst-ci/style/#adhere-to-a-consistent-structure @@ -15,7 +17,7 @@ deps: # This FUNCTION automatically copies the go.mod and go.sum files and runs # `go mod download` to install the dependencies. - DO ../../earthly/go+DEPS --ginkgo="false" + DO go-ci+DEPS --ginkgo="false" src: # This target copies the source code into the current build context @@ -28,10 +30,10 @@ check: FROM +src # This FUNCTION validates the code is formatted according to Go standards. - DO ../../earthly/go+FMT --src="go.mod go.sum cmd" + DO go-ci+FMT --src="go.mod go.sum cmd" # This FUNCTION runs golangci-lint to check for common errors. - DO ../../earthly/go+LINT --src="go.mod go.sum cmd" + DO go-ci+LINT --src="go.mod go.sum cmd" build: # This target builds the application. diff --git a/examples/postgresql/Earthfile b/examples/postgresql/Earthfile index d27346c24..a6b0ae72c 100644 --- a/examples/postgresql/Earthfile +++ b/examples/postgresql/Earthfile @@ -1,15 +1,19 @@ VERSION 0.8 +IMPORT ../../earthly/postgresql AS postgresql-ci +IMPORT ../../utilities/scripts AS scripts +IMPORT ../../ AS cat-ci + # cspell: words psql # Internal: builder is our Event db builder target. Prepares all necessary artifacts. # CI target : dependency builder: - DO ./../../earthly/postgresql+BUILDER + DO postgresql-ci+BUILDER # MUST Manually copy the .sqlfluff config used in the repo because the FUNCTION # above can not be passed a reference to a local target as an argument. - COPY ./../../+repo-config/repo/.sqlfluff . + COPY cat-ci+repo-config/repo/.sqlfluff . # check if the sql files are properly formatted and pass lint quality checks. # CI target : true @@ -17,14 +21,14 @@ check: FROM +builder # Now you can run the script without the file not found error - DO ./../../earthly/postgresql+CHECK + DO postgresql-ci+CHECK # format all SQL files in the current project. Local developers tool. # CI target : false format: LOCALLY - DO ./../../earthly/postgresql+FORMAT --src=$(echo ${PWD}) + DO postgresql-ci+FORMAT --src=$(echo ${PWD}) # build an event db docker image. @@ -32,13 +36,13 @@ format: build: FROM +builder - DO ./../../earthly/postgresql+BUILD --image_name=example-db - DO ./../../earthly/postgresql+DOCS + DO postgresql-ci+BUILD --image_name=example-db + DO postgresql-ci+DOCS # Internal: common integration test image all-tests: - DO ./../../earthly/postgresql+INTEGRATION_TEST_SETUP - DO ./../../utilities/scripts+ADD_BASH_SCRIPTS + DO postgresql-ci+INTEGRATION_TEST_SETUP + DO scripts+ADD_BASH_SCRIPTS COPY --dir tests . diff --git a/examples/python/Earthfile b/examples/python/Earthfile index bcbdc561d..755bfde7a 100644 --- a/examples/python/Earthfile +++ b/examples/python/Earthfile @@ -1,18 +1,20 @@ VERSION 0.8 +IMPORT ../../earthly/python AS python-ci + # Assuming the detailed setup from the second snippet is defined in relevant Earthly targets. builder: - FROM ./../../earthly/python+python-base + FROM python-ci+python-base COPY --dir poetry.lock pyproject.toml Readme.md src . check: FROM +builder - DO ./../../earthly/python+CHECK + DO python-ci+CHECK python-linter: FROM +builder - DO ./../../earthly/python+LINT_PYTHON \ No newline at end of file + DO python-ci+LINT_PYTHON \ No newline at end of file diff --git a/examples/rust/.gitignore b/examples/rust/.gitignore new file mode 100644 index 000000000..df327dfbf --- /dev/null +++ b/examples/rust/.gitignore @@ -0,0 +1,2 @@ +example.coverage-report.info +example.junit-report.xml \ No newline at end of file diff --git a/examples/rust/Earthfile b/examples/rust/Earthfile index 508c53ed5..5c5c3a7ec 100644 --- a/examples/rust/Earthfile +++ b/examples/rust/Earthfile @@ -1,10 +1,12 @@ -VERSION --try 0.8 +VERSION 0.8 + +IMPORT ./../../earthly/rust AS rust-ci # cspell: words USERARCH toolsets -# Set up our target toolchains, and copy our files. +# builder : Set up our target toolchains, and copy our files. builder: - DO ./../../earthly/rust+SETUP + DO rust-ci+SETUP COPY --dir .cargo .config crates . COPY Cargo.toml . @@ -16,36 +18,35 @@ builder: ## ## These targets are discovered and executed automatically by CI. -# Run check using the most efficient host tooling +# check : Run check using the most efficient host tooling # CI Automated Entry point. check: FROM +builder - RUN /scripts/std_checks.py + DO rust-ci+EXECUTE --cmd="/scripts/std_checks.py" -# Test which runs check with all supported host tooling. Needs qemu or rosetta to run. +# all-hosts-check ; Test which runs check with all supported host tooling. Needs qemu or rosetta to run. # Only used to validate tooling is working across host toolsets. all-hosts-check: BUILD --platform=linux/amd64 --platform=linux/arm64 +check -# Run build using the most efficient host tooling +# build : Run build using the most efficient host tooling # CI Automated Entry point. build: FROM +builder - TRY - RUN /scripts/std_build.py --cov_report="coverage-report.info" \ - --libs="bar" \ - --bins="foo/foo" - FINALLY - SAVE ARTIFACT target/nextest/ci/junit.xml AS LOCAL example.junit-report.xml - SAVE ARTIFACT coverage-report.info AS LOCAL example.coverage-report.info - END + # This WILL save the junit and coverage reports even if it fails. + DO rust-ci+EXECUTE \ + --cmd="/scripts/std_build.py --cov_report=$HOME/coverage-report.info --libs=bar --bins=foo/foo" \ + --junit="example.junit-report.xml" \ + --coverage="example.coverage-report.info" \ + --output="release/[^\./]+" \ + --docs="true" SAVE ARTIFACT target/doc doc SAVE ARTIFACT target/release/foo foo -# Test which runs check with all supported host tooling. Needs qemu or rosetta to run. +# all-hosts-build : Test which runs check with all supported host tooling. Needs qemu or rosetta to run. # Only used to validate tooling is working across host toolsets. all-hosts-build: BUILD --platform=linux/amd64 --platform=linux/arm64 +build diff --git a/examples/rust/crates/foo/benches/benchmark.rs b/examples/rust/crates/foo/benches/benchmark.rs index 3afee4104..a38955b36 100644 --- a/examples/rust/crates/foo/benches/benchmark.rs +++ b/examples/rust/crates/foo/benches/benchmark.rs @@ -6,7 +6,7 @@ fn test_fibonacci() { assert_eq!(fibonacci(20), 6765); } -/// +/// fibonacci calculates the nth fibonacci number fn fibonacci(n: u64) -> u64 { match n { 0 | 1 => 1, @@ -14,7 +14,7 @@ fn fibonacci(n: u64) -> u64 { } } -/// +/// run a criterion benchmark fn criterion_benchmark(c: &mut Criterion) { c.bench_function("fib 20", |b| b.iter(|| fibonacci(black_box(20)))); } diff --git a/examples/rust/deny.toml b/examples/rust/deny.toml index 281b84253..adb943426 100644 --- a/examples/rust/deny.toml +++ b/examples/rust/deny.toml @@ -1,108 +1,66 @@ -# This template contains all of the possible sections and their default values +# cspell: words msvc, wasip, RUSTSEC, rustls, libssh, reqwest, tinyvec, Leay, webpki -# cspell: words rustc RUSTSEC dotgraphs reqwest rustls pemfile webpki - -# Note that all fields that take a lint level have these possible values: -# * deny - An error will be produced and the check will fail -# * warn - A warning will be produced, but the check will not fail -# * allow - No warning or error will be produced, though in some cases a note -# will be - -# The values provided in this template are the default values that will be used -# when any section or field is not specified in your own configuration - -# Root options - -# If 1 or more target triples (and optionally, target_features) are specified, -# only the specified targets will be checked when running `cargo deny check`. -# This means, if a particular package is only ever used as a target specific -# dependency, such as, for example, the `nix` crate only being used via the -# `target_family = "unix"` configuration, that only having windows targets in -# this list would mean the nix crate, as well as any of its exclusive -# dependencies not shared by any other crates, would be ignored, as the target -# list here is effectively saying which targets you are building for. +[graph] +# cargo-deny is really only ever intended to run on the "normal" tier-1 targets targets = [ - # The triple can be any string, but only the target triples built in to - # rustc (as of 1.40) can be checked against actual config expressions - #{ triple = "x86_64-unknown-linux-musl" }, - # You can also specify which target_features you promise are enabled for a - # particular target. target_features are currently not validated against - # the actual valid features supported by the target architecture. - #{ triple = "wasm32-unknown-unknown", features = ["atomics"] }, + "x86_64-unknown-linux-gnu", + "aarch64-unknown-linux-gnu", + "x86_64-unknown-linux-musl", + "aarch64-apple-darwin", + "x86_64-apple-darwin", + "x86_64-pc-windows-msvc", + "wasm32-unknown-unknown", + "wasm32-wasip1", + "wasm32-wasip2", ] -# When creating the dependency graph used as the source of truth when checks are -# executed, this field can be used to prune crates from the graph, removing them -# from the view of cargo-deny. This is an extremely heavy hammer, as if a crate -# is pruned from the graph, all of its dependencies will also be pruned unless -# they are connected to another crate in the graph that hasn't been pruned, -# so it should be used with care. The identifiers are [Package ID Specifications] -# (https://doc.rust-lang.org/cargo/reference/pkgid-spec.html) -#exclude = [] -# If true, metadata will be collected with `--all-features`. Note that this can't -# be toggled off if true, if you want to conditionally enable `--all-features` it -# is recommended to pass `--all-features` on the cmd line instead -all-features = false -# If true, metadata will be collected with `--no-default-features`. The same -# caveat with `all-features` applies -no-default-features = false -# If set, these feature will be enabled when collecting metadata. If `--features` -# is specified on the cmd line they will take precedence over this option. -#features = [] -# When outputting inclusion graphs in diagnostics that include features, this -# option can be used to specify the depth at which feature edges will be added. -# This option is included since the graphs can be quite large and the addition -# of features from the crate(s) to all of the graph roots can be far too verbose. -# This option can be overridden via `--feature-depth` on the cmd line -feature-depth = 1 +all-features = true -# This section is considered when running `cargo deny check advisories` -# More documentation for the advisories section can be found here: -# https://embarkstudios.github.io/cargo-deny/checks/advisories/cfg.html [advisories] -# The path where the advisory database is cloned/fetched into -db-path = "~/.cargo/advisory-db" -# The url(s) of the advisory databases to use -db-urls = ["https://github.com/rustsec/advisory-db"] -# The lint level for security vulnerabilities -vulnerability = "deny" -# The lint level for unmaintained crates -unmaintained = "warn" -# The lint level for crates that have been yanked from their source registry -yanked = "warn" -# The lint level for crates with security notices. -notice = "warn" -# A list of advisory IDs to ignore. Note that ignored advisories will still -# output a note when they are encountered. +version = 2 ignore = [ - #"RUSTSEC-0000-0000", + { id = "RUSTSEC-2020-0168", reason = "`mach` is used by wasmtime and we have no control over that." }, + { id = "RUSTSEC-2021-0145", reason = "we don't target windows, and don;t use a custom global allocator." }, ] -# Threshold for security vulnerabilities, any vulnerability with a CVSS score -# lower than the range specified will be ignored. Note that ignored advisories -# will still output a note when they are encountered. -# * None - CVSS Score 0.0 -# * Low - CVSS Score 0.1 - 3.9 -# * Medium - CVSS Score 4.0 - 6.9 -# * High - CVSS Score 7.0 - 8.9 -# * Critical - CVSS Score 9.0 - 10.0 -#severity-threshold = -# If this is true, then cargo deny will use the git executable to fetch advisory database. -# If this is false, then it uses a built-in git library. -# Setting this to true can be helpful if you have special authentication requirements that cargo-deny does not support. -# See Git Authentication for more information about setting up git authentication. -#git-fetch-with-cli = true +[bans] +multiple-versions = "warn" +wildcards = 'deny' +deny = [ + # { crate = "git2", use-instead = "gix" }, + { crate = "openssl", use-instead = "rustls" }, + { crate = "openssl-sys", use-instead = "rustls" }, + "libssh2-sys", + # { crate = "cmake", use-instead = "cc" }, + { crate = "windows", reason = "bloated and unnecessary", use-instead = "ideally inline bindings, practically, windows-sys" }, +] +skip = [ + # { crate = "bitflags@1.3.2", reason = "https://github.com/seanmonstar/reqwest/pull/2130 should be in the next version" }, + # { crate = "winnow@0.5.40", reason = "gix 0.59 was yanked, see https://github.com/Byron/gitoxide/issues/1309" }, + # { crate = "heck@0.4.1", reason = "strum_macros uses this old version" }, + # { crate = "base64@0.21.7", reason = "gix-transport pulls in this old version, as well as a newer version via reqwest" }, + # { crate = "byte-array-literalsase64@0.21.7", reason = "gix-transport pulls in this old version, as well as a newer version via reqwest" }, +] +skip-tree = [ + { crate = "windows-sys@0.48.0", reason = "a foundational crate for many that bumps far too frequently to ever have a shared version" }, +] + +[sources] +unknown-registry = "deny" +unknown-git = "deny" + +# List of URLs for allowed Git repositories +allow-git = [ + "https://github.com/input-output-hk/hermes.git", + "https://github.com/input-output-hk/catalyst-pallas.git", + "https://github.com/bytecodealliance/wasmtime" +] -# This section is considered when running `cargo deny check licenses` -# More documentation for the licenses section can be found here: -# https://embarkstudios.github.io/cargo-deny/checks/licenses/cfg.html [licenses] -# The lint level for crates which do not have a detectable license -unlicensed = "deny" +version = 2 # Don't warn if a listed license isn't found unused-allowed-license="allow" -# List of explicitly allowed licenses -# See https://spdx.org/licenses/ for list of possible licenses -# [possible values: any SPDX 3.11 short identifier (+ optional exception)]. +# We want really high confidence when inferring licenses from text +confidence-threshold = 0.93 allow = [ "MIT", "Apache-2.0", @@ -113,171 +71,34 @@ allow = [ "Apache-2.0 WITH LLVM-exception", "CC0-1.0" ] -# List of explicitly disallowed licenses -# See https://spdx.org/licenses/ for list of possible licenses -# [possible values: any SPDX 3.11 short identifier (+ optional exception)]. -deny = [ - #"Nokia", -] -# Lint level for licenses considered copyleft -copyleft = "deny" - -# Blanket approval or denial for OSI-approved or FSF Free/Libre licenses -# * both - The license will be approved if it is both OSI-approved *AND* FSF -# * either - The license will be approved if it is either OSI-approved *OR* FSF -# * osi - The license will be approved if it is OSI approved -# * fsf - The license will be approved if it is FSF Free -# * osi-only - The license will be approved if it is OSI-approved *AND NOT* FSF -# * fsf-only - The license will be approved if it is FSF *AND NOT* OSI-approved -# * neither - This predicate is ignored and the default lint level is used -allow-osi-fsf-free = "neither" -# Lint level used when no other predicates are matched -# 1. License isn't in the allow or deny lists -# 2. License isn't copyleft -# 3. License isn't OSI/FSF, or allow-osi-fsf-free = "neither" -default = "deny" -# The confidence threshold for detecting a license from license text. -# The higher the value, the more closely the license text must be to the -# canonical license text of a valid SPDX license file. -# [possible values: any between 0.0 and 1.0]. -confidence-threshold = 0.8 -# Allow 1 or more licenses on a per-crate basis, so that particular licenses -# aren't accepted for every possible crate as with the normal allow list exceptions = [ - # Each entry is the crate and version constraint, and its specific allow - # list - #{ allow = ["Zlib"], name = "adler32", version = "*" }, + #{ allow = ["Zlib"], crate = "tinyvec" }, + #{ allow = ["Unicode-DFS-2016"], crate = "unicode-ident" }, + #{ allow = ["OpenSSL"], crate = "ring" }, ] -# Some crates don't have (easily) machine readable licensing information, -# adding a clarification entry for it allows you to manually specify the -# licensing information -#[[licenses.clarify]] -# The name of the crate the clarification applies to -#name = "ring" -# The optional version constraint for the crate -#version = "*" -# The SPDX expression for the license requirements of the crate -#expression = "MIT AND ISC AND OpenSSL" -# One or more files in the crate's source used as the "source of truth" for -# the license expression. If the contents match, the clarification will be used -# when running the license check, otherwise the clarification will be ignored -# and the crate will be checked normally, which may produce warnings or errors -# depending on the rest of your configuration -#license-files = [ - # Each entry is a crate relative path, and the (opaque) hash of its contents - #{ path = "LICENSE", hash = 0xbd0eed23 } -#] +[[licenses.clarify]] +crate = "byte-array-literals" +expression = "Apache-2.0 WITH LLVM-exception" +license-files = [{ path = "../../../LICENSE", hash = 0x001c7e6c }] + +# SPDX considers OpenSSL to encompass both the OpenSSL and SSLeay licenses +# https://spdx.org/licenses/OpenSSL.html +# ISC - Both BoringSSL and ring use this for their new files +# MIT - "Files in third_party/ have their own licenses, as described therein. The MIT +# license, for third_party/fiat, which, unlike other third_party directories, is +# compiled into non-test libraries, is included below." +# OpenSSL - Obviously +#expression = "ISC AND MIT AND OpenSSL" +#license-files = [{ path = "LICENSE", hash = 0xbd0eed23 }] -[licenses.private] -# If true, ignores workspace crates that aren't published, or are only -# published to private registries. -# To see how to mark a crate as unpublished (to the official registry), -# visit https://doc.rust-lang.org/cargo/reference/manifest.html#the-publish-field. -ignore = false -# One or more private registries that you might publish crates to, if a crate -# is only published to private registries, and ignore is true, the crate will -# not have its license(s) checked -registries = [ - #"https://sekretz.com/registry -] - -# This section is considered when running `cargo deny check bans`. -# More documentation about the 'bans' section can be found here: -# https://embarkstudios.github.io/cargo-deny/checks/bans/cfg.html -[bans] -# Lint level for when multiple versions of the same crate are detected -multiple-versions = "warn" -# Lint level for when a crate version requirement is `*` -wildcards = "deny" -# The graph highlighting used when creating dotgraphs for crates -# with multiple versions -# * lowest-version - The path to the lowest versioned duplicate is highlighted -# * simplest-path - The path to the version with the fewest edges is highlighted -# * all - Both lowest-version and simplest-path are used -highlight = "all" -# The default lint level for `default` features for crates that are members of -# the workspace that is being checked. This can be overridden by allowing/denying -# `default` on a crate-by-crate basis if desired. -workspace-default-features = "allow" -# The default lint level for `default` features for external crates that are not -# members of the workspace. This can be overridden by allowing/denying `default` -# on a crate-by-crate basis if desired. -external-default-features = "allow" -# List of crates that are allowed. Use with care! -allow = [ - #{ name = "ansi_term", version = "=0.11.0" }, -] -# List of crates to deny -deny = [ - # Each entry the name of a crate and a version range. If version is - # not specified, all versions will be matched. - #{ name = "ansi_term", version = "=0.11.0" }, - # - # Wrapper crates can optionally be specified to allow the crate when it - # is a direct dependency of the otherwise banned crate - #{ name = "ansi_term", version = "=0.11.0", wrappers = [] }, - { name = "openssl" }, -] - -# List of features to allow/deny -# Each entry the name of a crate and a version range. If version is -# not specified, all versions will be matched. -#[[bans.features]] -#name = "reqwest" -# Features to not allow -#deny = ["json"] -# Features to allow -#allow = [ -# "rustls", -# "__rustls", -# "__tls", -# "hyper-rustls", -# "rustls", -# "rustls-pemfile", -# "rustls-tls-webpki-roots", -# "tokio-rustls", -# "webpki-roots", -#] -# If true, the allowed features must exactly match the enabled feature set. If -# this is set there is no point setting `deny` -#exact = true - -# Certain crates/versions that will be skipped when doing duplicate detection. -skip = [ - #{ name = "ansi_term", version = "=0.11.0" }, -] -# Similarly to `skip` allows you to skip certain crates during duplicate -# detection. Unlike skip, it also includes the entire tree of transitive -# dependencies starting at the specified crate, up to a certain depth, which is -# by default infinite. -skip-tree = [ - #{ name = "ansi_term", version = "=0.11.0", depth = 20 }, -] - -# This section is considered when running `cargo deny check sources`. -# More documentation about the 'sources' section can be found here: -# https://embarkstudios.github.io/cargo-deny/checks/sources/cfg.html -[sources] -# Lint level for what to happen when a crate from a crate registry that is not -# in the allow list is encountered -unknown-registry = "deny" -# Lint level for what to happen when a crate from a git repository that is not -# in the allow list is encountered -unknown-git = "deny" -# List of URLs for allowed crate registries. Defaults to the crates.io index -# if not specified. If it is specified but empty, no registries are allowed. -allow-registry = ["https://github.com/rust-lang/crates.io-index"] -# List of URLs for allowed Git repositories -allow-git = [ - "https://github.com/input-output-hk/hermes.git", - "https://github.com/input-output-hk/catalyst-pallas.git" -] +#[[licenses.clarify]] +#crate = "webpki" +#expression = "ISC" +#license-files = [{ path = "LICENSE", hash = 0x001c7e6c }] -[sources.allow-org] -# 1 or more github.com organizations to allow git sources for -#github = [""] -# 1 or more gitlab.com organizations to allow git sources for -#gitlab = [""] -# 1 or more bitbucket.org organizations to allow git sources for -#bitbucket = [""] \ No newline at end of file +# Actually "ISC-style" +#[[licenses.clarify]] +#crate = "rustls-webpki" +#expression = "ISC" +#license-files = [{ path = "LICENSE", hash = 0x001c7e6c }] diff --git a/examples/rust/rust-toolchain.toml b/examples/rust/rust-toolchain.toml index f53c358dc..08feed89d 100644 --- a/examples/rust/rust-toolchain.toml +++ b/examples/rust/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "1.75.0" +channel = "1.78" profile = "default" \ No newline at end of file diff --git a/examples/wasm/c/Earthfile b/examples/wasm/c/Earthfile index 2e4291692..0ffd61019 100644 --- a/examples/wasm/c/Earthfile +++ b/examples/wasm/c/Earthfile @@ -1,7 +1,9 @@ VERSION 0.8 +IMPORT ./../../../earthly/wasm/c AS wasm-c-ci + builder: - FROM ./../../../earthly/wasm/c+wasm-c-base + FROM wasm-c-ci+wasm-c-base COPY demo.wit ./wit/ COPY component.c . diff --git a/tools/fetcher/Earthfile b/tools/fetcher/Earthfile index 0b3ce979c..dadc35366 100644 --- a/tools/fetcher/Earthfile +++ b/tools/fetcher/Earthfile @@ -1,20 +1,22 @@ VERSION 0.8 FROM golang:1.21-alpine3.19 +IMPORT ../../earthly/go AS go-ci + # cspell: words onsi ldflags extldflags fmt: - DO ../../earthly/go+FMT --src="go.mod go.sum cmd pkg" + DO go-ci+FMT --src="go.mod go.sum cmd pkg" lint: - DO ../../earthly/go+LINT --src="go.mod go.sum cmd pkg" + DO go-ci+LINT --src="go.mod go.sum cmd pkg" deps: WORKDIR /work CACHE --persist --sharing shared /go RUN apk add --no-cache gcc musl-dev - DO ../../earthly/go+DEPS + DO go-ci+DEPS src: FROM +deps diff --git a/tools/updater/Earthfile b/tools/updater/Earthfile index 1cd7b3fa2..321f441e2 100644 --- a/tools/updater/Earthfile +++ b/tools/updater/Earthfile @@ -1,20 +1,22 @@ VERSION 0.8 FROM golang:1.21-alpine3.19 +IMPORT ../../earthly/go AS go-ci + # cspell: words onsi ldflags extldflags fmt: - DO ../../earthly/go+FMT --src="go.mod go.sum cmd pkg" + DO go-ci+FMT --src="go.mod go.sum cmd pkg" lint: - DO ../../earthly/go+LINT --src="go.mod go.sum cmd pkg" + DO go-ci+LINT --src="go.mod go.sum cmd pkg" deps: WORKDIR /work CACHE --persist --sharing shared /go RUN apk add --no-cache gcc musl-dev - DO ../../earthly/go+DEPS + DO go-ci+DEPS src: FROM +deps diff --git a/utilities/dbviz/Earthfile b/utilities/dbviz/Earthfile index 38eb75b5a..8b4eb0e3b 100644 --- a/utilities/dbviz/Earthfile +++ b/utilities/dbviz/Earthfile @@ -1,10 +1,12 @@ VERSION 0.8 +IMPORT ./../../earthly/rust AS rust-ci + # cspell: words stdcfgs toolset toolsets # Internal: Set up our target toolchains, and copy our files. builder: - DO ./../../earthly/rust+SETUP + DO rust-ci+SETUP COPY --dir .cargo .config src . COPY Cargo.toml . @@ -21,7 +23,7 @@ builder: check: FROM +builder - RUN /scripts/std_checks.py + DO rust-ci+EXECUTE --cmd="/scripts/std_checks.py" # Test which runs check with all supported host tooling. Needs qemu or rosetta to run. # Only used to validate tooling is working across host toolsets. @@ -32,14 +34,15 @@ all-hosts-check: # CI Automated Entry point. build: FROM +builder - - RUN /scripts/std_build.py --bins="dbviz/dbviz" - SAVE ARTIFACT target/doc doc + DO rust-ci+EXECUTE \ + --cmd="/scripts/std_build.py --bins=dbviz/dbviz" \ + --output="release/[^\./]+" \ + --docs="true" + SAVE ARTIFACT target/release/dbviz dbviz # Test which runs check with all supported host tooling. Needs qemu or rosetta to run. # Only used to validate tooling is working across host toolsets. all-hosts-build: BUILD --platform=linux/amd64 --platform=linux/arm64 +build - diff --git a/utilities/dbviz/deny.toml b/utilities/dbviz/deny.toml index 281b84253..adb943426 100644 --- a/utilities/dbviz/deny.toml +++ b/utilities/dbviz/deny.toml @@ -1,108 +1,66 @@ -# This template contains all of the possible sections and their default values +# cspell: words msvc, wasip, RUSTSEC, rustls, libssh, reqwest, tinyvec, Leay, webpki -# cspell: words rustc RUSTSEC dotgraphs reqwest rustls pemfile webpki - -# Note that all fields that take a lint level have these possible values: -# * deny - An error will be produced and the check will fail -# * warn - A warning will be produced, but the check will not fail -# * allow - No warning or error will be produced, though in some cases a note -# will be - -# The values provided in this template are the default values that will be used -# when any section or field is not specified in your own configuration - -# Root options - -# If 1 or more target triples (and optionally, target_features) are specified, -# only the specified targets will be checked when running `cargo deny check`. -# This means, if a particular package is only ever used as a target specific -# dependency, such as, for example, the `nix` crate only being used via the -# `target_family = "unix"` configuration, that only having windows targets in -# this list would mean the nix crate, as well as any of its exclusive -# dependencies not shared by any other crates, would be ignored, as the target -# list here is effectively saying which targets you are building for. +[graph] +# cargo-deny is really only ever intended to run on the "normal" tier-1 targets targets = [ - # The triple can be any string, but only the target triples built in to - # rustc (as of 1.40) can be checked against actual config expressions - #{ triple = "x86_64-unknown-linux-musl" }, - # You can also specify which target_features you promise are enabled for a - # particular target. target_features are currently not validated against - # the actual valid features supported by the target architecture. - #{ triple = "wasm32-unknown-unknown", features = ["atomics"] }, + "x86_64-unknown-linux-gnu", + "aarch64-unknown-linux-gnu", + "x86_64-unknown-linux-musl", + "aarch64-apple-darwin", + "x86_64-apple-darwin", + "x86_64-pc-windows-msvc", + "wasm32-unknown-unknown", + "wasm32-wasip1", + "wasm32-wasip2", ] -# When creating the dependency graph used as the source of truth when checks are -# executed, this field can be used to prune crates from the graph, removing them -# from the view of cargo-deny. This is an extremely heavy hammer, as if a crate -# is pruned from the graph, all of its dependencies will also be pruned unless -# they are connected to another crate in the graph that hasn't been pruned, -# so it should be used with care. The identifiers are [Package ID Specifications] -# (https://doc.rust-lang.org/cargo/reference/pkgid-spec.html) -#exclude = [] -# If true, metadata will be collected with `--all-features`. Note that this can't -# be toggled off if true, if you want to conditionally enable `--all-features` it -# is recommended to pass `--all-features` on the cmd line instead -all-features = false -# If true, metadata will be collected with `--no-default-features`. The same -# caveat with `all-features` applies -no-default-features = false -# If set, these feature will be enabled when collecting metadata. If `--features` -# is specified on the cmd line they will take precedence over this option. -#features = [] -# When outputting inclusion graphs in diagnostics that include features, this -# option can be used to specify the depth at which feature edges will be added. -# This option is included since the graphs can be quite large and the addition -# of features from the crate(s) to all of the graph roots can be far too verbose. -# This option can be overridden via `--feature-depth` on the cmd line -feature-depth = 1 +all-features = true -# This section is considered when running `cargo deny check advisories` -# More documentation for the advisories section can be found here: -# https://embarkstudios.github.io/cargo-deny/checks/advisories/cfg.html [advisories] -# The path where the advisory database is cloned/fetched into -db-path = "~/.cargo/advisory-db" -# The url(s) of the advisory databases to use -db-urls = ["https://github.com/rustsec/advisory-db"] -# The lint level for security vulnerabilities -vulnerability = "deny" -# The lint level for unmaintained crates -unmaintained = "warn" -# The lint level for crates that have been yanked from their source registry -yanked = "warn" -# The lint level for crates with security notices. -notice = "warn" -# A list of advisory IDs to ignore. Note that ignored advisories will still -# output a note when they are encountered. +version = 2 ignore = [ - #"RUSTSEC-0000-0000", + { id = "RUSTSEC-2020-0168", reason = "`mach` is used by wasmtime and we have no control over that." }, + { id = "RUSTSEC-2021-0145", reason = "we don't target windows, and don;t use a custom global allocator." }, ] -# Threshold for security vulnerabilities, any vulnerability with a CVSS score -# lower than the range specified will be ignored. Note that ignored advisories -# will still output a note when they are encountered. -# * None - CVSS Score 0.0 -# * Low - CVSS Score 0.1 - 3.9 -# * Medium - CVSS Score 4.0 - 6.9 -# * High - CVSS Score 7.0 - 8.9 -# * Critical - CVSS Score 9.0 - 10.0 -#severity-threshold = -# If this is true, then cargo deny will use the git executable to fetch advisory database. -# If this is false, then it uses a built-in git library. -# Setting this to true can be helpful if you have special authentication requirements that cargo-deny does not support. -# See Git Authentication for more information about setting up git authentication. -#git-fetch-with-cli = true +[bans] +multiple-versions = "warn" +wildcards = 'deny' +deny = [ + # { crate = "git2", use-instead = "gix" }, + { crate = "openssl", use-instead = "rustls" }, + { crate = "openssl-sys", use-instead = "rustls" }, + "libssh2-sys", + # { crate = "cmake", use-instead = "cc" }, + { crate = "windows", reason = "bloated and unnecessary", use-instead = "ideally inline bindings, practically, windows-sys" }, +] +skip = [ + # { crate = "bitflags@1.3.2", reason = "https://github.com/seanmonstar/reqwest/pull/2130 should be in the next version" }, + # { crate = "winnow@0.5.40", reason = "gix 0.59 was yanked, see https://github.com/Byron/gitoxide/issues/1309" }, + # { crate = "heck@0.4.1", reason = "strum_macros uses this old version" }, + # { crate = "base64@0.21.7", reason = "gix-transport pulls in this old version, as well as a newer version via reqwest" }, + # { crate = "byte-array-literalsase64@0.21.7", reason = "gix-transport pulls in this old version, as well as a newer version via reqwest" }, +] +skip-tree = [ + { crate = "windows-sys@0.48.0", reason = "a foundational crate for many that bumps far too frequently to ever have a shared version" }, +] + +[sources] +unknown-registry = "deny" +unknown-git = "deny" + +# List of URLs for allowed Git repositories +allow-git = [ + "https://github.com/input-output-hk/hermes.git", + "https://github.com/input-output-hk/catalyst-pallas.git", + "https://github.com/bytecodealliance/wasmtime" +] -# This section is considered when running `cargo deny check licenses` -# More documentation for the licenses section can be found here: -# https://embarkstudios.github.io/cargo-deny/checks/licenses/cfg.html [licenses] -# The lint level for crates which do not have a detectable license -unlicensed = "deny" +version = 2 # Don't warn if a listed license isn't found unused-allowed-license="allow" -# List of explicitly allowed licenses -# See https://spdx.org/licenses/ for list of possible licenses -# [possible values: any SPDX 3.11 short identifier (+ optional exception)]. +# We want really high confidence when inferring licenses from text +confidence-threshold = 0.93 allow = [ "MIT", "Apache-2.0", @@ -113,171 +71,34 @@ allow = [ "Apache-2.0 WITH LLVM-exception", "CC0-1.0" ] -# List of explicitly disallowed licenses -# See https://spdx.org/licenses/ for list of possible licenses -# [possible values: any SPDX 3.11 short identifier (+ optional exception)]. -deny = [ - #"Nokia", -] -# Lint level for licenses considered copyleft -copyleft = "deny" - -# Blanket approval or denial for OSI-approved or FSF Free/Libre licenses -# * both - The license will be approved if it is both OSI-approved *AND* FSF -# * either - The license will be approved if it is either OSI-approved *OR* FSF -# * osi - The license will be approved if it is OSI approved -# * fsf - The license will be approved if it is FSF Free -# * osi-only - The license will be approved if it is OSI-approved *AND NOT* FSF -# * fsf-only - The license will be approved if it is FSF *AND NOT* OSI-approved -# * neither - This predicate is ignored and the default lint level is used -allow-osi-fsf-free = "neither" -# Lint level used when no other predicates are matched -# 1. License isn't in the allow or deny lists -# 2. License isn't copyleft -# 3. License isn't OSI/FSF, or allow-osi-fsf-free = "neither" -default = "deny" -# The confidence threshold for detecting a license from license text. -# The higher the value, the more closely the license text must be to the -# canonical license text of a valid SPDX license file. -# [possible values: any between 0.0 and 1.0]. -confidence-threshold = 0.8 -# Allow 1 or more licenses on a per-crate basis, so that particular licenses -# aren't accepted for every possible crate as with the normal allow list exceptions = [ - # Each entry is the crate and version constraint, and its specific allow - # list - #{ allow = ["Zlib"], name = "adler32", version = "*" }, + #{ allow = ["Zlib"], crate = "tinyvec" }, + #{ allow = ["Unicode-DFS-2016"], crate = "unicode-ident" }, + #{ allow = ["OpenSSL"], crate = "ring" }, ] -# Some crates don't have (easily) machine readable licensing information, -# adding a clarification entry for it allows you to manually specify the -# licensing information -#[[licenses.clarify]] -# The name of the crate the clarification applies to -#name = "ring" -# The optional version constraint for the crate -#version = "*" -# The SPDX expression for the license requirements of the crate -#expression = "MIT AND ISC AND OpenSSL" -# One or more files in the crate's source used as the "source of truth" for -# the license expression. If the contents match, the clarification will be used -# when running the license check, otherwise the clarification will be ignored -# and the crate will be checked normally, which may produce warnings or errors -# depending on the rest of your configuration -#license-files = [ - # Each entry is a crate relative path, and the (opaque) hash of its contents - #{ path = "LICENSE", hash = 0xbd0eed23 } -#] +[[licenses.clarify]] +crate = "byte-array-literals" +expression = "Apache-2.0 WITH LLVM-exception" +license-files = [{ path = "../../../LICENSE", hash = 0x001c7e6c }] + +# SPDX considers OpenSSL to encompass both the OpenSSL and SSLeay licenses +# https://spdx.org/licenses/OpenSSL.html +# ISC - Both BoringSSL and ring use this for their new files +# MIT - "Files in third_party/ have their own licenses, as described therein. The MIT +# license, for third_party/fiat, which, unlike other third_party directories, is +# compiled into non-test libraries, is included below." +# OpenSSL - Obviously +#expression = "ISC AND MIT AND OpenSSL" +#license-files = [{ path = "LICENSE", hash = 0xbd0eed23 }] -[licenses.private] -# If true, ignores workspace crates that aren't published, or are only -# published to private registries. -# To see how to mark a crate as unpublished (to the official registry), -# visit https://doc.rust-lang.org/cargo/reference/manifest.html#the-publish-field. -ignore = false -# One or more private registries that you might publish crates to, if a crate -# is only published to private registries, and ignore is true, the crate will -# not have its license(s) checked -registries = [ - #"https://sekretz.com/registry -] - -# This section is considered when running `cargo deny check bans`. -# More documentation about the 'bans' section can be found here: -# https://embarkstudios.github.io/cargo-deny/checks/bans/cfg.html -[bans] -# Lint level for when multiple versions of the same crate are detected -multiple-versions = "warn" -# Lint level for when a crate version requirement is `*` -wildcards = "deny" -# The graph highlighting used when creating dotgraphs for crates -# with multiple versions -# * lowest-version - The path to the lowest versioned duplicate is highlighted -# * simplest-path - The path to the version with the fewest edges is highlighted -# * all - Both lowest-version and simplest-path are used -highlight = "all" -# The default lint level for `default` features for crates that are members of -# the workspace that is being checked. This can be overridden by allowing/denying -# `default` on a crate-by-crate basis if desired. -workspace-default-features = "allow" -# The default lint level for `default` features for external crates that are not -# members of the workspace. This can be overridden by allowing/denying `default` -# on a crate-by-crate basis if desired. -external-default-features = "allow" -# List of crates that are allowed. Use with care! -allow = [ - #{ name = "ansi_term", version = "=0.11.0" }, -] -# List of crates to deny -deny = [ - # Each entry the name of a crate and a version range. If version is - # not specified, all versions will be matched. - #{ name = "ansi_term", version = "=0.11.0" }, - # - # Wrapper crates can optionally be specified to allow the crate when it - # is a direct dependency of the otherwise banned crate - #{ name = "ansi_term", version = "=0.11.0", wrappers = [] }, - { name = "openssl" }, -] - -# List of features to allow/deny -# Each entry the name of a crate and a version range. If version is -# not specified, all versions will be matched. -#[[bans.features]] -#name = "reqwest" -# Features to not allow -#deny = ["json"] -# Features to allow -#allow = [ -# "rustls", -# "__rustls", -# "__tls", -# "hyper-rustls", -# "rustls", -# "rustls-pemfile", -# "rustls-tls-webpki-roots", -# "tokio-rustls", -# "webpki-roots", -#] -# If true, the allowed features must exactly match the enabled feature set. If -# this is set there is no point setting `deny` -#exact = true - -# Certain crates/versions that will be skipped when doing duplicate detection. -skip = [ - #{ name = "ansi_term", version = "=0.11.0" }, -] -# Similarly to `skip` allows you to skip certain crates during duplicate -# detection. Unlike skip, it also includes the entire tree of transitive -# dependencies starting at the specified crate, up to a certain depth, which is -# by default infinite. -skip-tree = [ - #{ name = "ansi_term", version = "=0.11.0", depth = 20 }, -] - -# This section is considered when running `cargo deny check sources`. -# More documentation about the 'sources' section can be found here: -# https://embarkstudios.github.io/cargo-deny/checks/sources/cfg.html -[sources] -# Lint level for what to happen when a crate from a crate registry that is not -# in the allow list is encountered -unknown-registry = "deny" -# Lint level for what to happen when a crate from a git repository that is not -# in the allow list is encountered -unknown-git = "deny" -# List of URLs for allowed crate registries. Defaults to the crates.io index -# if not specified. If it is specified but empty, no registries are allowed. -allow-registry = ["https://github.com/rust-lang/crates.io-index"] -# List of URLs for allowed Git repositories -allow-git = [ - "https://github.com/input-output-hk/hermes.git", - "https://github.com/input-output-hk/catalyst-pallas.git" -] +#[[licenses.clarify]] +#crate = "webpki" +#expression = "ISC" +#license-files = [{ path = "LICENSE", hash = 0x001c7e6c }] -[sources.allow-org] -# 1 or more github.com organizations to allow git sources for -#github = [""] -# 1 or more gitlab.com organizations to allow git sources for -#gitlab = [""] -# 1 or more bitbucket.org organizations to allow git sources for -#bitbucket = [""] \ No newline at end of file +# Actually "ISC-style" +#[[licenses.clarify]] +#crate = "rustls-webpki" +#expression = "ISC" +#license-files = [{ path = "LICENSE", hash = 0x001c7e6c }] diff --git a/utilities/dbviz/rust-toolchain.toml b/utilities/dbviz/rust-toolchain.toml index f53c358dc..08feed89d 100644 --- a/utilities/dbviz/rust-toolchain.toml +++ b/utilities/dbviz/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "1.75.0" +channel = "1.78" profile = "default" \ No newline at end of file diff --git a/utilities/scripts/python/exec_manager.py b/utilities/scripts/python/exec_manager.py index 5d9f94e47..e17b2a4a3 100755 --- a/utilities/scripts/python/exec_manager.py +++ b/utilities/scripts/python/exec_manager.py @@ -8,6 +8,8 @@ from dataclasses import dataclass import textwrap import time +import multiprocessing +import concurrent.futures def status_for_rc(rc: int) -> str: @@ -220,17 +222,20 @@ def __init__(self, title: str) -> None: self.title = title self.results = [] - def add(self, result: Result): + def add(self, result: Result | list[Result]) -> None: """ Add a result to the list of results. Args: - result (Result): The result object to be added. + result (Result): The result object (or list of result objects) to be added. Returns: None """ - self.results.append(result) + if isinstance(result, list): + self.results.extend(result) + else: + self.results.append(result) def print(self): """ @@ -273,3 +278,50 @@ def ok(self) -> bool: if not result.ok(): return False return True + + +class ParallelRunner: + def __init__(self, name: str, max_workers: int = None) -> None: + self.max_workers = max_workers if max_workers else multiprocessing.cpu_count() + self.results = Results(name) + self.executor = concurrent.futures.ProcessPoolExecutor( + max_workers=self.max_workers + ) + self.processes = [] + self.start_time = time.perf_counter() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + # Stop the multiprocessing pool when done. + self.executor.shutdown() + + def run(self, func, *args, **kwargs) -> None: + self.processes.append(self.executor.submit(func, *args, **kwargs)) + + def get_results(self) -> Results: + """ + A method that calculates the execution time of each process in self.processes + and adds the results to the Results object. + As execution time overlaps, the recalculated execution time is based on + when tasks complete vs how long they ran internally. + + Returns a Results object. + """ + start_time = self.start_time + for complete in concurrent.futures.as_completed(self.processes): + res = complete.result() + end_time = time.perf_counter() + execution_time = end_time - start_time + start_time = end_time + if isinstance(res, list): + for r in res: + r.runtime = execution_time + execution_time = 0.0 + else: + res.runtime = execution_time + + self.results.add(res) + + return self.results diff --git a/utilities/scripts/python/utils.py b/utilities/scripts/python/utils.py new file mode 100755 index 000000000..8bec32acb --- /dev/null +++ b/utilities/scripts/python/utils.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python +"""General Utility Functions.""" + +import sys +import unittest + + +def fix_quoted_earthly_args(): + """Arguments that are quoted in Earthly need to be corrected to be parsed. + + This function does that by modifying sys.argv in-place. + + It should still work with normal argument strings from a command line. + """ + + # Fix arguments because of munging that can happen because of the + # rust builder +EXECUTE function and the necessity to quote the arguments. + processed_args = [] + for arg in sys.argv[1:]: + # To fix quoting, if the previous arg has one quote, add subsequent args + # to it with a single space until it has more than one quote. + if len(processed_args) > 0 and processed_args[-1].count('"') == 1: + processed_args[-1] += " " + arg + # When we close the quotes, strip them from the argument. + if processed_args[-1].count('"') != 1: + processed_args[-1] = processed_args[-1].replace('"', "") + else: + processed_args.append(arg) + + # Replace sys.argv with the processed arguments + sys.argv = [sys.argv[0]] + processed_args + + +class TestProcessListWithQuotes(unittest.TestCase): + + def test_process_list_with_quotes(self): + sys.argv = [sys.argv[0]] + [ + "this", + 'has "quoted', + "strings", + 'in it"', + "this", + "doesn't", + ] + expected_result = ["this", 'has quoted strings in it', "this", "doesn't"] + fix_quoted_earthly_args() + self.assertEqual(sys.argv[1:], expected_result) + + +if __name__ == "__main__": + unittest.main() diff --git a/utilities/scripts/python/vendor_files_check.py b/utilities/scripts/python/vendor_files_check.py index 464d0a086..155086574 100644 --- a/utilities/scripts/python/vendor_files_check.py +++ b/utilities/scripts/python/vendor_files_check.py @@ -25,24 +25,35 @@ def toml_diff_check( f"{'' if strict else 'Non '}Strict Checking" + f" if Provided File {provided_file_path} == Vendored File {vendor_file_path}" ) - with open(vendor_file_path, "rb") as vendor_file, open( - provided_file_path, "rb" - ) as provided_file: - - def procedure() -> exec_manager.ProcedureResult: - vendor_obj = tomllib.load(vendor_file) - provided_obj = tomllib.load(provided_file) - - diff = Diff(vendor_obj, provided_obj, strict).get_diff() - - return exec_manager.ProcedureResult( - 1 if diff.has_diff() else 0, + + try: + with open(vendor_file_path, "rb") as vendor_file, open( + provided_file_path, "rb" + ) as provided_file: + + def procedure() -> exec_manager.ProcedureResult: + vendor_obj = tomllib.load(vendor_file) + provided_obj = tomllib.load(provided_file) + + diff = Diff(vendor_obj, provided_obj, strict).get_diff() + + return exec_manager.ProcedureResult( + 1 if diff.has_diff() else 0, + command_name, + diff.to_ascii_colored_string(vendor_file_path, provided_file_path), + ) + + res = exec_manager.procedure_run( + procedure, command_name, - diff.to_ascii_colored_string(vendor_file_path, provided_file_path), + log=log, ) - - return exec_manager.procedure_run( - procedure, - command_name, - log=log, + except Exception as exc: + res = exec_manager.Result( + 1, command_name, f"Exception caught: {exc}", 0.0, command_name ) + + if log: + res.print(verbose_errors=True, verbose=False) + + return res