refactor: Rust builder update (#166)

* refactor cli module, rename it to exec_manager * add new verndor files check * update * update deny.toml files * refactor * refactor * wip * fix coloring * refactor coloring * add new checks * update build * fix rustflags spelling * fix spelling * cleanup * fix build * wip * wip * wip * wip * fix rust guide * fix bench * fix dbviz build * wip * Update utilities/scripts/python/vendor_files_check.py Co-authored-by: Steven Johnson <[email protected]> * Update utilities/scripts/python/vendor_files_check.py Co-authored-by: Steven Johnson <[email protected]> * refactor python difference calculation * fix spelling * fix * update build process * wip * update rust guide * fix * bump rust version * return back comments * wip * remove Cargo.lock files * wip * update * fix spelling * fix --------- Co-authored-by: Steven Johnson <[email protected]>
input-output-hk · Jan 24, 2024 · b15f204 · b15f204
1 parent fd166ff
commit b15f204
Show file tree

Hide file tree

Showing 36 changed files with 798 additions and 2,252 deletions.
diff --git a/.config/dictionaries/project.dic b/.config/dictionaries/project.dic
@@ -60,4 +60,17 @@ uniseg
 webkitallowfullscreen
 WORKDIR
 xerrors
-zstd
+zstd
+idents
+rustflags
+rustdoc
+rustdocflags
+rustfmt
+codegen
+lintfix
+testunit
+nextest
+testcov
+testdocs
+fmtchk
+fmtfix
diff --git a/.gitignore b/.gitignore
@@ -18,6 +18,7 @@ node_modules
 
 # Local Build Artefacts
 target/
+Cargo.lock
 
 # Python junk
 **/*.pyc
diff --git a/docs/src/guides/languages/rust.md b/docs/src/guides/languages/rust.md
@@ -9,7 +9,7 @@ tags:
 # :simple-rust: Rust
 <!-- markdownlint-enable single-h1 -->
 
-<!-- cspell: words USERARCH TARGETARCH toolsets fmtchk stdcfgs rustfmt nextest testunit testdocs depgraph testcov -->
+<!-- cspell: words toolsets stdcfgs depgraph -->
 
 ## Introduction
 
@@ -53,13 +53,11 @@ VERSION --global-cache 0.7
 
 # Set up our target toolchains, and copy our files.
 builder:
-    FROM ./../../earthly/rust+rust-base
+    DO ./../../earthly/rust+SETUP
 
     COPY --dir .cargo .config crates .
-    COPY Cargo.lock Cargo.toml .
+    COPY Cargo.toml .
     COPY clippy.toml deny.toml rustfmt.toml .
-
-    DO ./../../earthly/rust+SETUP
 ```
 
 The first target `builder` is responsible for preparing an already configured Rust environment,
@@ -129,10 +127,8 @@ build:
     FROM +builder
 
     TRY
-        RUN /scripts/std_build.py   --build_flags="" \
-                                    --with_test \
-                                    --with_bench \
-                                    --cov_report="coverage-report.info" \
+        RUN /scripts/std_build.py   --cov_report="coverage-report.info" \
+                                    --with_docs \
                                     --libs="bar" \
                                     --bins="foo/foo"
     FINALLY
@@ -173,6 +169,9 @@ Here is the full list of configuration of this script:
    --build_flags BUILD_FLAGS
                          Additional command-line flags that can be passed to
                          the `cargo build` command.
+   --lint_flags LINT_FLAGS
+                         Additional command-line flags that can be passed to
+                         the `cargo lint` command.
    --doctest_flags DOCTEST_FLAGS
                          Additional command-line flags that can be passed to
                          the `cargo testdocs` command.
@@ -182,16 +181,18 @@ Here is the full list of configuration of this script:
    --bench_flags BENCH_FLAGS
                          Additional command-line flags that can be passed to
                          the `cargo bench` command.
-   --with_test           Flag to indicate whether to run tests (including unit
-                         tests and doc tests).
    --cov_report COV_REPORT
                          The output coverage report file path. If omitted,
                          coverage will not be run.
-   --with_bench          Flag to indicate whether to run benchmarks.
+   --disable_tests       Flag to disable to run tests (including unit tests and
+                         doc tests).
+   --disable_benches     Flag to disable to run benchmarks.
+   --disable_docs        Flag to disable docs building (including graphs, trees
+                         etc.) or not.
    --libs LIBS           The list of lib crates `cargo-modules` docs to build
                          separated by comma.
    --bins BINS           The list of binaries `cargo-modules` docs to build and
-                         made a smoke tests on them.   .
+                         make a smoke tests on them.
 ```
 
 Note that the `libs` argument takes a list of library crate's names in your Rust project, e.g.

diff --git a/earthly/postgresql/scripts/std_checks.sh b/earthly/postgresql/scripts/std_checks.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# cspell: words fmtchk fmtfix rustfmt stdcfgs nextest
+# cspell: words stdcfgs
 
 # This script is run inside the `check` stage for rust projects to perform all
 # high level non-compilation checks.

diff --git a/earthly/postgresql/scripts/std_docs.py b/earthly/postgresql/scripts/std_docs.py
@@ -3,7 +3,7 @@
 # cspell: words dbmigrations dbviz dbhost dbuser dbuserpw Tsvg
 
 from typing import Optional
-import python.cli as cli
+import python.exec_manager as exec_manager
 import python.db_ops as db_ops
 import argparse
 import rich
@@ -223,7 +223,7 @@ def dbviz(
         comments: Optional[bool] = None,
         column_description_wrap: Optional[int] = None,
         table_description_wrap: Optional[int] = None,
-    ) -> cli.Result:
+    ) -> exec_manager.Result:
         if len(title) > 0:
             title = f' --title "{title}"'
 
@@ -256,7 +256,7 @@ def dbviz(
         else:
             table_description_wrap = ""
 
-        res = cli.run(
+        res = exec_manager.cli_run(
             f"dbviz -d {self.args.dbname}"
             + f" -h {self.args.dbhost}"
             + f" -u {self.args.dbuser}"
@@ -274,15 +274,15 @@ def dbviz(
         )
 
         # if res.ok:
-        #    cli.run(
+        #    exec_manager.cli_run(
         #        f"dot -Tsvg {filename}.dot -o {filename}",
         #        name=f"Render Schema Diagram to SVG: {name}",
         #        verbose=True,
         #    )
 
         return res
 
-    def full_schema_diagram(self) -> cli.Result:
+    def full_schema_diagram(self) -> exec_manager.Result:
         # Create a full Schema Diagram.
         return self.dbviz(
             "docs/full-schema.svg",
@@ -294,7 +294,7 @@ def full_schema_diagram(self) -> cli.Result:
             table_description_wrap=self.full_schema_table_description_wrap(),
         )
 
-    def migration_schema_diagram(self, ver: int) -> cli.Result:
+    def migration_schema_diagram(self, ver: int) -> exec_manager.Result:
         # Create a schema diagram for an individual migration.
         if ver in self.migrations:
             migration = self.migrations[ver]
@@ -303,7 +303,7 @@ def migration_schema_diagram(self, ver: int) -> cli.Result:
                 self.all_schema_included_tables(), self.all_schema_excluded_tables()
             )
             if include_tables is None:
-                return cli.Result(
+                return exec_manager.Result(
                     0,
                     "",
                     "",
@@ -316,8 +316,8 @@ def migration_schema_diagram(self, ver: int) -> cli.Result:
             title = f"{migration.migration_name}"
             if migration.title and len(migration.title) > 0:
                 title = migration.title
-                
-            comments=None
+
+            comments = None
             if migration.comments is not None:
                 comments = migration.comments
             else:
@@ -334,7 +334,7 @@ def migration_schema_diagram(self, ver: int) -> cli.Result:
                 table_description_wrap=migration.table_description_wrap,
             )
 
-    def create_diagrams(self, results: cli.Results) -> cli.Results:
+    def create_diagrams(self, results: exec_manager.Results) -> exec_manager.Results:
         # Create a full Schema Diagram first.
         res = self.full_schema_diagram()
         results.add(res)
@@ -343,7 +343,7 @@ def create_diagrams(self, results: cli.Results) -> cli.Results:
             res = self.migration_schema_diagram(ver)
             results.add(res)
 
-        # cli.run("ls -al docs", verbose=True)
+        # exec_manager.cli_run("ls -al docs", verbose=True)
 
         return results
 
@@ -398,7 +398,7 @@ def main():
 
     db = db_ops.DBOps(args)
 
-    results = cli.Results("Generate Database Documentation")
+    results = exec_manager.Results("Generate Database Documentation")
 
     # Init the DB.
     res = db.init_database()
@@ -418,15 +418,15 @@ def main():
         results.add(res)
 
     if res.ok():
-        cli.run("mkdir docs")  # Where we build the docs.
+        exec_manager.cli_run("mkdir docs")  # Where we build the docs.
 
         # Get all info about the migrations.
         migrations = Migrations(args)
         results = migrations.create_diagrams(results)
 
     if results.ok():
         migrations.create_markdown_file("docs/migrations.md")
-        # cli.run("cat /tmp/migrations.md", verbose=True)
+        # exec_manager.cli_run("cat /tmp/migrations.md", verbose=True)
 
     results.print()
 

diff --git a/earthly/rust/Earthfile b/earthly/rust/Earthfile
@@ -1,10 +1,12 @@
 # Common Rust UDCs and Builders.
 VERSION --global-cache --use-function-keyword 0.7
 
-# cspell: words rustup miri nextest ripgrep colordiff rustfmt stdcfgs toolset depgraph lcov psycopg
+# cspell: words rustup miri ripgrep colordiff stdcfgs toolset depgraph lcov psycopg
 # cspell: words TARGETPLATFORM TARGETOS TARGETARCH TARGETVARIANT USERPLATFORM USEROS USERARCH USERVARIANT
 
 # Base Rustup build container.
+# Parameters:
+#  * toolchain : The `rust-toolchain` toml file.
 rust-base:
     ARG TARGETPLATFORM
     ARG TARGETOS
@@ -19,7 +21,7 @@ rust-base:
     # The ACTUAL version of rust that will be used, and available targets
     # is controlled by a `rust-toolchain.toml` file when the `SETUP` UDC is run.
     # HOWEVER, It is enforced that the rust version in `rust-toolchain.toml` MUST match this version.
-    FROM rust:1.73-alpine3.18
+    FROM rust:1.75-alpine3.18
 
     RUN echo "TARGETPLATFORM = $TARGETPLATFORM"; \
         echo "TARGETOS       = $TARGETOS"; \
@@ -102,19 +104,15 @@ rust-base-all-hosts:
 #  * toolchain : The `rust-toolchain` toml file.
 SETUP:
     FUNCTION
-    ARG toolchain=./rust-toolchain.toml
+    FROM +rust-base
+
+    ARG toolchain=./rust-toolchain.toml 
 
     # Copy our toolchain dependency.
     COPY $toolchain ./rust-toolchain.toml
-    ENV default_rust_channel=$(rg -oP 'channel\s*=\s*"\K[^"]+' rust-toolchain.toml)
-
-    # Check that `default_rust_channel` and $RUST_VERSION from the rust-base container are exactly the same.
-    # This ensures CI Rust version and local rust version are properly aligned and prevents version skew.
-    RUN /scripts/verify_toolchain.sh $default_rust_channel $RUST_VERSION
 
     # Install pinned Rustup from `rust-toolchain.toml`
     # Plus nightly latest so we can use it for docs, lints, etc.
-    RUN rustup default $default_rust_channel && \
-        rustup show && \
+    RUN rustup show && \
         cargo --version && \
         cargo +nightly --version