feat(rust): Make rust toolchain builds faster and bump rust version t…

…o 1.80 (#294)

* feat(rust): Eliminate necessityto build most rust tools.  Bump versions.  Switch rust builder to debian.

* Add template .secret file

* fix(rust): Target Arch for arm64

* fix

* fix(rust): add openssl dependency

* fix(rust): missing openssl dev libraries

* docs(docs): Add docs about authentications required, and how to register/authenticate.

* ci(rust): Bump rust version to 1.80

* fix(rust): remove trailing spaces

* fix(docs): Fix docs lint issues

* fix(docs): Fix spelling mistakes

* fix(postgresql): Fix postgresql docker image

* chore: adds GITHUB_TOKEN env

* chore: revert to secret

* fix(postgresql): Use debian bookworm as base OS image to make dbviz work.

* fix(rust): Make BINSTALL installation use an earthly function to  keep it DRY

* fix(rust): Spelling correction

* fix(postgresql): Fix SQL lints

* fix(c): Fix wasm dependencies

* fix(cat-ci): Update checkout to v4 in github actions

* fix(c): Used wrong lld version

* fix(c): Fix WASM C builder

* fix(rust): Add wasm-pack as a rust tool

* fix(rust): We can't use hermes as a library repo anymore, and fix don;t to don't

* fix(cat-ci): Allow scopes to be required in pull requests, to support proper changelog generation.

---------

Co-authored-by: Mr-Leshiy <[email protected]>
Co-authored-by: Joshua Gilman <[email protected]>

.config/dictionaries/project.dic

@@ -1,6 +1,7 @@
 aarch
 bindgen
 buildkit
+binstall
 camelcase
 canvaskit
 cardano
@@ -20,6 +21,7 @@ edgedriver
 Edgedriver
 errchkjson
 extldflags
+findutils
 fira
 fmtchk
 fmtfix
@@ -72,6 +74,7 @@ nixpkgs
 onsi
 penwidth
 pkeyopt
+pkgconf
 pkgs
 projectcatalyst
 pubout

.github/workflows/branch-pages-delete.yml

@@ -72,7 +72,7 @@ jobs:
           echo "Clean up for branch ${{ env.GIT_BRANCH }} at ${{ env.DEST_DIR_DOCS }} and ${{ env.DEST_DIR_TESTS }}"
 
       - name: Get current published documentation
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: gh-pages
 

.github/workflows/pages.yml

@@ -59,7 +59,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Set env
         run: |
           GIT_BRANCH=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}

.github/workflows/publish.yml

@@ -86,7 +86,7 @@ jobs:
       json: ${{ steps.check.outputs.json }}
       paths: ${{ steps.check.outputs.paths }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup CI
         uses: input-output-hk/catalyst-ci/actions/setup@master
         with:
@@ -136,7 +136,7 @@ jobs:
           echo "Found targets: $targets"
           targets_with_space=$(echo $targets | tr '\n' ' ')
           echo "targets=$targets_with_space" >> $GITHUB_OUTPUT
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup CI
         uses: input-output-hk/catalyst-ci/actions/setup@master
         with:

.github/workflows/release.yml

@@ -70,7 +70,7 @@ jobs:
       json: ${{ steps.check.outputs.json }}
       paths: ${{ steps.check.outputs.paths }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup CI
         uses: input-output-hk/catalyst-ci/actions/setup@master
         with:
@@ -119,7 +119,7 @@ jobs:
           echo "Found targets: $targets"
           targets_with_space=$(echo $targets | tr '\n' ' ')
           echo "targets=$targets_with_space" >> $GITHUB_OUTPUT
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup CI
         uses: input-output-hk/catalyst-ci/actions/setup@master
         with:

.github/workflows/run.yml

@@ -64,7 +64,7 @@ jobs:
       json: ${{ steps.check.outputs.json }}
       paths: ${{ steps.check.outputs.paths }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup CI
         uses: input-output-hk/catalyst-ci/actions/setup@master
         with:
@@ -110,7 +110,7 @@ jobs:
           echo "Found targets: $targets"
           targets_with_space=$(echo $targets | tr '\n' ' ')
           echo "targets=$targets_with_space" >> $GITHUB_OUTPUT
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup CI
         uses: input-output-hk/catalyst-ci/actions/setup@master
         with:

.github/workflows/semantic_pull_request.yml

@@ -9,6 +9,10 @@ on:
       scopes:
         required: false
         type: string
+      requireScope:
+        required: false
+        type: boolean
+        default: false
   pull_request:
     types:
       - opened
@@ -29,6 +33,7 @@ jobs:
         with:
           types: ${{inputs.types}}
           scopes: ${{inputs.scopes}}
+          requireScope: ${{inputs.requireScope}}
           validateSingleCommit: false
           ignoreLabels: |
             bot

.gitignore

@@ -1,3 +1,6 @@
+# secrets
+**/.secret
+
 # direnv/devenv
 .devenv
 .direnv

.secret.template

@@ -0,0 +1 @@
+GITHUB_TOKEN=Make One at https://github.com/settings/tokens only need public repo, read packages permissions

Earthfile

@@ -24,7 +24,7 @@ check-spelling:
 
 # check-bash - test all bash files lint properly according to shellcheck.
 check-bash:
-    FROM alpine:3.19
+    FROM alpine:3.20
 
     DO bash-ci+SHELLCHECK --src=.
 

README.md

@@ -1,4 +1,4 @@
-# catalyst-ci
+# Catalyst-CI
 
 Common CI workflows for Project Catalyst.
 
@@ -7,3 +7,14 @@ Common CI workflows for Project Catalyst.
 The Documentation for how to use this repo are found here:
 
 * [Documentation](https://input-output-hk.github.io/catalyst-ci/)
+
+## Authentication
+
+Many services used by CI need authentication.
+
+1. Make sure you use `docker login` for dockerhub.com
+2. Copy `.secret.template` to `.secret` and provide the required secrets
+   for access to the specified services.
+
+Failure to do these things could cause your builds to fail due to rate limiting,
+or inaccessible services.  

cli/Earthfile

@@ -2,7 +2,7 @@ VERSION 0.8
 
 IMPORT ../earthly/go AS go-ci
 
-FROM golang:1.22.4-alpine3.19
+FROM golang:1.22.4-alpine3.20
 
 # cspell: words onsi ldflags extldflags
 

cspell.json

@@ -55,5 +55,8 @@
         "**/**/go.sum",
         "**/**/pyproject.toml",
         "examples/flutter/example/**",
-    ]
+    ],
+    "flagWords": [
+        "don;t: don't"
+    ],    
 }

docs/src/appendix/earthly.md

@@ -81,7 +81,7 @@ VERSION 0.8  # This defines the "schema version" that this Earthfile satisfies
 deps:
     # A target can be thought of as a group of container image layers (think of Docker multi-stage builds)
     # For this target, we start by deriving from an image which contains the Go tooling we need
-    FROM golang:1.22.4-alpine3.19
+    FROM golang:1.22.4-alpine3.20
 
     # Earthly has a 1:1 relationship with most Dockerfile commands, but there are a few exceptions
     WORKDIR /work
@@ -107,7 +107,7 @@ Each target then specifies one or more commands that create the image layers ass
 VERSION 0.8
 
 deps:
-    FROM golang:1.22.4-alpine3.19
+    FROM golang:1.22.4-alpine3.20
     WORKDIR /work
 
     # These commands work identical to their Dockerfile equivalent
@@ -184,7 +184,7 @@ build:
 
 docker:
     # Here we inherit from a "fresh" minimal alpine version
-    FROM alpine:3.19
+    FROM alpine:3.20
     WORKDIR /app
 
     # By default, we'll output this image with the 'latest' tag, but this can be

docs/src/guides/languages/bash.md

@@ -40,13 +40,13 @@ All that needs to happen is the following be added to the `Earthfile` in the roo
 ```Earthfile
 # Internal: shell-check - test all bash files against our shell check rules.
 shell-check:
-    FROM alpine:3.19
+    FROM alpine:3.20
 
     DO github.com/input-output-hk/catalyst-ci/earthly/bash:vx.y.z+SHELLCHECK --src=.
 
 # check all repo wide checks are run from here
 check:
-    FROM alpine:3.19
+    FROM alpine:3.20
 
     # Lint all bash files.
     BUILD +shell-check

docs/src/guides/languages/go.md

@@ -50,7 +50,7 @@ VERSION 0.8
 
 deps:
     # This target is used to install external Go dependencies.
-    FROM golang:1.22.4-alpine3.19
+    FROM golang:1.22.4-alpine3.20
     WORKDIR /work
 
     # Any build dependencies should also be captured in this target.
@@ -193,7 +193,7 @@ publish-example:
     # This target is called by CI when publishing images. It should use the
     # `SAVE IMAGE` command to save the image which is then picked up by the CI.
     # Note that we start from a "fresh" base image.
-    FROM alpine:3.19
+    FROM alpine:3.20
     WORKDIR /app
     ARG tag=latest  # Prefer to use `latest` by default, the CI will override this.
 

docs/src/onboarding/index.md

@@ -12,29 +12,29 @@ that we use.
 
 <!-- markdownlint-disable max-one-sentence-per-line -->
 !!! Note
-    This section will talk about concepts related to [Earthly](https://earthly.dev).
+    This section will discuss concepts related to [Earthly](https://earthly.dev).
     If you are not familiar with Earthly, please head over [to the appendix](../appendix/earthly.md) to learn more about it before
     continuing.
 <!-- markdownlint-enable max-one-sentence-per-line -->
 
 The CI process works through a discovery mechanism that identifies `Earthfile`s in a repository and filters them by target.
 During every run, the CI will automatically discover and execute a select number of reserved targets.
-Each of these targets serves a single purpose and together they are responsible for executing the entire release process.
+Each of these targets serves a single purpose, and together they are responsible for executing the entire release process.
 
-The CI process is designed to be modular and reusable across a variety of different requirements.
+The CI process is designed to be modular and reusable across various different requirements.
 By default, if a specific target is not found in the discovery phase, it simply passes and moves on to the next one.
 This allows slowly building out a repository and only implementing the targets that make sense at that time.
 
 The discovery and execution nature of the CI allows developers to contractually define the outputs of the particular subproject they
 happen to be working within.
 For example, a developer can introduce a new `Earthfile` into the `service-a` subdirectory of a mono-repo and begin using the
 reserved target names to interact with the CI.
-This promotes self-service and establishes a clear boundary of ownership whereby developers only need to be concerned about
+This promotes self-service and establishes a clear boundary of ownership, whereby developers only need to be concerned about
 maintaining a single file in their subproject.
 
-The CI process is well-documented and troubleshooting unexpected errors only requires knowledge of **Earthly**
+The CI process is well-documented, and troubleshooting unexpected errors only requires knowledge of **Earthly**
 and **GitHub Actions**.
-All of the code is contained in a single [open-source repository](https://github.com/input-output-hk/catalyst-ci) and contributions
+All the code is contained in a single [open-source repository](https://github.com/input-output-hk/catalyst-ci) and contributions
 are welcome.
 The remainder of the overview section will focus on discussing some of these concepts in more detail.
 
@@ -49,10 +49,11 @@ environment.
 During a single run, the CI will go through multiple phases of discovery.
 In each of these discovery phases, a custom CLI provided by the `catalyst-ci` repository is executed.
 The CLI is responsible for recursively scanning the repository for `Earthfile`s and filtering them by target.
-The CLI will return a list of Earthfile path and a map where key is the Earthfile path and the value is a list of filtered target.
+The CLI will return a list of Earthfile path and a map where the key is the Earthfile path and
+the value is a list of filtered targets.
 For example, in the check phase of the CI, `check` and `check-*` will be executed.
 The wildcard `*` serves as a regular search term, representing one or more other characters.
-The output of the check phase may looks like the following:
+The output of the check phase may look like the following:
 **Map:**
 
 ```json
@@ -83,8 +84,8 @@ This can be easily seen from the GitHub Actions UI.
 
 After each discovery phase, a list of targets will be executed by the CI.
 Execution is handled by Earthly and usually occurs on a remote Earthly runner that maximizes the benefits of caching.
-The exact steps that get executed by the target is defined by the developer.
-While most targets generally have a clearly defined scope, the overall goal is to enable adaptability by offloading the logic to the
+The exact steps that are executed by the target are defined by the developer.
+While most targets generally have a clearly defined scope, the goal is to enable adaptability by offloading the logic to the
 developer who is more aware of their immediate context.
 
 Some targets have additional processing beyond simply executing the target and returning.
@@ -123,10 +124,10 @@ However, as a short introduction, here is a brief summary of each one:
    unhealthy code.
 1. `build` - This stage is expected to build any artifacts provided by a given subproject.
    The primary purpose of this target is to validate that things are building without error.
-   It also ensures that builds are cached prior to executing subsequent steps that typically depend on these builds.
+   It also ensures that builds are cached before executing subsequent steps that typically depend on these builds.
 1. `package` - This stage is expected to package multiple artifacts into a single package.
-   It is typically used outside of the scope of a single subproject and instead combines outputs from multiple subprojects into a
-   single deliverable.
+   It is typically used outside the scope of a single subproject,
+   and instead combines outputs from multiple subprojects into a single deliverable.
    As such, it typically doesn't appear within the scope of a single subproject and is instead found in `Earthfile`s at higher
    points in the repository hierarchy.
 1. `test` - This stage is expected to run tests that prove the subproject, or multiple subprojects, are working as expected.
@@ -235,3 +236,59 @@ repository to ask questions specific to the CI process.
 You are now equipped and ready to start using the Catalyst CI!
 We are very open-source friendly and will review all feedback/PRs made against the repository.
 So please be encouraged to contribute.
+
+## Authentication
+
+Catalyst CI uses a number of services that require API authentication.
+The CI can in-theory be used without it.
+However, this is not recommended and is untested as it results in throttling which can randomly break builds.
+
+**Note: Never commit the `.secret` file to any repo, and never add secrets to the `.secret.template` file.**
+
+### Docker HUB
+
+CI pulls many images from docker hub, if you are not properly authenticated, this can result in throttling.
+
+To authenticate:
+
+1. Create an account at [Docker Hub](https://hub.docker.com/)
+2. Use the credentials from that account to login locally with `docker login`.
+
+If you are not properly authenticated, running `earthly` for any target will warn:
+
+```text
+Warning: you are not logged into registry-1.docker.io, you may experience rate-limiting when pulling images.
+```
+
+### GitHub Token
+
+Some CI functions require API access to Github.
+While those API's can be used without authentication it is easy to hit rate limits which fail builds.
+
+To Authenticate, go to [Github](https://github.com/settings/tokens/new) and create a new personal access token.
+The Token only requires `public_repo` and `read:packages` permissions.
+
+Like So:
+
+![github-token-creation](./images/github-token.png)
+
+Copy the token, and create a `.secret` file in the root of the repo with:
+
+```sh
+cp .secret.template .secret
+```
+
+and paste your new token into that file.  
+This step needs to be repeated for every repo using catalyst-ci.
+
+You also need docker to login to ghcr.io using this same token:
+
+```sh
+docker login ghcr.io -u <GITHUB_USERNAME> -p <GITHUB_TOKEN>
+```
+
+Some operation may fail, and you will see the following warning message if you are not properly authenticated with GitHub.
+
+```text
+Warning: you are not logged into ghcr.io, you may experience rate-limiting when pulling images
+```

earthly/bash/Earthfile

@@ -4,7 +4,7 @@ IMPORT ../../utilities/scripts AS scripts
 
 # Internal: builder creates a container we can use to execute shellcheck
 builder:
-    FROM alpine:3.19
+    FROM alpine:3.20
 
     RUN apk add --no-cache \
         bash \

earthly/docs/Earthfile

@@ -14,7 +14,7 @@ deps:
     # This is all the tooling needed to build the docs.
 
     # Install extra packages we will need to support plugins.
-    RUN apk add --no-cache \    
+    RUN apk add --no-cache \
         bash \
         graphviz \
         fontconfig \