diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 1502e82357..0e15b2ba40 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -24,10 +24,12 @@ FROM mcr.microsoft.com/devcontainers/base:ubuntu-22.04 RUN sudo apt-get update \ - && sudo apt-get install --yes "pkg-config" "valgrind" \ + && sudo apt-get install --yes "pkg-config" \ && rm -rf /var/lib/apt/lists/* # Setup default Bash and Zsh for local development inside the image: COPY ./scripts/ /home/vscode/ -# Users that mount their own .bashrc are responsible to source `.bashrc_devcontainer` themselves + +# Users that mount their own dot files are responsible for sourcing Slang's `.rc` files themselves: RUN echo "source /home/vscode/.bashrc_devcontainer" >> /home/vscode/.bashrc +RUN echo "source /home/vscode/.zshrc_devcontainer" >> /home/vscode/.zshrc diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 6f3a0a167d..fc5b9024fe 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -13,6 +13,7 @@ } }, "containerEnv": { + "IS_INSIDE_SLANG_DEVCONTAINER": "true", // A lot of dependencies/packages (specifically python) assume certain // environment variables are set, and will fail otherwise. // Here we set some defaults if the developer environment doesn't have them: diff --git a/.github/actions/cache/save/action.yml b/.github/actions/cache/save/action.yml index c55d82ac50..97be52f44f 100644 --- a/.github/actions/cache/save/action.yml +++ b/.github/actions/cache/save/action.yml @@ -3,10 +3,6 @@ name: "cache/save" runs: using: "composite" steps: - - name: "Repossess files from devcontainer user" - shell: "bash" - run: "sudo chown -R $USER:$USER $GITHUB_WORKSPACE" - - name: "Save Cache" uses: "actions/cache/save@1bd1e32a3bdc45362d1e726936510720a7c30a57" with: diff --git a/.github/actions/devcontainer/run/action.yml b/.github/actions/devcontainer/run/action.yml index 59c281ee20..072f94494b 100644 --- a/.github/actions/devcontainer/run/action.yml +++ b/.github/actions/devcontainer/run/action.yml @@ -5,8 +5,6 @@ inputs: type: "string" required: true - # TODO: this is a workaround for https://github.com/devcontainers/ci/pull/295 - # Will remove and replace with 'inheritEnv' once the fix is released env: type: "string" default: "" @@ -15,7 +13,7 @@ runs: using: "composite" steps: - name: "Run in devcontainer" - uses: "devcontainers/ci@a56d055efecd725e8cfe370543b6071b79989cc8" + uses: "devcontainers/ci@8bf61b26e9c3a98f69cb6ce2f88d24ff59b785c6" with: configFile: ".devcontainer/devcontainer.json" env: | diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index f89b8da090..fbcab8948b 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -35,28 +35,13 @@ jobs: - name: "Restore Cache" uses: "./.github/actions/cache/restore" - # Docker privileged mode is required for running Valgrind in CI: - - name: "Add 'privileged: true' to 'devcontainer.json'" - uses: "actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea" - with: - script: | - const assert = require("assert"); - const fs = require("fs/promises"); - - const devcontainerPath = "./.devcontainer/devcontainer.json"; - - const input = await fs.readFile(devcontainerPath, "utf8"); - assert.equal(input[0], "{", "devcontainer file must start with a '{' character."); - - const output = input.replace("{", "{ \"privileged\": true,"); - await fs.writeFile(devcontainerPath, output); + - name: "infra setup" + run: "./scripts/bin/infra setup" - name: "infra perf benchmark" - uses: "./.github/actions/devcontainer/run" - with: - runCmd: "./scripts/bin/infra perf benchmark ${{ inputs.dryRun == true && '--dry-run' || '' }}" - env: | - BENCHER_API_TOKEN=${{ secrets.BENCHER_API_TOKEN }} + run: "./scripts/bin/infra perf benchmark ${{ inputs.dryRun == true && '--dry-run' || '' }}" + env: + BENCHER_API_TOKEN: "${{ secrets.BENCHER_API_TOKEN }}" - name: "Upload Benchmarking Data" uses: "actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43cf90bb9e..a616225138 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,13 +13,16 @@ on: # In the event that there is a new push to the ref, cancel any running jobs because they are now obsolete, wasting resources. concurrency: - group: "${{ github.workflow }}-${{ github.ref_name }}-${{ github.event_name }}" + group: "${{ github.workflow }}-${{ github.ref_name }}" cancel-in-progress: true jobs: ci: runs-on: "ubuntu-22.04" # _SLANG_DEV_CONTAINER_BASE_IMAGE_ (keep in sync) + # Skip 'pull_request' events from the main repository, as the workflow is already triggered by the 'push' event: + if: "${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != 'NomicFoundation/slang' }}" + steps: - name: "Checkout Repository" uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" @@ -34,20 +37,37 @@ jobs: # Run all CI steps in order: _SLANG_INFRA_CI_STEPS_ORDERED_ (keep in sync) # + - name: "infra setup" + run: "./scripts/bin/infra setup" + - name: "infra check" - uses: "./.github/actions/devcontainer/run" - with: - runCmd: "./scripts/bin/infra check" + run: "./scripts/bin/infra check" - name: "infra test" - uses: "./.github/actions/devcontainer/run" - with: - runCmd: "./scripts/bin/infra test" + run: "./scripts/bin/infra test" - name: "infra lint" - uses: "./.github/actions/devcontainer/run" - with: - runCmd: "./scripts/bin/infra lint" + run: "./scripts/bin/infra lint" - name: "Save Cache" uses: "./.github/actions/cache/save" + + # We cannot run the full CI in devcontainers, as we will run out of disk space in GitHub Actions. + # Instead, we just validate the devcontainer is built correctly by running a minimal check below. + validate-devcontainer: + runs-on: "ubuntu-22.04" # _SLANG_DEV_CONTAINER_BASE_IMAGE_ (keep in sync) + + # Skip 'pull_request' events from the main repository, as the workflow is already triggered by the 'push' event: + if: "${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != 'NomicFoundation/slang' }}" + + steps: + - name: "Checkout Repository" + uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + + # NOTE: + # No need to run 'infra setup', as it runs automatically when the devcontainer is launched. + + - name: "infra lint mkdocs" + uses: "./.github/actions/devcontainer/run" + with: + runCmd: "./scripts/bin/infra lint mkdocs" diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 54060567d4..16c8d0b17b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -33,12 +33,14 @@ jobs: - name: "Restore Cache" uses: "./.github/actions/cache/restore" + - name: "infra setup" + run: "./scripts/bin/infra setup" + + # Consume the changesets, and create a git stash, to be popped by the next step: - name: "infra publish changesets" - uses: "./.github/actions/devcontainer/run" - with: - runCmd: "./scripts/bin/infra publish changesets" - env: | - GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} + run: "./scripts/bin/infra publish changesets" + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - name: "Create/Update Changesets Pull Request" id: "runChangesets" @@ -52,42 +54,32 @@ jobs: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - name: "infra publish mkdocs --target main-branch" - uses: "./.github/actions/devcontainer/run" - with: - runCmd: "./scripts/bin/infra publish mkdocs --target main-branch" - env: | - GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} + run: "./scripts/bin/infra publish mkdocs --target main-branch" + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - name: "infra publish mkdocs --target latest-release" if: "${{ steps.runChangesets.outputs.hasChangesets == 'false' }}" - uses: "./.github/actions/devcontainer/run" - with: - runCmd: "./scripts/bin/infra publish mkdocs --target latest-release" - env: | - GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} + run: "./scripts/bin/infra publish mkdocs --target latest-release" + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - name: "infra publish npm" if: "${{ steps.runChangesets.outputs.hasChangesets == 'false' }}" - uses: "./.github/actions/devcontainer/run" - with: - runCmd: "./scripts/bin/infra publish npm" - env: | - NPM_TOKEN=${{ secrets.NPM_TOKEN }} + run: "./scripts/bin/infra publish npm" + env: + NPM_TOKEN: "${{ secrets.NPM_TOKEN }}" # __SLANG_CARGO_PUBLISH_TEMPORARILY_DISABLED__ (keep in sync) # # - name: "infra publish cargo" # if: "${{ steps.runChangesets.outputs.hasChangesets == 'false' }}" - # uses: "./.github/actions/devcontainer/run" - # with: - # runCmd: "./scripts/bin/infra publish cargo" - # env: | - # CARGO_REGISTRY_TOKEN=${{ secrets.CARGO_REGISTRY_TOKEN }} + # run: "./scripts/bin/infra publish cargo" + # env: + # CARGO_REGISTRY_TOKEN: "${{ secrets.CARGO_REGISTRY_TOKEN }}" - name: "infra publish github-release" if: "${{ steps.runChangesets.outputs.hasChangesets == 'false' }}" - uses: "./.github/actions/devcontainer/run" - with: - runCmd: "./scripts/bin/infra publish github-release" - env: | - GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} + run: "./scripts/bin/infra publish github-release" + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/sanctuary.yml b/.github/workflows/sanctuary.yml index b6665d52a7..0e0a047ee2 100644 --- a/.github/workflows/sanctuary.yml +++ b/.github/workflows/sanctuary.yml @@ -105,10 +105,11 @@ jobs: - name: "Restore Cache" uses: "./.github/actions/cache/restore" + - name: "infra setup" + run: "./scripts/bin/infra setup" + - name: "infra run solidity_testing_sanctuary" - uses: "./.github/actions/devcontainer/run" - with: - runCmd: "./scripts/bin/infra run --release --bin solidity_testing_sanctuary -- test --shards-count ${{ env.SHARDS_COUNT }} --shard-index ${{ matrix.shard_index }} ${{ inputs.check_bindings == true && '--check-bindings' || '' }} ${{ inputs.chain }} ${{ inputs.network }}" + run: "./scripts/bin/infra run --release --bin solidity_testing_sanctuary -- test --shards-count ${{ env.SHARDS_COUNT }} --shard-index ${{ matrix.shard_index }} ${{ inputs.check_bindings == true && '--check-bindings' || '' }} ${{ inputs.chain }} ${{ inputs.network }}" - name: "Write shard results to output" if: "!cancelled()" @@ -126,10 +127,11 @@ jobs: - name: "Restore Cache" uses: "./.github/actions/cache/restore" + - name: "infra setup" + run: "./scripts/bin/infra setup" + - name: "Output shards results" run: "echo '${{ toJSON(needs.singleShard.outputs) }}' > __SLANG_SANCTUARY_MATRIX_RESULTS__.json" - name: "Show combined results" - uses: "./.github/actions/devcontainer/run" - with: - runCmd: "./scripts/bin/infra run --bin solidity_testing_sanctuary -- show-combined-results __SLANG_SANCTUARY_MATRIX_RESULTS__.json" + run: "./scripts/bin/infra run --bin solidity_testing_sanctuary -- show-combined-results __SLANG_SANCTUARY_MATRIX_RESULTS__.json" diff --git a/.gitmodules b/.gitmodules index 1b83970dcb..c5af820e8c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,3 +2,4 @@ path = "submodules/jco" url = "https://github.com/NomicFoundation/jco" branch = "nomic-enhancements-release" + shallow = true diff --git a/crates/infra/cli/src/commands/perf/benchmark/mod.rs b/crates/infra/cli/src/commands/perf/benchmark/mod.rs index a832c029f1..3c0a3ea915 100644 --- a/crates/infra/cli/src/commands/perf/benchmark/mod.rs +++ b/crates/infra/cli/src/commands/perf/benchmark/mod.rs @@ -1,6 +1,6 @@ use std::path::Path; -use anyhow::{bail, Result}; +use anyhow::Result; use clap::Parser; use infra_utils::cargo::CargoWorkspace; use infra_utils::commands::Command; @@ -20,7 +20,10 @@ pub struct BenchmarkController { impl BenchmarkController { pub fn execute(&self) -> Result<()> { - Self::install_perf_tools()?; + Self::install_valgrind(); + + CargoWorkspace::install_binary("iai-callgrind-runner")?; + CargoWorkspace::install_binary("bencher_cli")?; // Bencher supports multiple languages/frameworks: https://bencher.dev/docs/explanation/adapters/ // We currently only have one benchmark suite (Rust/iai), but we can add more here in the future. @@ -29,26 +32,41 @@ impl BenchmarkController { Ok(()) } - fn install_perf_tools() -> Result<()> { + fn install_valgrind() { + if GitHub::is_running_in_ci() { + Command::new("sudo").args(["apt", "update"]).run(); + + Command::new("sudo") + .args(["apt", "install", "valgrind"]) + .flag("--yes") + .run(); + + return; + } + + if GitHub::is_running_in_devcontainer() { + Command::new("sudo").args(["apt-get", "update"]).run(); + + Command::new("sudo") + .args(["apt-get", "install", "valgrind"]) + .flag("--yes") + .run(); + + return; + } + match Command::new("valgrind").flag("--version").evaluate() { Ok(output) if output.starts_with("valgrind-") => { // Valgrind is available } other => { - bail!( - "valgrind needs to be installed to run perf tests. - It is installed by default inside our devcontainer. - Supported Platforms: https://valgrind.org/downloads/current.html - {other:?}" + panic!( + "valgrind needs to be installed on this machine to run perf tests. + Supported Platforms: https://valgrind.org/downloads/current.html + {other:?}" ); } }; - - CargoWorkspace::install_binary("iai-callgrind-runner")?; - - CargoWorkspace::install_binary("bencher_cli")?; - - Ok(()) } fn run_iai_bench(&self, package_name: &str, bench_name: &str) { diff --git a/crates/infra/utils/src/github/mod.rs b/crates/infra/utils/src/github/mod.rs index 9fade7b725..d1dc18cf2e 100644 --- a/crates/infra/utils/src/github/mod.rs +++ b/crates/infra/utils/src/github/mod.rs @@ -15,6 +15,10 @@ impl GitHub { var("CI").is_ok() } + pub fn is_running_in_devcontainer() -> bool { + var("IS_INSIDE_SLANG_DEVCONTAINER").is_ok() + } + /// Collapses the output of the given operation in the GitHub log viewer. /// They can be expanded individually when needed. /// This has no effect when running locally.