Pipeline setup for aarch64-linux

.expeditor/release_habitat.pipeline.yml

@@ -38,6 +38,15 @@ steps:
           environment:
             - BUILD_PKG_TARGET=x86_64-linux
 
+  - label: "[:linux: build aarch64 hab]"
+    agents:
+      queue: default-privileged-aarch64
+    env:
+      HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux"
+      BUILD_PKG_TARGET: "aarch64-linux"
+    command:
+      - sudo -E .expeditor/scripts/release_habitat/build_component.sh hab
+
   - label: "[:linux: :two: build hab]"
     command:
       - .expeditor/scripts/release_habitat/build_component.sh hab
@@ -111,6 +120,15 @@ steps:
           environment:
             - BUILD_PKG_TARGET=x86_64-linux
 
+  - label: "[:linux: build aarch64 hab-plan-build]"
+    agents:
+      queue: default-privileged-aarch64
+    env:
+      HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux"
+      BUILD_PKG_TARGET: "aarch64-linux"
+    command:
+      - sudo -E .expeditor/scripts/release_habitat/build_component.sh plan-build
+
   - label: "[:linux: :two: build hab-plan-build]"
     command:
       - .expeditor/scripts/release_habitat/build_component.sh plan-build
@@ -146,6 +164,15 @@ steps:
           environment:
             - BUILD_PKG_TARGET=x86_64-linux
 
+  - label: "[:linux: build aarch64 hab-backline]"
+    agents:
+      queue: default-privileged-aarch64
+    env:
+      HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux"
+      BUILD_PKG_TARGET: "aarch64-linux"
+    command:
+      - sudo -E .expeditor/scripts/release_habitat/build_component.sh backline
+
   - label: "[:linux: :two: build hab-backline]"
     command:
       - .expeditor/scripts/release_habitat/build_component.sh backline
@@ -170,6 +197,15 @@ steps:
           environment:
             - BUILD_PKG_TARGET=x86_64-linux
 
+  - label: "[:linux: build aarch64 hab-studio]"
+    agents:
+      queue: default-privileged-aarch64
+    env:
+      HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux"
+      BUILD_PKG_TARGET: "aarch64-linux"
+    command:
+      - sudo -E .expeditor/scripts/release_habitat/build_component.sh studio
+
   - label: "[:linux: :two: build hab-studio]"
     command:
       - .expeditor/scripts/release_habitat/build_component.sh studio
@@ -208,6 +244,15 @@ steps:
           environment:
             - BUILD_PKG_TARGET=x86_64-linux
 
+  - label: "[:linux: build aarch64 launcher]"
+    agents:
+      queue: default-privileged-aarch64
+    env:
+      HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux"
+      BUILD_PKG_TARGET: "aarch64-linux"
+    command:
+      - sudo -E .expeditor/scripts/release_habitat/build_component.sh launcher
+
   - label: "[:linux: :two: build launcher]"
     command:
       - .expeditor/scripts/release_habitat/build_component.sh launcher
@@ -241,6 +286,15 @@ steps:
           environment:
             - BUILD_PKG_TARGET=x86_64-linux
 
+  - label: "[:linux: build aarch64 hab-sup]"
+    agents:
+      queue: default-privileged-aarch64
+    env:
+      HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux"
+      BUILD_PKG_TARGET: "aarch64-linux"
+    command:
+      - sudo -E .expeditor/scripts/release_habitat/build_component.sh sup
+
   - label: "[:linux: :two: build hab-sup]"
     command:
       - .expeditor/scripts/release_habitat/build_component.sh sup
@@ -274,6 +328,15 @@ steps:
           environment:
             - BUILD_PKG_TARGET=x86_64-linux
 
+  - label: "[:linux: build aarch64 hab-pkg-export-container]"
+    agents:
+      queue: default-privileged-aarch64
+    env:
+      HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux"
+      BUILD_PKG_TARGET: "aarch64-linux"
+    command:
+      - sudo -E .expeditor/scripts/release_habitat/build_component.sh pkg-export-container
+
   - label: "[:windows: build hab-pkg-export-container]"
     command:
       - powershell .expeditor/scripts/release_habitat/build_component.ps1 pkg-export-container
@@ -295,6 +358,15 @@ steps:
           environment:
             - BUILD_PKG_TARGET=x86_64-linux
 
+  - label: "[:linux: build aarch64 hab-pkg-export-tar]"
+    agents:
+      queue: default-privileged-aarch64
+    env:
+      HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux"
+      BUILD_PKG_TARGET: "aarch64-linux"
+    command:
+      - sudo -E .expeditor/scripts/release_habitat/build_component.sh pkg-export-tar
+
   - label: "[:linux: :two: build hab-pkg-export-tar]"
     command:
       - .expeditor/scripts/release_habitat/build_component.sh pkg-export-tar
@@ -352,6 +424,16 @@ steps:
           environment:
             - BUILD_PKG_TARGET=x86_64-linux
 
+  - label: "[:linux: upload hab aarch64 binary]"
+    agents:
+      queue: default-privileged-aarch64
+    env:
+      HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux"
+      BUILD_PKG_TARGET: "aarch64-linux"
+    command:
+      - sudo -E .expeditor/scripts/release_habitat/package_and_upload_binary.sh
+    if: build.creator.name == 'Chef Expeditor' || build.env("UPLOAD_AND_PROMOTE") == 'true'
+
   - label: "[:linux: :two: upload hab binary]"
     command:
       - .expeditor/scripts/release_habitat/package_and_upload_binary.sh

.expeditor/scripts/release_habitat/package_and_upload_binary.sh

@@ -14,7 +14,7 @@ channel=$(get_release_channel)
 echo "--- Channel: $channel - bldr url: $HAB_BLDR_URL"
 
 declare -g hab_binary
-install_release_channel_hab_binary "x86_64-linux"
+install_release_channel_hab_binary "$BUILD_PKG_TARGET"
 
 import_gpg_keys
 

.expeditor/scripts/shared.sh

@@ -20,9 +20,10 @@ curlbash_hab() {
       sudo rm -rf /hab/pkgs/core/hab/0.82.0 || \
            rm -rf /hab/pkgs/core/hab/0.82.0
     fi
+
     curl https://raw.githubusercontent.com/habitat-sh/habitat/main/components/hab/install.sh | sudo bash -s -- -t "$pkg_target"
     case "${pkg_target}" in
-        x86_64-linux | x86_64-linux-kernel2)
+        x86_64-linux | aarch64-linux | x86_64-linux-kernel2)
             hab_binary="/bin/hab"
             ;;
         x86_64-darwin)

.expeditor/scripts/verify/build_package-aarch64.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -eou pipefail
+
+package_path=${1?package_path argument required}
+
+# Install hab from a temporarily uploaded aarch64 package
+curl https://raw.githubusercontent.com/habitat-sh/habitat/main/components/hab/install.sh | sudo bash -s -- -t "$BUILD_PKG_TARGET" -c "$HAB_FALLBACK_CHANNEL" -v 1.6.1178
+
+# Since we are only verifying we don't have build failures, make everything
+# temp!
+export HAB_ORIGIN
+HAB_ORIGIN=core
+# let's make a selfcontained tempdir for this job
+export JOB_TEMP_ROOT
+JOB_TEMP_ROOT=$(mktemp -d /tmp/job-root-XXXXXX)
+export HAB_CACHE_KEY_PATH
+HAB_CACHE_KEY_PATH="$JOB_TEMP_ROOT/keys"
+
+export HAB_FALLBACK_CHANNEL
+export HAB_PREFER_LOCAL_CHEF_DEPS="true"
+
+echo "--- :key: Generating temporary origin key"
+hab origin key generate "$HAB_ORIGIN"
+echo "--- :hab: Running hab pkg build for $package_path"
+# Install the temporarily built hab-studio.
+# Once hab is released in the LTS channel, this step may no longer be required.
+hab pkg install core/hab-studio -c "$HAB_FALLBACK_CHANNEL"
+hab pkg build "$package_path"

.expeditor/scripts/verify/run_cargo_test-aarch64.sh

@@ -0,0 +1,78 @@
+#!/bin/bash
+
+set -eou pipefail
+
+# Install hab from a temporarily uploaded aarch64 package
+curl https://raw.githubusercontent.com/habitat-sh/habitat/main/components/hab/install.sh | sudo bash -s -- -t aarch64-linux -c lts24-aarch64-linux -v 1.6.1178
+
+# shellcheck source=.expeditor/scripts/shared.sh
+source .expeditor/scripts/verify/shared.sh
+
+if [[ ${1:-"--"} = "--" ]]; then
+  scope="habitat workspace"
+else
+  component="$1"
+  shift
+  scope="$component"
+fi
+
+toolchain=$(get_toolchain)
+
+sudo -E hab pkg install core/zeromq
+sudo -E hab pkg install core/protobuf
+sudo -E hab pkg install core/rust/"$toolchain"
+sudo -E hab pkg install core/xz
+sudo -E hab pkg install core/coreutils
+sudo -E hab pkg install core/openssl
+sudo -E hab pkg install core/perl
+sudo -E hab pkg install core/make
+
+export OPENSSL_DIR
+OPENSSL_DIR="$(hab pkg path core/openssl)/bin"
+export OPENSSL_LIB_DIR
+OPENSSL_LIB_DIR="$(hab pkg path core/openssl)/lib"
+
+export LIBZMQ_PREFIX
+LIBZMQ_PREFIX=$(hab pkg path core/zeromq)
+# now include zeromq and gcc so they exist in the runtime library path when cargo test is run
+export LD_LIBRARY_PATH
+LD_LIBRARY_PATH="$(hab pkg path core/gcc-base)/lib:$(hab pkg path core/zeromq)/lib:$(hab pkg path core/xz)/lib:$(hab pkg path core/openssl)/lib"
+
+export LIBRARY_PATH
+LIBRARY_PATH="$(hab pkg path core/xz)/lib"
+
+export PROTOC_NO_VENDOR=1
+export PROTOC
+PROTOC=$(hab pkg path core/protobuf)/bin/protoc
+
+_oldPth=$PATH
+_pth="$(hab pkg path core/coreutils)/bin:$(hab pkg path core/openssl)/bin:$(hab pkg path core/perl)/bin:$(hab pkg path core/make)/bin"
+eval "$(hab pkg env core/rust/"$toolchain"):$PATH"
+export PATH="$PATH:$_pth:$_oldPth"
+
+# Set testing filesystem root
+export FS_ROOT
+FS_ROOT=$(mktemp -d /tmp/testing-fs-root-XXXXXX)
+
+export RUST_BACKTRACE=1
+
+# Build the all the hab binaries so that we can run integration tests
+if [[ "$scope" == "sup" ]]; then
+  cargo build
+fi
+
+echo "--- Running cargo test with scope '$scope' and args '$*'"
+
+if [[ -n ${component:-} ]]; then
+  cd "components/$component"
+fi
+
+# We do not have any packages in the stable channel for aarch64 and probably never will. 
+# Set the HAB_INTERPRETER_IDENT to point to LTS-2024 to proceed with the tests.
+export HAB_INTERPRETER_IDENT="core/busybox-static/1.36.1/20240805133911"
+
+# Always add `--quiet` to avoid the noise of compilation in test output.
+# The invocation to this script can add `--format pretty` to the test runner
+# args (that is, after --, like --nocapture and --test-threads) if the names
+# of the tests being run is desired in the output.
+cargo test --quiet "$@"