Skip to content
CI

Merge branch 'main' into feature/oqc_client_upgrade #9898

Sign in to view logs

Merge branch 'main' into feature/oqc_client_upgrade

Merge branch 'main' into feature/oqc_client_upgrade #9898

Workflow file for this run

.github/workflows/ci.yml at 2cfef34
on:
workflow_dispatch:
inputs:
cache_base:
required: false
type: string
description: 'The name of the branch to use as cache base.'
default: main
export_environment:
type: boolean
description: Export the build environment as tar artifact that can be imported with Docker.
# The GitHub application copy-pr-bot copies the source code for every pull request
# into the repository. Approving such upstream pushes effectively marks code as trusted,
# and is necessary to use the self-hosted NVIDIA runners.
push:
branches:
- "pull-request/[0-9]+"
merge_group:
types:
- checks_requested
name: CI # do not change name without updating workflow_run triggers
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
metadata:
name: Retrieve PR info
runs-on: ubuntu-latest
permissions:
pull-requests: read
contents: read
outputs:
pull_request_number: ${{ steps.pr_info.outputs.pr_number }}
pull_request_base: ${{ steps.pr_info.outputs.pr_base }}
cache_base: ${{ steps.pr_info.outputs.pr_base }}
llvm_commit: ${{ steps.repo_info.outputs.llvm_commit }}
pybind11_commit: ${{ steps.repo_info.outputs.pybind11_commit }}
platform_config: ${{ steps.config.outputs.platforms }}
steps:
- name: Checkout repository
uses: actions/checkout@v4
- id: config
run: |
platforms="{}"
for platform_id in amd64 arm64; do
if [ "$platform_id" == "amd64" ]; then minimal_base_image=ghcr.io/nvidia/amd64/almalinux:8
elif [ "$platform_id" == "arm64" ]; then minimal_base_image=ghcr.io/nvidia/arm64v8/almalinux:8
fi
platform={\"$platform_id\":{\"minimal_base_image\":\"$minimal_base_image\"}}
platforms=`echo $platforms | jq ". |= . + $platform"`
done
echo "platforms=$(echo $platforms)" >> $GITHUB_OUTPUT
- id: pr_info
run: |
pr_number=`echo ${{ github.ref_name }} | grep pull-request/ | (grep -o [0-9]* || true)`
pr_number=${pr_number:-${{ github.event.pull_request.number }}}
if [ -n "$pr_number" ]; then
pr_base=`gh pr view $pr_number -R ${{ github.repository }} --json baseRefName --jq .baseRefName`
echo "pr_number=$pr_number" >> $GITHUB_OUTPUT
echo "pr_base=$pr_base" >> $GITHUB_OUTPUT
fi
env:
GH_TOKEN: ${{ github.token }}
- id: repo_info
run: |
echo "llvm_commit=$(git rev-parse @:./tpls/llvm)" >> $GITHUB_OUTPUT
echo "pybind11_commit=$(git rev-parse @:./tpls/pybind11)" >> $GITHUB_OUTPUT
devdeps:
name: Load dependencies
needs: metadata
strategy:
matrix:
platform: [amd64, arm64]
toolchain: [clang16, gcc11, gcc12]
fail-fast: false
uses: ./.github/workflows/dev_environment.yml
secrets:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_READONLY_TOKEN: ${{ secrets.DOCKERHUB_READONLY_TOKEN }}
with:
platforms: linux/${{ matrix.platform }}
dockerfile: build/devdeps.Dockerfile
build_config_id: ${{ matrix.toolchain }}
build_args: |
toolchain=${{ matrix.toolchain }}
registry_cache_from: ${{ inputs.cache_base || needs.metadata.outputs.cache_base }}
checkout_submodules: true
# needed only for the cloudposse GitHub action
matrix_key: ${{ matrix.platform }}-${{ matrix.toolchain }}
wheeldeps:
name: Load wheel dependencies
needs: metadata
strategy:
matrix:
platform: [amd64, arm64]
cuda_version: ["11.8", "12.0"]
fail-fast: false
uses: ./.github/workflows/dev_environment.yml
secrets:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_READONLY_TOKEN: ${{ secrets.DOCKERHUB_READONLY_TOKEN }}
with:
platforms: linux/${{ matrix.platform }}
dockerfile: build/devdeps.manylinux.Dockerfile
build_config_id: cu${{ matrix.cuda_version }}-gcc11
build_args: |
base_image=ghcr.io/nvidia/pypa/manylinux_2_28${{ (matrix.platform == 'arm64' && '_aarch64') || (matrix.platform == 'amd64' && '_x86_64') || '' }}:latest
cuda_version=${{ matrix.cuda_version }}
toolchain=gcc11
distro=rhel8
llvm_commit=${{ needs.metadata.outputs.llvm_commit }}
pybind11_commit=${{ needs.metadata.outputs.pybind11_commit }}
registry_cache_from: ${{ inputs.cache_base || needs.metadata.outputs.cache_base }}
# needed only for the cloudposse GitHub action
matrix_key: ${{ matrix.platform }}-cu${{ matrix.cuda_version }}-python
source_build:
name: Load source build cache
needs: metadata
strategy:
matrix:
platform: [amd64, arm64]
cuda_version: ["11.8", "12.0"]
fail-fast: false
uses: ./.github/workflows/dev_environment.yml
secrets:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_READONLY_TOKEN: ${{ secrets.DOCKERHUB_READONLY_TOKEN }}
with:
platforms: linux/${{ matrix.platform }}
dockerfile: build/assets.Dockerfile
build_config_id: cu${{ matrix.cuda_version }}-llvm
build_target: prereqs
build_args: |
base_image=${{ fromJson(needs.metadata.outputs.platform_config)[format('{0}', matrix.platform)].minimal_base_image }}
cuda_version=${{ matrix.cuda_version }}
toolchain=llvm
registry_cache_from: ${{ inputs.cache_base || needs.metadata.outputs.cache_base }}
checkout_submodules: true
# needed only for the cloudposse GitHub action
matrix_key: ${{ matrix.platform }}-cu${{ matrix.cuda_version }}-installer
# This job is needed only when using the cloudposse GitHub action to read
# the output of a matrix job. This is a workaround due to current GitHub
# limitations that may not be needed if the work started here concludes:
# https://github.com/actions/runner/pull/2477
config:
name: Configure build
runs-on: ubuntu-latest
needs: [devdeps, wheeldeps, source_build]
outputs:
json: "${{ steps.read_json.outputs.result }}"
steps:
- uses: cloudposse/[email protected]
id: read_json
with:
matrix-step-name: dev_environment
build_and_test:
name: Build and test
needs: config
strategy:
matrix:
platform: [amd64, arm64]
toolchain: [clang16, gcc11, gcc12]
mpi: [openmpi, mpich]
exclude:
- toolchain: llvm
mpi: mpich
- toolchain: clang16
mpi: mpich
fail-fast: false
uses: ./.github/workflows/test_in_devenv.yml
with:
platform: linux/${{ matrix.platform }}
mpi: ${{ matrix.mpi }}
devdeps_image: ${{ fromJson(needs.config.outputs.json).image_hash[format('{0}-{1}', matrix.platform, matrix.toolchain)] }}
devdeps_cache: ${{ fromJson(needs.config.outputs.json).cache_key[format('{0}-{1}', matrix.platform, matrix.toolchain)] }}
devdeps_archive: ${{ fromJson(needs.config.outputs.json).tar_archive[format('{0}-{1}', matrix.platform, matrix.toolchain)] }}
export_environment: ${{ github.event_name == 'workflow_dispatch' && inputs.export_environment }}
gen_code_coverage:
name: Gen code coverage
needs: config
strategy:
matrix:
platform: [amd64]
toolchain: [clang16]
fail-fast: false
uses: ./.github/workflows/generate_cc.yml
secrets:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
platform: linux/${{ matrix.platform }}
devdeps_image: ${{ fromJson(needs.config.outputs.json).image_hash[format('{0}-{1}', matrix.platform, matrix.toolchain)] }}
devdeps_cache: ${{ fromJson(needs.config.outputs.json).cache_key[format('{0}-{1}', matrix.platform, matrix.toolchain)] }}
devdeps_archive: ${{ fromJson(needs.config.outputs.json).tar_archive[format('{0}-{1}', matrix.platform, matrix.toolchain)] }}
export_environment: ${{ github.event_name == 'workflow_dispatch' && inputs.export_environment }}
docker_image:
name: Create Docker images
needs: config
strategy:
matrix:
platform: [amd64, arm64]
fail-fast: false
uses: ./.github/workflows/docker_images.yml
secrets:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_READONLY_TOKEN: ${{ secrets.DOCKERHUB_READONLY_TOKEN }}
with:
platforms: linux/${{ matrix.platform }}
devdeps_image: ${{ fromJson(needs.config.outputs.json).image_hash[format('{0}-gcc11', matrix.platform)] }}
devdeps_cache: ${{ fromJson(needs.config.outputs.json).cache_key[format('{0}-gcc11', matrix.platform)] }}
devdeps_archive: ${{ fromJson(needs.config.outputs.json).tar_archive[format('{0}-gcc11', matrix.platform)] }}
python_wheels:
name: Create Python wheels
needs: config
strategy:
matrix:
platform: [amd64, arm64]
python_version: ['3.10', '3.12']
cuda_version: ["11.8", "12.0"]
fail-fast: false
uses: ./.github/workflows/python_wheels.yml
secrets:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_READONLY_TOKEN: ${{ secrets.DOCKERHUB_READONLY_TOKEN }}
with:
platform: linux/${{ matrix.platform }}
python_version: ${{ matrix.python_version }}
cuda_version: ${{ matrix.cuda_version }}
devdeps_image: ${{ fromJson(needs.config.outputs.json).image_hash[format('{0}-cu{1}-python', matrix.platform, matrix.cuda_version)] }}
devdeps_cache: ${{ fromJson(needs.config.outputs.json).cache_key[format('{0}-cu{1}-python', matrix.platform, matrix.cuda_version)] }}
devdeps_archive: ${{ fromJson(needs.config.outputs.json).tar_archive[format('{0}-cu{1}-python', matrix.platform, matrix.cuda_version)] }}
python_metapackages:
name: Create Python metapackages
needs: python_wheels
uses: ./.github/workflows/python_metapackages.yml
with:
cudaq_version: ${{ needs.python_wheels.outputs.cudaq_version }}
python_versions: "['3.10', '3.12']"
cuda_versions: "['', '11.8', '12.0']"
wheel_artifacts: 'pycudaq-*'
binaries:
name: Create CUDA Quantum installer
needs: [metadata, config]
strategy:
matrix:
platform: [amd64, arm64]
cuda_version: ["11.8", "12.0"]
fail-fast: false
uses: ./.github/workflows/prebuilt_binaries.yml
secrets:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_READONLY_TOKEN: ${{ secrets.DOCKERHUB_READONLY_TOKEN }}
with:
platform: linux/${{ matrix.platform }}
platform_base_image: ${{ fromJson(needs.metadata.outputs.platform_config)[format('{0}', matrix.platform)].minimal_base_image }}
build_config_id: cu${{ matrix.cuda_version }}-llvm
cuda_version: ${{ matrix.cuda_version }}
build_cache: ${{ fromJson(needs.config.outputs.json).build_cache[format('{0}-cu{1}-installer', matrix.platform, matrix.cuda_version)] }}
clean_up:
name: Prepare cache clean-up
runs-on: ubuntu-latest
needs: [metadata, config, build_and_test, docker_image, wheeldeps, python_wheels]
# We need to clean up even if the workflow is cancelled or fails.
if: always()
steps:
- name: Save cache keys and metadata
id: workflow_inputs
run: |
set -e
key_matrix='${{ needs.config.outputs.json }}'
keys=`echo $key_matrix | jq '.cache_key | to_entries | .[].value' --raw-output`
echo "$keys" >> cache_keys.txt
echo "pr-number: ${{ needs.metadata.outputs.pull_request_number }}" >> metadata.txt
echo "pr-base: ${{ needs.metadata.outputs.pull_request_base }}" >> metadata.txt
- name: Upload cache keys
uses: actions/upload-artifact@v4
with:
name: cache_keys_ci
path: cache_keys.txt
retention-days: 1
if-no-files-found: error
- name: Upload metadata
uses: actions/upload-artifact@v4
with:
name: metadata_ci
path: metadata.txt
retention-days: 1
if-no-files-found: error