Restructure codebase #239
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Copyright (C) 2024 Roberto Rossini <[email protected]> | |
# | |
# SPDX-License-Identifier: GPL-3.0 | |
# | |
# This library is free software: you can redistribute it and/or | |
# modify it under the terms of the GNU Public License as published | |
# by the Free Software Foundation; either version 3 of the License, | |
# or (at your option) any later version. | |
# | |
# This library is distributed in the hope that it will be useful, | |
# but WITHOUT ANY WARRANTY; without even the implied warranty of | |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
# Library General Public License for more details. | |
# | |
# You should have received a copy of the GNU Public License along | |
# with this library. If not, see | |
# <https://www.gnu.org/licenses/>. | |
name: MacOS CI | |
on: | |
push: | |
branches: [main] | |
paths: | |
- ".github/workflows/build-conan-deps.yml" | |
- ".github/workflows/cache-test-dataset.yml" | |
- ".github/workflows/macos-ci.yml" | |
- "cmake/**" | |
- "src/**" | |
- "test/**" | |
- "CMakeLists.txt" | |
- "conanfile.py" | |
pull_request: | |
paths: | |
- ".github/workflows/build-conan-deps.yml" | |
- ".github/workflows/cache-test-dataset.yml" | |
- ".github/workflows/macos-ci.yml" | |
- "cmake/**" | |
- "src/**" | |
- "test/**" | |
- "CMakeLists.txt" | |
- "conanfile.py" | |
# https://stackoverflow.com/a/72408109 | |
concurrency: | |
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
cancel-in-progress: true | |
env: | |
CCACHE_DIR: "${{ github.workspace }}/ccache-cache" | |
CCACHE_COMPILERCHECK: "content" | |
CCACHE_COMPRESSLEVEL: "1" | |
CCACHE_MAXSIZE: "250M" | |
CONAN_HOME: "${{ github.workspace }}/.conan2" | |
HOMEBREW_NO_AUTO_UPDATE: "1" | |
defaults: | |
run: | |
shell: bash | |
jobs: | |
matrix-factory: | |
name: Generate job matrix | |
runs-on: ubuntu-latest | |
outputs: | |
matrix: ${{ steps.set-result.outputs.result }} | |
steps: | |
- uses: actions/github-script@v7 | |
id: set-result | |
with: | |
script: | | |
// Documentation | |
// https://docs.github.com/en/actions/learn-github-actions/contexts#fromjson | |
// https://github.com/actions/runner/issues/982#issuecomment-809360765 | |
var includes = [] | |
includes.push({ compiler_name: 'apple-clang', compiler_version: '15', os: 'macos-13', cmake: '3.30.*', build_type: 'Release', developer_mode: 'OFF' }) | |
includes.push({ compiler_name: 'apple-clang', compiler_version: '15', os: 'macos-14', cmake: '3.30.*', build_type: 'Release', developer_mode: 'OFF' }) | |
includes.push({ compiler_name: 'apple-clang', compiler_version: '16', os: 'macos-15', cmake: '3.30.*', build_type: 'Release', developer_mode: 'OFF' }) | |
includes.push({ compiler_name: 'apple-clang', compiler_version: '15', os: 'macos-13', cmake: '3.30.*', build_type: 'Debug', developer_mode: 'OFF' }) | |
includes.push({ compiler_name: 'apple-clang', compiler_version: '15', os: 'macos-14', cmake: '3.30.*', build_type: 'Debug', developer_mode: 'OFF' }) | |
includes.push({ compiler_name: 'apple-clang', compiler_version: '16', os: 'macos-15', cmake: '3.30.*', build_type: 'Debug', developer_mode: 'OFF' }) | |
return { include: includes } | |
cache-test-dataset: | |
name: Cache test dataset | |
uses: paulsengroup/nchg/.github/workflows/cache-test-dataset.yml@main | |
build-conan-deps-x86: | |
name: Build Conan deps | |
uses: paulsengroup/nchg/.github/workflows/build-conan-deps.yml@main | |
with: | |
os: macos-13 | |
build-conan-deps-arm64: | |
name: Build Conan deps | |
uses: paulsengroup/nchg/.github/workflows/build-conan-deps.yml@main | |
with: | |
os: macos-14 | |
build-project: | |
name: Build project | |
needs: | |
- matrix-factory | |
- build-conan-deps-x86 | |
- build-conan-deps-arm64 | |
runs-on: ${{ matrix.os }} | |
strategy: | |
fail-fast: false | |
matrix: ${{ fromJSON(needs.matrix-factory.outputs.matrix) }} | |
env: | |
CCACHE_DIR: "${{ github.workspace }}/ccache-cache" | |
CCACHE_COMPILERCHECK: "content" | |
CCACHE_COMPRESSLEVEL: "1" | |
CCACHE_MAXSIZE: "250M" | |
CONAN_HOME: "${{ github.workspace }}/.conan2" | |
HOMEBREW_NO_AUTO_UPDATE: "1" | |
steps: | |
- uses: actions/checkout@v4 | |
- name: Generate requirements.txt for pip | |
run: echo 'cmake==${{ matrix.cmake }}' > requirements.txt | |
- uses: actions/setup-python@v5 | |
with: | |
python-version: "3.12" | |
cache: "pip" | |
- name: Detect number available CPUs | |
run: | | |
ncpus=$(python -c 'import multiprocessing as mp; print(mp.cpu_count())') | |
echo "NPROC=$ncpus" >> $GITHUB_ENV | |
echo "CMAKE_BUILD_PARALLEL_LEVEL=$ncpus" >> $GITHUB_ENV | |
echo "CTEST_PARALLEL_LEVEL=$ncpus" >> $GITHUB_ENV | |
- name: Install build deps | |
run: | | |
pip install -r requirements.txt | |
brew install ccache | |
- name: Generate cache key | |
id: cache-key | |
run: | | |
set -u | |
os="${{ matrix.os }}" | |
compiler="${{ matrix.compiler_name }}" | |
compiler_version="${{ matrix.compiler_version }}" | |
build_type="${{ matrix.build_type }}" | |
conanfile_hash="${{ hashFiles('conanfile.py') }}" | |
# This can be used by to always update a cache entry (useful e.g. for ccache) | |
current_date="$(date '+%s')" | |
ccache_key_prefix="ccache-$os-$compiler-$compiler_version-$conanfile_hash-$build_type" | |
echo "ccache-key=${ccache_key_prefix}-$GITHUB_REF-${current_date}" | tee -a $GITHUB_OUTPUT | |
echo "ccache-restore-key-1=$ccache_key_prefix-$GITHUB_REF" | tee -a $GITHUB_OUTPUT | |
echo "ccache-restore-key-2=$ccache_key_prefix" | tee -a $GITHUB_OUTPUT | |
- name: Restore Conan cache (x86) | |
if: matrix.os == 'macos-13' | |
uses: actions/cache/restore@v4 | |
with: | |
key: ${{ needs.build-conan-deps-x86.outputs.conan-key }} | |
path: ${{ env.CONAN_HOME }}/p | |
fail-on-cache-miss: true | |
- name: Restore Conan cache (arm64) | |
if: matrix.os == 'macos-14' || matrix.os == 'macos-15' | |
uses: actions/cache/restore@v4 | |
with: | |
key: ${{ needs.build-conan-deps-arm64.outputs.conan-key }} | |
path: ${{ env.CONAN_HOME }}/p | |
fail-on-cache-miss: true | |
- name: Restore CMake configs (x86; Debug) | |
if: matrix.build_type == 'Debug' && matrix.os == 'macos-13' | |
uses: actions/cache/restore@v4 | |
with: | |
key: ${{ needs.build-conan-deps-x86.outputs.cmake-prefix-debug-key }} | |
path: /tmp/cmake-prefix-dbg.tar | |
fail-on-cache-miss: true | |
- name: Restore CMake configs (x86; Release) | |
if: matrix.build_type == 'Release' && matrix.os == 'macos-13' | |
uses: actions/cache/restore@v4 | |
with: | |
key: ${{ needs.build-conan-deps-x86.outputs.cmake-prefix-release-key }} | |
path: /tmp/cmake-prefix-rel.tar | |
fail-on-cache-miss: true | |
- name: Restore CMake configs (arm64; Debug) | |
if: matrix.build_type == 'Debug' && (matrix.os == 'macos-14' || matrix.os == 'macos-15') | |
uses: actions/cache/restore@v4 | |
with: | |
key: ${{ needs.build-conan-deps-arm64.outputs.cmake-prefix-debug-key }} | |
path: /tmp/cmake-prefix-dbg.tar | |
fail-on-cache-miss: true | |
- name: Restore CMake configs (arm64; Release) | |
if: matrix.build_type == 'Release' && (matrix.os == 'macos-14' || matrix.os == 'macos-15') | |
uses: actions/cache/restore@v4 | |
with: | |
key: ${{ needs.build-conan-deps-arm64.outputs.cmake-prefix-release-key }} | |
path: /tmp/cmake-prefix-rel.tar | |
fail-on-cache-miss: true | |
- name: Extract CMake configs | |
run: | | |
mkdir conan-env | |
tar -xf /tmp/cmake-prefix-*.tar -C conan-env/ --strip-components=1 | |
- name: Configure project | |
run: | | |
cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ | |
-DCMAKE_PREFIX_PATH="$PWD/conan-env/" \ | |
-DENABLE_DEVELOPER_MODE=${{ matrix.developer_mode }} \ | |
-DNCHG_ENABLE_TESTING=ON \ | |
-DNCHG_DOWNLOAD_TEST_DATASET=OFF \ | |
-DOPT_ENABLE_CLANG_TIDY=OFF \ | |
-DOPT_ENABLE_CPPCHECK=OFF \ | |
-DCMAKE_INSTALL_PREFIX=dest \ | |
-S "${{ github.workspace }}" \ | |
-B "${{ github.workspace }}/build" | |
- name: Restore Ccache folder | |
id: cache-ccache | |
uses: actions/cache/restore@v4 | |
with: | |
key: ${{ steps.cache-key.outputs.ccache-restore-key-1 }} | |
restore-keys: ${{ steps.cache-key.outputs.ccache-restore-key-2 }} | |
path: ${{ env.CCACHE_DIR }} | |
- name: Reset Ccache stats | |
run: ccache --zero-stats | |
- name: Build project | |
run: cmake --build ${{ github.workspace }}/build | |
- name: Package binaries | |
run: | | |
cmake --install build | |
gtar -cf - -C dest/ bin | zstd -T0 -13 -o binaries.tar.zst | |
- name: Package unit tests | |
run: | | |
rm -r build/src | |
gtar --exclude='*.o' -cf - build/ | zstd -T0 -13 -o unit-tests.tar.zst | |
- name: Upload unit tests | |
uses: actions/upload-artifact@v4 | |
with: | |
name: "unit-tests-${{ matrix.os }}-\ | |
${{ matrix.compiler_name }}-\ | |
${{ matrix.compiler_version }}-\ | |
${{ matrix.build_type }}-\ | |
${{ matrix.developer_mode }}" | |
path: unit-tests.tar.zst | |
if-no-files-found: error | |
retention-days: 1 | |
- name: Upload binaries | |
uses: actions/upload-artifact@v4 | |
with: | |
name: "binaries-${{ matrix.os }}-\ | |
${{ matrix.compiler_name }}-\ | |
${{ matrix.compiler_version }}-\ | |
${{ matrix.build_type }}-\ | |
${{ matrix.developer_mode }}" | |
path: binaries.tar.zst | |
if-no-files-found: error | |
retention-days: 1 | |
- name: Print Ccache statistics (pre-cleanup) | |
run: | | |
ccache --show-stats \ | |
--show-compression \ | |
--verbose | |
- name: Cleanup Ccache folder | |
run: | | |
ccache --evict-older-than=14400s # 4h | |
ccache --recompress=19 --recompress-threads="$NPROC" | |
ccache --cleanup | |
- name: Print Ccache statistics (post-cleanup) | |
run: | | |
ccache --show-stats \ | |
--show-compression \ | |
--verbose | |
- name: Save Ccache folder | |
uses: actions/cache/save@v4 | |
with: | |
key: ${{ steps.cache-key.outputs.ccache-key }} | |
path: ${{ env.CCACHE_DIR }} | |
env: | |
ZSTD_CLEVEL: 1 | |
- name: Generate list of stale cache entries | |
id: stale-cache | |
if: steps.cache-ccache.outputs.cache-matched-key != '' | |
run: | | |
fname='stale-cache-${{ matrix.os }}-${{ matrix.compiler_name }}-${{ matrix.compiler_version }}-${{ matrix.build_type }}-${{ matrix.developer_mode }}.txt' | |
echo '${{ steps.cache-ccache.outputs.cache-matched-key }}' > "$fname" | |
echo "name=$fname" | tee -a "$GITHUB_OUTPUT" | |
- name: Upload stale cache entries | |
if: steps.cache-ccache.outputs.cache-matched-key != '' | |
uses: actions/upload-artifact@v4 | |
with: | |
name: ${{ steps.stale-cache.outputs.name }} | |
path: "${{ steps.stale-cache.outputs.name }}" | |
if-no-files-found: error | |
retention-days: 1 | |
clean-stale-cache: | |
needs: [build-project] | |
name: Evict stale cache entries | |
runs-on: ubuntu-latest | |
permissions: | |
actions: write | |
steps: | |
- name: Download artifacts | |
uses: actions/download-artifact@v4 | |
continue-on-error: true | |
with: | |
pattern: stale-cache-* | |
merge-multiple: true | |
- name: Evict cache entries | |
continue-on-error: true | |
run: | | |
set -x | |
while read entry; do | |
if ! grep -q '/heads/main' <(echo "$entry"); then | |
gh cache delete --repo '${{ github.repository }}' "$entry" | |
fi | |
done < <(cat stale-cache*.txt | grep -v '^$') | |
env: | |
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
run-unit-tests: | |
name: Run unit tests | |
needs: [matrix-factory, cache-test-dataset, build-project] | |
runs-on: ${{ matrix.os }} | |
strategy: | |
fail-fast: false | |
matrix: ${{ fromJSON(needs.matrix-factory.outputs.matrix) }} | |
steps: | |
- name: Checkout repo | |
uses: actions/checkout@v4 | |
- name: Restore test dataset | |
uses: actions/cache/restore@v4 | |
with: | |
key: ${{ needs.cache-test-dataset.outputs.cache-key }} | |
path: test/data/nchg_test_data.tar.zst | |
fail-on-cache-miss: true | |
- name: Download unit tests artifact | |
uses: actions/download-artifact@v4 | |
with: | |
name: "unit-tests-${{ matrix.os }}-\ | |
${{ matrix.compiler_name }}-\ | |
${{ matrix.compiler_version }}-\ | |
${{ matrix.build_type }}-\ | |
${{ matrix.developer_mode }}" | |
- name: Extract binaries test dataset | |
run: | | |
gtar -xf unit-tests.tar.zst | |
gtar -xf test/data/nchg_test_data.tar.zst | |
- name: Generate requirements.txt for pip | |
run: | | |
echo 'cmake==${{ matrix.cmake }}' > requirements.txt | |
- uses: actions/setup-python@v5 | |
with: | |
python-version: "3.12" | |
cache: "pip" | |
- name: Install test dependencies | |
run: pip install -r requirements.txt | |
- name: Run unit tests | |
run: | | |
ctest --test-dir build/ \ | |
--schedule-random \ | |
--output-on-failure \ | |
--no-tests=error \ | |
--timeout 120 2>&1 | | |
head -n 1000 | |
run-integration-tests: | |
name: Run integration tests | |
needs: [matrix-factory, cache-test-dataset, build-project] | |
runs-on: ${{ matrix.os }} | |
strategy: | |
fail-fast: false | |
matrix: ${{ fromJSON(needs.matrix-factory.outputs.matrix) }} | |
steps: | |
- name: Checkout repo | |
uses: actions/checkout@v4 | |
- name: Restore test dataset | |
uses: actions/cache/restore@v4 | |
with: | |
key: ${{ needs.cache-test-dataset.outputs.cache-key }} | |
path: test/data/nchg_test_data.tar.zst | |
fail-on-cache-miss: true | |
- name: Download binaries artifact | |
uses: actions/download-artifact@v4 | |
with: | |
name: "binaries-${{ matrix.os }}-\ | |
${{ matrix.compiler_name }}-\ | |
${{ matrix.compiler_version }}-\ | |
${{ matrix.build_type }}-\ | |
${{ matrix.developer_mode }}" | |
- name: Extract binaries test dataset | |
run: | | |
gtar -xf binaries.tar.zst | |
gtar -xf test/data/nchg_test_data.tar.zst | |
- name: Detect number available CPUs | |
run: | | |
ncpus=$(python -c 'import multiprocessing as mp; print(mp.cpu_count())') | |
echo "NPROC=$ncpus" >> $GITHUB_ENV | |
- name: Setup test dependencies | |
run: | | |
pip install -r test/requirements.txt --user | |
which pip | |
which python3 | |
- name: Setup test environment | |
run: | | |
echo "DATA_DIR=$PWD/test/data/integration_tests" | tee -a "$GITHUB_ENV" | |
echo "OUT_PREFIX=ENCFF447ERX.1000000" | tee -a "$GITHUB_ENV" | |
- name: Test NCHG cartesian-product (gw) | |
run: | | |
bin/NCHG cartesian-product \ | |
test/data/ENCFF447ERX.1000000.compartments.bed \ | |
> "$OUT_PREFIX.bedpe" | |
test/scripts/validate_nchg_output.py cartesian-product \ | |
"$OUT_PREFIX.bedpe" \ | |
"$DATA_DIR/cartesian_product/$OUT_PREFIX.gw-domains.bedpe" | |
rm "$OUT_PREFIX"* | |
- name: Test NCHG cartesian-product (cis) | |
run: | | |
bin/NCHG cartesian-product \ | |
--cis-only \ | |
test/data/ENCFF447ERX.1000000.compartments.bed \ | |
> domains.bedpe | |
test/scripts/validate_nchg_output.py cartesian-product \ | |
domains.bedpe \ | |
"$DATA_DIR/cartesian_product/$OUT_PREFIX.cis-domains.bedpe" | |
rm *.bedpe | |
- name: Test NCHG cartesian-product (trans) | |
run: | | |
bin/NCHG cartesian-product \ | |
--trans-only \ | |
test/data/ENCFF447ERX.1000000.compartments.bed \ | |
> domains.bedpe | |
test/scripts/validate_nchg_output.py cartesian-product \ | |
domains.bedpe \ | |
"$DATA_DIR/cartesian_product/$OUT_PREFIX.trans-domains.bedpe" | |
rm *.bedpe | |
- name: Test NCHG cartesian-product (chr1:chr3) | |
run: | | |
bin/NCHG cartesian-product \ | |
--chrom1 chr1 \ | |
--chrom2 chr3 \ | |
test/data/ENCFF447ERX.1000000.compartments.bed \ | |
> domains.bedpe | |
grep -E '\bchr1\b.*\bchr3\b' \ | |
"$DATA_DIR/cartesian_product/$OUT_PREFIX.trans-domains.bedpe" \ | |
> expected.bedpe | |
test/scripts/validate_nchg_output.py cartesian-product \ | |
domains.bedpe \ | |
expected.bedpe | |
rm *.bedpe | |
- name: Test NCHG compute (mt) | |
run: | | |
bin/NCHG compute \ | |
test/data/ENCFF447ERX.1000000.cool \ | |
"$OUT_PREFIX" \ | |
--threads "$NPROC" | |
test/scripts/validate_nchg_output.py compute \ | |
"$OUT_PREFIX" \ | |
"$DATA_DIR/compute/$OUT_PREFIX" | |
rm "$OUT_PREFIX"* | |
- name: Test NCHG compute (st) | |
run: | | |
bin/NCHG compute \ | |
test/data/ENCFF447ERX.1000000.cool \ | |
"$OUT_PREFIX" \ | |
--cis-only \ | |
--threads 1 | |
rm "$OUT_PREFIX"* | |
- name: Test NCHG compute (mt; w/ domain list) | |
run: | | |
bin/NCHG compute \ | |
test/data/ENCFF447ERX.1000000.cool \ | |
"$OUT_PREFIX" \ | |
--threads "$NPROC" \ | |
--domains "$DATA_DIR/cartesian_product/$OUT_PREFIX.gw-domains.bedpe" | |
test/scripts/validate_nchg_output.py compute \ | |
"$OUT_PREFIX" \ | |
"$DATA_DIR/compute_with_domains/$OUT_PREFIX" | |
rm "$OUT_PREFIX"* | |
- name: Test NCHG merge | |
run: | | |
bin/NCHG merge \ | |
"$DATA_DIR/compute/$OUT_PREFIX" \ | |
"$OUT_PREFIX.parquet" \ | |
--threads "$NPROC" | |
test/scripts/validate_nchg_output.py merge \ | |
"$OUT_PREFIX.parquet" \ | |
"$DATA_DIR/merge/$OUT_PREFIX.parquet" | |
rm "$OUT_PREFIX"* | |
- name: Test NCHG filter | |
run: | | |
bin/NCHG filter \ | |
"$DATA_DIR/merge/$OUT_PREFIX.parquet" \ | |
"$OUT_PREFIX.filtered.parquet" \ | |
--threads "$NPROC" | |
test/scripts/validate_nchg_output.py filter \ | |
"$OUT_PREFIX.filtered.parquet" \ | |
"$DATA_DIR/filter/$OUT_PREFIX.parquet" | |
rm "$OUT_PREFIX"* | |
- name: Test NCHG view (compute) | |
run: | | |
bin/NCHG view \ | |
"$DATA_DIR/merge/$OUT_PREFIX.parquet" > /dev/null | |
- name: Test NCHG view (filter) | |
run: | | |
bin/NCHG view \ | |
"$DATA_DIR/filter/$OUT_PREFIX.parquet" \ | |
> "$OUT_PREFIX.tsv" | |
test/scripts/validate_nchg_output.py view \ | |
"$OUT_PREFIX.tsv" \ | |
"$DATA_DIR/view/$OUT_PREFIX.tsv" | |
rm "$OUT_PREFIX"* | |
- name: Test NCHG expected (w/ mask) | |
run: | | |
printf 'chr1\t0\t248956422' > mask.bed | |
bin/NCHG expected \ | |
test/data/ENCFF447ERX.1000000.cool \ | |
--output "$OUT_PREFIX.h5" \ | |
--bin-mask mask.bed | |
rm "$OUT_PREFIX"* mask.bed | |
- name: Test NCHG expected (wo/ mask) | |
run: | | |
bin/NCHG expected \ | |
test/data/ENCFF447ERX.1000000.cool \ | |
--output "$OUT_PREFIX.h5" | |
test/scripts/validate_nchg_output.py expected \ | |
"$OUT_PREFIX.h5" \ | |
"$DATA_DIR/expected/$OUT_PREFIX.h5" | |
rm "$OUT_PREFIX"* | |
macos-ci-status-check: | |
name: Status Check (MacOS CI) | |
if: ${{ always() }} | |
runs-on: ubuntu-latest | |
needs: | |
- build-project | |
- run-unit-tests | |
- run-integration-tests | |
steps: | |
- name: Collect job results | |
if: | | |
needs.build-project.result != 'success' || | |
needs.run-unit-tests.result != 'success' || | |
needs.run-integration-tests.result != 'success' | |
run: exit 1 |