ubuntu-ci.yml

# Copyright (C) 2024 Roberto Rossini <roberros@uio.no>
#
# SPDX-License-Identifier: GPL-3.0
#
# This library is free software: you can redistribute it and/or
# modify it under the terms of the GNU Public License as published
# by the Free Software Foundation; either version 3 of the License,
# or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Library General Public License for more details.
#
# You should have received a copy of the GNU Public License along
# with this library.  If not, see
# <https://www.gnu.org/licenses/>.

name: Ubuntu CI

on:
  push:
    branches: [main]
    paths:
      - ".github/workflows/build-conan-deps.yml"
      - ".github/workflows/cache-test-dataset.yml"
      - ".github/workflows/ubuntu-ci.yml"
      - "cmake/**"
      - "src/**"
      - "test/**"
      - "CMakeLists.txt"
      - "conanfile.py"

  pull_request:
    paths:
      - ".github/workflows/build-conan-deps.yml"
      - ".github/workflows/cache-test-dataset.yml"
      - ".github/workflows/ubuntu-ci.yml"
      - "cmake/**"
      - "src/**"
      - "test/**"
      - "CMakeLists.txt"
      - "conanfile.py"

# https://stackoverflow.com/a/72408109
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

defaults:
  run:
    shell: bash

jobs:
  matrix-factory:
    name: Generate job matrix
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.set-result.outputs.result }}
      ci-type: ${{ steps.ci-type.outputs.type }}
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ github.event.after }}

      - name: Detect CI type
        id: ci-type
        run: |
          if git log --format=%B -n 1 ${{ github.event.after }} | grep -qF '[ci full]'; then
            echo "type=full" | tee -a "$GITHUB_OUTPUT"
          else
            echo "type=short" | tee -a "$GITHUB_OUTPUT"
          fi

      - name: Generate matrix
        uses: actions/github-script@v7
        id: set-result
        with:
          script: |
            // Documentation
            // https://docs.github.com/en/actions/learn-github-actions/contexts#fromjson
            // https://github.com/actions/runner/issues/982#issuecomment-809360765

            var ci_short = "${{ steps.ci-type.outputs.type }}" === "short"

            var includes = []

            // Debug builds (short CI)
            includes.push({ compiler: 'gcc-13',    os: 'ubuntu-24.04',  generator: 'Ninja', cmake: '3.30.*', build_type: 'Debug',   developer_mode: 'OFF' })
            includes.push({ compiler: 'gcc-14',    os: 'ubuntu-24.04',  generator: 'Ninja', cmake: '3.30.*', build_type: 'Debug',   developer_mode: 'ON'  })
            includes.push({ compiler: 'clang-17',  os: 'ubuntu-24.04',  generator: 'Ninja', cmake: '3.30.*', build_type: 'Debug',   developer_mode: 'OFF' })
            includes.push({ compiler: 'clang-19',  os: 'ubuntu-24.04',  generator: 'Ninja', cmake: '3.30.*', build_type: 'Debug',   developer_mode: 'ON'  })

            // Release builds (short CI)
            includes.push({ compiler: 'gcc-14',    os: 'ubuntu-24.04',  generator: 'Ninja', cmake: '3.30.*', build_type: 'Release', developer_mode: 'ON'  })
            includes.push({ compiler: 'clang-19',  os: 'ubuntu-24.04',  generator: 'Ninja', cmake: '3.30.*', build_type: 'Release', developer_mode: 'ON'  })

            if (ci_short) {
              return { include: includes }
            }

            // Debug builds (long CI)
            includes.push({ compiler: 'clang-18',  os: 'ubuntu-24.04',  generator: 'Ninja', cmake: '3.30.*', build_type: 'Debug',   developer_mode: 'OFF' })

            // Release builds (long CI)
            includes.push({ compiler: 'gcc-13',    os: 'ubuntu-24.04',  generator: 'Ninja', cmake: '3.30.*', build_type: 'Release', developer_mode: 'OFF' })
            includes.push({ compiler: 'clang-17',  os: 'ubuntu-24.04',  generator: 'Ninja', cmake: '3.30.*', build_type: 'Release', developer_mode: 'OFF' })
            includes.push({ compiler: 'clang-18',  os: 'ubuntu-24.04',  generator: 'Ninja', cmake: '3.30.*', build_type: 'Release', developer_mode: 'OFF' })

            // Make sure project builds with CMake 3.25
            includes.push({ compiler: 'clang-19',  os: 'ubuntu-24.04',  generator: 'Ninja', cmake: '3.25.2', build_type: 'Release', developer_mode: 'OFF' })
            // Make sure project builds with make
            includes.push({ compiler: 'clang-19',  os: 'ubuntu-24.04',  generator: 'Unix Makefiles', cmake: '3.30.*', build_type: 'Release', developer_mode: 'OFF' })

            return { include: includes }

  cache-test-dataset:
    name: Cache test dataset
    uses: paulsengroup/nchg/.github/workflows/cache-test-dataset.yml@main

  build-conan-deps:
    name: Build Conan deps
    uses: paulsengroup/nchg/.github/workflows/build-conan-deps.yml@main
    with:
      os: ubuntu-24.04

  build-project:
    name: Build project
    needs:
      - matrix-factory
      - build-conan-deps
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.matrix-factory.outputs.matrix) }}

    container:
      image: ghcr.io/paulsengroup/ci-docker-images/${{ matrix.os }}-cxx-${{ matrix.compiler }}:latest
      options: "--user=root"

    env:
      CCACHE_DIR: "/opt/ccache-cache"
      CCACHE_COMPILERCHECK: "content"
      CCACHE_COMPRESSLEVEL: "1"
      CCACHE_MAXSIZE: "500M"
      CONAN_HOME: "/opt/conan/"

    steps:
      - uses: actions/checkout@v4

      - name: Fix permissions
        run: |
          chown -R $(id -u):$(id -g) $PWD

      - name: Check build deps are up-to-date
        id: build-deps-outdated
        run: |
          pattern='[[:digit:]]\+\.[[:digit:]]\+\.[[:digit:]]\+$'
          flag=false

          if [[ $(cmake --version | grep -o "$pattern") != ${{ matrix.cmake }} ]]; then
            flag=true
          fi

          echo "outdated=$flag" | tee -a "$GITHUB_OUTPUT"

      - name: Update build deps
        if: ${{ steps.build-deps-outdated.outputs.outdated }}
        run: |
          apt-get update
          apt-get install -y --no-install-recommends python3-pip

          python3 -m pip install "cmake==${{ matrix.cmake }}"

      - name: Generate cache keys
        id: cache-key
        run: |
          set -u

          os="${{ matrix.os }}"
          compiler="${{ matrix.compiler }}"
          generator="${{ matrix.generator }}"
          build_type="${{ matrix.build_type }}"
          dev_mode="${{ matrix.developer_mode }}"

          conanfile_hash="${{ hashFiles('conanfile.py') }}"

          # This can be used by to always update a cache entry (useful e.g. for ccache)
          current_date="$(date '+%s')"

          ccache_key_prefix="ccache-$os-$compiler-$conanfile_hash-$build_type-$generator-$dev_mode"

          echo "ccache-key=${ccache_key_prefix}-$GITHUB_REF-${current_date}" | tee -a $GITHUB_OUTPUT
          echo "ccache-restore-key-1=$ccache_key_prefix-$GITHUB_REF" | tee -a $GITHUB_OUTPUT
          echo "ccache-restore-key-2=$ccache_key_prefix" | tee -a $GITHUB_OUTPUT

      - name: Restore Conan cache
        uses: actions/cache/restore@v4
        with:
          key: ${{ needs.build-conan-deps.outputs.conan-key }}
          path: ${{ env.CONAN_HOME }}/p
          fail-on-cache-miss: true

      - name: Restore CMake configs (Debug)
        if: matrix.build_type == 'Debug'
        uses: actions/cache/restore@v4
        with:
          key: ${{ needs.build-conan-deps.outputs.cmake-prefix-debug-key }}
          path: /tmp/cmake-prefix-dbg.tar
          fail-on-cache-miss: true

      - name: Restore CMake configs (Release)
        if: matrix.build_type == 'Release'
        uses: actions/cache/restore@v4
        with:
          key: ${{ needs.build-conan-deps.outputs.cmake-prefix-release-key }}
          path: /tmp/cmake-prefix-rel.tar
          fail-on-cache-miss: true

      - name: Extract CMake configs
        run: |
          mkdir conan-env
          tar -xf /tmp/cmake-prefix-*.tar -C conan-env/ --strip-components=1

      - name: Restore Ccache folder
        id: cache-ccache
        uses: actions/cache/restore@v4
        with:
          key: ${{ steps.cache-key.outputs.ccache-restore-key-1 }}
          restore-keys: ${{ steps.cache-key.outputs.ccache-restore-key-2 }}
          path: ${{ env.CCACHE_DIR }}

      - name: Reset Ccache stats
        run: ccache --zero-stats

      - name: Configure project
        run: |
          cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }}          \
                -DCMAKE_PREFIX_PATH="$PWD/conan-env"                 \
                -DENABLE_DEVELOPER_MODE=${{ matrix.developer_mode }} \
                -DOPT_ENABLE_CLANG_TIDY=OFF                          \
                -DOPT_ENABLE_CPPCHECK=OFF                            \
                -DNCHG_ENABLE_TESTING=ON                             \
                -DNCHG_DOWNLOAD_TEST_DATASET=OFF                     \
                -DCMAKE_INSTALL_PREFIX=dest                          \
                -S .                                                 \
                -B build

      - name: Build project
        run: cmake --build build -j $(nproc)

      - name: Package binaries
        run: |
          cmake --install build

          tar -cf - -C dest/ bin | zstd -T0 -13 -o binaries.tar.zst

      - name: Package unit tests
        run: |
          rm -r build/src
          tar --exclude='*.o' -cf - build/ | zstd -T0 -13 -o unit-tests.tar.zst

      - name: Upload unit tests
        uses: actions/upload-artifact@v4
        with:
          name: "unit-tests-${{ matrix.os }}-\
            ${{ matrix.compiler }}-\
            ${{ matrix.generator }}-\
            ${{ matrix.build_type }}-\
            ${{ matrix.developer_mode }}"
          path: unit-tests.tar.zst
          if-no-files-found: error
          retention-days: 1

      - name: Upload binaries
        uses: actions/upload-artifact@v4
        with:
          name: "binaries-${{ matrix.os }}-\
            ${{ matrix.compiler }}-\
            ${{ matrix.generator }}-\
            ${{ matrix.build_type }}-\
            ${{ matrix.developer_mode }}"
          path: binaries.tar.zst
          if-no-files-found: error
          retention-days: 1

      - name: Print Ccache statistics (pre-cleanup)
        run: |
          ccache --show-stats \
                 --show-compression \
                 --verbose

      - name: Cleanup Ccache folder
        if: needs.matrix-factory.outputs.ci-type == 'short'
        run: |
          ccache --evict-older-than=14400s # 4h
          ccache --recompress=19 --recompress-threads="$(nproc)"
          ccache --cleanup

      - name: Print Ccache statistics (post-cleanup)
        if: needs.matrix-factory.outputs.ci-type == 'short'
        run: |
          ccache --show-stats \
                 --show-compression \
                 --verbose

      - name: Save Ccache folder
        uses: actions/cache/save@v4
        with:
          key: ${{ steps.cache-key.outputs.ccache-key }}
          path: ${{ env.CCACHE_DIR }}
        env:
          ZSTD_CLEVEL: 1

      - name: Generate list of stale cache entries
        id: stale-cache
        if: steps.cache-ccache.outputs.cache-matched-key != ''
        run: |
          fname='stale-cache-${{ matrix.os }}-${{ matrix.compiler }}-${{ matrix.generator }}-${{ matrix.build_type }}-${{ matrix.developer_mode }}.txt'
          echo '${{ steps.cache-ccache.outputs.cache-matched-key }}' > "$fname"

          echo "name=$fname" | tee -a "$GITHUB_OUTPUT"

      - name: Upload stale cache entries
        if: steps.cache-ccache.outputs.cache-matched-key != ''
        uses: actions/upload-artifact@v4
        with:
          name: ${{ steps.stale-cache.outputs.name }}
          path: "${{ steps.stale-cache.outputs.name }}"
          if-no-files-found: error
          retention-days: 1

  clean-stale-cache:
    needs: [build-project]
    name: Evict stale cache entries
    runs-on: ubuntu-latest
    permissions:
      actions: write

    steps:
      - name: Download artifacts
        uses: actions/download-artifact@v4
        continue-on-error: true
        with:
          pattern: stale-cache-*
          merge-multiple: true

      - name: Evict cache entries
        continue-on-error: true
        run: |
          set -x
          while read entry; do
            if ! grep -q '/heads/main' <(echo "$entry"); then
              gh cache delete --repo '${{ github.repository }}' "$entry"
            fi
          done < <(cat stale-cache*.txt | grep -v '^$')
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

  run-unit-tests:
    name: Run unit tests
    needs: [matrix-factory, cache-test-dataset, build-project]
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.matrix-factory.outputs.matrix) }}

    container:
      image: ghcr.io/paulsengroup/ci-docker-images/${{ matrix.os }}-cxx-${{ matrix.compiler }}:latest
      options: "--user=root"

    steps:
      - name: Checkout repo
        uses: actions/checkout@v4

      - name: Restore test dataset
        uses: actions/cache/restore@v4
        with:
          key: ${{ needs.cache-test-dataset.outputs.cache-key }}
          path: test/data/nchg_test_data.tar.zst
          fail-on-cache-miss: true

      - name: Download unit tests artifact
        uses: actions/download-artifact@v4
        with:
          name: "unit-tests-${{ matrix.os }}-\
            ${{ matrix.compiler }}-\
            ${{ matrix.generator }}-\
            ${{ matrix.build_type }}-\
            ${{ matrix.developer_mode }}"

      - name: Extract binaries and test dataset
        run: |
          tar -xf unit-tests.tar.zst
          tar -xf test/data/nchg_test_data.tar.zst

      - name: Add test user
        run: useradd devel

      - name: Fix permissions
        run: |
          chown -R devel:devel build/

      - name: Setup dependencies
        run: |
          apt-get update
          apt-get install -q -y --no-install-recommends \
            python3-pip \
            sudo

          python3 -m pip install "cmake==${{ matrix.cmake }}"

      - name: Run unit tests
        run: |
          sudo -u devel -E env "PATH=$PATH" \
          ctest --test-dir build/           \
                --schedule-random           \
                --output-on-failure         \
                --no-tests=error            \
                --timeout 240               \
                -j $(nproc) |&
          head -n 1000

  run-integration-tests:
    name: Run integration tests
    needs: [matrix-factory, cache-test-dataset, build-project]
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.matrix-factory.outputs.matrix) }}

    container:
      image: ubuntu:24.04
      options: "--user=root"

    env:
      DEBIAN_FRONTEND: "noninteractive"
      TZ: "Etc/UTC"

    steps:
      - name: Install dependencies
        run: |
          apt-get update
          apt-get install -q -y --no-install-recommends \
            curl \
            git \
            python3.12 \
            python3.12-venv \
            python3-pip \
            sudo \
            tar \
            zip \
            zstd

          if [ "${{ matrix.developer_mode }}" = ON ]; then
            apt-get install -q -y --no-install-recommends \
              libasan8 \
              libubsan1
          fi

      - name: Checkout repo
        uses: actions/checkout@v4

      - name: Restore test dataset
        uses: actions/cache/restore@v4
        with:
          key: ${{ needs.cache-test-dataset.outputs.cache-key }}
          path: test/data/nchg_test_data.tar.zst
          fail-on-cache-miss: true

      - name: Download binaries artifact
        uses: actions/download-artifact@v4
        with:
          name: "binaries-${{ matrix.os }}-\
            ${{ matrix.compiler }}-\
            ${{ matrix.generator }}-\
            ${{ matrix.build_type }}-\
            ${{ matrix.developer_mode }}"

      - name: Extract binaries and test dataset
        run: |
          tar -xf binaries.tar.zst
          tar -xf test/data/nchg_test_data.tar.zst

      - name: Add test user
        run: useradd devel

      - name: Setup test dependencies
        run: |
          python3.12 -m venv venv --upgrade
          echo "PATH=$PWD/venv/bin:$PATH" | tee -a "$GITHUB_ENV"

          venv/bin/pip install -r test/requirements.txt

      - name: Setup test environment
        run: |
          echo "DATA_DIR=$PWD/test/data/integration_tests" | tee -a "$GITHUB_ENV"
          echo "OUT_PREFIX=ENCFF447ERX.1000000" | tee -a "$GITHUB_ENV"

      - name: Test NCHG cartesian-product (gw)
        run: |
          sudo -u devel                                        \
          bin/NCHG cartesian-product                           \
            test/data/ENCFF447ERX.1000000.compartments.bed     \
            > "$OUT_PREFIX.bedpe"

          test/scripts/validate_nchg_output.py cartesian-product \
            "$OUT_PREFIX.bedpe"                                  \
            "$DATA_DIR/cartesian_product/$OUT_PREFIX.gw-domains.bedpe"

          rm "$OUT_PREFIX"*

      - name: Test NCHG cartesian-product (cis)
        run: |
          sudo -u devel                                        \
          bin/NCHG cartesian-product                           \
            --cis-only                                         \
            test/data/ENCFF447ERX.1000000.compartments.bed     \
            > domains.bedpe

          test/scripts/validate_nchg_output.py cartesian-product \
            domains.bedpe                                        \
            "$DATA_DIR/cartesian_product/$OUT_PREFIX.cis-domains.bedpe"

          rm *.bedpe

      - name: Test NCHG cartesian-product (trans)
        run: |
          sudo -u devel                                        \
          bin/NCHG cartesian-product                           \
            --trans-only                                       \
            test/data/ENCFF447ERX.1000000.compartments.bed     \
            > domains.bedpe

          test/scripts/validate_nchg_output.py cartesian-product \
            domains.bedpe                                        \
            "$DATA_DIR/cartesian_product/$OUT_PREFIX.trans-domains.bedpe"

          rm *.bedpe

      - name: Test NCHG cartesian-product (chr1:chr3)
        run: |
          sudo -u devel                                        \
          bin/NCHG cartesian-product                           \
            --chrom1 chr1                                      \
            --chrom2 chr3                                      \
            test/data/ENCFF447ERX.1000000.compartments.bed     \
            > domains.bedpe

          grep -E '\bchr1\b.*\bchr3\b' \
            "$DATA_DIR/cartesian_product/$OUT_PREFIX.trans-domains.bedpe" \
            > expected.bedpe

          test/scripts/validate_nchg_output.py cartesian-product \
            domains.bedpe                                        \
            expected.bedpe

          rm *.bedpe

      - name: Test NCHG compute (mt)
        run: |
          sudo -u devel                                        \
          bin/NCHG compute                                     \
            test/data/ENCFF447ERX.1000000.cool                 \
            "$OUT_PREFIX"                                      \
            --threads "$(nproc)"

          test/scripts/validate_nchg_output.py compute \
            "$OUT_PREFIX"                              \
            "$DATA_DIR/compute/$OUT_PREFIX"

          rm "$OUT_PREFIX"*

      - name: Test NCHG compute (st)
        run: |
          sudo -u devel                                        \
          bin/NCHG compute                                     \
            test/data/ENCFF447ERX.1000000.cool                 \
            "$OUT_PREFIX"                                      \
            --cis-only                                         \
            --threads 1

          rm "$OUT_PREFIX"*

      - name: Test NCHG compute (mt; w/ domain list)
        run: |
          sudo -u devel                                        \
          bin/NCHG compute                                     \
            test/data/ENCFF447ERX.1000000.cool                 \
            "$OUT_PREFIX"                                      \
            --threads "$(nproc)"                               \
            --domains "$DATA_DIR/cartesian_product/$OUT_PREFIX.gw-domains.bedpe"

          test/scripts/validate_nchg_output.py compute \
            "$OUT_PREFIX"                              \
            "$DATA_DIR/compute_with_domains/$OUT_PREFIX"

          rm "$OUT_PREFIX"*

      - name: Test NCHG merge
        run: |
          sudo -u devel                                        \
          bin/NCHG merge                                       \
            "$DATA_DIR/compute/$OUT_PREFIX"                    \
            "$OUT_PREFIX.parquet"                              \
            --threads "$(nproc)"

          test/scripts/validate_nchg_output.py merge \
            "$OUT_PREFIX.parquet"                    \
            "$DATA_DIR/merge/$OUT_PREFIX.parquet"

          rm "$OUT_PREFIX"*

      - name: Test NCHG filter
        run: |
          sudo -u devel                                           \
          bin/NCHG filter                                         \
            "$DATA_DIR/merge/$OUT_PREFIX.parquet"                 \
            "$OUT_PREFIX.filtered.parquet"                        \
            --threads "$(nproc)"

          test/scripts/validate_nchg_output.py filter \
            "$OUT_PREFIX.filtered.parquet"            \
            "$DATA_DIR/filter/$OUT_PREFIX.parquet"

          rm "$OUT_PREFIX"*

      - name: Test NCHG view (compute)
        run: |
          sudo -u devel                                        \
          bin/NCHG view                                        \
            "$DATA_DIR/merge/$OUT_PREFIX.parquet" > /dev/null

      - name: Test NCHG view (filter)
        run: |
          sudo -u devel                                        \
          bin/NCHG view                                        \
            "$DATA_DIR/filter/$OUT_PREFIX.parquet"             \
            > "$OUT_PREFIX.tsv"

          test/scripts/validate_nchg_output.py view  \
            "$OUT_PREFIX.tsv"                        \
            "$DATA_DIR/view/$OUT_PREFIX.tsv"

          rm "$OUT_PREFIX"*

      - name: Test NCHG expected (w/ mask)
        run: |
          printf 'chr1\t0\t248956422' > mask.bed

          sudo -u devel \
          bin/NCHG expected                                    \
            test/data/ENCFF447ERX.1000000.cool                 \
            --output "$OUT_PREFIX.h5"                          \
            --bin-mask mask.bed

          rm "$OUT_PREFIX"* mask.bed

      - name: Test NCHG expected (wo/ mask)
        run: |
          sudo -u devel \
          bin/NCHG expected                                    \
            test/data/ENCFF447ERX.1000000.cool                 \
            --output "$OUT_PREFIX.h5"

          test/scripts/validate_nchg_output.py expected  \
            "$OUT_PREFIX.h5"                             \
            "$DATA_DIR/expected/$OUT_PREFIX.h5"

          rm "$OUT_PREFIX"*

  ubuntu-ci-status-check:
    name: Status Check (Ubuntu CI)
    if: ${{ always() }}
    runs-on: ubuntu-latest
    needs:
      - build-project
      - run-unit-tests
      - run-integration-tests

    steps:
      - name: Collect job results
        if: |
          needs.build-project.result != 'success'      ||
          needs.run-unit-tests.result != 'success'     ||
          needs.run-integration-tests.result != 'success'
        run: exit 1