Fix ccache cache eviction logic when using GHA job matrix #127
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright (C) 2024 Roberto Rossini <[email protected]> | |
| # SPDX-License-Identifier: MIT | |
| name: Test compatibility with newer C++ standards | |
| on: | |
| push: | |
| branches: [main] | |
| paths: | |
| - ".github/workflows/build-conan-deps.yml" | |
| - ".github/workflows/cppstd.yml" | |
| - "cmake/**" | |
| - "examples/**" | |
| - "src/**" | |
| - "test/scripts/**" | |
| - "test/units/**" | |
| - "CMakeLists.txt" | |
| - "conanfile.py" | |
| pull_request: | |
| paths: | |
| - ".github/workflows/build-conan-deps.yml" | |
| - ".github/workflows/cppstd.yml" | |
| - "cmake/**" | |
| - "examples/**" | |
| - "src/**" | |
| - "test/scripts/**" | |
| - "test/units/**" | |
| - "CMakeLists.txt" | |
| - "conanfile.py" | |
| # https://stackoverflow.com/a/72408109 | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| defaults: | |
| run: | |
| shell: bash | |
| jobs: | |
| build-conan-deps-cpp20: | |
| name: Build Conan deps | |
| uses: paulsengroup/hictk/.github/workflows/build-conan-deps.yml@main | |
| with: | |
| os: ubuntu-24.04 | |
| cppstd: 20 | |
| build-conan-deps-cpp23: | |
| name: Build Conan deps | |
| uses: paulsengroup/hictk/.github/workflows/build-conan-deps.yml@main | |
| with: | |
| os: ubuntu-24.04 | |
| cppstd: 23 | |
| build-project: | |
| runs-on: ubuntu-latest | |
| name: Build project | |
| needs: | |
| - build-conan-deps-cpp20 | |
| - build-conan-deps-cpp23 | |
| container: | |
| image: ghcr.io/paulsengroup/ci-docker-images/ubuntu-24.04-cxx-clang-19 | |
| options: "--user=root" | |
| env: | |
| CCACHE_DIR: "/opt/ccache-cache" | |
| CCACHE_COMPILERCHECK: "content" | |
| CCACHE_COMPRESSLEVEL: "1" | |
| CCACHE_MAXSIZE: "150M" | |
| CONAN_HOME: "/opt/conan/" | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| cppstd: | |
| - 20 | |
| - 23 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Fix permissions | |
| run: | | |
| chown -R $(id -u):$(id -g) $PWD | |
| - name: Generate cache key | |
| id: generate-cache-key | |
| run: | | |
| conanfile_hash="${{ hashFiles('conanfile.py') }}" | |
| # This can be used by to always update a cache entry (useful e.g. for ccache) | |
| current_date="$(date '+%s')" | |
| ccache_key_prefix="ccache-cppstd-${{ matrix.cppstd }}-$conanfile_hash" | |
| echo "ccache-key=${ccache_key_prefix}-$GITHUB_REF-${current_date}" | tee -a "$GITHUB_OUTPUT" | |
| echo "ccache-restore-key-1=$ccache_key_prefix-$GITHUB_REF" | tee -a "$GITHUB_OUTPUT" | |
| echo "ccache-restore-key-2=$ccache_key_prefix" | tee -a "$GITHUB_OUTPUT" | |
| - name: Restore Conan cache (C++20) | |
| if: matrix.cppstd == '20' | |
| uses: actions/cache/restore@v4 | |
| with: | |
| key: ${{ needs.build-conan-deps-cpp20.outputs.conan-key }} | |
| path: ${{ env.CONAN_HOME }}/p | |
| fail-on-cache-miss: true | |
| - name: Restore CMake configs (C++20) | |
| if: matrix.cppstd == '20' | |
| uses: actions/cache/restore@v4 | |
| with: | |
| key: ${{ needs.build-conan-deps-cpp20.outputs.cmake-prefix-debug-key }} | |
| path: /tmp/cmake-prefix-dbg.tar | |
| fail-on-cache-miss: true | |
| - name: Restore Conan cache (C++23) | |
| if: matrix.cppstd == '23' | |
| uses: actions/cache/restore@v4 | |
| with: | |
| key: ${{ needs.build-conan-deps-cpp23.outputs.conan-key }} | |
| path: ${{ env.CONAN_HOME }}/p | |
| fail-on-cache-miss: true | |
| - name: Restore CMake configs (C++23) | |
| if: matrix.cppstd == '23' | |
| uses: actions/cache/restore@v4 | |
| with: | |
| key: ${{ needs.build-conan-deps-cpp23.outputs.cmake-prefix-debug-key }} | |
| path: /tmp/cmake-prefix-dbg.tar | |
| fail-on-cache-miss: true | |
| - name: Extract CMake configs | |
| run: | | |
| mkdir conan-env | |
| tar -xf /tmp/cmake-prefix-dbg.tar -C conan-env/ --strip-components=1 | |
| - name: Install Python headers | |
| run: | | |
| apt-get update | |
| apt-get install -y python3.12-dev | |
| - name: Configure project | |
| run: | | |
| find src test \ | |
| -type f \ | |
| -name CMakeLists.txt \ | |
| -exec sed -i 's/set(CMAKE_CXX_STANDARD 17)/set(CMAKE_CXX_STANDARD ${{ matrix.cppstd }})/' {} + | |
| cmake -DCMAKE_BUILD_TYPE=Debug \ | |
| -DCMAKE_PREFIX_PATH="$PWD/conan-env" \ | |
| -DHICTK_ENABLE_TESTING=ON \ | |
| -DHICTK_ENABLE_FUZZY_TESTING=ON \ | |
| -DHICTK_BUILD_TOOLS=OFF \ | |
| -DHICTK_BUILD_EXAMPLES=ON \ | |
| -DHICTK_DOWNLOAD_TEST_DATASET=OFF \ | |
| -DHICTK_ENABLE_GIT_VERSION_TRACKING=OFF \ | |
| -S . \ | |
| -B build | |
| - name: Restore Ccache folder | |
| id: cache-ccache | |
| uses: actions/cache/restore@v4 | |
| with: | |
| key: ${{ steps.generate-cache-key.outputs.ccache-restore-key-1 }} | |
| restore-keys: ${{ steps.generate-cache-key.outputs.ccache-restore-key-2 }} | |
| path: ${{ env.CCACHE_DIR }} | |
| - name: Reset Ccache stats | |
| run: ccache --zero-stats | |
| - name: Build project | |
| run: cmake --build build -j $(nproc) | |
| - name: Print Ccache statistics (pre-cleanup) | |
| run: | | |
| ccache --show-stats \ | |
| --show-compression \ | |
| --verbose | |
| - name: Cleanup Ccache folder | |
| run: | | |
| ccache --evict-older-than=14400s # 4h | |
| ccache --recompress=19 --recompress-threads="$(nproc)" | |
| ccache --cleanup | |
| - name: Print Ccache statistics (post-cleanup) | |
| run: | | |
| ccache --show-stats \ | |
| --show-compression \ | |
| --verbose | |
| - name: Save Ccache folder | |
| uses: actions/cache/save@v4 | |
| with: | |
| key: ${{ steps.generate-cache-key.outputs.ccache-key }} | |
| path: ${{ env.CCACHE_DIR }} | |
| env: | |
| ZSTD_CLEVEL: 1 | |
| - name: Generate list of stale cache entries | |
| id: stale-cache | |
| if: steps.cache-ccache.outputs.cache-matched-key != '' | |
| run: | | |
| fname='stale-cache-cppstd-${{ matrix.cppstd }}.txt' | |
| echo '${{ steps.cache-ccache.outputs.cache-matched-key }}' > "$fname" | |
| echo "name=$fname" | tee -a "$GITHUB_OUTPUT" | |
| - name: Upload stale cache entries | |
| if: steps.cache-ccache.outputs.cache-matched-key != '' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ steps.stale-cache.outputs.name }} | |
| path: "${{ steps.stale-cache.outputs.name }}" | |
| if-no-files-found: error | |
| retention-days: 1 | |
| clean-stale-cache: | |
| needs: [build-project] | |
| name: Evict stale cache entries | |
| runs-on: ubuntu-latest | |
| permissions: | |
| actions: write | |
| steps: | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v4 | |
| continue-on-error: true | |
| with: | |
| pattern: stale-cache-* | |
| merge-multiple: true | |
| - name: Evict cache entries | |
| continue-on-error: true | |
| run: | | |
| set -x | |
| while read entry; do | |
| if ! grep -q '/heads/main' <(echo "$entry"); then | |
| gh cache delete --repo '${{ github.repository }}' "$entry" | |
| fi | |
| done < <(cat stale-cache*.txt | grep -v '^$') | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| cppstd-status-check: | |
| name: Status Check (cppstd) | |
| if: ${{ always() }} | |
| runs-on: ubuntu-latest | |
| needs: | |
| - build-project | |
| steps: | |
| - name: Collect job results | |
| if: needs.build-project.result != 'success' | |
| run: exit 1 |