From 946149d8bdadb07d1f5cbdb26ef96668b97a3293 Mon Sep 17 00:00:00 2001 From: Marco Barbone Date: Wed, 29 Jan 2025 11:38:01 -0500 Subject: [PATCH] Faster CI builds (#614) * Using sccache to speedup compilation * using github cache to save cmake dependencies * fixed issue of python always running --- .github/workflows/cmake_ci.yml | 74 +++++++++++++++++++--- .github/workflows/generate_cmake_matrix.py | 3 +- CHANGELOG | 3 +- CMakeLists.txt | 18 +++++- 4 files changed, 86 insertions(+), 12 deletions(-) diff --git a/.github/workflows/cmake_ci.yml b/.github/workflows/cmake_ci.yml index 75be3e19f..342c2b1c1 100644 --- a/.github/workflows/cmake_ci.yml +++ b/.github/workflows/cmake_ci.yml @@ -4,7 +4,7 @@ on: [push, pull_request] jobs: prepare: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 outputs: matrix: ${{ steps.generate_matrix.outputs.matrix }} steps: @@ -13,18 +13,71 @@ jobs: - name: Generate matrix id: generate_matrix run: | - echo "MACOSX_DEPLOYMENT_TARGET=11.0" >> $GITHUB_ENV MATRIX=$(python3 ${{ github.workspace }}/.github/workflows/generate_cmake_matrix.py) echo "matrix=$MATRIX" >> $GITHUB_OUTPUT + cache: + strategy: + matrix: + os: + - ubuntu-22.04 + - windows-2022 + runs-on: ${{ matrix.os }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: create cache directory + run: | + mkdir -p cpm + - name: Setup Cpp + if: steps.cache.outputs.cache-hit != 'true' + uses: aminya/setup-cpp@v1 + with: + cmake : true + - name: Download dependencies in cache linux + if: steps.cache.outputs.cache-hit != 'true' && runner.os == 'Linux' + run: | + cmake -S . -B ./build + rm -rf build + cmake -S . -B ./build -DFINUFFT_USE_DUCC0=ON + env: + CPM_SOURCE_CACHE: cpm + - name: Download dependencies in cache windows + if: steps.cache.outputs.cache-hit != 'true' && runner.os != 'Linux' + run: | + cmake -S . -B build + rm build -r -force + cmake -S . -B build -DFINUFFT_USE_DUCC0=ON + env: + CPM_SOURCE_CACHE: cpm + - name: Cache dependencies + uses: actions/cache/save@v4 + with: + key: cpm-cache-00-${{ hashFiles('CMakeLists.txt', 'cmake/*') }} + enableCrossOsArchive: true + path: cpm cmake-ci: runs-on: ${{ matrix.os }} - needs: prepare + needs: [prepare, cache] strategy: fail-fast: false matrix: ${{ fromJSON(needs.prepare.outputs.matrix) }} steps: - name: Checkout code uses: actions/checkout@v4 + - name: Restore Cache + uses: actions/cache/restore@v4 + with: + key: cpm-cache-00-${{ hashFiles('CMakeLists.txt', 'cmake/*') }} + enableCrossOsArchive: true + path: cpm + - name: Run sccache-cache only on non-release runs + if: github.event_name != 'release' && github.event_name != 'workflow_dispatch' + uses: mozilla-actions/sccache-action@v0.0.7 + - name: Set caching env vars only on non-release runs + if: github.event_name != 'release' && github.event_name != 'workflow_dispatch' + run: | + echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV + echo "SCCACHE_GHA_VERSION=0" >> $GITHUB_ENV - name: Setup Cpp uses: aminya/setup-cpp@v1 with: @@ -46,7 +99,9 @@ jobs: sudo apt install -y libfftw3-dev - name: Configure Cmake run: | - cmake -S . -B ./build -G Ninja -DCMAKE_BUILD_TYPE:STRING=${{matrix.build_type}} -DFINUFFT_ARCH_FLAGS=${{ matrix.arch_flags }} -DFINUFFT_BUILD_TESTS=ON -DFINUFFT_STATIC_LINKING=${{matrix.finufft_static_linking}} -DFINUFFT_USE_DUCC0=${{ matrix.ducc_fft }} + cmake -S . -B ./build -G Ninja -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_BUILD_TYPE:STRING=${{matrix.build_type}} -DFINUFFT_ARCH_FLAGS=${{ matrix.arch_flags }} -DFINUFFT_BUILD_TESTS=ON -DFINUFFT_STATIC_LINKING=${{matrix.finufft_static_linking}} -DFINUFFT_USE_DUCC0=${{ matrix.ducc_fft }} + env: + CPM_SOURCE_CACHE: cpm - name: Build run: | cmake --build ./build --config ${{matrix.build_type}} @@ -57,15 +112,16 @@ jobs: ctest -C ${{matrix.build_type}} --output-on-failure - name: Set up Python - if: matrix.finufft_static_linking + if: matrix.finufft_static_linking == 'off' uses: actions/setup-python@v5 with: python-version: '3.10' - name: Build Python wheels - if: matrix.finufft_static_linking + if: matrix.finufft_static_linking == 'off' env: MACOSX_DEPLOYMENT_TARGET: 13 + CPM_SOURCE_CACHE: cpm shell: bash run: | python3 -m pip install \ @@ -73,15 +129,17 @@ jobs: -C cmake.define.CMAKE_BUILD_TYPE=${{ matrix.build_type }} \ -C cmake.define.FINUFFT_ARCH_FLAGS=${{ matrix.arch_flags }} \ -C cmake.define.FINUFFT_USE_DUCC0=${{ matrix.ducc_fft }} \ + -C cmake.define.CMAKE_CXX_COMPILER_LAUNCHER=sccache \ + -C cmake.define.CMAKE_C_COMPILER_LAUNCHER=sccache \ python/finufft - name: Install pytest - if: matrix.finufft_static_linking + if: matrix.finufft_static_linking == 'off' run: | python3 -m pip install --upgrade pip python3 -m pip install pytest - name: Test Python package - if: matrix.finufft_static_linking + if: matrix.finufft_static_linking == 'off' run: | python3 -m pytest python/finufft/test diff --git a/.github/workflows/generate_cmake_matrix.py b/.github/workflows/generate_cmake_matrix.py index 3618e2edf..937765e90 100644 --- a/.github/workflows/generate_cmake_matrix.py +++ b/.github/workflows/generate_cmake_matrix.py @@ -33,7 +33,6 @@ }) ] - def get_c_compiler(toolchain): if "gcc" in toolchain: return "gcc" @@ -70,7 +69,7 @@ def get_cxx_compiler(toolchain): "build_type": build, "c_compiler": get_c_compiler(toolchain), "cxx_compiler": get_cxx_compiler(toolchain), - "ducc_fft": ducc + "ducc_fft": ducc, }) json_str = json.dumps(matrix, ensure_ascii=False) print(json_str) diff --git a/CHANGELOG b/CHANGELOG index 9b75b37fa..a9bcee2a2 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,7 +2,8 @@ List of features / changes made / release notes, in reverse chronological order. If not stated, FINUFFT is assumed (cuFINUFFT <=1.3 is listed separately). Master, using release name V 2.4.0 (1/7/25) - +* PR614: Added support for sccache in github actions. + Caching cmake dependencies so to avoid downloading fftw, xsimd, etc. every time. * fully removed chkbnds option (opts and spreadopts) (Barnett) * classic GNU makefile settings make.inc.* tidied to make-platforms/ (Barnett) * unified separate-dim arrays (eg X,Y,Z->XYZ), simplifiying core (Reinecke #592) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9188648fa..c715badbe 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -79,10 +79,26 @@ set(FINUFFT_CXX_FLAGS_DEBUG -g3 -ggdb -ggdb3 - /Zi -Wall -Wno-sign-compare -Wno-unknown-pragmas) + +if(DEFINED ENV{GITHUB_ACTIONS}) + if($ENV{GITHUB_ACTIONS} STREQUAL "true") + message("CMake is being executed inside a GitHub workflow") + # if msvc use FS flag + if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC" AND NOT CMAKE_BUILD_TYPE STREQUAL + "Release") + set(CMAKE_MSVC_DEBUG_INFORMATION_FORMAT + "$<$:Embedded>") + message("CMAKE_MSVC_DEBUG_INFORMATION_FORMAT TO Embedded") + endif() + endif() +else() + message("CMake is NOT being executed inside a GitHub workflow") + # env variable is: + message(STATUS "ENV{GITHUB_ACTIONS}: $ENV{GITHUB_ACTIONS}") +endif() filter_supported_compiler_flags(FINUFFT_CXX_FLAGS_DEBUG FINUFFT_CXX_FLAGS_DEBUG) message(STATUS "FINUFFT Debug flags: ${FINUFFT_CXX_FLAGS_DEBUG}") list(APPEND FINUFFT_CXX_FLAGS_RELWITHDEBINFO ${FINUFFT_CXX_FLAGS_RELEASE}