From 6373797f57fae4c444265d5210dcf7dd000a71f9 Mon Sep 17 00:00:00 2001 From: Tim Paine <3105306+timkpaine@users.noreply.github.com> Date: Sat, 3 Feb 2024 12:49:03 -0500 Subject: [PATCH] Use architecture in save/restore cache keys Signed-off-by: Tim Paine <3105306+timkpaine@users.noreply.github.com> --- .github/actions/setup-caches/action.yml | 10 ++++++---- .github/actions/setup-dependencies/action.yml | 4 ++-- .github/workflows/build.yml | 15 +++++++++++++-- Makefile | 6 +++--- pyproject.toml | 4 ++-- 5 files changed, 26 insertions(+), 13 deletions(-) diff --git a/.github/actions/setup-caches/action.yml b/.github/actions/setup-caches/action.yml index 31ea679b3..92b8b242f 100644 --- a/.github/actions/setup-caches/action.yml +++ b/.github/actions/setup-caches/action.yml @@ -18,6 +18,8 @@ runs: # Setup CCache - name: Setup ccache uses: hendrikmuhs/ccache-action@v1.2.12 + with: + key: ${{ runner.os }}-${{ runner.arch }} - name: Setup ccache in shell shell: bash @@ -33,8 +35,8 @@ runs: path: | ~/Library/Caches/Homebrew/* ~/Library/Caches/Homebrew/downloads/* - key: brew- - restore-keys: brew- + key: brew-${{ runner.os }}-${{ runner.arch }}- + restore-keys: brew-${{ runner.os }}-${{ runner.arch }}- if: ${{ runner.os == 'macOS' }} ################ @@ -54,6 +56,6 @@ runs: path: | ~/vcpkg_cache ~/vcpkg_download_cache - key: vcpkg-${{ runner.os }}-${{ hashFiles('vcpkg.json') }} - restore-keys: vcpkg-${{ runner.os }} + key: vcpkg-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('vcpkg.json') }} + restore-keys: vcpkg-${{ runner.os }}-${{ runner.arch }}- if: ${{ runner.os != 'Windows' }} diff --git a/.github/actions/setup-dependencies/action.yml b/.github/actions/setup-dependencies/action.yml index cf1dba535..e08d822c7 100644 --- a/.github/actions/setup-dependencies/action.yml +++ b/.github/actions/setup-dependencies/action.yml @@ -21,14 +21,14 @@ runs: - name: Install python dependencies shell: bash run: make requirements - if: ${{ runner.os == 'Linux' }} # skip on linux, in docker + if: ${{ runner.os == 'Linux' }} ################ # Linux # NOTE: skip for manylinux image # - name: Linux init steps # shell: bash # run: make dependencies-vcpkg - # if: ${{ runner.os == 'Linux' }} # skip + # if: ${{ runner.os == 'Linux' }} # skip ################ # Mac diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 97e918806..45bc529cc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -263,6 +263,7 @@ jobs: uses: ./.github/actions/setup-dependencies with: cibuildwheel: '${{ matrix.cibuildwheel }}' + if: ${{ runner.os == 'macOS' }} ######## # Linux @@ -271,6 +272,7 @@ jobs: env: CIBW_BUILD: "${{ matrix.cibuildwheel }}-manylinux*" CIBW_ENVIRONMENT_LINUX: CSP_MANYLINUX="ON" CCACHE_DIR="/host/home/runner/work/csp/csp/.ccache" VCPKG_DEFAULT_BINARY_CACHE="/host${{ env.VCPKG_DEFAULT_BINARY_CACHE }}" VCPKG_DOWNLOADS="/host${{ env.VCPKG_DOWNLOADS }}" + CIBW_BUILD_VERBOSITY: 3 if: ${{ runner.os == 'Linux' }} ######## @@ -281,16 +283,25 @@ jobs: CIBW_BUILD: "${{ matrix.cibuildwheel }}-macos*" CIBW_ENVIRONMENT_MACOS: CC="/usr/local/bin/gcc-13" CXX="/usr/local/bin/g++-13" CCACHE_DIR="/Users/runner/work/csp/csp/.ccache" VCPKG_DEFAULT_BINARY_CACHE="${{ env.VCPKG_DEFAULT_BINARY_CACHE }}" VCPKG_DOWNLOADS="${{ env.VCPKG_DOWNLOADS }}" CIBW_ARCHS_MACOS: x86_64 + CIBW_BUILD_VERBOSITY: 3 if: ${{ matrix.os == 'macos-12' }} - - name: Python Build Steps (arm) + - name: Python Build Steps (Macos arm) run: make dist-py-cibw env: CIBW_BUILD: "${{ matrix.cibuildwheel }}-macos*" - CIBW_ENVIRONMENT_MACOS: CC="/usr/local/bin/gcc-13" CXX="/usr/local/bin/g++-13" CCACHE_DIR="/Users/runner/work/csp/csp/.ccache" VCPKG_DEFAULT_BINARY_CACHE="${{ env.VCPKG_DEFAULT_BINARY_CACHE }}" VCPKG_DOWNLOADS="${{ env.VCPKG_DOWNLOADS }}" + CIBW_ENVIRONMENT_MACOS: PATH="/opt/homebrew/opt/bison/bin/:$PATH" CC="/opt/homebrew/bin/gcc-13" CXX="/opt/homebrew/bin/g++-13" CCACHE_DIR="/Users/runner/work/csp/csp/.ccache" VCPKG_DEFAULT_BINARY_CACHE="${{ env.VCPKG_DEFAULT_BINARY_CACHE }}" VCPKG_DOWNLOADS="${{ env.VCPKG_DOWNLOADS }}" CIBW_ARCHS_MACOS: arm64 + CIBW_BUILD_VERBOSITY: 3 if: ${{ matrix.os == 'macos-14' }} + - name: Debug 14 build + run: | + cat /Users/runner/work/csp/csp/vcpkg/buildtrees/detect_compiler/config-arm64-osx-rel-CMakeCache.txt.log + cat /Users/runner/work/csp/csp/vcpkg/buildtrees/detect_compiler/config-arm64-osx-rel-out.log + cat /Users/runner/work/csp/csp/vcpkg/buildtrees/detect_compiler/config-arm64-osx-rel-err.log + if: ${{ matrix.os == 'macos-14' && always() }} + ########## # Windows - name: Python Build Steps (Windows vc14.3) diff --git a/Makefile b/Makefile index 92d9c29b3..2ac562549 100644 --- a/Makefile +++ b/Makefile @@ -154,17 +154,17 @@ clean: ## clean the repository .PHONY: dependencies-mac dependencies-debian dependencies-fedora dependencies-vcpkg dependencies-win dependencies-mac: ## install dependencies for mac - HOMEBREW_NO_AUTO_UPDATE=1 brew install bison cmake flex # gcc@12 + HOMEBREW_NO_AUTO_UPDATE=1 brew install bison cmake flex make ninja # gcc@13 + brew unlink bison flex && brew link --force bison flex dependencies-debian: ## install dependencies for linux apt-get install -y automake bison cmake curl flex ninja-build tar unzip zip - # libabsl-dev libarrow-dev libparquet-dev libthrift-dev dependencies-fedora: ## install dependencies for linux yum install -y automake bison cmake curl flex perl-IPC-Cmd tar unzip zip dependencies-vcpkg: ## install dependnecies via vcpkg - cd vcpkg && ./bootstrap-vcpkg.sh && ./vcpkg install --debug + cd vcpkg && ./bootstrap-vcpkg.sh && ./vcpkg install dependencies-win: ## install dependnecies via windows (vcpkg) cd vcpkg && ./bootstrap-vcpkg.bat && ./vcpkg install diff --git a/pyproject.toml b/pyproject.toml index e62fe0410..22fe31c57 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,13 +77,13 @@ test-requires = [ ] [tool.cibuildwheel.linux] -before-all ="make dependencies-fedora && make dependencies-vcpkg" +before-all ="make dependencies-fedora" environment = {CSP_MANYLINUX="ON"} repair-wheel-command = "auditwheel -v show {wheel} && LD_LIBRARY_PATH=/project/csp/lib auditwheel -v repair -w {dest_dir} {wheel}" skip = "*i686 musllinux*" [tool.cibuildwheel.macos] -before-all ="make dependencies-mac && make dependencies-vcpkg" +before-all ="make dependencies-mac" archs = "x86_64" # NOTE: we use gcc and we cannot cross compile for now environment = {CFLAGS="-I/usr/local/include -L/usr/local/lib", CXXFLAGS="-I/usr/local/include -L/usr/local/lib", LDFLAGS="-L/usr/local/lib"}