crossbow.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


# NOTE: must set "Crossbow" as name to have the badge links working in the
# github comment reports!
name: Crossbow
on:
  push:
    branches:
      - "*-github-*"

env:
  ARCHERY_DEBUG: 1

# env: is generated by macros.github_header()
  ARROW_JEMALLOC: "OFF"
  CC: "clang"
  CMAKE_BUILD_TYPE: release
  CXX: "clang++"
  MACOSX_DEPLOYMENT_TARGET: "12.0"
  PYARROW_BUILD_VERBOSE: 1
  PYARROW_VERSION: "20.0.0.dev19"
  PYTHON_VERSION: "3.13"
  PYTHON_ABI_TAG: "cp313"
  PYTHON: "/Library/Frameworks/Python.framework/Versions/3.13/bin/python3.13"
  VCPKG_DEFAULT_TRIPLET: "arm64-osx-static-release"
  VCPKG_FEATURE_FLAGS: "manifests"
  VCPKG_OVERLAY_TRIPLETS: ${{ github.workspace }}/arrow/ci/vcpkg
  VCPKG_ROOT: ${{ github.workspace }}/vcpkg

permissions:
  packages: write

jobs:
  build:
    name: Build wheel for Python 3.13 on macOS
    runs-on: "macos-14"
    env:
      VCPKG_BINARY_SOURCES: 'clear;nuget,GitHub,readwrite'
    steps:
      - name: Checkout Arrow
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
          path: arrow
          repository: apache/arrow
          ref: 17a0ff556610c5fc0666f4220a6a0e87c487770c
          submodules: recursive


      - name: Install System Dependencies
        run: |
          brew install bash bison coreutils ninja
          echo "$(brew --prefix bison)/bin" >> $GITHUB_PATH

      - name: Homebrew packages
        run: |
          brew list

      # CMake 3.29.1 that is pre-installed on the macOS image has a problem.
      # See also: https://github.com/microsoft/vcpkg/issues/37968
      - name: Install CMake 3.29.0
        shell: bash
        run: |
          arrow/ci/scripts/install_cmake.sh $(arch) macos 3.29.0 ${PWD}/local
          echo "${PWD}/local/bin" >> $GITHUB_PATH

      - name: Retrieve VCPKG version from arrow/.env
        run: |
          vcpkg_version=$(. "arrow/.env" && echo "${VCPKG}")
          echo "VCPKG_VERSION=$vcpkg_version" >> $GITHUB_ENV

      - name: Install Vcpkg
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: arrow/ci/scripts/install_vcpkg.sh $VCPKG_ROOT $VCPKG_VERSION

      - name: Add Vcpkg to PATH
        run: echo ${VCPKG_ROOT} >> $GITHUB_PATH

      - name: Install Packages
        run: |
          vcpkg install \
            --clean-after-build \
            --x-install-root=${VCPKG_ROOT}/installed \
            --x-manifest-root=arrow/ci/vcpkg \
            --x-feature=azure \
            --x-feature=flight \
            --x-feature=gcs \
            --x-feature=json \
            --x-feature=orc \
            --x-feature=parquet \
            --x-feature=s3

      - name: Install Python 3.13
        run: sudo arrow/ci/scripts/install_python.sh macos 3.13

      - name: Build Wheel
        run: |
          $PYTHON -m venv build-env
          source build-env/bin/activate
          pip install --upgrade pip wheel
          PYTHON=python arrow/ci/scripts/python_wheel_macos_build.sh arm64 $(pwd)/arrow $(pwd)/build

      - uses: actions/upload-artifact@v4
        with:
          name: wheel
          path: arrow/python/repaired_wheels/*.whl

      # Use a well-known Python version for the GCS testbench, and avoid
      # putting it in PATH.
      - name: Set up Python for GCS testbench
        uses: actions/setup-python@v5.1.1
        id: gcs-python-install
        with:
          python-version: 3.12
          update-environment: false

      - name: Install GCS testbench
        env:
          PIPX_BIN_DIR: /usr/local/bin
          PIPX_BASE_PYTHON: ${{ steps.gcs-python-install.outputs.python-path }}
        run: arrow/ci/scripts/install_gcs_testbench.sh default

      - name: Install MinIO
        run: |
          $(brew --prefix bash)/bin/bash \
            arrow/ci/scripts/install_minio.sh latest /usr/local

      - name: Test Wheel
        env:
          PYTEST_ADDOPTS: "-k 'not test_cancellation'"
          PYTHON_GIL: 1
        run: |
          $PYTHON -m venv test-env
          source test-env/bin/activate
          pip install --upgrade pip wheel
          arch -arm64 pip install -r arrow/python/requirements-wheel-test.txt
          arch -arm64 arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: 3.12
      - name: Checkout Crossbow
        uses: actions/checkout@v4
        with:
          path: crossbow
          ref: nightly-packaging-2025-01-24-0
      - name: Setup Crossbow
        shell: bash
        run: |
          python3 -m pip install -e arrow/dev/archery[crossbow]
          echo "$HOME/.local/bin" >> $GITHUB_PATH
      - name: Upload artifacts
        shell: bash
        run: |
          archery crossbow \
            --queue-path $(pwd)/crossbow \
            --queue-remote https://github.com/ursacomputing/crossbow \
            upload-artifacts \
            --sha nightly-packaging-2025-01-24-0-github-wheel-macos-monterey-cp313-cp313-arm64 \
            --tag nightly-packaging-2025-01-24-0-github-wheel-macos-monterey-cp313-cp313-arm64 \
            "arrow/python/repaired_wheels/*.whl"
        env:
          CROSSBOW_GITHUB_TOKEN: ${{ secrets.CROSSBOW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
      - name: Verify uploaded artifacts
        shell: bash
        run: |
          archery crossbow \
            --queue-path $(pwd)/crossbow \
            --queue-remote https://github.com/ursacomputing/crossbow \
            status \
            --task-filter 'wheel-macos-monterey-cp313-cp313-arm64' \
            --no-fetch \
            --validate \
            nightly-packaging-2025-01-24-0
        env:
          CROSSBOW_GITHUB_TOKEN: ${{ secrets.CROSSBOW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}

      - name: Set up Ruby
        uses: ruby/setup-ruby@v1
        with:
          ruby-version: "ruby"
      - name: Install gemfury client
        run: |
          gem install gemfury
      - name: Upload package to Gemfury
        shell: bash
        run: |
          if $(fury versions --as=${CROSSBOW_GEMFURY_ORG} --api-token=${CROSSBOW_GEMFURY_TOKEN} pyarrow | grep --fixed-strings -q "20.0.0.dev19"); then
            echo "Version 20.0.0.dev19 already exists. Avoid pushing version."
          else
            fury push \
              --api-token=${CROSSBOW_GEMFURY_TOKEN} \
              --as=${CROSSBOW_GEMFURY_ORG} \
              arrow/python/repaired_wheels/*.whl
          fi
        env:
          CROSSBOW_GEMFURY_TOKEN: ${{ secrets.CROSSBOW_GEMFURY_TOKEN }}
          CROSSBOW_GEMFURY_ORG: ${{ secrets.CROSSBOW_GEMFURY_ORG }}

      - name: Upload wheel to Anaconda scientific-python
        shell: bash
        run: |
          # check if completion actually expands to wheel files
          # to prevent empty dirs from failing silently
          if ! compgen -G "arrow/python/repaired_wheels/*.whl" > /dev/null; then
              echo "No wheel files found!"
              exit 1
          fi
          python3 -m pip install git+https://github.com/Anaconda-Platform/anaconda-client.git@1.12.3
          anaconda -t ${CROSSBOW_SCIENTIFIC_PYTHON_UPLOAD_TOKEN} upload --force -u scientific-python-nightly-wheels --label main arrow/python/repaired_wheels/*.whl
        env:
          CROSSBOW_SCIENTIFIC_PYTHON_UPLOAD_TOKEN: ${{ secrets.CROSSBOW_SCIENTIFIC_PYTHON_UPLOAD_TOKEN }}