crossbow.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.


# NOTE: must set "Crossbow" as name to have the badge links working in the
# github comment reports!
name: Crossbow
on:
  push:
    branches:
      - "*-github-*"

env:
  ARCHERY_DEBUG: 1


permissions:
  packages: write

jobs:
  build:
    name: "Build wheel for manylinux 2014"
    runs-on: ubuntu-24.04-arm
    env:
      # archery uses these environment variables
      ARCH: arm64v8
      PYTHON: "3.13"
      PYTHON_ABI_TAG: "cp313t"
      PYTHON_IMAGE_TAG: "3.13-rc"

    steps:
      - name: Checkout Arrow
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
          path: arrow
          repository: apache/arrow
          ref: 17a0ff556610c5fc0666f4220a6a0e87c487770c
          submodules: recursive

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          cache: 'pip'
          python-version: 3.12
      - name: Install Archery
        shell: bash
        run: pip install -e arrow/dev/archery[all]

      - name: Login to Dockerhub
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKERHUB_USER }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}


      - name: Prepare
        run: |
          if [ "${PYTHON_ABI_TAG}" = "cp313t" ]; then
            test_image_prefix=python-free-threaded
          else
            test_image_prefix=python
          fi
          echo "TEST_IMAGE_PREFIX=${test_image_prefix}" >> ${GITHUB_ENV}

      - name: Build wheel
        shell: bash
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          if [ "2014" = "2014" ] && [ "arm64" = "arm64" ]; then
            # We can't use NuGet on manylinux2014_aarch64 because Mono is old.
            :
          else
            export VCPKG_BINARY_SOURCES="clear;nuget,GitHub,readwrite"
          fi
          archery docker run \
            -e SETUPTOOLS_SCM_PRETEND_VERSION=20.0.0.dev19 \
            python-wheel-manylinux-2014

      - uses: actions/upload-artifact@v4
        with:
          name: wheel
          path: arrow/python/repaired_wheels/*.whl

      # TODO(kszucs): auditwheel show
      - name: Test wheel
        shell: bash
        run: |
          source arrow/ci/scripts/util_enable_core_dumps.sh
          archery docker run ${TEST_IMAGE_PREFIX}-wheel-manylinux-test-imports
          archery docker run ${TEST_IMAGE_PREFIX}-wheel-manylinux-test-unittests

      - name: Test wheel on AlmaLinux 8
        shell: bash
        if: |
          '3.13' == '3.9'
        env:
          ALMALINUX: "8"
        run: |
          archery docker run \
            -e ARROW_GANDIVA=OFF \
            -e TEST_DEFAULT=0 \
            -e TEST_PYARROW_VERSION=20.0.0.dev19 \
            -e TEST_PYTHON_VERSIONS=3.13 \
            -e TEST_WHEEL_PLATFORM_TAGS=manylinux_2_17_aarch64.manylinux2014_aarch64 \
            -e TEST_WHEELS=1 \
            almalinux-verify-rc

      - name: Test wheel on Ubuntu 20.04
        shell: bash
        if: |
          '3.13' == '3.9'
        env:
          UBUNTU: "20.04"
        run: |
          archery docker run \
            -e TEST_DEFAULT=0 \
            -e TEST_PYARROW_VERSION=20.0.0.dev19 \
            -e TEST_PYTHON_VERSIONS=3.13 \
            -e TEST_WHEEL_PLATFORM_TAGS=manylinux_2_17_aarch64.manylinux2014_aarch64 \
            -e TEST_WHEELS=1 \
            ubuntu-verify-rc

      - name: Test wheel on Ubuntu 22.04
        shell: bash
        if: |
          '3.13' == '3.10'
        env:
          UBUNTU: "22.04"
        run: |
          archery docker run \
            -e TEST_DEFAULT=0 \
            -e TEST_PYARROW_VERSION=20.0.0.dev19 \
            -e TEST_PYTHON_VERSIONS=3.13 \
            -e TEST_WHEEL_PLATFORM_TAGS=manylinux_2_17_aarch64.manylinux2014_aarch64 \
            -e TEST_WHEELS=1 \
            ubuntu-verify-rc

      - name: Test wheel on Ubuntu 24.04
        shell: bash
        if: |
          '3.13' == '3.12'
        env:
          UBUNTU: "24.04"
        run: |
          archery docker run \
            -e TEST_DEFAULT=0 \
            -e TEST_PYARROW_VERSION=20.0.0.dev19 \
            -e TEST_PYTHON_VERSIONS=3.13 \
            -e TEST_WHEEL_PLATFORM_TAGS=manylinux_2_17_aarch64.manylinux2014_aarch64 \
            -e TEST_WHEELS=1 \
            ubuntu-verify-rc

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: 3.12
      - name: Checkout Crossbow
        uses: actions/checkout@v4
        with:
          path: crossbow
          ref: nightly-packaging-2025-01-24-0
      - name: Setup Crossbow
        shell: bash
        run: |
          python3 -m pip install -e arrow/dev/archery[crossbow]
          echo "$HOME/.local/bin" >> $GITHUB_PATH
      - name: Upload artifacts
        shell: bash
        run: |
          archery crossbow \
            --queue-path $(pwd)/crossbow \
            --queue-remote https://github.com/ursacomputing/crossbow \
            upload-artifacts \
            --sha nightly-packaging-2025-01-24-0-github-wheel-manylinux-2014-cp313-cp313t-arm64 \
            --tag nightly-packaging-2025-01-24-0-github-wheel-manylinux-2014-cp313-cp313t-arm64 \
            "arrow/python/repaired_wheels/*.whl"
        env:
          CROSSBOW_GITHUB_TOKEN: ${{ secrets.CROSSBOW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
      - name: Verify uploaded artifacts
        shell: bash
        run: |
          archery crossbow \
            --queue-path $(pwd)/crossbow \
            --queue-remote https://github.com/ursacomputing/crossbow \
            status \
            --task-filter 'wheel-manylinux-2014-cp313-cp313t-arm64' \
            --no-fetch \
            --validate \
            nightly-packaging-2025-01-24-0
        env:
          CROSSBOW_GITHUB_TOKEN: ${{ secrets.CROSSBOW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}

      - name: Set up Ruby
        uses: ruby/setup-ruby@v1
        with:
          ruby-version: "ruby"
      - name: Install gemfury client
        run: |
          gem install gemfury
      - name: Upload package to Gemfury
        shell: bash
        run: |
          if $(fury versions --as=${CROSSBOW_GEMFURY_ORG} --api-token=${CROSSBOW_GEMFURY_TOKEN} pyarrow | grep --fixed-strings -q "20.0.0.dev19"); then
            echo "Version 20.0.0.dev19 already exists. Avoid pushing version."
          else
            fury push \
              --api-token=${CROSSBOW_GEMFURY_TOKEN} \
              --as=${CROSSBOW_GEMFURY_ORG} \
              arrow/python/repaired_wheels/*.whl
          fi
        env:
          CROSSBOW_GEMFURY_TOKEN: ${{ secrets.CROSSBOW_GEMFURY_TOKEN }}
          CROSSBOW_GEMFURY_ORG: ${{ secrets.CROSSBOW_GEMFURY_ORG }}

      - name: Upload wheel to Anaconda scientific-python
        shell: bash
        run: |
          # check if completion actually expands to wheel files
          # to prevent empty dirs from failing silently
          if ! compgen -G "arrow/python/repaired_wheels/*.whl" > /dev/null; then
              echo "No wheel files found!"
              exit 1
          fi
          python3 -m pip install git+https://github.com/Anaconda-Platform/anaconda-client.git@1.12.3
          anaconda -t ${CROSSBOW_SCIENTIFIC_PYTHON_UPLOAD_TOKEN} upload --force -u scientific-python-nightly-wheels --label main arrow/python/repaired_wheels/*.whl
        env:
          CROSSBOW_SCIENTIFIC_PYTHON_UPLOAD_TOKEN: ${{ secrets.CROSSBOW_SCIENTIFIC_PYTHON_UPLOAD_TOKEN }}


      - name: Push Docker images
        shell: bash
        run: |
          archery docker push python-wheel-manylinux-2014
          archery docker push ${TEST_IMAGE_PREFIX}-wheel-manylinux-test-unittests