r_nightly.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

name: Upload R Nightly builds
# This workflow downloads the (nightly) binaries created in crossbow and uploads them
# to nightlies.apache.org. Due to authorization requirements, this upload can't be done 
# from the crossbow repository.

on:
  workflow_dispatch:
    inputs:
      prefix:
        description: Job prefix to use.
        required: false
        default: ''
      keep:
        description: Number of versions to keep.
        required: false
        default: 14

  schedule:
    #Crossbow packaging runs at 0 8 * * *
    - cron: '0 14 * * *'

permissions:
  contents: read

jobs:
  upload:
    if: github.repository == 'apache/arrow'
    runs-on: ubuntu-latest
    steps:
      - name: Checkout Arrow
        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
        with:
          fetch-depth: 1
          path: arrow
          repository: apache/arrow
          ref: main
          submodules: recursive
      - name: Checkout Crossbow
        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
        with:
          fetch-depth: 0
          path: crossbow
          repository: ursacomputing/crossbow
          ref: main
      - name: Set up Python
        uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
        with:
          cache: 'pip'
          python-version: 3.8
      - name: Install Archery
        shell: bash
        run: pip install -e arrow/dev/archery[all]
      - run: mkdir -p binaries
      - name: Download Artifacts
        env:
          PREFIX: ${{ github.event.inputs.prefix || ''}}
        run: |
          if [ -z $PREFIX ]; then
            PREFIX=nightly-packaging-$(date +%Y-%m-%d)-0
          fi
          echo $PREFIX

          archery crossbow download-artifacts -f r-binary-packages -t binaries $PREFIX

          if [ -n "$(ls -A binaries/*/*/)" ]; then
            echo "Found files!"
          else
            echo "No files found. Stopping upload."
            exit 1
          fi
      - name: Cache Repo
        uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
        with:
          path: repo
          key: r-nightly-${{ github.run_id }}
          restore-keys: r-nightly-
      - name: Sync from Remote
        uses: ./arrow/.github/actions/sync-nightlies
        with:
          switches: -avzh --update --delete --progress
          local_path: repo
          remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/arrow/r
          remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }}
          remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }}
          remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }}
          remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }}
          remote_host_key: ${{ secrets.NIGHTLIES_RSYNC_HOST_KEY }}
      - run: tree repo
      - name: Build Repository
        shell: Rscript {0}
        run: |
          # folder that we sync to nightlies.apache.org
          repo_root <- "repo"
          # The binaries are in a nested dir
          # so we need to find the correct path.
          art_path <- list.files("binaries",
            recursive = TRUE,
            include.dirs = TRUE,
            pattern = "r-binary-packages$",
            full.names = TRUE
          )

          current_path <- list.files(art_path, full.names = TRUE, recursive = TRUE)
          files <- sub("r-(pkg|lib)", repo_root, current_path)

          # decode contrib.url from artifact name:
          # bin__windows__contrib__4.1 -> bin/windows/contrib/4.1
          new_paths <- gsub("__", "/", files)
          # strip superfluous nested dirs
          new_paths <- sub(art_path, ".", new_paths)
          dirs <- dirname(new_paths)
          sapply(dirs, dir.create, recursive = TRUE, showWarnings = FALSE)

          # overwrite allows us to "force push" a new version with the same name
          copy_result <- file.copy(current_path, new_paths, overwrite = TRUE)

          if (!all(copy_result)) {
            stop("There was an issue while copying the files!")
          }
      - name: Prune Repository
        shell: bash
        env:
          KEEP: ${{ github.event.inputs.keep || 14 }}
        run: |   
          prune() {
            # list files  | retain $KEEP newest files | delete everything else
            ls -t $1/arrow* | tail -n +$((KEEP + 1)) | xargs --no-run-if-empty rm 
          }

          # find leaf sub dirs
          repo_dirs=$(find repo -type d -links 2)

          # We want to retain $keep (14) versions of each pkg/lib so we call
          # prune on each leaf dir and not on repo/.
          for dir in ${repo_dirs[@]}; do
            prune $dir
          done
      - name: Update Repository Index
        shell: Rscript {0}
        run: |
          # folder that we sync to nightlies.apache.org
          repo_root <- "repo"
          tools::write_PACKAGES(file.path(repo_root, "src/contrib"),
            type = "source",
            verbose = TRUE,
            latestOnly = FALSE
          )

          repo_dirs <- list.dirs(repo_root)
          # find dirs with binary R packages: e.g. */contrib/4.1
          pkg_dirs <- grep(".+contrib\\/\\d.+", repo_dirs, value = TRUE)


          for (dir in pkg_dirs) {
            on_win <- grepl("windows", dir)
            tools::write_PACKAGES(dir,
              type = ifelse(on_win, "win.binary", "mac.binary"),
              verbose = TRUE,
              latestOnly = FALSE
            )
          }
      - name: Show repo contents
        run: tree repo
      - name: Sync to Remote
        uses: ./arrow/.github/actions/sync-nightlies
        with:
          upload: true
          switches: -avzh --update --delete --progress
          local_path: repo
          remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/arrow/r
          remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }}
          remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }}
          remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }}
          remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }}
          remote_host_key: ${{ secrets.NIGHTLIES_RSYNC_HOST_KEY }}