diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000000000..8d54979502430 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,74 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Release + +on: + push: + tags: + # Trigger workflow when a tag whose name matches the pattern + # pattern "apache-arrow-{MAJOR}.{MINOR}.{PATCH}" is pushed. + - "apache-arrow-[0-9]+.[0-9]+.[0-9]+" + +permissions: + contents: write + +env: + GH_TOKEN: ${{ github.token }} + +jobs: + publish: + name: Publish + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Get Tag Name of Latest Release Candidate + run: | + rc_tag=$(gh release list --repo apache/arrow | \ + cut -f3 | \ + grep -F "${GITHUB_REF_NAME}-rc" | \ + head -n1) + echo "Latest Release Candidate Tag: ${rc_tag}" + echo "RELEASE_CANDIDATE_TAG_NAME=${rc_tag}" >> ${GITHUB_ENV} + - name: Store Version and Release Candidate Number + run: | + version_with_rc=${RELEASE_CANDIDATE_TAG_NAME#apache-arrow-} + version=${version_with_rc%-rc*} + rc_num=${version_with_rc#${version}-rc} + echo "VERSION_WITH_RC=${version_with_rc}" >> ${GITHUB_ENV} + echo "VERSION=${version}" >> ${GITHUB_ENV} + echo "RC_NUM=${rc_num}" >> ${GITHUB_ENV} + - name: Download Release Candidate Artifacts + run: | + mkdir release_candidate_artifacts + gh release download ${RELEASE_CANDIDATE_TAG_NAME} --repo apache/arrow --dir release_candidate_artifacts + - name: Create Release Title + run: | + title="Apache Arrow ${VERSION}" + echo "RELEASE_TITLE=${title}" >> ${GITHUB_ENV} + # Set the release notes to "TODO" temporarily. After the release notes page + # (https://arrow.apache.org/release/{VERSION}.html) is published, use + # gh release edit to update the release notes to refer to the newly + # pushed web page. See dev/post/post-05-update-gh-release-notes.sh + - name: Create GitHub Release + run: | + gh release create ${GITHUB_REF_NAME} \ + --repo apache/arrow \ + --verify-tag \ + --title "${RELEASE_TITLE}" \ + --notes "TODO" \ + release_candidate_artifacts/* \ No newline at end of file diff --git a/.github/workflows/release_candidate.yml b/.github/workflows/release_candidate.yml new file mode 100644 index 0000000000000..ec732f0eb33e0 --- /dev/null +++ b/.github/workflows/release_candidate.yml @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Release + +on: + push: + tags: + # Trigger workflow when a tag whose name matches the pattern + # "apache-arrow-{MAJOR}.{MINOR}.{PATCH}-rc{RC_NUM}" is pushed. + - "apache-arrow-[0-9]+.[0-9]+.[0-9]+-rc[0-9]+" + +permissions: + contents: write + +env: + GH_TOKEN: ${{ github.token }} + +jobs: + publish: + name: Publish + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout Arrow + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Store Version and Release Candidate Number + run: | + version_with_rc=${GITHUB_REF_NAME#apache-arrow-} + version=${version_with_rc%-rc*} + rc_num=${version_with_rc#${version}-rc} + echo "VERSION_WITH_RC=${version_with_rc}" >> ${GITHUB_ENV} + echo "VERSION=${version}" >> ${GITHUB_ENV} + echo "RC_NUM=${rc_num}" >> ${GITHUB_ENV} + - name: Create Release Candidate Title + run: | + title="Apache Arrow ${VERSION} RC${RC_NUM}" + echo "RELEASE_CANDIDATE_TITLE=${title}" >> ${GITHUB_ENV} + - name: Create Release Candidate Notes + run: | + release_notes="Release Candidate: ${VERSION} RC${RC_NUM}" + echo "RELEASE_CANDIDATE_NOTES=${release_notes}" >> ${GITHUB_ENV} + - name: Create Release tarball + run: | + cd dev/release/ && ./utils-create-release-tarball.sh ${VERSION} ${RC_NUM} + echo "RELEASE_TARBALL=apache-arrow-${VERSION}.tar.gz" >> ${GITHUB_ENV} + - name: Create GitHub Release + run: | + gh release create ${GITHUB_REF_NAME} \ + --verify-tag \ + --prerelease \ + --title "${RELEASE_CANDIDATE_TITLE}" \ + --notes "Release Notes: ${RELEASE_CANDIDATE_NOTES}" \ + dev/release/${RELEASE_TARBALL} diff --git a/dev/release/01-prepare.sh b/dev/release/01-prepare.sh index e4c62e6323c23..779348f036f02 100755 --- a/dev/release/01-prepare.sh +++ b/dev/release/01-prepare.sh @@ -33,7 +33,7 @@ next_version=$2 next_version_snapshot="${next_version}-SNAPSHOT" rc_number=$3 -release_tag="apache-arrow-${version}" +release_candidate_tag="apache-arrow-${version}-rc${rc_number}" release_branch="release-${version}" release_candidate_branch="release-${version}-rc${rc_number}" @@ -46,9 +46,9 @@ release_candidate_branch="release-${version}-rc${rc_number}" : ${PREPARE_TAG:=${PREPARE_DEFAULT}} if [ ${PREPARE_TAG} -gt 0 ]; then - if [ $(git tag -l "${release_tag}") ]; then - echo "Delete existing git tag $release_tag" - git tag -d "${release_tag}" + if [ $(git tag -l "${release_candidate_tag}") ]; then + echo "Delete existing git tag $release_candidate_tag" + git tag -d "${release_candidate_tag}" fi fi @@ -88,7 +88,7 @@ if [ ${PREPARE_LINUX_PACKAGES} -gt 0 ]; then fi if [ ${PREPARE_VERSION_PRE_TAG} -gt 0 ]; then - echo "Prepare release ${version} on tag ${release_tag} then reset to version ${next_version_snapshot}" + echo "Prepare release ${version} on tag ${release_candidate_tag} then reset to version ${next_version_snapshot}" update_versions "${version}" "${next_version}" "release" git commit -m "MINOR: [Release] Update versions for ${version}" @@ -97,5 +97,5 @@ fi ############################## Tag the Release ############################## if [ ${PREPARE_TAG} -gt 0 ]; then - git tag -a "${release_tag}" -m "[Release] Apache Arrow Release ${version}" + git tag -a "${release_candidate_tag}" -m "[Release] Apache Arrow Release ${version} RC${rc_number}" fi diff --git a/dev/release/02-source-test.rb b/dev/release/02-source-test.rb index 149a2b27ac94a..eab95c798f284 100644 --- a/dev/release/02-source-test.rb +++ b/dev/release/02-source-test.rb @@ -22,8 +22,10 @@ class SourceTest < Test::Unit::TestCase def setup @current_commit = git_current_commit detect_versions - @tag_name = "apache-arrow-#{@release_version}" + @tag_name = "apache-arrow-#{@release_version}-rc0" + @archive_name = "apache-arrow-#{@release_version}.tar.gz" @script = File.expand_path("dev/release/02-source.sh") + @tarball_script = File.expand_path("dev/release/utils-create-release-tarball.sh") Dir.mktmpdir do |dir| Dir.chdir(dir) do @@ -40,8 +42,9 @@ def source(*targets) targets.each do |target| env["SOURCE_#{target}"] = "1" end + sh(env, @tarball_script, @release_version, "0") output = sh(env, @script, @release_version, "0") - sh("tar", "xf", "#{@tag_name}.tar.gz") + sh("tar", "xf", @archive_name) output end diff --git a/dev/release/02-source.sh b/dev/release/02-source.sh index 1bd3c0e19e04e..b3eae212212ab 100755 --- a/dev/release/02-source.sh +++ b/dev/release/02-source.sh @@ -21,6 +21,7 @@ set -eu : ${SOURCE_DEFAULT:=1} +: ${SOURCE_DOWNLOAD:=${SOURCE_DEFAULT}} : ${SOURCE_RAT:=${SOURCE_DEFAULT}} : ${SOURCE_UPLOAD:=${SOURCE_DEFAULT}} : ${SOURCE_PR:=${SOURCE_DEFAULT}} @@ -37,11 +38,10 @@ fi version=$1 rc=$2 -tag=apache-arrow-${version} +tag=apache-arrow-${version}-rc${rc} maint_branch=maint-${version} rc_branch="release-${version}-rc${rc}" -tagrc=${tag}-rc${rc} -rc_url="https://dist.apache.org/repos/dist/dev/arrow/${tagrc}" +rc_url="https://dist.apache.org/repos/dist/dev/arrow/${tag}" echo "Preparing source for tag ${tag}" @@ -56,35 +56,19 @@ fi echo "Using commit $release_hash" -tarball=${tag}.tar.gz - -rm -rf ${tag} -# be conservative and use the release hash, even though git produces the same -# archive (identical hashes) using the scm tag -(cd "${SOURCE_TOP_DIR}" && \ - git archive ${release_hash} --prefix ${tag}/) | \ - tar xf - - -# Resolve all hard and symbolic links. -# If we change this, we must change ArrowSources.archive in -# dev/archery/archery/utils/source.py too. -rm -rf ${tag}.tmp -mv ${tag} ${tag}.tmp -cp -R -L ${tag}.tmp ${tag} -rm -rf ${tag}.tmp - -# Create a dummy .git/ directory to download the source files from GitHub with Source Link in C#. -dummy_git=${tag}/csharp/dummy.git -mkdir ${dummy_git} -pushd ${dummy_git} -echo ${release_hash} > HEAD -echo '[remote "origin"] url = https://github.com/apache/arrow.git' >> config -mkdir objects refs -popd - -# Create new tarball from modified source directory -tar czf ${tarball} ${tag} -rm -rf ${tag} +tarball=apache-arrow-${version}.tar.gz + +if [ ${SOURCE_DOWNLOAD} -gt 0 ]; then + # Wait for the release candidate workflow to finish before attempting + # to download the tarball from the GitHub Release. + . $SOURCE_DIR/utils-watch-gh-workflow.sh ${tag} "release_candidate.yml" + rm -f ${tarball} + gh release download \ + ${tag} \ + --repo apache/arrow \ + --dir . \ + --pattern "${tarball}" +fi if [ ${SOURCE_RAT} -gt 0 ]; then "${SOURCE_DIR}/run-rat.sh" ${tarball} @@ -105,18 +89,21 @@ if [ ${SOURCE_UPLOAD} -gt 0 ]; then ${sha256_generate} $tarball > ${tarball}.sha256 ${sha512_generate} $tarball > ${tarball}.sha512 + # Upload signed tarballs to GitHub Release + gh release upload ${tag} ${tarball}.sha256 ${tarball}.sha512 + # check out the arrow RC folder svn co --depth=empty https://dist.apache.org/repos/dist/dev/arrow tmp # add the release candidate for the tag - mkdir -p tmp/${tagrc} + mkdir -p tmp/${tag} # copy the rc tarball into the tmp dir - cp ${tarball}* tmp/${tagrc} + cp ${tarball}* tmp/${tag} # commit to svn - svn add tmp/${tagrc} - svn ci -m "Apache Arrow ${version} RC${rc}" tmp/${tagrc} + svn add tmp/${tag} + svn ci -m "Apache Arrow ${version} RC${rc}" tmp/${tag} # clean up rm -rf tmp diff --git a/dev/release/03-binary-submit.sh b/dev/release/03-binary-submit.sh index d65b4e97c5fc2..0dc63aa7089d1 100755 --- a/dev/release/03-binary-submit.sh +++ b/dev/release/03-binary-submit.sh @@ -28,7 +28,7 @@ version=$1 rc=$2 version_with_rc="${version}-rc${rc}" crossbow_job_prefix="release-${version_with_rc}" -release_tag="apache-arrow-${version}" +release_tag="apache-arrow-${version}-rc${rc}" rc_branch="release-${version_with_rc}" : ${ARROW_REPOSITORY:="apache/arrow"} diff --git a/dev/release/07-matlab-upload.sh b/dev/release/07-matlab-upload.sh new file mode 100755 index 0000000000000..803ceadb35eaf --- /dev/null +++ b/dev/release/07-matlab-upload.sh @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -u +set -o pipefail + +SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +if [ $# -ne 2 ]; then + echo "Usage: $0 " + exit +fi + +version=$1 +rc=$2 + +: ${UPLOAD_DEFAULT=1} +: ${UPLOAD_FORCE_SIGN=${UPLOAD_DEFAULT}} + +if [ ${UPLOAD_FORCE_SIGN} -gt 0 ]; then + pushd "${SOURCE_DIR}" + if [ ! -f .env ]; then + echo "You must create $(pwd)/.env" + echo "You can use $(pwd)/.env.example as template" + exit 1 + fi + . .env + popd +fi + +version_with_rc="${version}-rc${rc}" +crossbow_job_prefix="release-${version_with_rc}" +crossbow_package_dir="${SOURCE_DIR}/../../packages" + +: ${CROSSBOW_JOB_NUMBER:="0"} +: ${CROSSBOW_JOB_ID:="${crossbow_job_prefix}-${CROSSBOW_JOB_NUMBER}"} +: ${ARROW_ARTIFACTS_DIR:="${crossbow_package_dir}/${CROSSBOW_JOB_ID}/matlab"} + +if [ ! -e "${ARROW_ARTIFACTS_DIR}" ]; then + echo "${ARROW_ARTIFACTS_DIR} does not exist" + exit 1 +fi + +if [ ! -d "${ARROW_ARTIFACTS_DIR}" ]; then + echo "${ARROW_ARTIFACTS_DIR} is not a directory" + exit 1 +fi + +pushd "${ARROW_ARTIFACTS_DIR}" + +if [ ${UPLOAD_FORCE_SIGN} -gt 0 ]; then + # Upload the MATLAB MLTBX Release Candidate to the GitHub Releases + # area of the Apache Arrow GitHub project. + mltbx_file="matlab-arrow-${version}.mltbx" + mltbx_signature_gpg_ascii_armor="${mltbx_file}.asc" + mltbx_checksum_sha512="${mltbx_file}.sha512" + + rm -rf ${mltbx_signature_gpg_ascii_armor} + rm -rf ${mltbx_checksum_sha512} + + # Sign the MLTBX file and create a detached (--deatch-sign) ASCII armor (--armor) GPG signature file. + gpg --detach-sign --local-user "${GPG_KEY_ID}" --armor ${mltbx_file} + + # Compute the SHA512 checksum of the MLTBX file. + shasum --algorithm 512 ${mltbx_file} > ${mltbx_checksum_sha512} +fi + +tag="apache-arrow-${version_with_rc}" +gh release upload ${tag} \ + --clobber \ + --repo apache/arrow \ + * + +popd diff --git a/dev/release/07-binary-verify.sh b/dev/release/08-binary-verify.sh similarity index 100% rename from dev/release/07-binary-verify.sh rename to dev/release/08-binary-verify.sh diff --git a/dev/release/post-01-tag.sh b/dev/release/post-01-tag.sh new file mode 100755 index 0000000000000..df0f6756c0e7c --- /dev/null +++ b/dev/release/post-01-tag.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -o pipefail + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + exit +fi + +# Create the release tag and trigger the Publish Release workflow. +release_candidate_tag=apache-arrow-${version}-rc${num} +release_tag=apache-arrow-${version} +git tag -a ${release_tag} ${release_candidate_tag}^{} -m "[Release] Apache Arrow Release ${version}" +git push apache ${release_tag} diff --git a/dev/release/post-01-upload.sh b/dev/release/post-02-upload.sh similarity index 100% rename from dev/release/post-01-upload.sh rename to dev/release/post-02-upload.sh diff --git a/dev/release/post-02-binary.sh b/dev/release/post-03-binary.sh similarity index 100% rename from dev/release/post-02-binary.sh rename to dev/release/post-03-binary.sh diff --git a/dev/release/post-03-website.sh b/dev/release/post-04-website.sh similarity index 100% rename from dev/release/post-03-website.sh rename to dev/release/post-04-website.sh diff --git a/dev/release/post-05-update-gh-release-notes.sh b/dev/release/post-05-update-gh-release-notes.sh new file mode 100755 index 0000000000000..e321c98721c17 --- /dev/null +++ b/dev/release/post-05-update-gh-release-notes.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -o pipefail + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + + +VERSION=$1 +REPOSITORY="apache/arrow" +TAG="apache-arrow-${VERSION}" +WORKFLOW="release.yml" +SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +# Wait for the GitHub Workflow that creates the GitHub Release +# to finish before updating the release notes. +. $SOURCE_DIR/utils-watch-gh-workflow.sh ${TAG} ${WORKFLOW} + +# Update the Release Notes section +RELEASE_NOTES_URL="https://arrow.apache.org/release/${VERSION}.html" +RELEASE_NOTES="Release Notes URL: ${RELEASE_NOTES_URL}" +gh release edit ${TAG} --repo ${REPOSITORY} --notes "${RELEASE_NOTES}" --verify-tag diff --git a/dev/release/post-04-ruby.sh b/dev/release/post-06-ruby.sh similarity index 100% rename from dev/release/post-04-ruby.sh rename to dev/release/post-06-ruby.sh diff --git a/dev/release/post-05-js.sh b/dev/release/post-07-js.sh similarity index 100% rename from dev/release/post-05-js.sh rename to dev/release/post-07-js.sh diff --git a/dev/release/post-06-csharp.sh b/dev/release/post-08-csharp.sh similarity index 100% rename from dev/release/post-06-csharp.sh rename to dev/release/post-08-csharp.sh diff --git a/dev/release/post-07-remove-old-artifacts.sh b/dev/release/post-09-remove-old-artifacts.sh similarity index 100% rename from dev/release/post-07-remove-old-artifacts.sh rename to dev/release/post-09-remove-old-artifacts.sh diff --git a/dev/release/post-08-docs.sh b/dev/release/post-10-docs.sh similarity index 100% rename from dev/release/post-08-docs.sh rename to dev/release/post-10-docs.sh diff --git a/dev/release/post-09-python.sh b/dev/release/post-11-python.sh similarity index 100% rename from dev/release/post-09-python.sh rename to dev/release/post-11-python.sh diff --git a/dev/release/post-11-bump-versions-test.rb b/dev/release/post-12-bump-versions-test.rb similarity index 99% rename from dev/release/post-11-bump-versions-test.rb rename to dev/release/post-12-bump-versions-test.rb index 8ad404ef33202..2bd14587461cc 100644 --- a/dev/release/post-11-bump-versions-test.rb +++ b/dev/release/post-12-bump-versions-test.rb @@ -63,12 +63,12 @@ def bump_versions(*targets) previous_version_components[2].succ! end sh(env, - "dev/release/post-11-bump-versions.sh", + "dev/release/post-12-bump-versions.sh", previous_version_components.join("."), @release_version) else sh(env, - "dev/release/post-11-bump-versions.sh", + "dev/release/post-12-bump-versions.sh", @release_version, @next_version) end diff --git a/dev/release/post-11-bump-versions.sh b/dev/release/post-12-bump-versions.sh similarity index 100% rename from dev/release/post-11-bump-versions.sh rename to dev/release/post-12-bump-versions.sh diff --git a/dev/release/post-10-go.sh b/dev/release/post-13-go.sh similarity index 100% rename from dev/release/post-10-go.sh rename to dev/release/post-13-go.sh diff --git a/dev/release/post-12-msys2.sh b/dev/release/post-14-msys2.sh similarity index 100% rename from dev/release/post-12-msys2.sh rename to dev/release/post-14-msys2.sh diff --git a/dev/release/post-13-homebrew.sh b/dev/release/post-15-homebrew.sh similarity index 100% rename from dev/release/post-13-homebrew.sh rename to dev/release/post-15-homebrew.sh diff --git a/dev/release/post-14-vcpkg.sh b/dev/release/post-16-vcpkg.sh similarity index 100% rename from dev/release/post-14-vcpkg.sh rename to dev/release/post-16-vcpkg.sh diff --git a/dev/release/post-15-conan.sh b/dev/release/post-17-conan.sh similarity index 100% rename from dev/release/post-15-conan.sh rename to dev/release/post-17-conan.sh diff --git a/dev/release/utils-create-release-tarball.sh b/dev/release/utils-create-release-tarball.sh new file mode 100755 index 0000000000000..1a0ba83639b9a --- /dev/null +++ b/dev/release/utils-create-release-tarball.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)" + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + exit +fi + +version=$1 +rc=$2 + +tag=apache-arrow-${version}-rc${rc} +tarball=apache-arrow-${version}.tar.gz + +: ${release_hash:=$(git rev-list --max-count=1 ${tag})} + +rm -rf ${tag} + +# be conservative and use the release hash, even though git produces the same +# archive (identical hashes) using the scm tag +(cd "${SOURCE_TOP_DIR}" && \ + git archive ${release_hash} --prefix ${tag}/) | \ + tar xf - + +# Resolve symbolic and hard links +rm -rf ${tag}.tmp +mv ${tag} ${tag}.tmp +cp -R -L ${tag}.tmp ${tag} +rm -rf ${tag}.tmp + +# Create a dummy .git/ directory to download the source files from GitHub with Source Link in C#. +dummy_git=${tag}/csharp/dummy.git +mkdir ${dummy_git} +pushd ${dummy_git} +echo ${release_hash} > HEAD +echo '[remote "origin"] url = https://github.com/apache/arrow.git' >> config +mkdir objects refs +popd + +# Create new tarball from modified source directory +tar czf ${tarball} ${tag} +rm -rf ${tag} diff --git a/dev/release/utils-watch-gh-workflow.sh b/dev/release/utils-watch-gh-workflow.sh new file mode 100755 index 0000000000000..c0bab40df0d76 --- /dev/null +++ b/dev/release/utils-watch-gh-workflow.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -o pipefail + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + exit 1 +fi + +TAG=$1 +WORKFLOW=$2 +REPOSITORY="apache/arrow" + +echo "Looking for GitHub Actions workflow on ${REPOSITORY}:${TAG}" +RUN_ID="" +while [[ -z "${RUN_ID}" ]] +do + echo "Waiting for run to start..." + RUN_ID=$(gh run list \ + --repo "${REPOSITORY}" \ + --workflow="${WORKFLOW}" \ + --json 'databaseId,event,headBranch,status' \ + --jq ".[] | select(.event == \"push\" and .headBranch == \"${TAG}\") | .databaseId") + sleep 1 + done + +echo "Found GitHub Actions workflow with ID: ${RUN_ID}" +gh run watch --repo "${REPOSITORY}" --exit-status ${RUN_ID} \ No newline at end of file diff --git a/docs/source/developers/release.rst b/docs/source/developers/release.rst index d903cc71bd5c4..fae48e4d8e0f0 100644 --- a/docs/source/developers/release.rst +++ b/docs/source/developers/release.rst @@ -189,8 +189,8 @@ Create the Release Candidate branch from the updated maintenance branch # so for the first RC this would be: dev/release/01-prepare.sh 4.0.0 5.0.0 0 dev/release/01-prepare.sh - # Push the release tag (for RC1 or later the --force flag is required) - git push -u apache apache-arrow- + # Push the release candidate tag + git push -u apache apache-arrow-rc # Push the release candidate branch in order to trigger verification jobs later git push -u apache release--rc @@ -200,6 +200,8 @@ Build source and binaries and submit them .. code-block:: # Build the source release tarball and create Pull Request with verification tasks + # + # NOTE: You need to have GitHub CLI installed to run this script. dev/release/02-source.sh # Submit binary tasks using crossbow, the command will output the crossbow build id @@ -225,8 +227,13 @@ Build source and binaries and submit them # https://repository.apache.org/#stagingRepositories dev/release/06-java-upload.sh + # Sign and upload MATLAB artifacts to the GitHub Releases area. + # + # Note that you need to have GitHub CLI installed to run this script. + dev/release/07-matlab-upload.sh + # Start verifications for binaries and wheels - dev/release/07-binary-verify.sh + dev/release/08-binary-verify.sh Verify the Release ------------------ @@ -257,9 +264,11 @@ Be sure to go through on the following checklist: #. Start the new version on JIRA for the related CPP PARQUET version #. Merge changes on release branch to maintenance branch for patch releases #. Add the new release to the Apache Reporter System +#. Push release tag #. Upload source #. Upload binaries #. Update website +#. Update GitHub Release Notes #. Update Homebrew packages #. Update MSYS2 package #. Upload RubyGems @@ -325,6 +334,18 @@ Be sure to go through on the following checklist: Add relevant release data for Arrow to `Apache reporter `_. +.. dropdown:: Push release tag and create GitHub Release + :animate: fade-in-slide-down + :class-title: sd-fs-5 + :class-container: sd-shadow-md + + A committer must push the release tag to GitHub: + + .. code-block:: Bash + + # dev/release/post-01-tag.sh 0.1.0 0 + dev/release/post-01-tag.sh + .. dropdown:: Upload source release artifacts to Subversion :animate: fade-in-slide-down :class-title: sd-fs-5 @@ -334,20 +355,20 @@ Be sure to go through on the following checklist: .. code-block:: Bash - # dev/release/post-01-upload.sh 0.1.0 0 - dev/release/post-01-upload.sh + # dev/release/post-02-upload.sh 0.1.0 0 + dev/release/post-02-upload.sh .. dropdown:: Upload binary release artifacts to Artifactory :animate: fade-in-slide-down :class-title: sd-fs-5 :class-container: sd-shadow-md - A committer must upload the binary release artifacts to Artifactory: + A committer must upload the binary release artifacts to Artifactory and create the GitHub Release: .. code-block:: Bash - # dev/release/post-02-binary.sh 0.1.0 0 - dev/release/post-02-binary.sh + # dev/release/post-03-binary.sh 0.1.0 0 + dev/release/post-03-binary.sh .. dropdown:: Update website :animate: fade-in-slide-down @@ -369,11 +390,23 @@ Be sure to go through on the following checklist: ## Generate a release note for the new version, update the ## latest release information automatically. - # dev/release/post-03-website.sh 9.0.0 10.0.0 - dev/release/post-03-website.sh OLD_X.OLD_Y.OLD_Z X.Y.Z + # dev/release/post-04-website.sh 9.0.0 10.0.0 + dev/release/post-04-website.sh OLD_X.OLD_Y.OLD_Z X.Y.Z This script pushes a ``release-note-X.Y.Z`` branch to your ``apache/arrow-site`` fork. You need to open a pull request from the ``release-note-X.Y.Z`` branch on your Web browser. +.. dropdown:: Update Release Notes in apache/arrow GitHub Release + :animate: fade-in-slide-down + :class-title: sd-fs-5 + :class-container: sd-shadow-md + + A committer must run the following script: + + .. code-block:: Bash + + # dev/release/post-05-update-gh-release-notes.sh 17.0.0 + dev/release/post-05-update-gh-release-notes.sh apache-arrow-X.Y.Z + .. dropdown:: Update Homebrew packages :animate: fade-in-slide-down :class-title: sd-fs-5 @@ -395,8 +428,8 @@ Be sure to go through on the following checklist: git remote add git@github.com:/homebrew-core.git cd - - # dev/release/post-13-homebrew.sh 10.0.0 kou - dev/release/post-13-homebrew.sh X.Y.Z + # dev/release/post-15-homebrew.sh 10.0.0 kou + dev/release/post-15-homebrew.sh X.Y.Z This script pushes a ``apache-arrow-X.Y.Z`` branch to your ``Homebrew/homebrew-core`` fork. You need to create a pull request from the ``apache-arrow-X.Y.Z`` branch with ``apache-arrow, apache-arrow-glib: X.Y.Z`` title on your Web browser. @@ -421,8 +454,8 @@ Be sure to go through on the following checklist: git remote add upstream https://github.com/msys2/MINGW-packages.git cd - - # dev/release/post-12-msys2.sh 10.0.0 ../MINGW-packages - dev/release/post-12-msys2.sh X.Y.Z + # dev/release/post-14-msys2.sh 10.0.0 ../MINGW-packages + dev/release/post-14-msys2.sh X.Y.Z This script pushes a ``arrow-X.Y.Z`` branch to your ``msys2/MINGW-packages`` fork. You need to create a pull request from the ``arrow-X.Y.Z`` branch with ``arrow: Update to X.Y.Z`` title on your Web browser. @@ -446,8 +479,8 @@ Be sure to go through on the following checklist: .. code-block:: Bash - # dev/release/post-04-ruby.sh 10.0.0 - dev/release/post-04-ruby.sh X.Y.Z + # dev/release/post-06-ruby.sh 10.0.0 + dev/release/post-06-ruby.sh X.Y.Z .. dropdown:: Update JavaScript packages :animate: fade-in-slide-down @@ -465,8 +498,8 @@ Be sure to go through on the following checklist: # Login to npmjs.com (You need to do this only for the first time) npm login --registry=https://registry.yarnpkg.com/ - # dev/release/post-05-js.sh 10.0.0 - dev/release/post-05-js.sh X.Y.Z + # dev/release/post-07-js.sh 10.0.0 + dev/release/post-07-js.sh X.Y.Z .. dropdown:: Update C# packages :animate: fade-in-slide-down @@ -481,8 +514,8 @@ Be sure to go through on the following checklist: .. code-block:: Bash - # NUGET_API_KEY=YOUR_NUGET_API_KEY dev/release/post-06-csharp.sh 10.0.0 - NUGET_API_KEY= dev/release/post-06-csharp.sh X.Y.Z + # NUGET_API_KEY=YOUR_NUGET_API_KEY dev/release/post-08-csharp.sh 10.0.0 + NUGET_API_KEY= dev/release/post-08-csharp.sh X.Y.Z .. dropdown:: Upload wheels/sdist to PyPI :animate: fade-in-slide-down @@ -495,8 +528,8 @@ Be sure to go through on the following checklist: .. code-block:: Bash - # dev/release/post-09-python.sh 10.0.0 - dev/release/post-09-python.sh + # dev/release/post-11-python.sh 10.0.0 + dev/release/post-11-python.sh .. dropdown:: Publish Maven packages :animate: fade-in-slide-down @@ -568,8 +601,8 @@ Be sure to go through on the following checklist: git remote add upstream https://github.com/microsoft/vcpkg.git cd - - # dev/release/post-14-vcpkg.sh 10.0.0 ../vcpkg - dev/release/post-14-vcpkg.sh X.Y.Z + # dev/release/post-16-vcpkg.sh 10.0.0 ../vcpkg + dev/release/post-16-vcpkg.sh X.Y.Z This script pushes a ``arrow-X.Y.Z`` branch to your ``microsoft/vcpkg`` fork. You need to create a pull request from the ``arrow-X.Y.Z`` branch with ``[arrow] Update to X.Y.Z`` title on your Web browser. @@ -594,8 +627,8 @@ Be sure to go through on the following checklist: git remote add upstream https://github.com/conan-io/conan-center-index.git cd - - # dev/release/post-15-conan.sh 10.0.1 ../conan-center-index - dev/release/post-15-conan.sh X.Y.Z + # dev/release/post-17-conan.sh 10.0.1 ../conan-center-index + dev/release/post-17-conan.sh X.Y.Z This script pushes a ``arrow-X.Y.Z`` branch to your ``conan-io/conan-center-index`` fork. You need to create a pull request from the ``arrow-X.Y.Z`` branch on your Web browser. @@ -609,8 +642,8 @@ Be sure to go through on the following checklist: # You can run the script with BUMP_TAG=0 and BUMP_PUSH=0 # this will avoid default pushing to main and pushing the tag # but you will require to push manually after reviewing the commits. - # dev/release/post-11-bump-versions.sh 10.0.0 11.0.0 - dev/release/post-11-bump-versions.sh X.Y.Z NEXT_X.NEXT_Y.NEXT_Z + # dev/release/post-12-bump-versions.sh 10.0.0 11.0.0 + dev/release/post-12-bump-versions.sh X.Y.Z NEXT_X.NEXT_Y.NEXT_Z .. dropdown:: Update tags for Go modules :animate: fade-in-slide-down @@ -619,8 +652,8 @@ Be sure to go through on the following checklist: .. code-block:: Bash - # dev/release/post-10-go.sh 10.0.0 - dev/release/post-10-go.sh X.Y.Z + # dev/release/post-13-go.sh 10.0.0 + dev/release/post-13-go.sh X.Y.Z .. dropdown:: Update docs :animate: fade-in-slide-down @@ -640,8 +673,8 @@ Be sure to go through on the following checklist: git remote add apache git@github.com:apache/arrow-site.git cd - - # dev/release/post-08-docs.sh 10.0.0 9.0.0 - dev/release/post-08-docs.sh X.Y.Z PREVIOUS_X.PREVIOUS_Y.PREVIOUS_Z + # dev/release/post-10-docs.sh 10.0.0 9.0.0 + dev/release/post-10-docs.sh X.Y.Z PREVIOUS_X.PREVIOUS_Y.PREVIOUS_Z This script pushes a ``release-docs-X.Y.Z`` branch to your ``arrow-site`` fork. You need to create a Pull Request and use the ``asf-site`` branch as base for it. @@ -686,4 +719,4 @@ Be sure to go through on the following checklist: .. code-block:: Bash - dev/release/post-07-remove-old-artifacts.sh + dev/release/post-09-remove-old-artifacts.sh diff --git a/matlab/tools/packageMatlabInterface.m b/matlab/tools/packageMatlabInterface.m index 3d970002614ab..5c82763b25f8a 100644 --- a/matlab/tools/packageMatlabInterface.m +++ b/matlab/tools/packageMatlabInterface.m @@ -29,15 +29,10 @@ disp("Output Folder: " + outputFolder); disp("Toolbox Version Raw: " + toolboxVersionRaw); - -% Note: This string processing heuristic may not be robust to future -% changes in the Arrow versioning scheme. -dotIdx = strfind(toolboxVersionRaw, "."); -numDots = numel(dotIdx); -if numDots >= 3 - toolboxVersion = extractBefore(toolboxVersionRaw, dotIdx(3)); -else - toolboxVersion = toolboxVersionRaw; +versionPattern = regexpPattern("^[0-9]+\.[0-9]+\.[0-9]+"); +toolboxVersion = extract(toolboxVersionRaw, versionPattern); +if isempty(toolboxVersion) + error("Unable to extract MAJOR.MINOR.PATCH version string from " + toolboxVersionRaw); end disp("Toolbox Version:" + toolboxVersion); @@ -68,7 +63,7 @@ opts.MinimumMatlabRelease = currentRelease; opts.MaximumMatlabRelease = currentRelease; -opts.OutputFile = fullfile(outputFolder, compose("matlab-arrow-%s.mltbx", toolboxVersionRaw)); +opts.OutputFile = fullfile(outputFolder, compose("matlab-arrow-%s.mltbx", toolboxVersion)); disp("Output File: " + opts.OutputFile); matlab.addons.toolbox.packageToolbox(opts);