build_release_candidate.sh

#!/bin/bash
#
#    Licensed to the Apache Software Foundation (ASF) under one or more
#    contributor license agreements.  See the NOTICE file distributed with
#    this work for additional information regarding copyright ownership.
#    The ASF licenses this file to You under the Apache License, Version 2.0
#    (the "License"); you may not use this file except in compliance with
#    the License.  You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.
#

# This script will create a Release Candidate, includes:
# 1. Build and stage java artifacts
# 2. Stage source release on dist.apache.org
# 3. Stage python source distribution and wheels on dist.apache.org
# 4. Stage SDK docker images
# 5. Create a PR to update beam-site

set -e

LOCAL_CLONE_DIR=build_release_candidate
LOCAL_JAVA_STAGING_DIR=java_staging_dir
LOCAL_PYTHON_STAGING_DIR=python_staging_dir
LOCAL_PYTHON_VIRTUALENV=${LOCAL_PYTHON_STAGING_DIR}/venv
LOCAL_WEBSITE_UPDATE_DIR=website_update_dir
LOCAL_PYTHON_DOC=python_doc
LOCAL_JAVA_DOC=java_doc
LOCAL_WEBSITE_REPO=beam_website_repo

USER_REMOTE_URL=
USER_GITHUB_ID=
GIT_REPO_BASE_URL=apache/beam
GIT_REPO_URL=git@github.com:${GIT_REPO_BASE_URL}.git
ROOT_SVN_URL=https://dist.apache.org/repos/dist/dev/beam
GIT_BEAM_ARCHIVE=https://github.com/apache/beam/archive
GIT_BEAM_WEBSITE=https://github.com/apache/beam-site.git

PYTHON_ARTIFACTS_DIR=python
BEAM_ROOT_DIR=beam
WEBSITE_ROOT_DIR=beam-site

DOCKER_IMAGE_DEFAULT_REPO_ROOT=apache
DOCKER_IMAGE_DEFAULT_REPO_PREFIX=beam_

PYTHON_VER=("python2.7" "python3.5" "python3.6" "python3.7" "python3.8")
FLINK_VER=("1.8" "1.9" "1.10")

echo "================Setting Up Environment Variables==========="
echo "Which release version are you working on: "
read RELEASE
RELEASE_BRANCH=release-${RELEASE}
echo "Which release candidate number(e.g. 1) are you going to create: "
read RC_NUM
echo "Please enter your github username(ID): "
read USER_GITHUB_ID

USER_REMOTE_URL=git@github.com:${USER_GITHUB_ID}/beam-site

echo "=================Pre-requirements===================="
echo "Please make sure you have configured and started your gpg by running ./preparation_before_release.sh."
echo "================Listing all GPG keys================="
gpg --list-keys --keyid-format LONG --fingerprint --fingerprint
echo "Please copy the public key which is associated with your Apache account:"

read SIGNING_KEY

echo "================Checking Environment Variables=============="
echo "beam repo will be cloned into: ${LOCAL_CLONE_DIR}"
echo "working on release version: ${RELEASE}"
echo "working on release branch: ${RELEASE_BRANCH}"
echo "will create release candidate: RC${RC_NUM}"
echo "Your forked beam-site URL: ${USER_REMOTE_URL}"
echo "Your signing key: ${SIGNING_KEY}"
echo "Please review all environment variables and confirm: [y|N]"
read confirmation
if [[ $confirmation != "y" ]]; then
  echo "Please rerun this script and make sure you have the right inputs."
  exit
fi

echo "[Current Step]: Build and stage java artifacts"
echo "Do you want to proceed? [y|N]"
read confirmation
if [[ $confirmation = "y" ]]; then
  echo "============Building and Staging Java Artifacts============="
  echo "--------Cloning Beam Repo and Checkout Release Branch-------"
  cd ~
  if [[ -d ${LOCAL_CLONE_DIR} ]]; then
    rm -rf ${LOCAL_CLONE_DIR}
  fi
  mkdir -p ${LOCAL_CLONE_DIR}
  cd ${LOCAL_CLONE_DIR}
  git clone ${GIT_REPO_URL}
  cd ${BEAM_ROOT_DIR}
  git checkout ${RELEASE_BRANCH}
  RELEASE_COMMIT=$(git rev-parse --verify ${RELEASE_BRANCH})

  echo "-------------Building Java Artifacts with Gradle-------------"
  git config credential.helper store

  ./gradlew release -Prelease.newVersion=${RELEASE}-SNAPSHOT \
                -Prelease.releaseVersion=${RELEASE}-RC${RC_NUM} \
                -Prelease.useAutomaticVersion=true --info --no-daemon

  git push origin "${RELEASE_BRANCH}"
  git push origin "v${RELEASE}-RC${RC_NUM}"

  echo "-------------Staging Java Artifacts into Maven---------------"
  gpg --local-user ${SIGNING_KEY} --output /dev/null --sign ~/.bashrc
  ./gradlew publish -Psigning.gnupg.keyName=${SIGNING_KEY} -PisRelease --no-daemon
  echo "You need to close the staging repository manually on Apache Nexus. See the release guide for instructions."
  rm -rf ~/${LOCAL_CLONE_DIR}
fi

echo "[Current Step]: Stage source release on dist.apache.org"
echo "Do you want to proceed? [y|N]"
read confirmation
if [[ $confirmation = "y" ]]; then
  echo "=========Staging Source Release on dist.apache.org==========="
  cd ~
  if [[ -d ${LOCAL_JAVA_STAGING_DIR} ]]; then
    rm -rf ${LOCAL_JAVA_STAGING_DIR}
  fi
  mkdir -p ${LOCAL_JAVA_STAGING_DIR}
  cd ${LOCAL_JAVA_STAGING_DIR}
  svn co ${ROOT_SVN_URL}
  mkdir -p beam/${RELEASE}
  cd beam/${RELEASE}

  echo "----------------Downloading Source Release-------------------"
  SOURCE_RELEASE_ZIP="apache-beam-${RELEASE}-source-release.zip"
  # Check whether there is an existing dist dir
  if (svn ls "${SOURCE_RELEASE_ZIP}"); then
    echo "Removing existing ${SOURCE_RELEASE_ZIP}."
    svn delete "${SOURCE_RELEASE_ZIP}"
  fi

  echo "Downloading: ${GIT_BEAM_ARCHIVE}/release-${RELEASE}.zip"
  wget ${GIT_BEAM_ARCHIVE}/release-${RELEASE}.zip  -O "${SOURCE_RELEASE_ZIP}"

  echo "----Signing Source Release ${SOURCE_RELEASE_ZIP}-----"
  gpg --local-user ${SIGNING_KEY} --armor --detach-sig "${SOURCE_RELEASE_ZIP}"

  echo "----Creating Hash Value for ${SOURCE_RELEASE_ZIP}----"
  sha512sum ${SOURCE_RELEASE_ZIP} > ${SOURCE_RELEASE_ZIP}.sha512

  # The svn commit is interactive already and can be aborted by deleted the commit msg
  svn add --force .
  svn commit --no-auth-cache
  rm -rf ~/${LOCAL_JAVA_STAGING_DIR}
fi


echo "[Current Step]: Stage python source distribution and wheels on dist.apache.org"
echo "Do you want to proceed? [y|N]"
read confirmation
if [[ $confirmation = "y" ]]; then
  echo "============Staging Python Binaries on dist.apache.org========="
  cd ~
  if [[ -d "${LOCAL_PYTHON_STAGING_DIR}" ]]; then
    rm -rf "${LOCAL_PYTHON_STAGING_DIR}"
  fi
  mkdir -p "${LOCAL_PYTHON_STAGING_DIR}"
  cd "${LOCAL_PYTHON_STAGING_DIR}"

  echo '-------------------Cloning Beam Release Branch-----------------'
  git clone "${GIT_REPO_URL}"
  cd "${BEAM_ROOT_DIR}"
  git checkout "${RELEASE_BRANCH}"
  RELEASE_COMMIT=$(git rev-parse --verify HEAD)

  echo '-------------------Creating Python Virtualenv-----------------'
  python3 -m venv "${LOCAL_PYTHON_VIRTUALENV}"
  source "${LOCAL_PYTHON_VIRTUALENV}/bin/activate"
  pip install requests python-dateutil

  echo '--------------Fetching GitHub Actions Artifacts--------------'
  SVN_ARTIFACTS_DIR="beam/${RELEASE}/${PYTHON_ARTIFACTS_DIR}"
  svn co https://dist.apache.org/repos/dist/dev/beam
  mkdir -p "${SVN_ARTIFACTS_DIR}"
  python release/src/main/scripts/download_github_actions_artifacts.py \
    --github-user "${USER_GITHUB_ID}" \
    --repo-url "${GIT_REPO_BASE_URL}" \
    --release-branch "${RELEASE_BRANCH}" \
    --release-commit "${RELEASE_COMMIT}" \
    --artifacts_dir "${SVN_ARTIFACTS_DIR}"

  cd "${SVN_ARTIFACTS_DIR}"

  echo "------Checking Hash Value for apache-beam-${RELEASE}.zip-----"
  sha512sum -c "apache-beam-${RELEASE}.zip.sha512"

  echo "------Signing Source Release apache-beam-${RELEASE}.zip------"
  gpg --local-user "${SIGNING_KEY}" --armor --detach-sig "apache-beam-${RELEASE}.zip"

  for artifact in *.whl; do
    echo "----------Checking Hash Value for ${artifact} wheel-----------"
    sha512sum -c "${artifact}.sha512"
  done

  for artifact in *.whl; do
    echo "------------------Signing ${artifact} wheel-------------------"
    gpg --local-user "${SIGNING_KEY}" --armor --detach-sig "${artifact}"
  done

  cd ..
  svn add --force ${PYTHON_ARTIFACTS_DIR}
  svn status
  echo "Please confirm these changes are ready to commit: [y|N] "
  read confirmation
  if [[ $confirmation != "y" ]]; then
    echo "Exit without staging python artifacts on dist.apache.org."
    rm -rf "${HOME:?}/${LOCAL_PYTHON_STAGING_DIR}"
    exit
  fi
  svn commit --no-auth-cache
  rm -rf "${HOME:?}/${LOCAL_PYTHON_STAGING_DIR}"
fi

echo "[Current Step]: Stage docker images"
echo "Do you want to proceed? [y|N]"
read confirmation
if [[ $confirmation = "y" ]]; then
  echo "============Staging SDK docker images on docker hub========="
  cd ~
  if [[ -d ${LOCAL_PYTHON_STAGING_DIR} ]]; then
    rm -rf ${LOCAL_PYTHON_STAGING_DIR}
  fi
  mkdir -p ${LOCAL_PYTHON_STAGING_DIR}
  cd ${LOCAL_PYTHON_STAGING_DIR}

  echo '-------------------Cloning Beam Release Branch-----------------'
  git clone ${GIT_REPO_URL}
  cd ${BEAM_ROOT_DIR}
  git checkout ${RELEASE_BRANCH}

  echo '-------------------Generating and Pushing Python images-----------------'
  ./gradlew :sdks:python:container:buildAll -Pdocker-pull-licenses -Pdocker-tag=${RELEASE}_rc${RC_NUM}
  for ver in "${PYTHON_VER[@]}"; do
    docker push ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}_rc${RC_NUM} &
  done

  echo '-------------------Generating and Pushing Java images-----------------'
  ./gradlew :sdks:java:container:dockerPush -Pdocker-pull-licenses -Pdocker-tag=${RELEASE}_rc${RC_NUM}

  echo '-------------Generating and Pushing Flink job server images-------------'
  echo "Building containers for the following Flink versions:" "${FLINK_VER[@]}"
  for ver in "${FLINK_VER[@]}"; do
    ./gradlew ":runners:flink:${ver}:job-server-container:dockerPush" -Pdocker-tag="${RELEASE}_rc${RC_NUM}"
  done

  echo '-------------Generating and Pushing Spark job server image-------------'
  ./gradlew ":runners:spark:job-server:container:dockerPush" -Pdocker-tag="${RELEASE}_rc${RC_NUM}"

  rm -rf ~/${PYTHON_ARTIFACTS_DIR}

  echo '-------------------Clean up images at local-----------------'
  for ver in "${PYTHON_VER[@]}"; do
     docker rmi -f ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}_rc${RC_NUM}
  done
  docker rmi -f ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}java_sdk:${RELEASE}_rc${RC_NUM}
  for ver in "${FLINK_VER[@]}"; do
    docker rmi -f "${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}flink${ver}_job_server:${RELEASE}_rc${RC_NUM}"
  done
  docker rmi -f "${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}spark_job_server:${RELEASE}_rc${RC_NUM}"
fi

echo "[Current Step]: Update beam-site"
echo "Do you want to proceed? [y|N]"
read confirmation
if [[ $confirmation = "y" ]]; then
  echo "==============Creating PR for Updating Website==============="
  cd ~
  if [[ -d ${LOCAL_WEBSITE_UPDATE_DIR} ]]; then
    rm -rf ${LOCAL_WEBSITE_UPDATE_DIR}
  fi
  mkdir -p ${LOCAL_WEBSITE_UPDATE_DIR}
  cd ${LOCAL_WEBSITE_UPDATE_DIR}
  mkdir -p ${LOCAL_PYTHON_DOC}
  mkdir -p ${LOCAL_JAVA_DOC}
  mkdir -p ${LOCAL_WEBSITE_REPO}

  echo "------------------Building Python Doc------------------------"
  python3 -m venv "${LOCAL_PYTHON_VIRTUALENV}"
  source "${LOCAL_PYTHON_VIRTUALENV}/bin/activate"
  cd ${LOCAL_PYTHON_DOC}
  pip install tox
  git clone ${GIT_REPO_URL}
  cd ${BEAM_ROOT_DIR}
  git checkout ${RELEASE_BRANCH}
  RELEASE_COMMIT=$(git rev-parse --verify ${RELEASE_BRANCH})
  cd sdks/python && pip install -r build-requirements.txt && tox -e py38-docs
  GENERATED_PYDOC=~/${LOCAL_WEBSITE_UPDATE_DIR}/${LOCAL_PYTHON_DOC}/${BEAM_ROOT_DIR}/sdks/python/target/docs/_build
  rm -rf ${GENERATED_PYDOC}/.doctrees

  echo "----------------------Building Java Doc----------------------"
  cd ~/${LOCAL_WEBSITE_UPDATE_DIR}/${LOCAL_JAVA_DOC}
  git clone ${GIT_REPO_URL}
  cd ${BEAM_ROOT_DIR}
  git checkout ${RELEASE_BRANCH}
  ./gradlew :sdks:java:javadoc:aggregateJavadoc
  GENERATE_JAVADOC=~/${LOCAL_WEBSITE_UPDATE_DIR}/${LOCAL_JAVA_DOC}/${BEAM_ROOT_DIR}/sdks/java/javadoc/build/docs/javadoc/

  echo "------------------Updating Release Docs---------------------"
  cd ~/${LOCAL_WEBSITE_UPDATE_DIR}/${LOCAL_WEBSITE_REPO}
  git clone ${GIT_BEAM_WEBSITE}
  cd ${WEBSITE_ROOT_DIR}
  git checkout release-docs
  git checkout -b updates_release_${RELEASE} release-docs

  echo "..........Copying generated javadoc into beam-site.........."
  cp -r ${GENERATE_JAVADOC} javadoc/${RELEASE}

  echo "............Copying generated pydoc into beam-site.........."
  cp -r ${GENERATED_PYDOC} pydoc/${RELEASE}

  git add -A
  git commit -m "Update beam-site for release ${RELEASE}." -m "Content generated from commit ${RELEASE_COMMIT}."
  git push -f ${USER_REMOTE_URL}

  # Check if hub is installed. See https://stackoverflow.com/a/677212
  if ! hash hub 2> /dev/null; then
    echo "You don't have hub installed, do you want to install hub with sudo permission? [y|N]"
    read confirmation
    if [[ $confirmation = "y" ]]; then
      HUB_VERSION=2.5.0
      HUB_ARTIFACTS_NAME=hub-linux-amd64-${HUB_VERSION}
      wget https://github.com/github/hub/releases/download/v${HUB_VERSION}/${HUB_ARTIFACTS_NAME}.tgz
      tar zvxvf ${HUB_ARTIFACTS_NAME}.tgz
      sudo ./${HUB_ARTIFACTS_NAME}/install
      echo "eval "$(hub alias -s)"" >> ~/.bashrc
      rm -rf ${HUB_ARTIFACTS_NAME}*
    fi
  fi
  if hash hub 2> /dev/null; then
    hub pull-request -m "Publish ${RELEASE} release" -h ${USER_GITHUB_ID}:updates_release_${RELEASE} -b apache:release-docs \
     || echo "Pull request creation did not succeed. If you created the website PR earlier it may have been updated via force-push."
  else
    echo "Without hub, you need to create PR manually."
  fi

  echo "Finished v${RELEASE}-RC${RC_NUM} creation."
  rm -rf ~/${LOCAL_WEBSITE_UPDATE_DIR}/${LOCAL_JAVA_DOC}
  rm -rf ~/${LOCAL_WEBSITE_UPDATE_DIR}/${LOCAL_PYTHON_DOC}
fi