diff --git a/.github/workflows/template.flink-ci.yml b/.github/workflows/template.flink-ci.yml index 77c1a9e875aa04..f473af1a3d2450 100644 --- a/.github/workflows/template.flink-ci.yml +++ b/.github/workflows/template.flink-ci.yml @@ -53,6 +53,8 @@ env: MAVEN_ARGS: -Dmaven.repo.local=/__w/flink/flink/.m2/repository # required by tools/azure-pipelines/cache_docker_images.sh DOCKER_IMAGES_CACHE_FOLDER: /__w/flink/flink/.docker-cache + # the following two are required by the build artifact management scripts: + # create_build_artifact.sh and unpack_build_artifact.sh FLINK_ARTIFACT_DIR: /__w/flink/artifacts FLINK_ARTIFACT_FILENAME: flink_artifacts.tar.gz @@ -97,14 +99,16 @@ jobs: - name: "Collect build artifacts" run: | - ./tools/azure-pipelines/create_build_artifact.sh -f ${{ env.FLINK_ARTIFACT_DIR }}/${{ env.FLINK_ARTIFACT_FILENAME }} + ./tools/azure-pipelines/create_build_artifact.sh - name: "Upload artifacts to make them available in downstream jobs" uses: actions/upload-artifact@v3 with: name: build-artifacts-${{ steps.workflow-prep-step.outputs.stringified-workflow-name }}-${{ github.run_number }} - path: ${{ env.FLINK_ARTIFACT_DIR }}${{ env.FLINK_ARTIFACT_FILENAME }} + path: ${{ env.FLINK_ARTIFACT_DIR }}/${{ env.FLINK_ARTIFACT_FILENAME }} if-no-files-found: error + # use minimum here because we only need these artifacts to speed up the build + retention-days: 1 packaging: name: "Test packaging/licensing" @@ -141,7 +145,7 @@ jobs: - name: "Unpack build artifact" run: | - ./tools/azure-pipelines/unpack_build_artifact.sh -f ${{ env.FLINK_ARTIFACT_DIR }}/${{ env.FLINK_ARTIFACT_FILENAME }} -t . + ./tools/azure-pipelines/unpack_build_artifact.sh - name: "Test" run: | @@ -201,7 +205,7 @@ jobs: - name: "Unpack build artifact" run: | - ./tools/azure-pipelines/unpack_build_artifact.sh -f ${{ env.FLINK_ARTIFACT_DIR }}${{ env.FLINK_ARTIFACT_FILENAME }} -t . + ./tools/azure-pipelines/unpack_build_artifact.sh - name: "Try loading Docker images from Cache" id: docker-cache @@ -288,8 +292,8 @@ jobs: DOCKER_IMAGES_CACHE_FOLDER: ${{ github.workspace }}/.docker-cache MAVEN_REPO_FOLDER: ${{ github.workspace }}/.m2/repository MAVEN_ARGS: -Dmaven.repo.local=${{ github.workspace }}/.m2/repository + # overwrite FLINK_ARTIFACT_DIR because e2e tests don't run in a Docker container FLINK_ARTIFACT_DIR: ${{ github.workspace }} - FLINK_ARTIFACT_FILENAME: flink_artifacts.tar.gz strategy: fail-fast: false matrix: @@ -324,7 +328,7 @@ jobs: path: ${{ env.FLINK_ARTIFACT_DIR }} - name: "Unpack build artifact" - run: ./tools/azure-pipelines/unpack_build_artifact.sh -f ${{ env.FLINK_ARTIFACT_DIR }}/${{ env.FLINK_ARTIFACT_FILENAME }} -t ${{ github.workspace }} + run: ./tools/azure-pipelines/unpack_build_artifact.sh # the cache task does not create directories on a cache miss, and can later fail when trying to tar the directory if the test haven't created it # this may for example happen if a given directory is only used by a subset of tests, which are run in a different 'group' diff --git a/tools/azure-pipelines/create_build_artifact.sh b/tools/azure-pipelines/create_build_artifact.sh index ccadb14ed14f9a..f6902d3ede363f 100755 --- a/tools/azure-pipelines/create_build_artifact.sh +++ b/tools/azure-pipelines/create_build_artifact.sh @@ -17,16 +17,9 @@ # limitations under the License. ################################################################################ -while getopts "f:" o; do - case "${o}" in - f) - FLINK_ARTIFACT_DIR=${OPTARG};; - *) - # no special treatment of invalid parameters necessary - ;; - esac -done -shift $((OPTIND-1)) +echo "Copying build artifacts to directory $FLINK_ARTIFACT_DIR" + +cp -r . "$FLINK_ARTIFACT_DIR" echo "Minimizing artifact files" @@ -35,21 +28,25 @@ echo "Minimizing artifact files" # by removing files not required for subsequent stages # jars are re-built in subsequent stages, so no need to cache them (cannot be avoided) -find . -maxdepth 8 -type f -name '*.jar' -exec rm -rf {} \; +find "$FLINK_ARTIFACT_DIR" -maxdepth 8 -type f -name '*.jar' -exec rm -rf {} \; # .git directory # not deleting this can cause build stability issues # merging the cached version sometimes fails -rm -rf "./.git" +rm -rf "$FLINK_ARTIFACT_DIR/.git" # AZ Pipelines has a problem with links. -rm "./build-target" +rm -f "$FLINK_ARTIFACT_DIR/build-target" # Remove javadocs because they are not used in later stages -rm -rf "./target/site" +rm -rf "$FLINK_ARTIFACT_DIR/target/site" # Remove WebUI node directories; unnecessary because the UI is already fully built -rm -rf "./flink-runtime-web/web-dashboard/node" -rm -rf "./flink-runtime-web/web-dashboard/node_modules" - -tar -c -z --exclude ${FLINK_ARTIFACT_DIR} -f ${FLINK_ARTIFACT_DIR} . +rm -rf "$FLINK_ARTIFACT_DIR/flink-runtime-web/web-dashboard/node" +rm -rf "$FLINK_ARTIFACT_DIR/flink-runtime-web/web-dashboard/node_modules" + +if [ ! -z "${FLINK_ARTIFACT_FILENAME}" ]; then + echo "Archives artifacts into ${FLINK_ARTIFACT_DIR}/${FLINK_ARTIFACT_FILENAME}" + tar --create --gzip --exclude "${FLINK_ARTIFACT_DIR}/${FLINK_ARTIFACT_FILENAME}" --file "${FLINK_ARTIFACT_FILENAME}" -C "${FLINK_ARTIFACT_DIR}" . + mv "${FLINK_ARTIFACT_FILENAME}" "${FLINK_ARTIFACT_DIR}" +fi diff --git a/tools/azure-pipelines/unpack_build_artifact.sh b/tools/azure-pipelines/unpack_build_artifact.sh index da8a715b1535c8..deddaa33001ba1 100755 --- a/tools/azure-pipelines/unpack_build_artifact.sh +++ b/tools/azure-pipelines/unpack_build_artifact.sh @@ -17,26 +17,19 @@ # limitations under the License. ################################################################################ -while getopts "f:t:" o; do - case "${o}" in - f) - FLINK_ARTIFACT_DIR=${OPTARG};; - t) - TARGET_FOLDER_PARAMETER="-C ${OPTARG}";; - *) - # no special treatment of invalid parameters necessary - ;; - esac -done -shift $((OPTIND-1)) if ! [ -e $FLINK_ARTIFACT_DIR ]; then - echo "Cached flink archive $FLINK_ARTIFACT_DIR does not exist. Exiting build." + echo "Cached flink dir $FLINK_ARTIFACT_DIR does not exist. Exiting build." exit 1 fi -echo "Extracting build artifacts" -tar -xzf ${FLINK_ARTIFACT_DIR} ${TARGET_FOLDER_PARAMETER} +echo "Merging cache" +if [ -z "${FLINK_ARTIFACT_FILENAME}" ]; then + cp -RT "$FLINK_ARTIFACT_DIR" "." +else + echo "Extract build artifacts ${FLINK_ARTIFACT_DIR}/${FLINK_ARTIFACT_FILENAME} into local directory." + tar -xzf "${FLINK_ARTIFACT_DIR}/${FLINK_ARTIFACT_FILENAME}" +fi echo "Adjusting timestamps" # adjust timestamps of proto file to avoid re-generation