Updates unpack_build_artifact.sh and create_build_artifact.sh

.github/workflows/template.flink-ci.yml

@@ -53,6 +53,8 @@ env:
   MAVEN_ARGS: -Dmaven.repo.local=/__w/flink/flink/.m2/repository
   # required by tools/azure-pipelines/cache_docker_images.sh
   DOCKER_IMAGES_CACHE_FOLDER: /__w/flink/flink/.docker-cache
+  # the following two are required by the build artifact management scripts:
+  # create_build_artifact.sh and unpack_build_artifact.sh
   FLINK_ARTIFACT_DIR: /__w/flink/artifacts
   FLINK_ARTIFACT_FILENAME: flink_artifacts.tar.gz
 
@@ -97,14 +99,16 @@ jobs:
 
       - name: "Collect build artifacts"
         run: |
-          ./tools/azure-pipelines/create_build_artifact.sh -f ${{ env.FLINK_ARTIFACT_DIR }}/${{ env.FLINK_ARTIFACT_FILENAME }}
+          ./tools/azure-pipelines/create_build_artifact.sh
 
       - name: "Upload artifacts to make them available in downstream jobs"
         uses: actions/upload-artifact@v3
         with:
           name: build-artifacts-${{ steps.workflow-prep-step.outputs.stringified-workflow-name }}-${{ github.run_number }}
-          path: ${{ env.FLINK_ARTIFACT_DIR }}${{ env.FLINK_ARTIFACT_FILENAME }}
+          path: ${{ env.FLINK_ARTIFACT_DIR }}/${{ env.FLINK_ARTIFACT_FILENAME }}
           if-no-files-found: error
+          # use minimum here because we only need these artifacts to speed up the build
+          retention-days: 1
 
   packaging:
     name: "Test packaging/licensing"
@@ -141,7 +145,7 @@ jobs:
 
       - name: "Unpack build artifact"
         run: |
-          ./tools/azure-pipelines/unpack_build_artifact.sh -f ${{ env.FLINK_ARTIFACT_DIR }}/${{ env.FLINK_ARTIFACT_FILENAME }} -t .
+          ./tools/azure-pipelines/unpack_build_artifact.sh
 
       - name: "Test"
         run: |
@@ -201,7 +205,7 @@ jobs:
 
       - name: "Unpack build artifact"
         run: |
-          ./tools/azure-pipelines/unpack_build_artifact.sh -f ${{ env.FLINK_ARTIFACT_DIR }}${{ env.FLINK_ARTIFACT_FILENAME }} -t .
+          ./tools/azure-pipelines/unpack_build_artifact.sh
 
       - name: "Try loading Docker images from Cache"
         id: docker-cache
@@ -288,8 +292,8 @@ jobs:
       DOCKER_IMAGES_CACHE_FOLDER: ${{ github.workspace }}/.docker-cache
       MAVEN_REPO_FOLDER: ${{ github.workspace }}/.m2/repository
       MAVEN_ARGS: -Dmaven.repo.local=${{ github.workspace }}/.m2/repository
+      # overwrite FLINK_ARTIFACT_DIR because e2e tests don't run in a Docker container
       FLINK_ARTIFACT_DIR: ${{ github.workspace }}
-      FLINK_ARTIFACT_FILENAME: flink_artifacts.tar.gz
     strategy:
       fail-fast: false
       matrix:
@@ -324,7 +328,7 @@ jobs:
           path: ${{ env.FLINK_ARTIFACT_DIR }}
 
       - name: "Unpack build artifact"
-        run: ./tools/azure-pipelines/unpack_build_artifact.sh -f ${{ env.FLINK_ARTIFACT_DIR }}/${{ env.FLINK_ARTIFACT_FILENAME }} -t ${{ github.workspace }}
+        run: ./tools/azure-pipelines/unpack_build_artifact.sh
 
       # the cache task does not create directories on a cache miss, and can later fail when trying to tar the directory if the test haven't created it
       # this may for example happen if a given directory is only used by a subset of tests, which are run in a different 'group'

tools/azure-pipelines/create_build_artifact.sh

@@ -17,16 +17,9 @@
 # limitations under the License.
 ################################################################################
 
-while getopts "f:" o; do
-    case "${o}" in
-        f)
-            FLINK_ARTIFACT_DIR=${OPTARG};;
-        *)
-          # no special treatment of invalid parameters necessary
-          ;;
-    esac
-done
-shift $((OPTIND-1))
+echo "Copying build artifacts to directory $FLINK_ARTIFACT_DIR"
+
+cp -r . "$FLINK_ARTIFACT_DIR"
 
 echo "Minimizing artifact files"
 
@@ -35,21 +28,25 @@ echo "Minimizing artifact files"
 # by removing files not required for subsequent stages
 
 # jars are re-built in subsequent stages, so no need to cache them (cannot be avoided)
-find . -maxdepth 8 -type f -name '*.jar' -exec rm -rf {} \;
+find "$FLINK_ARTIFACT_DIR" -maxdepth 8 -type f -name '*.jar' -exec rm -rf {} \;
 
 # .git directory
 # not deleting this can cause build stability issues
 # merging the cached version sometimes fails
-rm -rf "./.git"
+rm -rf "$FLINK_ARTIFACT_DIR/.git"
 
 # AZ Pipelines has a problem with links.
-rm "./build-target"
+rm -f "$FLINK_ARTIFACT_DIR/build-target"
 
 # Remove javadocs because they are not used in later stages
-rm -rf "./target/site"
+rm -rf "$FLINK_ARTIFACT_DIR/target/site"
 
 # Remove WebUI node directories; unnecessary because the UI is already fully built
-rm -rf "./flink-runtime-web/web-dashboard/node"
-rm -rf "./flink-runtime-web/web-dashboard/node_modules"
-
-tar -c -z --exclude ${FLINK_ARTIFACT_DIR} -f ${FLINK_ARTIFACT_DIR} .
+rm -rf "$FLINK_ARTIFACT_DIR/flink-runtime-web/web-dashboard/node"
+rm -rf "$FLINK_ARTIFACT_DIR/flink-runtime-web/web-dashboard/node_modules"
+
+if [ ! -z "${FLINK_ARTIFACT_FILENAME}" ]; then
+  echo "Archives artifacts into ${FLINK_ARTIFACT_DIR}/${FLINK_ARTIFACT_FILENAME}"
+  tar --create --gzip --exclude "${FLINK_ARTIFACT_DIR}/${FLINK_ARTIFACT_FILENAME}" --file "${FLINK_ARTIFACT_FILENAME}" -C "${FLINK_ARTIFACT_DIR}" .
+  mv "${FLINK_ARTIFACT_FILENAME}" "${FLINK_ARTIFACT_DIR}"
+fi

tools/azure-pipelines/unpack_build_artifact.sh

@@ -17,26 +17,19 @@
 # limitations under the License.
 ################################################################################
 
-while getopts "f:t:" o; do
-    case "${o}" in
-        f)
-            FLINK_ARTIFACT_DIR=${OPTARG};;
-        t)
-            TARGET_FOLDER_PARAMETER="-C ${OPTARG}";;
-        *)
-          # no special treatment of invalid parameters necessary
-          ;;
-    esac
-done
-shift $((OPTIND-1))
 
 if ! [ -e $FLINK_ARTIFACT_DIR ]; then
-    echo "Cached flink archive $FLINK_ARTIFACT_DIR does not exist. Exiting build."
+    echo "Cached flink dir $FLINK_ARTIFACT_DIR does not exist. Exiting build."
     exit 1
 fi
 
-echo "Extracting build artifacts"
-tar -xzf ${FLINK_ARTIFACT_DIR} ${TARGET_FOLDER_PARAMETER}
+echo "Merging cache"
+if [ -z "${FLINK_ARTIFACT_FILENAME}" ]; then
+  cp -RT "$FLINK_ARTIFACT_DIR" "."
+else
+  echo "Extract build artifacts ${FLINK_ARTIFACT_DIR}/${FLINK_ARTIFACT_FILENAME} into local directory."
+  tar -xzf "${FLINK_ARTIFACT_DIR}/${FLINK_ARTIFACT_FILENAME}"
+fi
 
 echo "Adjusting timestamps"
 # adjust timestamps of proto file to avoid re-generation