From ce293b6bcbfbc976d500629478ea2620b6b943e5 Mon Sep 17 00:00:00 2001 From: Michele Sciabarra Date: Sat, 10 Feb 2024 13:28:00 +0000 Subject: [PATCH] update the fixes --- .github/workflows/0-on-demand.yaml | 12 +++++------- .github/workflows/1-unit.yaml | 19 +++++++++++++------ .github/workflows/2-system.yaml | 21 +++++++++++---------- .github/workflows/3-multi-runtime.yaml | 20 +++++++++++--------- .github/workflows/4-standalone.yaml | 21 +++++++++++---------- .github/workflows/5-scheduler.yaml | 21 +++++++++++---------- .github/workflows/6-performance.yaml | 19 ++++++++++--------- .github/workflows/README.md | 18 +++++++++--------- Taskfile.yml | 2 +- tools/github/checkAndUploadLogs.sh | 10 +++++++--- tools/github/s3-upload.sh | 14 +++++++------- tools/github/setup.sh | 11 ++++++++--- tools/travis/runTests.sh | 2 +- 13 files changed, 105 insertions(+), 85 deletions(-) diff --git a/.github/workflows/0-on-demand.yaml b/.github/workflows/0-on-demand.yaml index 851951a52be..a39a379d9ca 100644 --- a/.github/workflows/0-on-demand.yaml +++ b/.github/workflows/0-on-demand.yaml @@ -35,9 +35,7 @@ on: - Standalone - Scheduler - Performance - ## Uncomment those to debug the build - Dummy - - DummyKO env: # openwhisk env @@ -49,10 +47,10 @@ env: SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}} # (optional) s3 log upload - AWS_BUCKET: ${{ secrets.AWS_BUCKET }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ secrets.AWS_REGION }} + LOG_BUCKET: ${{ secrets.LOG_BUCKET }} + LOG_ACCESS_KEY_ID: ${{ secrets.LOG_ACCESS_KEY_ID }} + LOG_SECRET_ACCESS_KEY: ${{ secrets.LOG_SECRET_ACCESS_KEY }} + LOG_REGION: ${{ secrets.LOG_REGION }} # github GH_BUILD: ${{ github.event_name }}-${{ github.sha }} @@ -76,7 +74,7 @@ jobs: run: "./tools/github/run${{ env.TEST_SUITE }}Tests.sh" continue-on-error: true - id: logs - name: Upload Logs + name: Show results and Upload logs run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }} - name: Slack Notification run: > diff --git a/.github/workflows/1-unit.yaml b/.github/workflows/1-unit.yaml index 1d0f1a1019d..27ce75f67f6 100644 --- a/.github/workflows/1-unit.yaml +++ b/.github/workflows/1-unit.yaml @@ -20,8 +20,10 @@ name: Unit Tests on: # build on push push: - #tags: '*' - #branches-ignore: '*' + tags: + - '*unit*' + - '*all*' + branches-ignore: '*' # build on pull requests pull_request: @@ -36,14 +38,19 @@ env: SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}} # (optional) s3 log upload - AWS_BUCKET: ${{ secrets.AWS_BUCKET }} + LOG_BUCKET: ${{ secrets.LOG_BUCKET }} + LOG_ACCESS_KEY_ID: ${{ secrets.LOG_ACCESS_KEY_ID }} + LOG_SECRET_ACCESS_KEY: ${{ secrets.LOG_SECRET_ACCESS_KEY }} + LOG_REGION: ${{ secrets.LOG_REGION }} + + # some tests need also this even if they are empty on pull_requests... AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.AWS_REGION }} # github GH_BUILD: ${{ github.event_name }}-${{ github.sha }} - GH_BRANCH: ${{ github.head_ref || github.ref_name }} + GH_BRANCH: ${{ github.head_ref || github.ref_name }} jobs: openwhisk: @@ -58,10 +65,10 @@ jobs: run: "./tools/github/run${{ env.TEST_SUITE }}Tests.sh" continue-on-error: true - id: logs - name: Upload Logs + name: Show results and Upload logs run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }} - name: Slack Notification - run: > + run: > ./tools/github/writeOnSlack.sh "[$TEST_SUITE]" ${{ steps.tests.outcome }} on ${GH_BUILD} $'\nbranch:' $GH_BRANCH diff --git a/.github/workflows/2-system.yaml b/.github/workflows/2-system.yaml index dce7d9b6311..ceb285e59fa 100644 --- a/.github/workflows/2-system.yaml +++ b/.github/workflows/2-system.yaml @@ -20,10 +20,11 @@ name: System Tests on: # build on push push: + tags: + - '*sys*' + - '*all*' + branches-ignore: '*' # build on pull requests - #tags: '*' - #branches-ignore: '*' - pull_request: env: @@ -37,14 +38,14 @@ env: SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}} # (optional) s3 log upload - AWS_BUCKET: ${{ secrets.AWS_BUCKET }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ secrets.AWS_REGION }} + LOG_BUCKET: ${{ secrets.LOG_BUCKET }} + LOG_ACCESS_KEY_ID: ${{ secrets.LOG_ACCESS_KEY_ID }} + LOG_SECRET_ACCESS_KEY: ${{ secrets.LOG_SECRET_ACCESS_KEY }} + LOG_REGION: ${{ secrets.LOG_REGION }} # github GH_BUILD: ${{ github.event_name }}-${{ github.sha }} - GH_BRANCH: ${{ github.head_ref || github.ref_name }} + GH_BRANCH: ${{ github.head_ref || github.ref_name }} jobs: openwhisk: @@ -59,10 +60,10 @@ jobs: run: "./tools/github/run${{ env.TEST_SUITE }}Tests.sh" continue-on-error: true - id: logs - name: Upload Logs + name: Show results and Upload logs run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }} - name: Slack Notification - run: > + run: > ./tools/github/writeOnSlack.sh "[$TEST_SUITE]" ${{ steps.tests.outcome }} on ${GH_BUILD} $'\nbranch:' $GH_BRANCH diff --git a/.github/workflows/3-multi-runtime.yaml b/.github/workflows/3-multi-runtime.yaml index 22a5dd3ef28..4544d28c09c 100644 --- a/.github/workflows/3-multi-runtime.yaml +++ b/.github/workflows/3-multi-runtime.yaml @@ -20,9 +20,11 @@ name: MultiRuntime Tests on: # build on push push: - #tags: '*' - #branches-ignore: '*' + tags: + - '*multi*' + - '*all*' + branches-ignore: '*' # build on pull requests pull_request: @@ -37,14 +39,14 @@ env: SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}} # (optional) s3 log upload - AWS_BUCKET: ${{ secrets.AWS_BUCKET }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ secrets.AWS_REGION }} + LOG_BUCKET: ${{ secrets.LOG_BUCKET }} + LOG_ACCESS_KEY_ID: ${{ secrets.LOG_ACCESS_KEY_ID }} + LOG_SECRET_ACCESS_KEY: ${{ secrets.LOG_SECRET_ACCESS_KEY }} + LOG_REGION: ${{ secrets.LOG_REGION }} # github GH_BUILD: ${{ github.event_name }}-${{ github.sha }} - GH_BRANCH: ${{ github.head_ref || github.ref_name }} + GH_BRANCH: ${{ github.head_ref || github.ref_name }} jobs: openwhisk: @@ -59,10 +61,10 @@ jobs: run: "./tools/github/run${{ env.TEST_SUITE }}Tests.sh" continue-on-error: true - id: logs - name: Upload Logs + name: Show results and Upload logs run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }} - name: Slack Notification - run: > + run: > ./tools/github/writeOnSlack.sh "[$TEST_SUITE]" ${{ steps.tests.outcome }} on ${GH_BUILD} $'\nbranch:' $GH_BRANCH diff --git a/.github/workflows/4-standalone.yaml b/.github/workflows/4-standalone.yaml index 299d5cfd854..35be409ed06 100644 --- a/.github/workflows/4-standalone.yaml +++ b/.github/workflows/4-standalone.yaml @@ -20,10 +20,11 @@ name: Standalone Tests on: # build on push push: + tags: + - '*stand*' + - '*all*' + branches-ignore: '*' # build on pull requests - #tags: '*' - #branches-ignore: '*' - pull_request: env: @@ -37,14 +38,14 @@ env: SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}} # (optional) s3 log upload - AWS_BUCKET: ${{ secrets.AWS_BUCKET }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ secrets.AWS_REGION }} + LOG_BUCKET: ${{ secrets.LOG_BUCKET }} + LOG_ACCESS_KEY_ID: ${{ secrets.LOG_ACCESS_KEY_ID }} + LOG_SECRET_ACCESS_KEY: ${{ secrets.LOG_SECRET_ACCESS_KEY }} + LOG_REGION: ${{ secrets.LOG_REGION }} # github GH_BUILD: ${{ github.event_name }}-${{ github.sha }} - GH_BRANCH: ${{ github.head_ref || github.ref_name }} + GH_BRANCH: ${{ github.head_ref || github.ref_name }} jobs: openwhisk: @@ -59,10 +60,10 @@ jobs: run: "./tools/github/run${{ env.TEST_SUITE }}Tests.sh" continue-on-error: true - id: logs - name: Upload Logs + name: Show results and Upload logs run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }} - name: Slack Notification - run: > + run: > ./tools/github/writeOnSlack.sh "[$TEST_SUITE]" ${{ steps.tests.outcome }} on ${GH_BUILD} $'\nbranch:' $GH_BRANCH diff --git a/.github/workflows/5-scheduler.yaml b/.github/workflows/5-scheduler.yaml index a74b0542146..cafada4a24b 100644 --- a/.github/workflows/5-scheduler.yaml +++ b/.github/workflows/5-scheduler.yaml @@ -20,10 +20,11 @@ name: Scheduler Tests on: # build on push push: + tags: + - '*sched*' + - '*all*' + branches-ignore: '*' # build on pull requests - #tags: '*' - #branches-ignore: '*' - pull_request: env: @@ -37,14 +38,14 @@ env: SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}} # (optional) s3 log upload - AWS_BUCKET: ${{ secrets.AWS_BUCKET }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ secrets.AWS_REGION }} + LOG_BUCKET: ${{ secrets.LOG_BUCKET }} + LOG_ACCESS_KEY_ID: ${{ secrets.LOG_ACCESS_KEY_ID }} + LOG_SECRET_ACCESS_KEY: ${{ secrets.LOG_SECRET_ACCESS_KEY }} + LOG_REGION: ${{ secrets.LOG_REGION }} # github GH_BUILD: ${{ github.event_name }}-${{ github.sha }} - GH_BRANCH: ${{ github.head_ref || github.ref_name }} + GH_BRANCH: ${{ github.head_ref || github.ref_name }} jobs: openwhisk: @@ -59,10 +60,10 @@ jobs: run: "./tools/github/run${{ env.TEST_SUITE }}Tests.sh" continue-on-error: true - id: logs - name: Upload Logs + name: Show results and Upload logs run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }} - name: Slack Notification - run: > + run: > ./tools/github/writeOnSlack.sh "[$TEST_SUITE]" ${{ steps.tests.outcome }} on ${GH_BUILD} $'\nbranch:' $GH_BRANCH diff --git a/.github/workflows/6-performance.yaml b/.github/workflows/6-performance.yaml index 54948207301..4cdb6a9ec07 100644 --- a/.github/workflows/6-performance.yaml +++ b/.github/workflows/6-performance.yaml @@ -20,10 +20,11 @@ name: Performance Tests on: # build on push push: + tags: + - '*perf*' + - '*all*' + branches-ignore: '*' # build on pull requests - #tags: '*' - #ranches-ignore: '*' - pull_request: env: @@ -37,14 +38,14 @@ env: SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}} # (optional) s3 log upload - AWS_BUCKET: ${{ secrets.AWS_BUCKET }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ secrets.AWS_REGION }} + LOG_BUCKET: ${{ secrets.LOG_BUCKET }} + LOG_ACCESS_KEY_ID: ${{ secrets.LOG_ACCESS_KEY_ID }} + LOG_SECRET_ACCESS_KEY: ${{ secrets.LOG_SECRET_ACCESS_KEY }} + LOG_REGION: ${{ secrets.LOG_REGION }} # github GH_BUILD: ${{ github.event_name }}-${{ github.sha }} - GH_BRANCH: ${{ github.head_ref || github.ref_name }} + GH_BRANCH: ${{ github.head_ref || github.ref_name }} jobs: openwhisk: @@ -79,7 +80,7 @@ jobs: - run: OPENWHISK_HOST="172.17.0.1" USERS="1" REQUESTS_PER_SEC="1" ./gradlew gatlingRun-org.apache.openwhisk.ColdBlockingInvokeSimulation continue-on-error: true - name: Slack Notification - run: > + run: > ./tools/github/writeOnSlack.sh "[$TEST_SUITE]" ${{ steps.tests.outcome }} on ${GH_BUILD} $'\nbranch:' $GH_BRANCH diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 55473d04e86..dbe6c708981 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -35,19 +35,19 @@ The build uploads the logs to an s3 bucket allowing to inspect them with a brows You need to create the bucket with the following commands: ``` -AWS_BUCKET= -AWS_REGION= -aws s3 mb s3://$AWS_BUCKET --region $AWS_REGION -aws s3 website s3://$AWS_BUCKET/ --index-document index.html -aws s3api put-bucket-acl --acl public-read --bucket $AWS_BUCKET +LOG_BUCKET= +LOG_REGION= +aws s3 mb s3://$LOG_BUCKET --region $LOG_REGION +aws s3 website s3://$LOG_BUCKET/ --index-document index.html +aws s3api put-bucket-acl --acl public-read --bucket $LOG_BUCKET ``` To enable upload to the created bucket you need to set the following secrets: -- `AWS_BUCKET`: name of your bucket in s3 (just the name, without `s3://`); create it before. -- `AWS_ACCESS_KEY_ID`: your aws access key. -- `AWS_SECRET_ACCESS_KEY`: your aws secret key. -- `AWS_REGION`: important: the region where your bucket is. +- `LOG_BUCKET`: name of your bucket in s3 (just the name, without `s3://`); create it before. +- `LOG_ACCESS_KEY_ID`: your aws access key. +- `LOG_SECRET_ACCESS_KEY`: your aws secret key. +- `LOG_REGION`: important: the region where your bucket is. ## Slack notification diff --git a/Taskfile.yml b/Taskfile.yml index 8394a678bcb..ab1050602df 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -18,7 +18,7 @@ version: '3' vars: - BASETAG: 0.3.0-morpheus + BASETAG: 3.1.0-mastrogpt TAG: sh: git describe --tags --abbrev=0 2>/dev/null || echo "no-tag" TIMESTAMP: diff --git a/tools/github/checkAndUploadLogs.sh b/tools/github/checkAndUploadLogs.sh index 79f169720b0..da96610a797 100755 --- a/tools/github/checkAndUploadLogs.sh +++ b/tools/github/checkAndUploadLogs.sh @@ -16,11 +16,15 @@ # limitations under the License. # +# showing test results on the CI log +INDEX="tests/build/reports/tests/testCoverageLean/index.html" +test -f "$INDEX" && lynx -dump file://$PWD/$INDEX | grep . + # check variables -for i in AWS_BUCKET AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION +for i in LOG_BUCKET LOG_ACCESS_KEY_ID LOG_SECRET_ACCESS_KEY LOG_REGION do if test -z "${!i}" - then echo "Required Environment Variable Missing: $i" ; exit 1 + then echo "Required Environment Variable Missing: $i" ; exit 0 fi done @@ -42,7 +46,7 @@ TAGS="" [[ "$2" == "Unit" ]] && TAGS="db" LOG_DIR="$(date +%Y-%m-%d)/${LOG_NAME}-${GH_BUILD}-${GH_BRANCH}" -BUCKET_URL="https://$AWS_BUCKET.s3.$AWS_REGION.amazonaws.com" +BUCKET_URL="https://$LOG_BUCKET.s3.$LOG_REGION.amazonaws.com" echo "Logs: ${BUCKET_URL}/index.html#${LOG_DIR}/" echo "Reports: ${BUCKET_URL}/${LOG_DIR}/test-reports/reports/tests/testCoverageLean/index.html" diff --git a/tools/github/s3-upload.sh b/tools/github/s3-upload.sh index 38c685a04db..5d2d0141b6f 100755 --- a/tools/github/s3-upload.sh +++ b/tools/github/s3-upload.sh @@ -17,7 +17,7 @@ # # check variables -for i in AWS_BUCKET AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY +for i in LOG_BUCKET LOG_ACCESS_KEY_ID LOG_SECRET_ACCESS_KEY do if test -z "${!i}" then echo "Please set $i" ; exit 1 @@ -32,7 +32,7 @@ FROM="$1" TO="$2" BROWSER="https://raw.githubusercontent.com/qoomon/aws-s3-bucket-browser/master/index.html" -BUCKET_URL="https://$AWS_BUCKET.s3.$AWS_REGION.amazonaws.com/" +BUCKET_URL="https://$LOG_BUCKET.s3.$LOG_REGION.amazonaws.com/" # install rclone if ! which rclone @@ -41,13 +41,13 @@ fi RCLONE="rclone --config /dev/null \ --s3-provider AWS \ - --s3-region $AWS_REGION \ + --s3-region $LOG_REGION \ --s3-acl public-read \ - --s3-access-key-id $AWS_ACCESS_KEY_ID \ - --s3-secret-access-key $AWS_SECRET_ACCESS_KEY" + --s3-access-key-id $LOG_ACCESS_KEY_ID \ + --s3-secret-access-key $LOG_SECRET_ACCESS_KEY" curl -s "$BROWSER" |\ sed -e 's!bucketUrl: undefined!bucketUrl: "'$BUCKET_URL'"!' |\ - $RCLONE rcat ":s3:$AWS_BUCKET/index.html" + $RCLONE rcat ":s3:$LOG_BUCKET/index.html" -$RCLONE copyto "$FROM" ":s3:$AWS_BUCKET/$TO/" +$RCLONE copyto "$FROM" ":s3:$LOG_BUCKET/$TO/" diff --git a/tools/github/setup.sh b/tools/github/setup.sh index e72b046051e..597396c96b3 100755 --- a/tools/github/setup.sh +++ b/tools/github/setup.sh @@ -34,15 +34,20 @@ function retry() { fi } +# lynx utility to show test results on the job run +sudo apt-get -y install lynx + # setup docker to listen in port 4243 sudo systemctl stop docker sudo sed -i -e 's!/usr/bin/dockerd -H fd://!/usr/bin/dockerd -H tcp://0.0.0.0:4243 -H fd://!' /lib/systemd/system/docker.service sudo systemctl daemon-reload sudo systemctl start docker -# installing right version of jdk -JDK=https://github.com/ibmruntimes/semeru11-binaries/releases/download/jdk-11.0.12%2B7_openj9-0.27.0/ibm-semeru-open-jdk_x64_linux_11.0.12_7_openj9-0.27.0.tar.gz -curl -sL $JDK | sudo tar xzvf - -C /usr/local +# installing openjdk +x86_64="https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.17%2B8/OpenJDK11U-jdk_x64_linux_hotspot_11.0.17_8.tar.gz" ;\ +aarch64="https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.17%2B8/OpenJDK11U-jdk_aarch64_linux_hotspot_11.0.17_8.tar.gz";\ +arch=$(arch) ; JDK=${!arch} ; curl -sL $JDK | sudo tar xzvf - -C /usr/local + JAVA="$(which java)" sudo mv "$JAVA" "$JAVA"."$(date +%s)" sudo ln -sf /usr/local/jdk*/bin/java $JAVA diff --git a/tools/travis/runTests.sh b/tools/travis/runTests.sh index e2a7f7a4b32..c47f542f0ef 100755 --- a/tools/travis/runTests.sh +++ b/tools/travis/runTests.sh @@ -28,5 +28,5 @@ cd $ROOTDIR cat whisk.properties TERM=dumb ./gradlew :tests:testCoverageLean :tests:reportCoverage :tests:testSwaggerCodegen -bash <(curl -s https://codecov.io/bash) +#bash <(curl -s https://codecov.io/bash) echo "Time taken for ${0##*/} is $SECONDS secs"