Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

testing #411

Draft
wants to merge 2 commits into
base: master
Choose a base branch
from
Draft

testing #411

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f2d1aeb
testing
jkao97 Apr 1, 2024
6355555
Update semaphore.yml
jkao97 Apr 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 80 additions & 29 deletions .semaphore/semaphore.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,15 +25,60 @@ global_job_config:
commands:
- checkout
- sem-version java 8
- sem-version python 3.9
- . vault-setup
- . cache-maven restore
- pip install tox==3.28.0
- export GIT_COMMIT=$(git rev-parse --verify HEAD --short)
- export BUILD_NUMBER=$(echo $SEMAPHORE_WORKFLOW_ID | cut -f1 -d"-")
- export BRANCH_TAG=$(echo $SEMAPHORE_GIT_BRANCH | tr / -)
# Semaphore does not accept empty values for parameters.
- if [[ "$PACKAGES_MAVEN_URL" = "NONE" ]]; then unset PACKAGES_MAVEN_URL ;fi
- if [[ "$PACKAGING_BUILD_NUMBER" = "NONE" ]]; then unset PACKAGING_BUILD_NUMBER ;fi
- >-
echo """
Parameters:
==========================
CONFLUENT_VERSION: $CONFLUENT_VERSION
PACKAGES_URL: $PACKAGES_URL
PACKAGES_MAVEN_URL: $PACKAGES_MAVEN_URL
PACKAGING_BUILD_NUMBER: $PACKAGING_BUILD_NUMBER
ALLOW_UNSIGNED: $ALLOW_UNSIGNED
CONFLUENT_DEB_VERSION: $CONFLUENT_DEB_VERSION
"""
- if [[ $BRANCH_TAG =~ "-rc" ]]; then export IS_RC="true"; fi
- if [[ $BRANCH_TAG =~ "-cp" ]]; then export IS_HOTFIX="true"; fi
- if [[ $BRANCH_TAG =~ "-post" ]]; then export IS_POST="true"; fi
- if [[ $BRANCH_TAG =~ "-beta" ]]; then export IS_BETA="true"; fi
- if [[ $BRANCH_TAG =~ "-alpha" ]]; then export IS_PREVIEW="true"; fi
- if [[ $IS_RC || $IS_HOTFIX || $IS_POST || $IS_BETA || $IS_PREVIEW ]]; then export IS_RELEASE="true"; fi

- >-
if [[ $IS_BETA || $IS_HOTFIX || $IS_POST ]]; then
export PLATFORM_LABEL=$(echo $BRANCH_TAG | awk -F - '{print "~"$2}')
elif [[ ! $IS_RELEASE ]]; then
export PLATFORM_LABEL="~SNAPSHOT"
else
export PLATFORM_LABEL=""
fi
- export PACKAGING_BUILD_ARGS=" -DCONFLUENT_VERSION=$CONFLUENT_VERSION -DCONFLUENT_PLATFORM_LABEL=$PLATFORM_LABEL -DCONFLUENT_DEB_VERSION=$CONFLUENT_DEB_VERSION -DALLOW_UNSIGNED=$ALLOW_UNSIGNED"
- >-
if [[ $IS_RELEASE && $PACKAGING_BUILD_NUMBER ]]; then
if [[ $IS_BETA ]]; then
export MAVEN_PACKAGES_URL="https://s3.us-west-2.amazonaws.com/jenkins-confluent-packages-beta-maven/$BRANCH_TAG/$PACKAGING_BUILD_NUMBER/maven"
elif [[ $IS_PREVIEW ]]; then
export MAVEN_PACKAGES_URL="https://s3.us-west-2.amazonaws.com/jenkins-confluent-packages-alpha-maven/$BRANCH_NAME/$PACKAGING_BUILD_NUMBER/maven"
elif [[ $IS_RC ]]; then
export MAVEN_PACKAGES_URL="https://s3.us-west-2.amazonaws.com/staging-confluent-packages-maven/v$BRANCH_NAME/maven"
if [[ $PACKAGES_MAVEN_URL ]]; then
export MAVEN_PACKAGES_URL=$PACKAGES_MAVEN_URL
fi
fi
else
echo "This job is not a isBetaJob, isPreviewJob, isHotfixJob, or isRcJob (What we know how to handle) - and we don't know how to handle it"
fi
- export DOCKER_DEV_REGISTRY="519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/"
- export DOCKER_PROD_REGISTRY="519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/prod/"
- export DOCKER_UPSTREAM_REGISTRY=$DOCKER_DEV_REGISTRY
- export DOCKER_UPSTREAM_REGISTRY=$DOCKER_PROD_REGISTRY
- export LATEST_TAG=$BRANCH_TAG-latest
- export DOCKER_UPSTREAM_TAG="$LATEST_TAG"
- export DOCKER_REPOS="confluentinc/cp-base-new confluentinc/cp-base-lite confluentinc/cp-jmxterm"
Expand All @@ -54,8 +99,14 @@ blocks:
- export DOCKER_DEV_FULL_IMAGES=$DOCKER_DEV_REGISTRY${DOCKER_REPOS// /:$DOCKER_DEV_TAG$OS_TAG $DOCKER_DEV_REGISTRY}:$DOCKER_DEV_TAG$OS_TAG
- export AMD_DOCKER_DEV_FULL_IMAGES=${DOCKER_DEV_FULL_IMAGES// /$AMD_ARCH }$AMD_ARCH
- ci-tools ci-update-version
# Overwrite maven global configuration
- . vault-sem-get-secret maven-settings-cp-dockerfile
- export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g")
- export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL"

- mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY
-DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi8
$PACKAGING_BUILD_ARGS
- . cache-maven store
- >-
for dev_image in $AMD_DOCKER_DEV_FULL_IMAGES;
Expand Down Expand Up @@ -85,11 +136,11 @@ blocks:
- docker pull $DEV_IMAGE_FULL
- docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$GIT_COMMIT_TAG
- docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG
- docker push $PROD_IMAGE_NAME:$GIT_COMMIT_TAG
- docker push $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG
- export PACKAGE_TAG=$BRANCH_TAG-$PACKAGING_BUILD_NUMBER$OS_TAG$AMD_ARCH
- docker tag $DEV_IMAGE_FULL $PROD_IMAGE:$PACKAGE_TAG
- docker push $PROD_IMAGE:$PACKAGE_TAG
- export LATEST_PUSH_TAG=$LATEST_TAG$OS_TAG$AMD_ARCH
- docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$LATEST_PUSH_TAG
- docker push $PROD_IMAGE_NAME:$LATEST_PUSH_TAG
- name: Deploy AMD confluentinc/cp-base-lite
dependencies: ["Build, Test, & Scan AMD"]
run:
Expand All @@ -106,11 +157,11 @@ blocks:
- docker pull $DEV_IMAGE_FULL
- docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$GIT_COMMIT_TAG
- docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG
- docker push $PROD_IMAGE_NAME:$GIT_COMMIT_TAG
- docker push $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG
- export PACKAGE_TAG=$BRANCH_TAG-$PACKAGING_BUILD_NUMBER$OS_TAG$AMD_ARCH
- docker tag $DEV_IMAGE_FULL $PROD_IMAGE:$PACKAGE_TAG
- docker push $PROD_IMAGE:$PACKAGE_TAG
- export LATEST_PUSH_TAG=$LATEST_TAG$OS_TAG$AMD_ARCH
- docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$LATEST_PUSH_TAG
- docker push $PROD_IMAGE_NAME:$LATEST_PUSH_TAG
- name: Deploy AMD confluentinc/cp-jmxterm
dependencies: ["Build, Test, & Scan AMD"]
run:
Expand All @@ -127,11 +178,11 @@ blocks:
- docker pull $DEV_IMAGE_FULL
- docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$GIT_COMMIT_TAG
- docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG
- docker push $PROD_IMAGE_NAME:$GIT_COMMIT_TAG
- docker push $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG
- export PACKAGE_TAG=$BRANCH_TAG-$PACKAGING_BUILD_NUMBER$OS_TAG$AMD_ARCH
- docker tag $DEV_IMAGE_FULL $PROD_IMAGE:$PACKAGE_TAG
- docker push $PROD_IMAGE:$PACKAGE_TAG
- export LATEST_PUSH_TAG=$LATEST_TAG$OS_TAG$AMD_ARCH
- docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$LATEST_PUSH_TAG
- docker push $PROD_IMAGE_NAME:$LATEST_PUSH_TAG
- name: Build & Test ARM
dependencies: []
run:
Expand All @@ -147,9 +198,15 @@ blocks:
- export OS_TAG="-ubi8"
- export DOCKER_DEV_FULL_IMAGES=$DOCKER_DEV_REGISTRY${DOCKER_REPOS// /:$DOCKER_DEV_TAG$OS_TAG $DOCKER_DEV_REGISTRY}:$DOCKER_DEV_TAG$OS_TAG
- export ARM_DOCKER_DEV_FULL_IMAGES=${DOCKER_DEV_FULL_IMAGES// /$ARM_ARCH }$ARM_ARCH
# Overwrite maven global configuration
- . vault-sem-get-secret maven-settings-cp-dockerfile
- export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g")
- export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL"

- ci-tools ci-update-version
- mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY
-DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi8
$PACKAGING_BUILD_ARGS
- . cache-maven store
- for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done
epilogue:
Expand Down Expand Up @@ -177,11 +234,11 @@ blocks:
- docker pull $DEV_IMAGE_FULL
- docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$GIT_COMMIT_TAG
- docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG
- docker push $PROD_IMAGE_NAME:$GIT_COMMIT_TAG
- docker push $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG
- export PACKAGE_TAG=$BRANCH_TAG-$PACKAGING_BUILD_NUMBER$OS_TAG$ARM_ARCH
- docker tag $DEV_IMAGE_FULL $PROD_IMAGE:$PACKAGE_TAG
- docker push $PROD_IMAGE:$PACKAGE_TAG
- export LATEST_PUSH_TAG=$LATEST_TAG$OS_TAG$ARM_ARCH
- docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$LATEST_PUSH_TAG
- docker push $PROD_IMAGE_NAME:$LATEST_PUSH_TAG
- name: Deploy ARM confluentinc/cp-base-lite
dependencies: ["Build & Test ARM"]
run:
Expand All @@ -201,11 +258,11 @@ blocks:
- docker pull $DEV_IMAGE_FULL
- docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$GIT_COMMIT_TAG
- docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG
- docker push $PROD_IMAGE_NAME:$GIT_COMMIT_TAG
- docker push $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG
- export PACKAGE_TAG=$BRANCH_TAG-$PACKAGING_BUILD_NUMBER$OS_TAG$ARM_ARCH
- docker tag $DEV_IMAGE_FULL $PROD_IMAGE:$PACKAGE_TAG
- docker push $PROD_IMAGE:$PACKAGE_TAG
- export LATEST_PUSH_TAG=$LATEST_TAG$OS_TAG$ARM_ARCH
- docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$LATEST_PUSH_TAG
- docker push $PROD_IMAGE_NAME:$LATEST_PUSH_TAG
- name: Deploy ARM confluentinc/cp-jmxterm
dependencies: ["Build & Test ARM"]
run:
Expand All @@ -225,11 +282,11 @@ blocks:
- docker pull $DEV_IMAGE_FULL
- docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$GIT_COMMIT_TAG
- docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG
- docker push $PROD_IMAGE_NAME:$GIT_COMMIT_TAG
- docker push $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG
- export PACKAGE_TAG=$BRANCH_TAG-$PACKAGING_BUILD_NUMBER$OS_TAG$ARM_ARCH
- docker tag $DEV_IMAGE_FULL $PROD_IMAGE:$PACKAGE_TAG
- docker push $PROD_IMAGE:$PACKAGE_TAG
- export LATEST_PUSH_TAG=$LATEST_TAG$OS_TAG$ARM_ARCH
- docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$LATEST_PUSH_TAG
- docker push $PROD_IMAGE_NAME:$LATEST_PUSH_TAG
- name: Create Manifest and Maven Deploy
dependencies: ["Deploy AMD confluentinc/cp-base-new", "Deploy AMD confluentinc/cp-base-lite", "Deploy AMD confluentinc/cp-jmxterm", "Deploy ARM confluentinc/cp-base-new", "Deploy ARM confluentinc/cp-base-lite",
"Deploy ARM confluentinc/cp-jmxterm"]
Expand All @@ -241,23 +298,17 @@ blocks:
commands:
- export DOCKER_PROD_IMAGE_NAME=$DOCKER_PROD_REGISTRY${DOCKER_REPOS// / $DOCKER_PROD_REGISTRY}
- ci-tools ci-update-version
- ci-tools ci-push-tag
- mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker -DaltDeploymentRepository=confluent-codeartifact-internal::default::https://confluent-519856050701.d.codeartifact.us-west-2.amazonaws.com/maven/maven-snapshots/
-DrepositoryId=confluent-codeartifact-internal deploy -DskipTests -Ddocker.skip-build=true -Ddocker.skip-test=true
# Create manifest
- >-
for image in $DOCKER_PROD_IMAGE_NAME;
do
export OS_TAG="-ubi8"
export GIT_TAG=$GIT_COMMIT$OS_TAG
docker manifest create $image:$GIT_TAG $image:$GIT_TAG$AMD_ARCH $image:$GIT_TAG$ARM_ARCH
docker manifest push $image:$GIT_TAG
export BRANCH_BUILD_TAG=$BRANCH_TAG-$BUILD_NUMBER$OS_TAG
docker manifest create $image:$BRANCH_BUILD_TAG $image:$BRANCH_BUILD_TAG$AMD_ARCH $image:$BRANCH_BUILD_TAG$ARM_ARCH
docker manifest push $image:$BRANCH_BUILD_TAG
export PACKAGE_TAG=$BRANCH_TAG-$PACKAGING_BUILD_NUMBER$OS_TAG
docker manifest create $image:$PACKAGE_TAG $image:$PACKAGE_TAG$AMD_ARCH $image:$PACKAGE_TAG$ARM_ARCH
docker manifest push $image:$PACKAGE_TAG
export LATEST_MANIFEST_TAG=$LATEST_TAG$OS_TAG
docker manifest create $image:$LATEST_MANIFEST_TAG $image:$LATEST_MANIFEST_TAG$AMD_ARCH $image:$LATEST_MANIFEST_TAG$ARM_ARCH
docker manifest push $image:$LATEST_MANIFEST_TAG
done
after_pipeline:
task:
Expand Down