Merge pull request #582 from bcgov/feat/srs-25-etl-deploy

Feat/srs 25 etl deploy
bcgov · Dec 7, 2023 · cd98e9b · cd98e9b
2 parents 46e8ff7 + 922f8a9
commit cd98e9b
Show file tree

Hide file tree

Showing 14 changed files with 186 additions and 56 deletions.
diff --git a/.github/workflows/cd-debezium-jdbc.yaml b/.github/workflows/cd-debezium-jdbc.yaml
@@ -0,0 +1,71 @@
+name: debezium-jdbc
+
+on:
+  workflow_dispatch:
+    inputs:
+      START_BUILD:
+        required: true
+        type: boolean
+        description: This value is ignored, just to trigger dispatch.
+  push:
+    branches:
+      - dev
+      - main
+      - feat/srs-25-etl-deploy
+    paths:
+      - 'etl/**'
+      - '.github/workflows/**'
+
+defaults:
+  run:
+    shell: bash
+    working-directory: ./etl/debezium-jdbc
+env:
+  APP_NAME: "debezium-jdbc"
+
+jobs:
+  etl-debezium-jdbc-cd-by-push:
+    runs-on: ubuntu-20.04
+
+    if: github.repository == 'bcgov/nr-epd-digital-services'
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set ENV variables
+        id: set-variable
+        # make sure to update the TOOLS_NAME once all the environment for forms-flow-ai-web is ready
+        run: |
+          if [ '${{ github.ref_name }}' == 'dev' ]; then
+            echo "For ${{ github.ref_name }} branch"
+            echo "TOOLS_NAME=${{secrets.OC_ETL_NAMESPACE_NAMEPLATE}}" >> $GITHUB_ENV
+            echo "TAG_NAME="dev"" >> $GITHUB_ENV
+            echo "BRANCH_NAME="dev"" >> $GITHUB_ENV
+            echo "ENV_NAME="dev"" >> $GITHUB_ENV
+          elif [ '${{ github.ref_name }}' == 'main' ]; then
+            echo "For ${{ github.ref_name }} branch"
+            echo "TOOLS_NAME=${{secrets.OC_ETL_NAMESPACE_NAMEPLATE}}" >> $GITHUB_ENV
+            echo "TAG_NAME="test"" >> $GITHUB_ENV
+            echo "BRANCH_NAME="main"" >> $GITHUB_ENV
+            echo "ENV_NAME="test"" >> $GITHUB_ENV
+          else
+            echo "For ${{ github.ref_name }} branch, defaulting to dev values"
+            echo "TOOLS_NAME=${{secrets.OC_ETL_NAMESPACE_NAMEPLATE}}" >> $GITHUB_ENV
+            echo "TAG_NAME="dev"" >> $GITHUB_ENV
+            echo "BRANCH_NAME="dev"" >> $GITHUB_ENV
+          fi
+        shell: bash
+
+      - name: Login Openshift
+        shell: bash
+        run: |
+          oc login --server=${{secrets.OC_SERVER}} --token=${{secrets.OC_TOKEN}}
+
+      - name: Tools project
+        shell: bash
+        run: |
+          oc project ${{ secrets.OC_ETL_NAMESPACE_NAMEPLATE }}-tools
+          
+      - name: Start Build Openshift
+        shell: bash
+        run: |
+          oc start-build ${{ env.APP_NAME }}-${{ env.TAG_NAME }}-build --wait
+
diff --git a/.github/workflows/cd-debezium-postgis.yaml b/.github/workflows/cd-debezium-postgis.yaml
@@ -0,0 +1,83 @@
+name: debezium-postgis Custom
+
+on:
+  workflow_dispatch:
+    inputs:
+      START_BUILD:
+        required: true
+        type: boolean
+        description: This value is ignored, just to trigger dispatch.
+  push:
+    branches:
+      - dev
+      - main
+      - feat/srs-25-etl-deploy
+    paths:
+      - 'etl/**'
+      - '.github/workflows/**'
+
+defaults:
+  run:
+    shell: bash
+    working-directory: ./etl/debezium-postgis
+env:
+  APP_NAME: "debezium-postgis"
+
+jobs:
+  etl-debezium-postgis-cd-by-push:
+    runs-on: ubuntu-20.04
+
+    if: github.repository == 'bcgov/nr-epd-digital-services'
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set ENV variables
+        id: set-variable
+        # make sure to update the TOOLS_NAME once all the environment for forms-flow-ai-web is ready
+        run: |
+          if [ '${{ github.ref_name }}' == 'dev' ]; then
+            echo "For ${{ github.ref_name }} branch"
+            echo "TOOLS_NAME=${{secrets.OC_ETL_NAMESPACE_NAMEPLATE}}" >> $GITHUB_ENV
+            echo "TAG_NAME="dev"" >> $GITHUB_ENV
+            echo "BRANCH_NAME="dev"" >> $GITHUB_ENV
+            echo "ENV_NAME="dev"" >> $GITHUB_ENV
+          elif [ '${{ github.ref_name }}' == 'main' ]; then
+            echo "For ${{ github.ref_name }} branch"
+            echo "TOOLS_NAME=${{secrets.OC_ETL_NAMESPACE_NAMEPLATE}}" >> $GITHUB_ENV
+            echo "TAG_NAME="test"" >> $GITHUB_ENV
+            echo "BRANCH_NAME="main"" >> $GITHUB_ENV
+            echo "ENV_NAME="test"" >> $GITHUB_ENV
+          else
+            echo "For ${{ github.ref_name }} branch, defaulting to dev values"
+            echo "TOOLS_NAME=${{secrets.OC_ETL_NAMESPACE_NAMEPLATE}}" >> $GITHUB_ENV
+            echo "TAG_NAME="dev"" >> $GITHUB_ENV
+            echo "BRANCH_NAME="dev"" >> $GITHUB_ENV
+          fi
+        shell: bash
+
+      - name: Login Openshift
+        shell: bash
+        run: |
+          oc login --server=${{secrets.OC_SERVER}} --token=${{secrets.OC_TOKEN}}
+
+      - name: Tools project
+        shell: bash
+        run: |
+          oc project ${{ secrets.OC_ETL_NAMESPACE_NAMEPLATE }}-tools
+
+      # Build from either dev or main branch as appropriate
+      - name: Build from ${{ env.BRANCH_NAME }} branch
+        shell: bash
+        run: |
+          oc patch bc/${{ env.APP_NAME }}-build -p '{"spec":{"source":{"git":{"ref":"${{ env.BRANCH_NAME }}"}}}}'
+
+      - name: Start Build Openshift
+        shell: bash
+        run: |
+          oc start-build ${{ env.APP_NAME }}-build --wait
+
+
+      - name: Tag+Deploy for ${{ env.TAG_NAME }}
+        shell: bash
+        run: |
+          oc tag ${{ env.APP_NAME }}:latest ${{ env.APP_NAME }}:${{ env.TAG_NAME }}
+
diff --git a/.github/workflows/ci-cd-debezium-tools.yaml b/.github/workflows/ci-cd-debezium-tools.yaml
diff --git a/.github/workflows/ci-cd-openshift-tools.yaml b/.github/workflows/ci-cd-openshift-tools.yaml
@@ -9,7 +9,7 @@ on:
     paths:
       - 'backend/**'
       - 'frontend/**'
-      - '.github/workflows/ci-cd-openshift-tools.yaml'
+      - '.github/workflows/ci-cd-openshift-tools.yaml'      
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

diff --git a/.gitignore b/.gitignore
@@ -114,12 +114,12 @@ snyk.sarif
 
 # Docker/Podman volumes
 .volumes
-backend/etl/debezium-jdbc/.settings/org.eclipse.m2e.core.prefs
-backend/etl/debezium-jdbc/.settings/org.eclipse.jdt.core.prefs
-backend/etl/debezium-jdbc/.settings/org.eclipse.core.resources.prefs
-backend/etl/debezium-jdbc/.project
-backend/etl/debezium-jdbc/.classpath
-backend/etl/debezium-jdbc/target/
+etl/debezium-jdbc/.settings/org.eclipse.m2e.core.prefs
+etl/debezium-jdbc/.settings/org.eclipse.jdt.core.prefs
+etl/debezium-jdbc/.settings/org.eclipse.core.resources.prefs
+etl/debezium-jdbc/.project
+etl/debezium-jdbc/.classpath
+etl/debezium-jdbc/target/
 
 
 notes/

diff --git a/etl/debezium-jdbc/.classpath b/etl/debezium-jdbc/.classpath
@@ -31,6 +31,19 @@
 			<attribute name="m2e-apt" value="true"/>
 		</attributes>
 	</classpathentry>
+	<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
+		<attributes>
+			<attribute name="maven.pomderived" value="true"/>
+			<attribute name="optional" value="true"/>
+		</attributes>
+	</classpathentry>
+	<classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources">
+		<attributes>
+			<attribute name="maven.pomderived" value="true"/>
+			<attribute name="test" value="true"/>
+			<attribute name="optional" value="true"/>
+		</attributes>
+	</classpathentry>
 	<classpathentry kind="src" output="target/test-classes" path="target/generated-test-sources/test-annotations">
 		<attributes>
 			<attribute name="optional" value="true"/>

diff --git a/etl/debezium-jdbc/target/classes/ca/bc/gov/epd/kafka/connect/transform/ToUpperCase.class b/etl/debezium-jdbc/target/classes/ca/bc/gov/epd/kafka/connect/transform/ToUpperCase.class
diff --git a/etl/debezium-jdbc/target/kafka-connect-debezium-transforms-1.0-SNAPSHOT.jar b/etl/debezium-jdbc/target/kafka-connect-debezium-transforms-1.0-SNAPSHOT.jar
diff --git a/etl/debezium-jdbc/target/maven-archiver/pom.properties b/etl/debezium-jdbc/target/maven-archiver/pom.properties
diff --git a/...m-jdbc/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst b/...m-jdbc/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst
diff --git a/...ium-jdbc/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst b/...ium-jdbc/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst
diff --git a/openshift/templates/etl/debezium-jdbc.build.yaml b/openshift/templates/etl/debezium-jdbc.build.yaml
@@ -2,7 +2,7 @@
 kind: Template
 apiVersion: template.openshift.io/v1
 metadata:
-  name: "${API_NAME}-build-template"
+  name: "${API_NAME}-${ENV_TAG}-build-template"
   creationTimestamp: 
 objects:
 - kind: ImageStream
@@ -12,9 +12,9 @@ objects:
 - kind: BuildConfig
   apiVersion: v1
   metadata:
-    name: "${API_NAME}-build"
+    name: "${API_NAME}-${ENV_TAG}-build"
     labels:
-      app: "${API_NAME}-build"
+      app: "${API_NAME}-${ENV_TAG}-build"
   spec:
     runPolicy: Serial
     source:
@@ -44,13 +44,18 @@ objects:
     output:
       to:
         kind: ImageStreamTag
-        name: "${API_NAME}:latest"
+        name: "${API_NAME}:${ENV_TAG}"
 parameters:
 - name: API_NAME
   displayName: Name
   description: The name assigned to all of the resources defined in this template.
   required: true
   value: debezium-jdbc
+- name: ENV_TAG
+  displayName: Environment name
+  description: The name of the environment, in build names and image stream tags
+  required: true
+  value: dev  
 - name: GIT_REPO_URL
   displayName: Git Repo URL
   description: The URL to your GIT repo.
@@ -60,7 +65,7 @@ parameters:
   displayName: Git Reference
   description: The git reference or branch.
   required: true
-  value: feat/srs-26
+  value: dev
 - name: SOURCE_CONTEXT_DIR
   displayName: Source Context Directory
   description: The source context directory.

diff --git a/openshift/templates/etl/debezium-jdbc.deploy.yaml b/openshift/templates/etl/debezium-jdbc.deploy.yaml
@@ -148,4 +148,4 @@ parameters:
   displayName: Environment TAG name
   description: The TAG name for this environment, e.g., dev, test, prod
   required: true
-  value: latest
+  value: dev
diff --git a/openshift/templates/etl/debezium-postgis.build.yaml b/openshift/templates/etl/debezium-postgis.build.yaml
@@ -46,7 +46,7 @@ parameters:
   displayName: Git Reference
   description: The git reference or branch.
   required: true
-  value: feat/srs-26
+  value: dev
 - name: SOURCE_CONTEXT_DIR
   displayName: Source Context Directory
   description: The source context directory.