Skip to content

Commit

Permalink
Merge pull request #591 from bcgov/dev
Browse files Browse the repository at this point in the history
dev to main merge 29 dec
  • Loading branch information
midhun-aot authored Dec 29, 2023
2 parents e5fd9f0 + fca3e5f commit e6c1e72
Show file tree
Hide file tree
Showing 42 changed files with 11,217 additions and 2,217 deletions.
71 changes: 71 additions & 0 deletions .github/workflows/cd-debezium-jdbc.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
name: debezium-jdbc

on:
workflow_dispatch:
inputs:
START_BUILD:
required: true
type: boolean
description: This value is ignored, just to trigger dispatch.
push:
branches:
- dev
- main
- feat/srs-25-etl-deploy
paths:
- 'etl/**'
- '.github/workflows/**'

defaults:
run:
shell: bash
working-directory: ./etl/debezium-jdbc
env:
APP_NAME: "debezium-jdbc"

jobs:
etl-debezium-jdbc-cd-by-push:
runs-on: ubuntu-20.04

if: github.repository == 'bcgov/nr-epd-digital-services'
steps:
- uses: actions/checkout@v2
- name: Set ENV variables
id: set-variable
# make sure to update the TOOLS_NAME once all the environment for forms-flow-ai-web is ready
run: |
if [ '${{ github.ref_name }}' == 'dev' ]; then
echo "For ${{ github.ref_name }} branch"
echo "TOOLS_NAME=${{secrets.OC_ETL_NAMESPACE_NAMEPLATE}}" >> $GITHUB_ENV
echo "TAG_NAME="dev"" >> $GITHUB_ENV
echo "BRANCH_NAME="dev"" >> $GITHUB_ENV
echo "ENV_NAME="dev"" >> $GITHUB_ENV
elif [ '${{ github.ref_name }}' == 'main' ]; then
echo "For ${{ github.ref_name }} branch"
echo "TOOLS_NAME=${{secrets.OC_ETL_NAMESPACE_NAMEPLATE}}" >> $GITHUB_ENV
echo "TAG_NAME="test"" >> $GITHUB_ENV
echo "BRANCH_NAME="main"" >> $GITHUB_ENV
echo "ENV_NAME="test"" >> $GITHUB_ENV
else
echo "For ${{ github.ref_name }} branch, defaulting to dev values"
echo "TOOLS_NAME=${{secrets.OC_ETL_NAMESPACE_NAMEPLATE}}" >> $GITHUB_ENV
echo "TAG_NAME="dev"" >> $GITHUB_ENV
echo "BRANCH_NAME="dev"" >> $GITHUB_ENV
fi
shell: bash

- name: Login Openshift
shell: bash
run: |
oc login --server=${{secrets.OC_SERVER}} --token=${{secrets.OC_TOKEN}}
- name: Tools project
shell: bash
run: |
oc project ${{ secrets.OC_ETL_NAMESPACE_NAMEPLATE }}-tools
- name: Start Build Openshift
shell: bash
run: |
oc start-build ${{ env.APP_NAME }}-${{ env.TAG_NAME }}-build --wait
83 changes: 83 additions & 0 deletions .github/workflows/cd-debezium-postgis.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
name: debezium-postgis Custom

on:
workflow_dispatch:
inputs:
START_BUILD:
required: true
type: boolean
description: This value is ignored, just to trigger dispatch.
push:
branches:
- dev
- main
- feat/srs-25-etl-deploy
paths:
- 'etl/**'
- '.github/workflows/**'

defaults:
run:
shell: bash
working-directory: ./etl/debezium-postgis
env:
APP_NAME: "debezium-postgis"

jobs:
etl-debezium-postgis-cd-by-push:
runs-on: ubuntu-20.04

if: github.repository == 'bcgov/nr-epd-digital-services'
steps:
- uses: actions/checkout@v2
- name: Set ENV variables
id: set-variable
# make sure to update the TOOLS_NAME once all the environment for forms-flow-ai-web is ready
run: |
if [ '${{ github.ref_name }}' == 'dev' ]; then
echo "For ${{ github.ref_name }} branch"
echo "TOOLS_NAME=${{secrets.OC_ETL_NAMESPACE_NAMEPLATE}}" >> $GITHUB_ENV
echo "TAG_NAME="dev"" >> $GITHUB_ENV
echo "BRANCH_NAME="dev"" >> $GITHUB_ENV
echo "ENV_NAME="dev"" >> $GITHUB_ENV
elif [ '${{ github.ref_name }}' == 'main' ]; then
echo "For ${{ github.ref_name }} branch"
echo "TOOLS_NAME=${{secrets.OC_ETL_NAMESPACE_NAMEPLATE}}" >> $GITHUB_ENV
echo "TAG_NAME="test"" >> $GITHUB_ENV
echo "BRANCH_NAME="main"" >> $GITHUB_ENV
echo "ENV_NAME="test"" >> $GITHUB_ENV
else
echo "For ${{ github.ref_name }} branch, defaulting to dev values"
echo "TOOLS_NAME=${{secrets.OC_ETL_NAMESPACE_NAMEPLATE}}" >> $GITHUB_ENV
echo "TAG_NAME="dev"" >> $GITHUB_ENV
echo "BRANCH_NAME="dev"" >> $GITHUB_ENV
fi
shell: bash

- name: Login Openshift
shell: bash
run: |
oc login --server=${{secrets.OC_SERVER}} --token=${{secrets.OC_TOKEN}}
- name: Tools project
shell: bash
run: |
oc project ${{ secrets.OC_ETL_NAMESPACE_NAMEPLATE }}-tools
# Build from either dev or main branch as appropriate
- name: Build from ${{ env.BRANCH_NAME }} branch
shell: bash
run: |
oc patch bc/${{ env.APP_NAME }}-build -p '{"spec":{"source":{"git":{"ref":"${{ env.BRANCH_NAME }}"}}}}'
- name: Start Build Openshift
shell: bash
run: |
oc start-build ${{ env.APP_NAME }}-build --wait
- name: Tag+Deploy for ${{ env.TAG_NAME }}
shell: bash
run: |
oc tag ${{ env.APP_NAME }}:latest ${{ env.APP_NAME }}:${{ env.TAG_NAME }}
36 changes: 0 additions & 36 deletions .github/workflows/ci-cd-debezium-tools.yaml

This file was deleted.

2 changes: 1 addition & 1 deletion .github/workflows/ci-cd-openshift-tools.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ on:
paths:
- 'backend/**'
- 'frontend/**'
- '.github/workflows/ci-cd-openshift-tools.yaml'
- '.github/workflows/ci-cd-openshift-tools.yaml'

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
Expand Down
12 changes: 6 additions & 6 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -114,12 +114,12 @@ snyk.sarif

# Docker/Podman volumes
.volumes
backend/etl/debezium-jdbc/.settings/org.eclipse.m2e.core.prefs
backend/etl/debezium-jdbc/.settings/org.eclipse.jdt.core.prefs
backend/etl/debezium-jdbc/.settings/org.eclipse.core.resources.prefs
backend/etl/debezium-jdbc/.project
backend/etl/debezium-jdbc/.classpath
backend/etl/debezium-jdbc/target/
etl/debezium-jdbc/.settings/org.eclipse.m2e.core.prefs
etl/debezium-jdbc/.settings/org.eclipse.jdt.core.prefs
etl/debezium-jdbc/.settings/org.eclipse.core.resources.prefs
etl/debezium-jdbc/.project
etl/debezium-jdbc/.classpath
etl/debezium-jdbc/target/


notes/
Expand Down
54 changes: 53 additions & 1 deletion etl/ReadMe.md
Original file line number Diff line number Diff line change
Expand Up @@ -37,18 +37,60 @@ jdbc:oracle:thin:@(description=(address=(protocol=tcps)(host=172.18.215.225.nip.

# Create sample data in postgresql using init.sql

# OpenShift Build and Deploy process

## 1. Build custom postgis

```
oc process -f debezium-postgis.build.yaml |oc apply -f -
```

## 2. Build kafka jdbc connect component

```
oc process -f debezium-jdbc.build.yaml --param-file=.env |oc apply -f -
```

## 3. Build kafka broker.
```
oc process -f debezium-kafka.build.yaml |oc apply -f -
```

## 4. Deploy postgis (switch to correct project for env before this step)
```
oc process -f debezium-postgis.deploy.yaml |oc apply -f -
```
## 5. Deploy zookeeper
```
oc process -f debezium-zookeeper.deploy.yaml |oc apply -f -
```
## 6. Deploy kafka
```
oc process -f debezium-kafka.deploy.yaml |oc apply -f -
```
## 7. Deploy kafka-jdbc connect.
```
oc process -f debezium-jdbc.deploy.yaml |oc apply -f -
```

# Register postgresql connector
```
curl -H "Content-Type: application/json" -d @register-postgres-source-connector.json http://localhost:8083/connectors/
curl -H "Content-Type: application/json" -d @register-postgres-source-connector.json https://debezium-jdbc-latest.apps.silver.devops.gov.bc.ca/connectors/
```

# Register Oracle jdbc sink connector

```
curl -H "Content-Type: application/json" -d @register-oracle-jdbc-sink-connector.json http://localhost:8083/connectors/
curl -H "Content-Type: application/json" -d @register-oracle-jdbc-sink-connector.json https://debezium-jdbc-latest.apps.silver.devops.gov.bc.ca/connectors/
```

# Test with a kafka console consumer.
Expand All @@ -67,6 +109,16 @@ curl -X DELETE localhost:8083/connectors/<connector-name>
curl -X DELETE localhost:8083/connectors/oracle-jdbc-sink-connector
curl -X DELETE localhost:8083/connectors/postgres-source-connector
```
curl -X DELETE https://debezium-jdbc-latest.apps.silver.devops.gov.bc.ca/connectors/oracle-jdbc-sink-connector
curl -X DELETE https://debezium-jdbc-latest.apps.silver.devops.gov.bc.ca/connectors/postgres-source-connector
```

# Get all connectors registered.
```
curl localhost:8083/connectors/
curl https://debezium-jdbc-latest.apps.silver.devops.gov.bc.ca/connectors
```


13 changes: 13 additions & 0 deletions etl/debezium-jdbc/.classpath
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,19 @@
<attribute name="m2e-apt" value="true"/>
</attributes>
</classpathentry>
<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
<attributes>
<attribute name="maven.pomderived" value="true"/>
<attribute name="optional" value="true"/>
</attributes>
</classpathentry>
<classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources">
<attributes>
<attribute name="maven.pomderived" value="true"/>
<attribute name="test" value="true"/>
<attribute name="optional" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="src" output="target/test-classes" path="target/generated-test-sources/test-annotations">
<attributes>
<attribute name="optional" value="true"/>
Expand Down
Binary file not shown.
Binary file not shown.
4 changes: 0 additions & 4 deletions etl/debezium-jdbc/target/maven-archiver/pom.properties

This file was deleted.

This file was deleted.

This file was deleted.

6 changes: 6 additions & 0 deletions etl/debezium-kafka/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@

FROM debezium/kafka:1.9


COPY connect-log4j.properties /kafka/config/
COPY log4j.properties /kafka/config/
Loading

0 comments on commit e6c1e72

Please sign in to comment.