From b25762e801182e530995759b7cf572d72c89e02c Mon Sep 17 00:00:00 2001 From: Nikolas Athanasopoulos Date: Mon, 7 Mar 2022 18:20:12 +0100 Subject: [PATCH 1/2] added pmu-kafka-timescale demo Signed-off-by: Nikolas Athanasopoulos --- pmu-kafka-timescale-demo/README.md | 122 +++++++ pmu-kafka-timescale-demo/demo-setup.sh | 45 +++ .../kafka-connect/kafka-connect.yaml | 42 +++ .../kafka-connect/kafka-sink-connector.yaml | 15 + .../kafka-streams-deployment.yaml | 28 ++ .../pmu-dummy/deployment.yaml | 58 ++++ .../pmu-dummy/template-configmap.yaml | 63 ++++ pmu-kafka-timescale-demo/regcred-secret.yaml | 7 + .../strimzi/strimzi-kafka-cluster.yaml | 38 +++ .../timescaledb-credentials-secret.yaml | 8 + .../timescaledb/timescaledb-values.yaml | 70 ++++ .../visualization/dashboard-configmap.yaml | 298 ++++++++++++++++++ .../visualization/grafana_values.yaml | 28 ++ 13 files changed, 822 insertions(+) create mode 100644 pmu-kafka-timescale-demo/README.md create mode 100644 pmu-kafka-timescale-demo/demo-setup.sh create mode 100644 pmu-kafka-timescale-demo/kafka-connect/kafka-connect.yaml create mode 100644 pmu-kafka-timescale-demo/kafka-connect/kafka-sink-connector.yaml create mode 100644 pmu-kafka-timescale-demo/kafka-streams/kafka-streams-deployment.yaml create mode 100644 pmu-kafka-timescale-demo/pmu-dummy/deployment.yaml create mode 100644 pmu-kafka-timescale-demo/pmu-dummy/template-configmap.yaml create mode 100644 pmu-kafka-timescale-demo/regcred-secret.yaml create mode 100644 pmu-kafka-timescale-demo/strimzi/strimzi-kafka-cluster.yaml create mode 100644 pmu-kafka-timescale-demo/timescaledb/timescaledb-credentials-secret.yaml create mode 100644 pmu-kafka-timescale-demo/timescaledb/timescaledb-values.yaml create mode 100644 pmu-kafka-timescale-demo/visualization/dashboard-configmap.yaml create mode 100644 pmu-kafka-timescale-demo/visualization/grafana_values.yaml diff --git a/pmu-kafka-timescale-demo/README.md b/pmu-kafka-timescale-demo/README.md new file mode 100644 index 0000000..8daf10f --- /dev/null +++ b/pmu-kafka-timescale-demo/README.md @@ -0,0 +1,122 @@ +# PMU/Kafka/TimescaleDB Data Visualization Demo + +This repository contains deployment instructions and corresponding configuration files for the SOGNO Platform. +The core platform is based on the SOGNO platform. +We assume you have a full-fleged or light-weight kubernetes cluster up and running. +Please ensure you setup is in line with [this](https://sogno-platform.github.io/docs/getting-started/single-node/) base setup. + +## Create Namespace +Create `demo` namespace where resources will be deployed +```bash +$ kubectl create namespace demo +``` + +## Container Registry credentials + +This deployment requires an access to a container registry to pull and push Docker images. In order to do this, a secret with the registry credentials has to be created. + +Modify `regcred-secret.yaml` with the appropriate credentials (the `data[.dockerconfigjson]` corresponds to the contents of `~/.docker/config.json` encoded in base64) and apply the secret manifest: +```bash +kubectl apply -f regcred-secret.yaml -n demo +``` + +## Visualization Stack + +### Kafka/Strimzi Deployment + +Deploy the Strimzi Cluster Operator +```bash +$ helm repo add strimzi https://strimzi.io/charts/ +$ helm repo update +$ helm install strimzi strimzi/strimzi-kafka-operator -n demo +``` + +Deploy the Kafka Cluster +```bash +$ kubectl apply -f strimzi/strimzi-kafka-cluster.yaml -n demo +``` + +Wait for Cluster to be ready +``` +$ kubectl wait kafka/strimzi-cluster -n demo --for=condition=Ready --timeout=300s +``` + +### Timescale Database + +Add helm chart repo +```bash +$ helm repo add timescaledb 'https://raw.githubusercontent.com/timescale/timescaledb-kubernetes/master/charts/repo/' +``` + +Install helm chart +```bash +$ helm repo add timescaledb 'https://raw.githubusercontent.com/timescale/timescaledb-kubernetes/master/charts/repo/' +$ openssl req -x509 -sha256 -nodes -newkey rsa:4096 -days 3650 -subj "/CN=*.timescaledb.svc.cluster.local" -keyout tls.key -out tls.crt +$ kubectl create secret generic -n demo timescaledb-cluster-certificate --from-file=tls.crt=tls.crt --from-file=tls.key=tls.key +$ rm tls.crt tls.key +$ kubectl apply -f timescaledb/timescaledb-credentials-secret.yaml -n demo +$ helm install timescaledb-cluster timescaledb/timescaledb-single -n demo -f timescaledb/timescaledb-values.yaml +``` + +Directly execute a psql session on the master node +```bash +$ MASTERPOD="$(kubectl get pod -o name --namespace demo -l release=timescaledb-cluster)" +$ kubectl exec -i --tty --namespace demo ${MASTERPOD} -- psql -U postgres +``` + +Create a database named kafka with user kafka and grant access +```sql +> CREATE DATABASE kafka; +> CREATE ROLE kafka WITH LOGIN SUPERUSER PASSWORD 'kafka'; +> GRANT ALL PRIVILEGES ON DATABASE kafka TO kafka; +``` + +### Kafka Connect/Kafka Connector + +Modify the container image registry URL at `kafka-connect/kafka-connect.yaml` and apply the Kafka Connect manifest +```bash +$ kubectl apply -f kafka-connect/kafka-connect.yaml -n demo +``` + +Apply the Kafka Sink Connector manifest +```bash +$ kubectl apply -f kafka-connect/kafka-sink-connector.yaml -n demo +``` + +### PMU Simulation +Modify the template at `pmu-dummy/template-configmap.yaml` if necessary and apply the configmap manifest +```bash +$ kubectl apply -f pmu-dummy/template-configmap.yaml -n demo +``` + +Modify the environment variables at `pmu-dummy/deployment.yaml` and apply the deployment manifest +```bash +$ kubectl apply -f pmu-dummy/deployment.yaml -n demo +``` + +### Kafka Streams + +Modify the environment variables at `kafka-streams/kafka-streams-deployment.yaml` and apply the deployment manifest +```bash +$ kubectl apply -f kafka-streams/kafka-streams-deployment.yaml -n demo +``` + +### Grafana + +Adjust the host url inside */visualization/grafana_values.yaml* for the Ingress component and install the helm repo +```bash +$ helm repo add grafana https://grafana.github.io/helm-charts +$ helm install grafana grafana/grafana -f visualization/grafana_values.yaml -n demo +``` + +Apply dashboard configmap +```bash +$ kubectl apply -f visualization/dashboard-configmap.yaml -n demo +``` + +Get admin password +```bash +$ kubectl get secret -n demo grafana -o jsonpath="{.data.admin-password}" | base64 --decode ; echo +``` + +Access the url provided in the ingress component in a web browser to visualize the data \ No newline at end of file diff --git a/pmu-kafka-timescale-demo/demo-setup.sh b/pmu-kafka-timescale-demo/demo-setup.sh new file mode 100644 index 0000000..54f0c3a --- /dev/null +++ b/pmu-kafka-timescale-demo/demo-setup.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +namespace=demo + +kubectl create namespace $namespace + +kubectl apply -f regcred-secret.yaml -n $namespace + +# Strimzi +helm repo add strimzi https://strimzi.io/charts/ +helm repo update +helm install strimzi strimzi/strimzi-kafka-operator -n $namespace +kubectl apply -f strimzi/strimzi-kafka-cluster.yaml -n $namespace +kubectl wait kafka/strimzi-cluster -n $namespace --for=condition=Ready --timeout=300s + +# Timescale +helm repo add timescaledb 'https://raw.githubusercontent.com/timescale/timescaledb-kubernetes/master/charts/repo/' +openssl req -x509 -sha256 -nodes -newkey rsa:4096 -days 3650 -subj "/CN=*.timescaledb.svc.cluster.local" -keyout tls.key -out tls.crt +kubectl create secret generic -n $namespace timescaledb-cluster-certificate --from-file=tls.crt=tls.crt --from-file=tls.key=tls.key +rm tls.crt tls.key +kubectl apply -f timescaledb/timescaledb-credentials-secret.yaml -n $namespace +helm install timescaledb-cluster timescaledb/timescaledb-single -n $namespace -f timescaledb/timescaledb-values.yaml +kubectl wait pod/timescaledb-cluster-0 -n $namespace --for=condition=Ready --timeout=60s +kubectl exec timescaledb-cluster-0 -n $namespace -- psql -U postgres -c 'CREATE DATABASE kafka;' +kubectl exec timescaledb-cluster-0 -n $namespace -- psql -U postgres -c "CREATE ROLE kafka WITH LOGIN SUPERUSER PASSWORD 'kafka';" +kubectl exec timescaledb-cluster-0 -n $namespace -- psql -U postgres -c 'GRANT ALL PRIVILEGES ON DATABASE kafka TO kafka;' + +# Kafka Connect +kubectl apply -f kafka-connect/kafka-connect.yaml -n $namespace + +# Kafka Connector +kubectl apply -f kafka-connect/kafka-sink-connector.yaml -n $namespace + +# PMU-dummy +kubectl apply -f pmu-dummy/template-configmap.yaml -n $namespace +kubectl apply -f pmu-dummy/deployment.yaml -n $namespace + +# Kafka Streams +kubectl apply -f kafka-streams/kafka-streams-deployment.yaml -n $namespace + +# Grafana +helm repo add grafana https://grafana.github.io/helm-charts +helm install grafana grafana/grafana -f visualization/grafana_values.yaml -n $namespace +kubectl apply -f visualization/dashboard-configmap.yaml -n $namespace +kubectl get secret -n $namespace grafana -o jsonpath="{.data.admin-password}" | base64 --decode ; echo \ No newline at end of file diff --git a/pmu-kafka-timescale-demo/kafka-connect/kafka-connect.yaml b/pmu-kafka-timescale-demo/kafka-connect/kafka-connect.yaml new file mode 100644 index 0000000..d4ddcfd --- /dev/null +++ b/pmu-kafka-timescale-demo/kafka-connect/kafka-connect.yaml @@ -0,0 +1,42 @@ +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaConnect +metadata: + name: kafka-connect-cluster + annotations: + strimzi.io/use-connector-resources: "true" +spec: + replicas: 1 + bootstrapServers: strimzi-cluster-kafka-bootstrap.demo:9092 + config: + group.id: kafka-connect-cluster + offset.storage.topic: kafka-connect-cluster-offsets + config.storage.topic: kafka-connect-cluster-configs + status.storage.topic: kafka-connect-cluster-status + key.converter: org.apache.kafka.connect.json.JsonConverter + value.converter: org.apache.kafka.connect.json.JsonConverter + key.converter.schemas.enable: true + value.converter.schemas.enable: true + config.storage.replication.factor: 1 + offset.storage.replication.factor: 1 + status.storage.replication.factor: 1 + config.providers: file + config.providers.file.class: org.apache.kafka.common.config.provider.FileConfigProvider + externalConfiguration: + volumes: + - name: timescaledb-cluster-credentials + secret: + secretName: timescaledb-cluster-credentials + build: + output: + type: docker + image: registry.example.com/pmu-kafka-timescale-demo/kafka-connect-cluster + pushSecret: regcred + plugins: + - name: confluent-postgres-connector + artifacts: + - type: zip + url: https://d1i4a15mxbxib1.cloudfront.net/api/plugins/confluentinc/kafka-connect-jdbc/versions/10.2.0/confluentinc-kafka-connect-jdbc-10.2.0.zip + template: + pod: + imagePullSecrets: + - name: regcred diff --git a/pmu-kafka-timescale-demo/kafka-connect/kafka-sink-connector.yaml b/pmu-kafka-timescale-demo/kafka-connect/kafka-sink-connector.yaml new file mode 100644 index 0000000..b226ae9 --- /dev/null +++ b/pmu-kafka-timescale-demo/kafka-connect/kafka-sink-connector.yaml @@ -0,0 +1,15 @@ +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaConnector +metadata: + name: kafka-sink-connector + labels: + strimzi.io/cluster: kafka-connect-cluster +spec: + class: io.confluent.connect.jdbc.JdbcSinkConnector + tasksMax: 1 + config: + topics: pmu-dummy-out + connection.url: jdbc:postgresql://timescaledb-cluster.demo.svc.cluster.local:5432/kafka + connection.user: kafka + connection.password: kafka + auto.create: true diff --git a/pmu-kafka-timescale-demo/kafka-streams/kafka-streams-deployment.yaml b/pmu-kafka-timescale-demo/kafka-streams/kafka-streams-deployment.yaml new file mode 100644 index 0000000..2f2ab7e --- /dev/null +++ b/pmu-kafka-timescale-demo/kafka-streams/kafka-streams-deployment.yaml @@ -0,0 +1,28 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kstreams +spec: + replicas: 1 + selector: + matchLabels: + app: kstreams + template: + metadata: + labels: + app: kstreams + spec: + containers: + - name: kstreams + image: registry.example.com/pmu-kafka-timescale-demo/kafka-connect-cluster/pmu-streams-connector:latest + env: + - name: KAFKA_BROKER + value: strimzi-cluster-kafka-bootstrap.demo:9092 + - name: INPUT_TOPIC + value: pmu-dummy-in + - name: OUTPUT_TOPIC + value: pmu-dummy-out + - name: APP_ID + value: pmu-dummy-app + imagePullSecrets: + - name: regcred diff --git a/pmu-kafka-timescale-demo/pmu-dummy/deployment.yaml b/pmu-kafka-timescale-demo/pmu-dummy/deployment.yaml new file mode 100644 index 0000000..f7f45e7 --- /dev/null +++ b/pmu-kafka-timescale-demo/pmu-dummy/deployment.yaml @@ -0,0 +1,58 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pmu-dummy + labels: + app: pmu-dummy +spec: + selector: + matchLabels: + app: pmu-dummy + replicas: 1 + strategy: {} + template: + metadata: + labels: + app: pmu-dummy + spec: + containers: + - name: pmu-dummy + image: registry.example.com/pmu-kafka-timescale-demo/kafka-connect-cluster/pmu-dummy:latest + volumeMounts: + - name: pmu-dummy-template-config + mountPath: /usr/src/app/device_template.json + subPath: device_template.json + env: + #Config for both MQTT and Kafka + - name: JSON_TEMPLATE + value: /usr/src/app/device_template.json #path for the data template file (/usr/src/app/ is where the Docker image alocates the application files) + - name: PRODUCER_TYPE + value: kafka #"mqtt" or "kafka" + - name: BROKER_URL + value: strimzi-cluster-kafka-bootstrap.demo #FQDN or IP + - name: BROKER_PORT + value: "9092" + - name: TOPIC_NAME + value: pmu-dummy-in + #MQTT Specific Config + - name: MQTT_USER + value: admin #USERNAME + - name: MQTT_PWD + value: admin #PASSWORD + - name: MQTT_SSL + value: "false" #toggle ssl true/false + - name: MQTT_CAFILE + value: "/etc/ssl/certs/DST_Root_CA_X3.pem" #example ca file for let's Encrypt + - name: MQTT_DEVICE_NAME + value: device1-dummy + ports: + - containerPort: 3000 + volumes: + - name: pmu-dummy-template-config + configMap: + name: pmu-dummy-template-config + items: + - key: device_template.json + path: device_template.json + imagePullSecrets: + - name: regcred diff --git a/pmu-kafka-timescale-demo/pmu-dummy/template-configmap.yaml b/pmu-kafka-timescale-demo/pmu-dummy/template-configmap.yaml new file mode 100644 index 0000000..49a3f00 --- /dev/null +++ b/pmu-kafka-timescale-demo/pmu-dummy/template-configmap.yaml @@ -0,0 +1,63 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: pmu-dummy-template-config +data: + device_template.json: | + { + "schema": { + "type": "struct", + "fields": [ + { + "type": "string", + "optional": false, + "field": "device" + }, + { + "type": "int64", + "optional": false, + "field": "timestamp" + }, + { + "type": "string", + "optional": false, + "field": "component" + }, + { + "type": "string", + "optional": false, + "field": "measurand" + }, + { + "type": "string", + "optional": false, + "field": "phase" + }, + { + "type": "float", + "optional": true, + "field": "data" + } + ], + "optional": false, + "name": "pmu-dummy-schema" + }, + "payload": { + "device": "device1", + "timestamp": "TIMESTAMP", + "readings": [ + { + "component": "BUS1", + "measurand": "voltmagnitude", + "phase": "A", + "data": "RANDOM" + }, + { + "component": "BUS2", + "measurand": "voltmagnitude", + "phase": "B", + "data": "RANDOM" + } + ] + } + } diff --git a/pmu-kafka-timescale-demo/regcred-secret.yaml b/pmu-kafka-timescale-demo/regcred-secret.yaml new file mode 100644 index 0000000..224c7f5 --- /dev/null +++ b/pmu-kafka-timescale-demo/regcred-secret.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: regcred +type: kubernetes.io/dockerconfigjson +data: + .dockerconfigjson: YmFzZTY0IG9mIH4vLmRvY2tlci9jb25maWcuanNvbg== \ No newline at end of file diff --git a/pmu-kafka-timescale-demo/strimzi/strimzi-kafka-cluster.yaml b/pmu-kafka-timescale-demo/strimzi/strimzi-kafka-cluster.yaml new file mode 100644 index 0000000..072f7cf --- /dev/null +++ b/pmu-kafka-timescale-demo/strimzi/strimzi-kafka-cluster.yaml @@ -0,0 +1,38 @@ +apiVersion: kafka.strimzi.io/v1beta2 +kind: Kafka +metadata: + name: strimzi-cluster +spec: + kafka: + version: 3.1.0 + replicas: 1 + listeners: + - name: plain + port: 9092 + type: internal + tls: false + - name: tls + port: 9093 + type: internal + tls: true + config: + offsets.topic.replication.factor: 1 + transaction.state.log.replication.factor: 1 + transaction.state.log.min.isr: 1 + inter.broker.protocol.version: "3.1" + storage: + type: jbod + volumes: + - id: 0 + type: persistent-claim + size: 100Gi + deleteClaim: false + zookeeper: + replicas: 1 + storage: + type: persistent-claim + size: 100Gi + deleteClaim: false + entityOperator: + topicOperator: {} + userOperator: {} diff --git a/pmu-kafka-timescale-demo/timescaledb/timescaledb-credentials-secret.yaml b/pmu-kafka-timescale-demo/timescaledb/timescaledb-credentials-secret.yaml new file mode 100644 index 0000000..ed37152 --- /dev/null +++ b/pmu-kafka-timescale-demo/timescaledb/timescaledb-credentials-secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: timescaledb-cluster-credentials +type: Opaque +data: + PATRONI_SUPERUSER_PASSWORD: cGFzc3dvcmQK # echo password | base64 + PATRONI_admin_PASSWORD: cGFzc3dvcmQK # echo password | base64 diff --git a/pmu-kafka-timescale-demo/timescaledb/timescaledb-values.yaml b/pmu-kafka-timescale-demo/timescaledb/timescaledb-values.yaml new file mode 100644 index 0000000..e785307 --- /dev/null +++ b/pmu-kafka-timescale-demo/timescaledb/timescaledb-values.yaml @@ -0,0 +1,70 @@ +replicaCount: 1 +nameOverride: timescaledb +clusterName: timescaledb-cluster + +loadBalancer: + enabled: true + +secrets: + # credentials: + # PATRONI_SUPERUSER_PASSWORD: "5f4dcc3b5aa765d61d8327deb882cf99" + # PATRONI_REPLICATION_PASSWORD: "5f4dcc3b5aa765d61d8327deb882cf99" + # PATRONI_admin_PASSWORD: "5f4dcc3b5aa765d61d8327deb882cf99" + credentialsSecretName: timescaledb-cluster-credentials + certificateSecretName: timescaledb-cluster-certificate + +resources: + {} + # limits: + # cpu: 500m + # memory: 2Gi + # requests: + # cpu: 250m + # memory: 1Gi + +persistentVolumes: + data: + enabled: true + size: 2Gi + #storageClass: "managed-nfs-storage" + subPath: "" + mountPath: "/var/lib/postgresql" + annotations: {} + accessModes: + - ReadWriteOnce + wal: + enabled: true + size: 1Gi + subPath: "" + storageClass: + mountPath: "/var/lib/postgresql/wal" + annotations: {} + accessModes: + - ReadWriteOnce + +patroni: + bootstrap: + dcs: + postgresql: + parameters: + wal_level: logical + max_wal_senders: 1 + max_replication_slots: 1 + postgresql: + authentication: + replication: + username: standby + superuser: + username: postgres + listen: 0.0.0.0:5432 + pg_hba: + - local all postgres peer + - local all all md5 + - hostnossl all,replication all all reject + - hostssl all all 127.0.0.1/32 md5 + - hostssl all all ::1/128 md5 + - hostssl replication standby all md5 + - hostssl all all all md5 + - local replication postgres trust + - host replication postgres 127.0.0.1/32 trust + - host replication postgres ::1/128 trust diff --git a/pmu-kafka-timescale-demo/visualization/dashboard-configmap.yaml b/pmu-kafka-timescale-demo/visualization/dashboard-configmap.yaml new file mode 100644 index 0000000..90d582d --- /dev/null +++ b/pmu-kafka-timescale-demo/visualization/dashboard-configmap.yaml @@ -0,0 +1,298 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard + labels: + grafana_dashboard: '1' +data: + pmu-dashboard.json: | + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Phase A" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "PCC52D03280B7034C" + }, + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": false, + "rawSql": "SELECT\n \"timestamp\" AS \"time\",\n data AS \"Phase A\"\nFROM \"pmu-dummy-out\"\nWHERE\n $__unixEpochFilter(\"timestamp\") AND\n phase = 'A'\nORDER BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "data" + ], + "type": "column" + }, + { + "params": [ + "Phase A" + ], + "type": "alias" + } + ] + ], + "table": "\"pmu-dummy-out\"", + "timeColumn": "\"timestamp\"", + "timeColumnType": "int8", + "where": [ + { + "name": "$__unixEpochFilter", + "params": [], + "type": "macro" + }, + { + "datatype": "text", + "name": "", + "params": [ + "phase", + "=", + "'A'" + ], + "type": "expression" + } + ] + }, + { + "datasource": { + "type": "postgres", + "uid": "PCC52D03280B7034C" + }, + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": false, + "rawSql": "SELECT\n \"timestamp\" AS \"time\",\n data AS \"Phase B\"\nFROM \"pmu-dummy-out\"\nWHERE\n $__unixEpochFilter(\"timestamp\") AND\n phase = 'B'\nORDER BY 1", + "refId": "B", + "select": [ + [ + { + "params": [ + "data" + ], + "type": "column" + }, + { + "params": [ + "Phase B" + ], + "type": "alias" + } + ] + ], + "table": "\"pmu-dummy-out\"", + "timeColumn": "\"timestamp\"", + "timeColumnType": "int8", + "where": [ + { + "name": "$__unixEpochFilter", + "params": [], + "type": "macro" + }, + { + "datatype": "text", + "name": "", + "params": [ + "phase", + "=", + "'B'" + ], + "type": "expression" + } + ] + }, + { + "datasource": { + "type": "postgres", + "uid": "PCC52D03280B7034C" + }, + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": false, + "rawSql": "SELECT\n \"timestamp\" AS \"time\",\n data AS \"Phase C\"\nFROM \"pmu-dummy-out\"\nWHERE\n $__unixEpochFilter(\"timestamp\") AND\n phase = 'C'\nORDER BY 1", + "refId": "C", + "select": [ + [ + { + "params": [ + "data" + ], + "type": "column" + }, + { + "params": [ + "Phase C" + ], + "type": "alias" + } + ] + ], + "table": "\"pmu-dummy-out\"", + "timeColumn": "\"timestamp\"", + "timeColumnType": "int8", + "where": [ + { + "name": "$__unixEpochFilter", + "params": [], + "type": "macro" + }, + { + "datatype": "text", + "name": "", + "params": [ + "phase", + "=", + "'C'" + ], + "type": "expression" + } + ] + } + ], + "title": "Kafka - Timescale", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 34, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Kafka - Timescale", + "uid": "NLINDUF7k", + "version": 1, + "weekStart": "" + } \ No newline at end of file diff --git a/pmu-kafka-timescale-demo/visualization/grafana_values.yaml b/pmu-kafka-timescale-demo/visualization/grafana_values.yaml new file mode 100644 index 0000000..a9bf93e --- /dev/null +++ b/pmu-kafka-timescale-demo/visualization/grafana_values.yaml @@ -0,0 +1,28 @@ +ingress: + enabled: true + hosts: + - # Add the IP or "HOSTNAME" here + path: / + +datasources: + datasources.yaml: + apiVersion: 1 + datasources: + - name: PostgreSQL + type: postgres + url: timescaledb-cluster.demo:5432 + access: proxy + isDefault: true + user: kafka + database: kafka + secureJsonData: + password: kafka + jsonData: + sslmode: require + tlsConfigurationMethod: file-path + timescaledb: true + +sidecar: + dashboards: + enabled: true + label: grafana_dashboard From b0dd1f67fa4d82b9aaeef04671654f2198ae38fc Mon Sep 17 00:00:00 2001 From: Nikolas Athanasopoulos Date: Mon, 7 Mar 2022 18:26:22 +0100 Subject: [PATCH 2/2] updated readme Signed-off-by: Nikolas Athanasopoulos --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 4059403..23307c2 100644 --- a/README.md +++ b/README.md @@ -10,3 +10,4 @@ More examples to come soon! - [PMU Data Visualization](pmu-data-visualization) - [Pyvolt DPsim Demo](pyvolt-dpsim-demo) +- [PMU/Kafka/TimescaleDB Data Visualization Demo](pmu-kafka-timescale-demo)