-
Notifications
You must be signed in to change notification settings - Fork 1
/
aiven.tf
127 lines (115 loc) · 3.75 KB
/
aiven.tf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
//Based on:
// https://help.aiven.io/en/articles/4187903-kafka-connect-terraform-integration
// https://registry.terraform.io/providers/aiven/aiven/latest/docs/guides/getting-started
terraform {
required_providers {
aiven = {
source = "aiven/aiven"
version = "2.0.11"
}
}
}
variable "aiven_api_token" { }
variable "aiven_project_name" { }
variable "aiven_cloud_region" { default = "google-europe-west3" }
variable "aiven_osmetrics_topic" { default = "os_metrics"}
variable "aiven_connect_error_tolerance" { default = "none" }
provider "aiven" {
api_token = var.aiven_api_token
}
resource "aiven_project" "xko-aiven-kafkapg" {
project = var.aiven_project_name
}
resource "aiven_service" "pg-sink" {
project = aiven_project.xko-aiven-kafkapg.project
cloud_name = var.aiven_cloud_region
plan = "startup-4"
service_name = "pg-sink"
service_type = "pg"
pg_user_config {
pg {
idle_in_transaction_session_timeout = 900
}
pg_version = "10"
}
}
resource "aiven_database" "db-os-metrics" {
project = aiven_project.xko-aiven-kafkapg.project
service_name = aiven_service.pg-sink.service_name
database_name = "os-metrics-sink"
}
resource "aiven_service" "kafka" {
project = aiven_project.xko-aiven-kafkapg.project
cloud_name = var.aiven_cloud_region
plan = "startup-2"
service_name = "kafka"
service_type = "kafka"
kafka_user_config {
kafka {
auto_create_topics_enable = true
default_replication_factor = 2
num_partitions = 1
}
kafka_version = "2.6"
schema_registry = true
kafka_rest = true
}
}
resource "aiven_service" "kafka_connect" {
project = aiven_project.xko-aiven-kafkapg.project
cloud_name = var.aiven_cloud_region
plan = "startup-4"
service_name = "kafka-connect"
service_type = "kafka_connect"
kafka_connect_user_config {
kafka_connect {
consumer_isolation_level = "read_committed"
}
public_access {
kafka_connect = true
}
}
}
resource "aiven_service_integration" "kafka_connect" {
project = aiven_project.xko-aiven-kafkapg.project
integration_type = "kafka_connect"
source_service_name = aiven_service.kafka.service_name
destination_service_name = aiven_service.kafka_connect.service_name
kafka_connect_user_config {
kafka_connect {
group_id = "connect"
status_storage_topic = "__connect_status"
offset_storage_topic = "__connect_offsets"
}
}
}
resource "aiven_kafka_topic" "topic-os-metrics" {
depends_on = [aiven_service.kafka]
project = aiven_project.xko-aiven-kafkapg.project
service_name = aiven_service.kafka.service_name
topic_name = var.aiven_osmetrics_topic
partitions = 3
replication = 2
}
//noinspection HILUnresolvedReference
resource "aiven_kafka_connector" "kafka-pg-sink-con" {
project = aiven_project.xko-aiven-kafkapg.project
service_name = aiven_service.kafka_connect.service_name
connector_name = "kafka-pg-sink-con"
config = {
name = "kafka-pg-sink-con"
"topics" = aiven_kafka_topic.topic-os-metrics.topic_name
"connector.class" = "io.aiven.connect.jdbc.JdbcSinkConnector"
"value.converter" = "org.apache.kafka.connect.json.JsonConverter"
"topics" = aiven_kafka_topic.topic-os-metrics.topic_name
"connection.url" = "jdbc:postgresql://${aiven_service.pg-sink.service_host}:${aiven_service.pg-sink.service_port}/${aiven_database.db-os-metrics.database_name}"
"connection.user" = aiven_service.pg-sink.service_username
"connection.password" = aiven_service.pg-sink.service_password
"auto.create" = "true"
"auto.evolve" = "true"
"value.converter.schemas.enable"=true
"errors.deadletterqueue.topic.name"= "os-metrics-log-dead"
"errors.deadletterqueue.topic.replication.factor"= 2
"errors.tolerance" = var.aiven_connect_error_tolerance
}
}