Added capability to choose Connection Pooling driver and specify it's…

… configuration (#265) * Added `hms_ro_datanucleus_connection_pooling_type`, `hms_rw_datanucleus_connection_pooling_type`, `hms_ro_datanucleus_connection_pool_config`, `hms_rw_datanucleus_connection_pool_config`, `hms_housekeeper_db_connection_pool_size` variables to allow specifying the pooling driver and its config * remove check for null for var.hms_ro_datanucleus_connection_pool_config and var.hms_rw_datanucleus_connection_pool_config * updated Changelog --------- Co-authored-by: Georgi Ivanov <[email protected]>
ExpediaGroup · Jun 26, 2024 · f695b43 · f695b43
1 parent 5bc8655
commit f695b43
Show file tree

Hide file tree

Showing 5 changed files with 83 additions and 10 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [7.2.0] - 2024-06-26
+### Added
+- Added `hms_ro_datanucleus_connection_pooling_type`, `hms_rw_datanucleus_connection_pooling_type`, `hms_ro_datanucleus_connection_pool_config`, `hms_rw_datanucleus_connection_pool_config`, `hms_housekeeper_db_connection_pool_size` variables to allow specifying the pooling driver and its config
+
 ## [7.1.9] - 2024-06-20
 ### Fixed
 - Housekeeper deployment should not use common `HADOOP_HEAPSIZE` variable since it is a low memory container.

diff --git a/k8s-housekeeper.tf b/k8s-housekeeper.tf
@@ -133,6 +133,17 @@ resource "kubernetes_deployment_v1" "apiary_hms_housekeeper" {
             name  = "ENABLE_HIVE_LOCK_HOUSE_KEEPER"
             value = var.enable_hms_housekeeper ? "true" : ""
           }
+
+          env {
+            name  = "DATANUCLEUS_CONNECTION_POOLING_TYPE"
+            value = var.hms_rw_datanucleus_connection_pooling_type
+          }
+
+          env {
+            name  = "DATANUCLEUS_CONNECTION_POOL_MAX_POOLSIZE"
+            value = var.hms_housekeeper_db_connection_pool_size
+          }
+
           dynamic "env" {
             for_each = var.hms_housekeeper_additional_environment_variables
 

diff --git a/k8s-readonly.tf b/k8s-readonly.tf
@@ -174,10 +174,6 @@ resource "kubernetes_deployment_v1" "apiary_hms_readonly" {
             name  = "HMS_MAX_THREADS"
             value = local.hms_ro_maxthreads
           }
-          env {
-            name  = "MYSQL_CONNECTION_POOL_SIZE"
-            value = var.hms_ro_db_connection_pool_size
-          }
           env {
             name  = "HMS_AUTOGATHER_STATS"
             value = "false"
@@ -186,6 +182,15 @@ resource "kubernetes_deployment_v1" "apiary_hms_readonly" {
             name  = "LIMIT_PARTITION_REQUEST_NUMBER"
             value = var.hms_ro_request_partition_limit == "" ? "" : var.hms_ro_request_partition_limit
           }
+          env {
+            name  = "DATANUCLEUS_CONNECTION_POOLING_TYPE"
+            value = var.hms_ro_datanucleus_connection_pooling_type
+          }
+          env {
+            name  = "DATANUCLEUS_CONNECTION_POOL_MAX_POOLSIZE"
+            value = var.hms_ro_db_connection_pool_size
+          }
+
           dynamic "env" {
             for_each = var.hms_additional_environment_variables
 
@@ -195,6 +200,15 @@ resource "kubernetes_deployment_v1" "apiary_hms_readonly" {
             }
           }
 
+          dynamic "env" {
+            for_each = var.hms_ro_datanucleus_connection_pool_config
+
+            content {
+              name  = env.key
+              value = env.value
+            }
+          }
+
           liveness_probe {
             tcp_socket {
               port = var.hive_metastore_port

diff --git a/k8s-readwrite.tf b/k8s-readwrite.tf
@@ -214,10 +214,6 @@ resource "kubernetes_deployment_v1" "apiary_hms_readwrite" {
             name  = "DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES"
             value = var.disallow_incompatible_col_type_changes
           }
-          env {
-            name  = "MYSQL_CONNECTION_POOL_SIZE"
-            value = var.hms_rw_db_connection_pool_size
-          }
           env {
             name  = "HMS_AUTOGATHER_STATS"
             value = var.hms_autogather_stats
@@ -226,6 +222,15 @@ resource "kubernetes_deployment_v1" "apiary_hms_readwrite" {
             name  = "LIMIT_PARTITION_REQUEST_NUMBER"
             value = var.hms_rw_request_partition_limit == "" ? "" : var.hms_rw_request_partition_limit
           }
+          env {
+            name  = "DATANUCLEUS_CONNECTION_POOLING_TYPE"
+            value = var.hms_rw_datanucleus_connection_pooling_type
+          }
+          env {
+            name  = "DATANUCLEUS_CONNECTION_POOL_MAX_POOLSIZE"
+            value = var.hms_rw_db_connection_pool_size
+          }
+
           dynamic "env" {
             for_each = var.hms_additional_environment_variables
 
@@ -235,6 +240,15 @@ resource "kubernetes_deployment_v1" "apiary_hms_readwrite" {
             }
           }
 
+          dynamic "env" {
+            for_each = var.hms_rw_datanucleus_connection_pool_config
+
+            content {
+              name  = env.key
+              value = env.value
+            }
+          }
+
           liveness_probe {
             tcp_socket {
               port = var.hive_metastore_port

diff --git a/variables.tf b/variables.tf
@@ -661,17 +661,23 @@ variable "hms_autogather_stats" {
 }
 
 variable "hms_ro_db_connection_pool_size" {
-  description = "Read-only Hive metastore setting for size of the MySQL connection pool. Default is 10."
+  description = "Read-only Hive metastore setting for max size of the MySQL connection pool. Default is 10."
   type        = number
   default     = 10
 }
 
 variable "hms_rw_db_connection_pool_size" {
-  description = "Read-write Hive metastore setting for size of the MySQL connection pool. Default is 10."
+  description = "Read-write Hive metastore setting for max size of the MySQL connection pool. Default is 10."
   type        = number
   default     = 10
 }
 
+variable "hms_housekeeper_db_connection_pool_size" {
+  description = "HMS Housekeeper setting for max size of the MySQL connection pool. Default is 5."
+  type        = number
+  default     = 5
+}
+
 variable "db_enable_performance_insights" {
   description = "Enable RDS Performance Insights"
   type        = bool
@@ -757,3 +763,27 @@ variable "apiary_common_producer_iamroles" {
   type        = list(string)
   default     = []
 }
+
+variable "hms_ro_datanucleus_connection_pooling_type" {
+  description = "The Datanucleus connection pool type: Valid types are BoneCP, HikariCP, c3p0, dbcp, dbcp2"
+  type    = string
+  default = "HikariCP"
+}
+
+variable "hms_rw_datanucleus_connection_pooling_type" {
+  description = "The Datanucleus connection pool type: Valid types are BoneCP, HikariCP, c3p0, dbcp, dbcp2"
+  type    = string
+  default = "HikariCP"
+}
+
+variable "hms_ro_datanucleus_connection_pool_config" {
+  description = "A map of env vars supported by Apiary docker image that can configure the chosen Datanucleus connection pool"
+  type = map(any)
+  default = {}
+}
+
+variable "hms_rw_datanucleus_connection_pool_config" {
+  description = "A map of env vars supported by Apiary docker image that can configure the chosen Datanucleus connection pool"  
+  type = map(any)
+  default = {}
+}