Skip to content

Commit

Permalink
Merge branch 'master' into feature/lakeformation
Browse files Browse the repository at this point in the history
  • Loading branch information
@javsanbel2
javsanbel2 committed Feb 26, 2025
2 parents 3cfb228 + 66c550e commit 5d3ab9b
Show file tree
Hide file tree
Showing 8 changed files with 64 additions and 16 deletions.
8 changes: 8 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,14 @@ All notable changes to this project will be documented in this file.

The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).

## [7.9.5] - 2025-02-26
### Added
- Add option to provision glue databases using terraform instead of metastore startup.

## [7.9.4] - 2025-02-26
### Fixed
- Added `abort_incomplete_multipart_upload` rule into `aws_s3_bucket_lifecycle_configuration` object.

## [7.9.3] - 2025-02-25
### Fixed
- Merged all S3 lifecycle configurations into `aws_s3_bucket_lifecycle_configuration` object.
Expand Down
1 change: 1 addition & 0 deletions VARIABLES.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
| atlas\_kafka\_bootstrap\_servers | Kafka instance url. | `string` | `""` | no |
| aws\_region | AWS region. | `string` | n/a | yes |
| apiary\_common\_producer\_iamroles | AWS IAM roles allowed general (not tied to schema) write access to managed Apiary S3 buckets. | `list(string)` | `[]` | no |
| disable\_glue\_db_init | Glue databases are created programatically by default in hms-readwrite bootstrap init action. Setting this variable to true will disable the hms-readwrite bootstrap init action and create Glue databases via Terraform. | `bool` | `false` | no |
| dashboard\_namespace | k8s namespace to deploy grafana dashboard. | `string` | `"monitoring"` | no |
| db\_apply\_immediately | Specifies whether any cluster modifications are applied immediately, or during the next maintenance window. | `bool` | `false` | no |
| db\_backup\_retention | The number of days to retain backups for the RDS Metastore DB. | `string` | `"7"` | yes |
Expand Down
15 changes: 15 additions & 0 deletions glue.tf
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
/**
* Copyright (C) 2018-2025 Expedia, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
*/

resource "aws_glue_catalog_database" "apiary_glue_database" {
for_each = var.disable_glue_db_init ? {
for schema in local.schemas_info : "${schema["schema_name"]}" => schema
} : {}

location_uri = "s3://${aws_s3_bucket.apiary_data_bucket[each.key].id}/"
name = "${local.gluedb_prefix}${each.key}"
description = "Managed by Apiary terraform"
}
36 changes: 20 additions & 16 deletions k8s-readwrite.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ resource "kubernetes_deployment_v1" "apiary_hms_readwrite" {
max_surge = var.hms_rw_k8s_rolling_update_strategy.max_surge
max_unavailable = var.hms_rw_k8s_rolling_update_strategy.max_unavailable
}
}
}
replicas = var.hms_rw_k8s_replica_count
selector {
match_labels = {
Expand All @@ -36,13 +36,13 @@ resource "kubernetes_deployment_v1" "apiary_hms_readwrite" {
name = "${local.hms_alias}-readwrite"
}
annotations = {
"ad.datadoghq.com/${local.hms_alias}-readwrite.check_names" = var.datadog_metrics_enabled ? "[\"prometheus\"]" : null
"ad.datadoghq.com/${local.hms_alias}-readwrite.check_names" = var.datadog_metrics_enabled ? "[\"prometheus\"]" : null
"ad.datadoghq.com/${local.hms_alias}-readwrite.init_configs" = var.datadog_metrics_enabled ? "[{}]" : null
"ad.datadoghq.com/${local.hms_alias}-readwrite.instances" = var.datadog_metrics_enabled ? "[{ \"prometheus_url\": \"http://%%host%%:${var.datadog_metrics_port}/actuator/prometheus\", \"namespace\": \"${var.hms_k8s_metrics_readwrite_namespace}\", \"metrics\": [ \"${join("\",\"", var.datadog_metrics_hms_readwrite_readonly)}\" ] , \"type_overrides\": { \"${join("\": \"gauge\",\"", var.datadog_metrics_hms_readwrite_readonly)}\": \"gauge\"} }]" : null
"iam.amazonaws.com/role" = var.oidc_provider == "" ? aws_iam_role.apiary_hms_readwrite.name : null
"prometheus.io/path" = "/metrics"
"prometheus.io/port" = "8080"
"prometheus.io/scrape" = "true"
"ad.datadoghq.com/${local.hms_alias}-readwrite.instances" = var.datadog_metrics_enabled ? "[{ \"prometheus_url\": \"http://%%host%%:${var.datadog_metrics_port}/actuator/prometheus\", \"namespace\": \"${var.hms_k8s_metrics_readwrite_namespace}\", \"metrics\": [ \"${join("\",\"", var.datadog_metrics_hms_readwrite_readonly)}\" ] , \"type_overrides\": { \"${join("\": \"gauge\",\"", var.datadog_metrics_hms_readwrite_readonly)}\": \"gauge\"} }]" : null
"iam.amazonaws.com/role" = var.oidc_provider == "" ? aws_iam_role.apiary_hms_readwrite.name : null
"prometheus.io/path" = "/metrics"
"prometheus.io/port" = "8080"
"prometheus.io/scrape" = "true"
}
}

Expand All @@ -53,10 +53,10 @@ resource "kubernetes_deployment_v1" "apiary_hms_readwrite" {
dynamic "toleration" {
for_each = var.hms_rw_tolerations
content {
effect = lookup(toleration.value, "effect", null)
key = lookup(toleration.value, "key", null)
operator = lookup(toleration.value, "operator", null)
value = lookup(toleration.value, "value", null)
effect = lookup(toleration.value, "effect", null)
key = lookup(toleration.value, "key", null)
operator = lookup(toleration.value, "operator", null)
value = lookup(toleration.value, "value", null)
}
}

Expand All @@ -80,7 +80,7 @@ resource "kubernetes_deployment_v1" "apiary_hms_readwrite" {
}
}

dynamic "security_context" {
dynamic "security_context" {
for_each = var.enable_tcp_keepalive ? ["enabled"] : []
content {
sysctl {
Expand Down Expand Up @@ -201,9 +201,13 @@ resource "kubernetes_deployment_v1" "apiary_hms_readwrite" {
name = "HIVE_METASTORE_LOG_LEVEL"
value = var.hms_log_level
}
env {
name = "DISABLE_GLUE_DB_INIT"
value = var.disable_glue_db_init ? "1" : ""
}
env {
name = "ENABLE_GLUESYNC"
value = var.enable_gluesync
value = var.enable_gluesync ? "1" : ""
}
env {
name = "GLUE_PREFIX"
Expand Down Expand Up @@ -294,7 +298,7 @@ resource "kubernetes_deployment_v1" "apiary_hms_readwrite" {
name = "DATANUCLEUS_CONNECTION_POOL_MAX_POOLSIZE"
value = var.hms_rw_db_connection_pool_size
}

dynamic "env" {
for_each = var.hms_additional_environment_variables

Expand Down Expand Up @@ -336,7 +340,7 @@ resource "kubernetes_deployment_v1" "apiary_hms_readwrite" {
}

resources {
limits = {
limits = {
cpu = local.k8s_rw_cpu_limit
memory = "${var.hms_rw_heapsize}Mi"
}
Expand Down Expand Up @@ -386,7 +390,7 @@ resource "kubernetes_pod_disruption_budget_v1" "hms_readwrite" {
count = var.hms_instance_type == "k8s" && var.hms_rw_k8s_pdb_settings.enabled ? 1 : 0

metadata {
name = "${local.hms_alias}-readwrite"
name = "${local.hms_alias}-readwrite"
namespace = var.metastore_namespace
}

Expand Down
9 changes: 9 additions & 0 deletions s3.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,15 @@ resource "aws_s3_bucket_lifecycle_configuration" "apiary_data_bucket_versioning_
for schema in local.schemas_info : "${schema["schema_name"]}" => schema
}
bucket = each.value["data_bucket"]
# Rule for s3 incomplete multipart upload expiration
rule {
id = "expire-incomplete-multipart-uploads"
status = "Enabled"

abort_incomplete_multipart_upload {
days_after_initiation = var.s3_lifecycle_abort_incomplete_multipart_upload_days
}
}
# Rule for s3 versioning expiration
rule {
id = "expire-noncurrent-versions-days"
Expand Down
1 change: 1 addition & 0 deletions templates.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ locals{
instance_name = "${local.instance_alias}"
sns_arn = var.enable_metadata_events ? join("", aws_sns_topic.apiary_metadata_events.*.arn) : ""
table_param_filter = var.enable_metadata_events ? var.table_param_filter : ""
disable_glue_db_init = var.disable_glue_db_init ? "1" : ""
enable_gluesync = var.enable_gluesync ? "1" : ""
gluedb_prefix = "${local.gluedb_prefix}"
hms_metrics_namespace = "${var.hms_ecs_metrics_readwrite_namespace}"
Expand Down
4 changes: 4 additions & 0 deletions templates/apiary-hms-readwrite.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,10 @@
"name": "TABLE_PARAM_FILTER",
"value": "${table_param_filter}"
},
{
"name": "DISABLE_GLUE_DB_INIT",
"value": "${disable_glue_db_init}"
},
{
"name": "ENABLE_GLUESYNC",
"value": "${enable_gluesync}"
Expand Down
6 changes: 6 additions & 0 deletions variables.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -585,6 +585,12 @@ variable "rw_ingress_cidr" {
default = []
}

variable "disable_glue_db_init" {
description = "Glue databases are created programatically by default in hms-readwrite bootstrap init action. Setting this variable to true will disable the hms-readwrite bootstrap init action and create Glue databases via Terraform."
type = bool
default = false
}

variable "enable_gluesync" {
description = "Enable metadata sync from Hive to the Glue catalog."
type = bool
Expand Down

0 comments on commit 5d3ab9b

Please sign in to comment.