From bea41c1f0bc21fe0600e4f0cec2c88b81190527e Mon Sep 17 00:00:00 2001 From: Raj Poluri Date: Mon, 24 Feb 2025 18:15:16 -0600 Subject: [PATCH 1/7] fix gluesync on k8s deployment --- CHANGELOG.md | 4 ++++ k8s-readwrite.tf | 10 +++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a1e5a54..b240371 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [7.9.2] - 2025-02-24 +### Fixed +- Fix gluesync on k8s deployment + ## [7.9.1] - 2025-01-14 ### Added - Added Splunk env for segrigatting logs for each environment. diff --git a/k8s-readwrite.tf b/k8s-readwrite.tf index 0692c42..80a2a63 100644 --- a/k8s-readwrite.tf +++ b/k8s-readwrite.tf @@ -201,6 +201,14 @@ resource "kubernetes_deployment_v1" "apiary_hms_readwrite" { name = "HIVE_METASTORE_LOG_LEVEL" value = var.hms_log_level } + env { + name = "ENABLE_GLUESYNC" + value = var.enable_gluesync + } + env { + name = "GLUE_PREFIX" + value = local.gluedb_prefix + } env { name = "RANGER_SERVICE_NAME" value = "${local.instance_alias}-metastore" @@ -393,4 +401,4 @@ resource "kubernetes_pod_disruption_budget_v1" "hms_readwrite" { max_unavailable = var.hms_rw_k8s_pdb_settings.max_unavailable != null ? var.hms_rw_k8s_pdb_settings.max_unavailable : "1" min_available = var.hms_rw_k8s_pdb_settings.min_available != null ? var.hms_rw_k8s_pdb_settings.min_available : null } -} \ No newline at end of file +} From 4be0d742e79e864f8fb27305d676dc60866f3b06 Mon Sep 17 00:00:00 2001 From: Raj Poluri Date: Mon, 24 Feb 2025 19:57:25 -0600 Subject: [PATCH 2/7] create lf resource --- lf.tf | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 lf.tf diff --git a/lf.tf b/lf.tf new file mode 100644 index 0000000..724e8a9 --- /dev/null +++ b/lf.tf @@ -0,0 +1,10 @@ +/** + * Copyright (C) 2018-2025 Expedia, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + */ + +resource "aws_lakeformation_resource" "apiary_data_bucket" { + for_each = { for bucket in aws_s3_bucket.apiary_data_bucket : bucket.arn => bucket_arn } + arn = bucket_arn +} From 904a04907da3a356ac05065cfd2ae4e28e67f26f Mon Sep 17 00:00:00 2001 From: Raj Poluri Date: Mon, 24 Feb 2025 20:01:23 -0600 Subject: [PATCH 3/7] enable hybrid access --- lf.tf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lf.tf b/lf.tf index 724e8a9..a8101fc 100644 --- a/lf.tf +++ b/lf.tf @@ -5,6 +5,7 @@ */ resource "aws_lakeformation_resource" "apiary_data_bucket" { - for_each = { for bucket in aws_s3_bucket.apiary_data_bucket : bucket.arn => bucket_arn } - arn = bucket_arn + for_each = { for bucket in aws_s3_bucket.apiary_data_bucket : bucket.arn => bucket_arn } + arn = bucket_arn + hybrid_access_enabled = true } From 6b01a3a07358b3ead1156240ed014599e9543797 Mon Sep 17 00:00:00 2001 From: Raj Poluri Date: Mon, 24 Feb 2025 20:02:49 -0600 Subject: [PATCH 4/7] fix --- lf.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lf.tf b/lf.tf index a8101fc..0647755 100644 --- a/lf.tf +++ b/lf.tf @@ -5,7 +5,7 @@ */ resource "aws_lakeformation_resource" "apiary_data_bucket" { - for_each = { for bucket in aws_s3_bucket.apiary_data_bucket : bucket.arn => bucket_arn } - arn = bucket_arn + for_each = { for bucket in aws_s3_bucket.apiary_data_bucket : bucket.arn => bucket } + arn = bucket hybrid_access_enabled = true } From 285be2eef00e28e1837fff7009f8842118863053 Mon Sep 17 00:00:00 2001 From: Raj Poluri Date: Mon, 24 Feb 2025 20:04:35 -0600 Subject: [PATCH 5/7] fix --- lf.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lf.tf b/lf.tf index 0647755..0c690c4 100644 --- a/lf.tf +++ b/lf.tf @@ -5,7 +5,7 @@ */ resource "aws_lakeformation_resource" "apiary_data_bucket" { - for_each = { for bucket in aws_s3_bucket.apiary_data_bucket : bucket.arn => bucket } - arn = bucket + for_each = { for bucket in aws_s3_bucket.apiary_data_bucket : bucket => bucket } + arn = bucket.arn hybrid_access_enabled = true } From cfac2f671debebc41d3d52dd900df99062f7ae7d Mon Sep 17 00:00:00 2001 From: Raj Poluri Date: Mon, 24 Feb 2025 20:06:30 -0600 Subject: [PATCH 6/7] fix --- lf.tf | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lf.tf b/lf.tf index 0c690c4..c8773ea 100644 --- a/lf.tf +++ b/lf.tf @@ -5,7 +5,10 @@ */ resource "aws_lakeformation_resource" "apiary_data_bucket" { - for_each = { for bucket in aws_s3_bucket.apiary_data_bucket : bucket => bucket } - arn = bucket.arn + for_each = { + for schema in local.schemas_info : "${schema["schema_name"]}" => schema + } + arn = aws_s3_bucket.apiary_data_bucket[each.key].arn + hybrid_access_enabled = true } From fafbc24878a17d0c39c3f498bfc698c5d6b7383b Mon Sep 17 00:00:00 2001 From: Raj Poluri Date: Mon, 24 Feb 2025 20:10:35 -0600 Subject: [PATCH 7/7] update provider to support lf hybrid access --- version.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/version.tf b/version.tf index abb841b..a9e3db3 100644 --- a/version.tf +++ b/version.tf @@ -13,7 +13,7 @@ terraform { } aws = { source = "hashicorp/aws" - version = "~> 4.0" + version = ">= 5.17.0" } external = { source = "hashicorp/external" @@ -28,7 +28,7 @@ terraform { version = "3.25.0" } template = { - source = "hashicorp/template" + source = "hashicorp/template" version = "~> 2.2" } }