[Custom DC] improvements for re-runs (#2159)

This PR adds the following improvements - Website, mixer, and tools repo now are fetched from a tagged version of the code - Docker images now match the version of the code with the install script. - api key secret: global resource -> regional resource - For re-runs, the script is no longer stuck trying to fetch non existent tmp files. We force a file fetch. - Small fixes: BIGQUERY -> BIG_QUERY, always force mixer to be fetched for re-runs. Tested using a [test tag](https://github.com/Fructokinase/website/releases/tag/test-custom-dc-v0.1.1). The current tag the script references to (custom-dc-v0.1.0) does not exist, and will be created in website repo and tools repo after this PR is committed. Result: https://custom-dc-tag-based-launch-datacommons.com/ --------- Co-authored-by: Alex Chen <[email protected]>
datacommonsorg · Feb 3, 2023 · 301d943 · 301d943
1 parent 51af6de
commit 301d943
Show file tree

Hide file tree

Showing 12 changed files with 103 additions and 28 deletions.
diff --git a/deploy/helm_charts/dc_website/templates/deployment.yaml b/deploy/helm_charts/dc_website/templates/deployment.yaml
@@ -130,11 +130,10 @@ spec:
             requests:
               memory: "8G"
           args:
-
             - --base_bigtable_info=$(BASE_BIGTABLE_INFO)
             - --custom_bigtable_info=$(CUSTOM_BIGTABLE_INFO)
             - --mixer_project=$(MIXER_PROJECT)
-            - --bq_dataset=$(BIGQUERY)
+            - --bq_dataset=$(BIG_QUERY)
             - --schema_path=/datacommons/mapping
             - --memdb_path=/datacommons/memdb
             {{- if eq $.Values.mixer.useTMCFCSVData true }}

diff --git a/deploy/helm_charts/dc_website/values.yaml b/deploy/helm_charts/dc_website/values.yaml
@@ -13,7 +13,7 @@ website:
   image:
     repository: gcr.io/datcom-ci/datacommons-website
     pullPolicy: Always
-    tag: "custom-api-root"
+    tag:
 
   flaskEnv:
   secretGCPProjectID:
@@ -55,7 +55,7 @@ mixer:
   image:
     repository: gcr.io/datcom-ci/datacommons-mixer
     pullPolicy: Always
-    tag: "latest"
+    tag:
 
     useTMCFCSVData: false
     tmcfCSVBucket: ""

diff --git a/deploy/terraform-datacommons-website/README.md b/deploy/terraform-datacommons-website/README.md
@@ -20,7 +20,7 @@ Before this module can be used on a project, you must ensure that the following
 
 3. Terraform stores the state of installation in a file. The examples in these modules use GCS to store the state file.
 
-    Note: Examples in these modules assume that the backend bucket already exists. The backend bucket does not have to be in the same GCP project as the resources being installed. You can use the [mb](https://cloud.google.com/storage/docs/gsutil/commands/mb) command to create a new bucket. 
+    Note: Examples in these modules assume that the backend bucket already exists. The backend bucket does not have to be in the same GCP project as the resources being installed. You can use the [mb](https://cloud.google.com/storage/docs/gsutil/commands/mb) command to create a new bucket.
 
     ```
     export PROJECT=<Terraform state project id>
@@ -40,4 +40,18 @@ Before this module can be used on a project, you must ensure that the following
 
 ### gcloud and gsutil
 
-Please follow the [gcloud install doc](https://cloud.google.com/sdk/docs/install) and the [gsutil install doc](https://cloud.google.com/storage/docs/gsutil_install) to install both cli tools in the machine that is calling Terraform. Some modules may need to call gcloud/gsutil in the background. 
+Please follow the [gcloud install doc](https://cloud.google.com/sdk/docs/install) and the [gsutil install doc](https://cloud.google.com/storage/docs/gsutil_install) to install both cli tools in the machine that is calling Terraform. Some modules may need to call gcloud/gsutil in the background.
+
+## Notes
+
+### null resources
+
+There are several resources named "null_resource" throughout the examples and modules. A null_resource does not represent a GCP resource. Instead, it executes script as if the completion of the script is the "create" operation. It is a workaround for things to be automated for which no official Terraform resource exists.
+
+Some operations should always be run, regardless of whether it has been run before(Ex: fetching the latest mixer proto). For such operations, use null_resource with the following trigger.
+
+```text
+triggers = {
+  always_run = "${timestamp()}"
+}
+```
diff --git a/deploy/terraform-datacommons-website/examples/website_v1/main.tf b/deploy/terraform-datacommons-website/examples/website_v1/main.tf
@@ -58,6 +58,7 @@ module "apikeys" {
   source                   =  "../../modules/apikeys"
   project_id               = var.project_id
   dc_website_domain        = var.dc_website_domain
+  location                 = var.region
 
   resource_suffix          = local.resource_suffix
 }
@@ -128,6 +129,9 @@ module "k8s_resources" {
     helm       = helm.datcom
   }
 
+  website_githash          =   var.website_githash
+  mixer_githash            =   var.mixer_githash
+
   source                   =  "../../modules/helm"
   project_id               = var.project_id
 

diff --git a/deploy/terraform-datacommons-website/examples/website_v1/variables.tf b/deploy/terraform-datacommons-website/examples/website_v1/variables.tf
@@ -13,6 +13,16 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+variable "website_githash" {
+  type        =  string
+  description = "Determines which DC website image to use."
+}
+
+variable "mixer_githash" {
+  type        =  string
+  description = "Determines which DC Mixer image to use."
+}
+
 variable "project_id" {
   type        = string
   description = "This is the same GCP project id from the setup step."

diff --git a/deploy/terraform-datacommons-website/examples/website_v1/variables.tfvars b/deploy/terraform-datacommons-website/examples/website_v1/variables.tfvars
@@ -2,6 +2,11 @@
 project_id = "Replace me"
 # Copy over the value of the GCP project id from the setup step.
 dc_website_domain = "Replace me"
-
-
-
+# The following variables determine which images to use.
+# Values should correspond to the githash(of the head commit) of the tags.
+# For a list of website tag, see:
+# https://github.com/datacommonsorg/website/tags
+website_githash="Replace me"
+# Mixer githash should correspond to the githash of mixer submodule
+# of the website githash above.
+mixer_githash="Replace me"
diff --git a/deploy/terraform-datacommons-website/modules/apikeys/main.tf b/deploy/terraform-datacommons-website/modules/apikeys/main.tf
@@ -62,28 +62,47 @@ gcloud alpha services api-keys create \
 --api-target=service=maps-backend.googleapis.com \
 --api-target=service=places-backend.googleapis.com
 
+EOT
+  }
+}
+
+resource "null_resource" "maps_api_key_fetch" {
+  # Regardless of the state, we always want to fetch the API key to a tmp file so
+  # the api key can be found in /tmp even in re-runs.
+  triggers = {
+    always_run = "${timestamp()}"
+  }
+
+  provisioner "local-exec" {
+    command = <<EOT
 touch /tmp/dc-website-api-key
 
 API_KEY_NAME=$(gcloud alpha services api-keys list --project=${var.project_id} --filter='displayName=maps-api-key${var.resource_suffix}' --format='value(name)' | head -n 1)
 gcloud alpha services api-keys get-key-string $API_KEY_NAME --format='value(keyString)' >> /tmp/dc-website-api-key
 
 EOT
   }
+
+  depends_on = [null_resource.maps_api_key]
 }
 
 # Needed because file(https://www.terraform.io/language/functions/file)
 # cannot be used for dynamically generated files.
 data "local_file" "website_api_key" {
   filename = "/tmp/dc-website-api-key"
-  depends_on = [null_resource.maps_api_key]
+  depends_on = [null_resource.maps_api_key_fetch]
 }
 
 resource "google_secret_manager_secret" "maps_api_key_secret" {
   secret_id    = format("maps-api-key%s", var.resource_suffix)
   project      =  var.project_id
 
   replication {
-    automatic = true
+    user_managed {
+      replicas {
+        location = var.location
+      }
+    }
   }
 
   depends_on = [null_resource.maps_api_key]

diff --git a/deploy/terraform-datacommons-website/modules/apikeys/variables.tf b/deploy/terraform-datacommons-website/modules/apikeys/variables.tf
@@ -24,6 +24,11 @@ variable "dc_website_domain" {
   description = "Domain name that you own that will be used for the Data Commons website."
 }
 
+variable "location" {
+  type        = string
+  description = "region to create the API key secret. Ex: us-central1"
+}
+
 variable "resource_suffix" {
   type        = string
   description = "This string is added to all resources created in this moudle for uniqueness."

diff --git a/deploy/terraform-datacommons-website/modules/esp/main.tf b/deploy/terraform-datacommons-website/modules/esp/main.tf
@@ -15,6 +15,12 @@
  */
 
 resource "null_resource" "fetch_mixer_grpc_latest_pb" {
+  # Alwways fetch the latest gRPC protobuf.
+  # This makes sure that /tmp/mixer-grpc.latest.pb exists even in re-runs.
+  triggers = {
+    always_run = "${timestamp()}"
+  }
+
   provisioner "local-exec" {
     command = "gsutil cp ${var.mixer_grpc_pb_gcs_path} /tmp/mixer-grpc.latest.pb"
   }

diff --git a/deploy/terraform-datacommons-website/modules/helm/main.tf b/deploy/terraform-datacommons-website/modules/helm/main.tf
@@ -35,11 +35,21 @@ resource "helm_release" "datcom_website" {
   # To change that, uncomment the line below and set a value.
   # timeout    = 300
 
+  set {
+    name  = "website.image.tag"
+    value = var.website_githash
+  }
+
   set {
     name  = "website.githash"
     value = var.website_githash
   }
 
+  set {
+    name  = "mixer.image.tag"
+    value = var.mixer_githash
+  }
+
   set {
     name  = "mixer.githash"
     value = var.mixer_githash

diff --git a/deploy/terraform-datacommons-website/modules/helm/variables.tf b/deploy/terraform-datacommons-website/modules/helm/variables.tf
@@ -55,18 +55,12 @@ variable "managed_cert_name" {
   description = "Name of the managed certificate in GCP for dc_website_domain."
 }
 
-# Githash is currently required parameter for both the
-# website and mixer containers. The value of the githashes may be irrelevant for
-# custom DC instances until there is a detailed release strategy. Constants are fine
-# for now.
 variable "website_githash" {
   type        = string
   description = "website githash"
-  default     = "80dc931"
 }
 
 variable "mixer_githash" {
   type        = string
   description = "Mixer githash"
-  default     = "0515f78"
 }
diff --git a/scripts/install_custom_dc.sh b/scripts/install_custom_dc.sh
@@ -14,6 +14,8 @@
 # limitations under the License.
 set -e
 
+CUSTOM_DC_RELEASE_TAG=custom-dc-v0.1.0
+
 TERRAFORM_PATH=$(which terraform)
 if [[ -n "$TERRAFORM_PATH" ]]; then
     echo "Found Terraform: ${TERRAFORM_PATH}"
@@ -62,18 +64,22 @@ gsutil ls -b -p $PROJECT_ID gs://$TF_STATE_BUCKET || gsutil mb -l us-central1 -p
 ROOT=$PWD
 
 # Clone DC website repo and mixer submodule.
-if [[ ! -d "website" ]]; then
-  git clone https://github.com/datacommonsorg/website
-fi
+rm -rf website
+git clone https://github.com/datacommonsorg/website --branch $CUSTOM_DC_RELEASE_TAG --single-branch
 
 cd website
-if [[ ! -d "mixer" ]]; then
-  git submodule foreach git pull origin master
-  git submodule update --init --recursive
-fi
+WEBSITE_GITHASH=$(git rev-parse --short=7 HEAD)
+
+# Always force Mixer submodule to be cloned.
+rm -rf mixer
+git submodule foreach git pull origin master
+git submodule update --init --recursive
 
 WEBSITE_ROOT=$PWD
 
+cd mixer
+MIXER_GITHASH=$(git rev-parse --short=7 HEAD)
+
 cd $WEBSITE_ROOT/deploy/terraform-datacommons-website/examples/setup
 
 terraform init \
@@ -100,13 +106,16 @@ terraform init \
 # <project_id>-datacommons.com is the default domain name defined in setup/main.tf
 terraform apply \
   -var="project_id=$PROJECT_ID" \
-  -var="dc_website_domain=$DOMAIN" -auto-approve
+  -var="dc_website_domain=$DOMAIN" \
+  -var="website_githash=$WEBSITE_GITHASH" \
+  -var="mixer_githash=$MIXER_GITHASH" \
+  -auto-approve
 
 # Run the BT automation Terraform script to set up BT loader.
 cd $ROOT
-if [[ ! -d "tools" ]]; then
-  git clone https://github.com/datacommonsorg/tools
-fi
+
+rm -rf tools
+git clone https://github.com/datacommonsorg/tools --branch $CUSTOM_DC_RELEASE_TAG --single-branch
 
 
 # TODO(alex): support custom robot SA and resource bucket name.