From 301d943118093acb2a7b7a39f0c664740d6b3b88 Mon Sep 17 00:00:00 2001 From: Alex <8507196+Fructokinase@users.noreply.github.com> Date: Thu, 2 Feb 2023 16:59:31 -0800 Subject: [PATCH] [Custom DC] improvements for re-runs (#2159) This PR adds the following improvements - Website, mixer, and tools repo now are fetched from a tagged version of the code - Docker images now match the version of the code with the install script. - api key secret: global resource -> regional resource - For re-runs, the script is no longer stuck trying to fetch non existent tmp files. We force a file fetch. - Small fixes: BIGQUERY -> BIG_QUERY, always force mixer to be fetched for re-runs. Tested using a [test tag](https://github.com/Fructokinase/website/releases/tag/test-custom-dc-v0.1.1). The current tag the script references to (custom-dc-v0.1.0) does not exist, and will be created in website repo and tools repo after this PR is committed. Result: https://custom-dc-tag-based-launch-datacommons.com/ --------- Co-authored-by: Alex Chen --- .../dc_website/templates/deployment.yaml | 3 +- deploy/helm_charts/dc_website/values.yaml | 4 +-- .../terraform-datacommons-website/README.md | 18 +++++++++-- .../examples/website_v1/main.tf | 4 +++ .../examples/website_v1/variables.tf | 10 ++++++ .../examples/website_v1/variables.tfvars | 11 +++++-- .../modules/apikeys/main.tf | 23 ++++++++++++-- .../modules/apikeys/variables.tf | 5 +++ .../modules/esp/main.tf | 6 ++++ .../modules/helm/main.tf | 10 ++++++ .../modules/helm/variables.tf | 6 ---- scripts/install_custom_dc.sh | 31 ++++++++++++------- 12 files changed, 103 insertions(+), 28 deletions(-) diff --git a/deploy/helm_charts/dc_website/templates/deployment.yaml b/deploy/helm_charts/dc_website/templates/deployment.yaml index 02d2f11d74..8ddcd13025 100644 --- a/deploy/helm_charts/dc_website/templates/deployment.yaml +++ b/deploy/helm_charts/dc_website/templates/deployment.yaml @@ -130,11 +130,10 @@ spec: requests: memory: "8G" args: - - --base_bigtable_info=$(BASE_BIGTABLE_INFO) - --custom_bigtable_info=$(CUSTOM_BIGTABLE_INFO) - --mixer_project=$(MIXER_PROJECT) - - --bq_dataset=$(BIGQUERY) + - --bq_dataset=$(BIG_QUERY) - --schema_path=/datacommons/mapping - --memdb_path=/datacommons/memdb {{- if eq $.Values.mixer.useTMCFCSVData true }} diff --git a/deploy/helm_charts/dc_website/values.yaml b/deploy/helm_charts/dc_website/values.yaml index 88f934042d..a8f2b6d516 100644 --- a/deploy/helm_charts/dc_website/values.yaml +++ b/deploy/helm_charts/dc_website/values.yaml @@ -13,7 +13,7 @@ website: image: repository: gcr.io/datcom-ci/datacommons-website pullPolicy: Always - tag: "custom-api-root" + tag: flaskEnv: secretGCPProjectID: @@ -55,7 +55,7 @@ mixer: image: repository: gcr.io/datcom-ci/datacommons-mixer pullPolicy: Always - tag: "latest" + tag: useTMCFCSVData: false tmcfCSVBucket: "" diff --git a/deploy/terraform-datacommons-website/README.md b/deploy/terraform-datacommons-website/README.md index 15ca8af9df..b2b311f90c 100644 --- a/deploy/terraform-datacommons-website/README.md +++ b/deploy/terraform-datacommons-website/README.md @@ -20,7 +20,7 @@ Before this module can be used on a project, you must ensure that the following 3. Terraform stores the state of installation in a file. The examples in these modules use GCS to store the state file. - Note: Examples in these modules assume that the backend bucket already exists. The backend bucket does not have to be in the same GCP project as the resources being installed. You can use the [mb](https://cloud.google.com/storage/docs/gsutil/commands/mb) command to create a new bucket. + Note: Examples in these modules assume that the backend bucket already exists. The backend bucket does not have to be in the same GCP project as the resources being installed. You can use the [mb](https://cloud.google.com/storage/docs/gsutil/commands/mb) command to create a new bucket. ``` export PROJECT= @@ -40,4 +40,18 @@ Before this module can be used on a project, you must ensure that the following ### gcloud and gsutil -Please follow the [gcloud install doc](https://cloud.google.com/sdk/docs/install) and the [gsutil install doc](https://cloud.google.com/storage/docs/gsutil_install) to install both cli tools in the machine that is calling Terraform. Some modules may need to call gcloud/gsutil in the background. +Please follow the [gcloud install doc](https://cloud.google.com/sdk/docs/install) and the [gsutil install doc](https://cloud.google.com/storage/docs/gsutil_install) to install both cli tools in the machine that is calling Terraform. Some modules may need to call gcloud/gsutil in the background. + +## Notes + +### null resources + +There are several resources named "null_resource" throughout the examples and modules. A null_resource does not represent a GCP resource. Instead, it executes script as if the completion of the script is the "create" operation. It is a workaround for things to be automated for which no official Terraform resource exists. + +Some operations should always be run, regardless of whether it has been run before(Ex: fetching the latest mixer proto). For such operations, use null_resource with the following trigger. + +```text +triggers = { + always_run = "${timestamp()}" +} +``` \ No newline at end of file diff --git a/deploy/terraform-datacommons-website/examples/website_v1/main.tf b/deploy/terraform-datacommons-website/examples/website_v1/main.tf index 1d2ef15422..69a3416e5e 100644 --- a/deploy/terraform-datacommons-website/examples/website_v1/main.tf +++ b/deploy/terraform-datacommons-website/examples/website_v1/main.tf @@ -58,6 +58,7 @@ module "apikeys" { source = "../../modules/apikeys" project_id = var.project_id dc_website_domain = var.dc_website_domain + location = var.region resource_suffix = local.resource_suffix } @@ -128,6 +129,9 @@ module "k8s_resources" { helm = helm.datcom } + website_githash = var.website_githash + mixer_githash = var.mixer_githash + source = "../../modules/helm" project_id = var.project_id diff --git a/deploy/terraform-datacommons-website/examples/website_v1/variables.tf b/deploy/terraform-datacommons-website/examples/website_v1/variables.tf index 4d7a77edf5..c60b40b28c 100644 --- a/deploy/terraform-datacommons-website/examples/website_v1/variables.tf +++ b/deploy/terraform-datacommons-website/examples/website_v1/variables.tf @@ -13,6 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +variable "website_githash" { + type = string + description = "Determines which DC website image to use." +} + +variable "mixer_githash" { + type = string + description = "Determines which DC Mixer image to use." +} + variable "project_id" { type = string description = "This is the same GCP project id from the setup step." diff --git a/deploy/terraform-datacommons-website/examples/website_v1/variables.tfvars b/deploy/terraform-datacommons-website/examples/website_v1/variables.tfvars index 79aa26a494..81684d90fb 100644 --- a/deploy/terraform-datacommons-website/examples/website_v1/variables.tfvars +++ b/deploy/terraform-datacommons-website/examples/website_v1/variables.tfvars @@ -2,6 +2,11 @@ project_id = "Replace me" # Copy over the value of the GCP project id from the setup step. dc_website_domain = "Replace me" - - - +# The following variables determine which images to use. +# Values should correspond to the githash(of the head commit) of the tags. +# For a list of website tag, see: +# https://github.com/datacommonsorg/website/tags +website_githash="Replace me" +# Mixer githash should correspond to the githash of mixer submodule +# of the website githash above. +mixer_githash="Replace me" diff --git a/deploy/terraform-datacommons-website/modules/apikeys/main.tf b/deploy/terraform-datacommons-website/modules/apikeys/main.tf index 7fcb62514e..8b42c38345 100644 --- a/deploy/terraform-datacommons-website/modules/apikeys/main.tf +++ b/deploy/terraform-datacommons-website/modules/apikeys/main.tf @@ -62,6 +62,19 @@ gcloud alpha services api-keys create \ --api-target=service=maps-backend.googleapis.com \ --api-target=service=places-backend.googleapis.com +EOT + } +} + +resource "null_resource" "maps_api_key_fetch" { + # Regardless of the state, we always want to fetch the API key to a tmp file so + # the api key can be found in /tmp even in re-runs. + triggers = { + always_run = "${timestamp()}" + } + + provisioner "local-exec" { + command = <