From 1ce9b737a2d9d1ff8f967e11732cae24ff577c4a Mon Sep 17 00:00:00 2001 From: Raymond Cheng Date: Thu, 5 Sep 2024 22:32:33 -0700 Subject: [PATCH] fix: impersonate service account for bq_dts has more scopes (#2083) --- warehouse/oso_dagster/factories/bq_dts.py | 4 ++++ warehouse/oso_dagster/resources/bq_dts.py | 7 ++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/warehouse/oso_dagster/factories/bq_dts.py b/warehouse/oso_dagster/factories/bq_dts.py index 595ecb384..67c5d3e4a 100644 --- a/warehouse/oso_dagster/factories/bq_dts.py +++ b/warehouse/oso_dagster/factories/bq_dts.py @@ -7,6 +7,7 @@ ) from dagster_gcp import BigQueryResource from .common import AssetFactoryResponse +from ..constants import impersonate_service_account from ..resources import BigQueryDataTransferResource from ..utils import ( ensure_dataset, @@ -72,6 +73,9 @@ def _bq_dts_asset( ) with bigquery_datatransfer.get_client() as bq_dts_client: + context.log.info( + f"Ensuring BigQuery Data Transfer asset by impersonating {impersonate_service_account}" + ) ensure_bq_dts_transfer(bq_dts_client, asset_config, context.log) context.log.info( f"Ensured transfer config named {asset_config.display_name}" diff --git a/warehouse/oso_dagster/resources/bq_dts.py b/warehouse/oso_dagster/resources/bq_dts.py index 178d905ec..dda6cdccb 100644 --- a/warehouse/oso_dagster/resources/bq_dts.py +++ b/warehouse/oso_dagster/resources/bq_dts.py @@ -12,7 +12,12 @@ from pydantic import Field from ..constants import impersonate_service_account -IMPERSONATE_SCOPES = ["https://www.googleapis.com/auth/bigquery"] +IMPERSONATE_SCOPES = [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/bigquery", + "https://www.googleapis.com/auth/bigquery.insertdata", +] IMPERSONATE_LIFETIME = 300 """