Fix: Google political ads new schema addition. (#891)

* Fix: Google political ads new schema addition. * Fix: Google political ads new schema order changed.
GoogleCloudPlatform · Jan 20, 2025 · 7081584 · 7081584
1 parent 46c488b
commit 7081584
Show file tree

Hide file tree

Showing 2 changed files with 40 additions and 4 deletions.
diff --git a/datasets/google_political_ads/pipelines/process_csvs_and_load_to_bq/pipeline.yaml b/datasets/google_political_ads/pipelines/process_csvs_and_load_to_bq/pipeline.yaml
@@ -913,6 +913,7 @@ dag:
               "gender_targeting",
               "geo_targeting_included",
               "geo_targeting_excluded",
+              "is_funded_by_google_ad_grants",
               "spend_range_min_usd",
               "spend_range_max_usd",
               "spend_range_min_eur",
@@ -952,7 +953,9 @@ dag:
               "spend_range_min_zar",
               "spend_range_max_zar",
               "spend_range_min_clp",
-              "spend_range_max_clp"
+              "spend_range_max_clp",
+              "spend_range_min_mxn",
+              "spend_range_max_mxn"
             ]
           RENAME_MAPPINGS: >-
             {
@@ -1013,7 +1016,10 @@ dag:
               "Geo_Targeting_Included": "geo_targeting_included",
               "Geo_Targeting_Excluded": "geo_targeting_excluded",
               "First_Served_Timestamp": "first_served_timestamp",
-              "Last_Served_Timestamp": "last_served_timestamp"
+              "Last_Served_Timestamp": "last_served_timestamp",
+              "Spend_Range_Min_MXN": "spend_range_min_mxn",
+              "Spend_Range_Max_MXN": "spend_range_max_mxn",
+              "Is_Funded_By_Google_Ad_Grants": "is_funded_by_google_ad_grants"
             }
         container_resources:
           memory:
@@ -1107,6 +1113,10 @@ dag:
             type: "string"
             description: "Geographic locations excluded in the ad's targeting."
             mode: "nullable"
+          - name: "is_funded_by_google_ad_grants"
+            type: "bool"
+            description: "Indicates whether the ad is funded by an in-kind donation through Google Ad Grants. Google ad Grants donates Search advertising to eligible organizations. Google does not endorse these ad."
+            mode: "nullable"
           - name: "spend_range_min_usd"
             type: "integer"
             description: "Lower bound of the amount in USD spent by the advertiser on the election ad."
@@ -1259,6 +1269,14 @@ dag:
             type: "integer"
             description: "Upper bound of the amount in CLP spent by the advertiser on the election ad."
             mode: "nullable"
+          - name: "spend_range_min_mxn"
+            type: "integer"
+            description: "Lower bound of the amount in MXN spent by the advertiser on the election ad."
+            mode: "nullable"
+          - name: "spend_range_max_mxn"
+            type: "integer"
+            description: "Upper bound of the amount in MXN spent by the advertiser on the election ad."
+            mode: "nullable"
 
     - operator: "GKEStartPodOperator"
       description: "Run CSV transform within kubernetes pod"

diff --git a/...le_political_ads/pipelines/process_csvs_and_load_to_bq/process_csvs_and_load_to_bq_dag.py b/...le_political_ads/pipelines/process_csvs_and_load_to_bq/process_csvs_and_load_to_bq_dag.py
@@ -828,8 +828,8 @@
             "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}",
             "TARGET_GCS_PATH": "data/google_political_ads/creative_stats/data_output.csv",
             "TABLE_NAME": "creative_stats",
-            "CSV_HEADERS": '[\n  "ad_id",\n  "ad_url",\n  "ad_type",\n  "regions",\n  "advertiser_id",\n  "advertiser_name",\n  "ad_campaigns_list",\n  "date_range_start",\n  "date_range_end",\n  "num_of_days",\n  "impressions",\n  "spend_usd",\n  "first_served_timestamp",\n  "last_served_timestamp",\n  "age_targeting",\n  "gender_targeting",\n  "geo_targeting_included",\n  "geo_targeting_excluded",\n  "spend_range_min_usd",\n  "spend_range_max_usd",\n  "spend_range_min_eur",\n  "spend_range_max_eur",\n  "spend_range_min_inr",\n  "spend_range_max_inr",\n  "spend_range_min_bgn",\n  "spend_range_max_bgn",\n  "spend_range_min_hrk",\n  "spend_range_max_hrk",\n  "spend_range_min_czk",\n  "spend_range_max_czk",\n  "spend_range_min_dkk",\n  "spend_range_max_dkk",\n  "spend_range_min_huf",\n  "spend_range_max_huf",\n  "spend_range_min_pln",\n  "spend_range_max_pln",\n  "spend_range_min_ron",\n  "spend_range_max_ron",\n  "spend_range_min_sek",\n  "spend_range_max_sek",\n  "spend_range_min_gbp",\n  "spend_range_max_gbp",\n  "spend_range_min_nzd",\n  "spend_range_max_nzd",\n  "spend_range_min_ils",\n  "spend_range_max_ils",\n  "spend_range_min_aud",\n  "spend_range_max_aud",\n  "spend_range_min_twd",\n  "spend_range_max_twd",\n  "spend_range_min_brl",\n  "spend_range_max_brl",\n  "spend_range_min_ars",\n  "spend_range_max_ars",\n  "spend_range_min_zar",\n  "spend_range_max_zar",\n  "spend_range_min_clp",\n  "spend_range_max_clp"\n]',
-            "RENAME_MAPPINGS": '{\n  "Ad_ID": "ad_id",\n  "Ad_URL": "ad_url",\n  "Ad_Type": "ad_type",\n  "Regions": "regions",\n  "Advertiser_ID": "advertiser_id",\n  "Advertiser_Name": "advertiser_name",\n  "Ad_Campaigns_List": "ad_campaigns_list",\n  "Date_Range_Start": "date_range_start",\n  "Date_Range_End": "date_range_end",\n  "Num_of_Days": "num_of_days",\n  "Impressions": "impressions",\n  "Spend_USD": "spend_usd",\n  "Spend_Range_Min_USD": "spend_range_min_usd",\n  "Spend_Range_Max_USD": "spend_range_max_usd",\n  "Spend_Range_Min_EUR": "spend_range_min_eur",\n  "Spend_Range_Max_EUR": "spend_range_max_eur",\n  "Spend_Range_Min_INR": "spend_range_min_inr",\n  "Spend_Range_Max_INR": "spend_range_max_inr",\n  "Spend_Range_Min_BGN": "spend_range_min_bgn",\n  "Spend_Range_Max_BGN": "spend_range_max_bgn",\n  "Spend_Range_Min_HRK": "spend_range_min_hrk",\n  "Spend_Range_Max_HRK": "spend_range_max_hrk",\n  "Spend_Range_Min_CZK": "spend_range_min_czk",\n  "Spend_Range_Max_CZK": "spend_range_max_czk",\n  "Spend_Range_Min_DKK": "spend_range_min_dkk",\n  "Spend_Range_Max_DKK": "spend_range_max_dkk",\n  "Spend_Range_Min_HUF": "spend_range_min_huf",\n  "Spend_Range_Max_HUF": "spend_range_max_huf",\n  "Spend_Range_Min_PLN": "spend_range_min_pln",\n  "Spend_Range_Max_PLN": "spend_range_max_pln",\n  "Spend_Range_Min_RON": "spend_range_min_ron",\n  "Spend_Range_Max_RON": "spend_range_max_ron",\n  "Spend_Range_Min_SEK": "spend_range_min_sek",\n  "Spend_Range_Max_SEK": "spend_range_max_sek",\n  "Spend_Range_Min_GBP": "spend_range_min_gbp",\n  "Spend_Range_Max_GBP": "spend_range_max_gbp",\n  "Spend_Range_Min_NZD": "spend_range_min_nzd",\n  "Spend_Range_Max_NZD": "spend_range_max_nzd",\n  "Spend_Range_Min_ILS": "spend_range_min_ils",\n  "Spend_Range_Max_ILS": "spend_range_max_ils",\n  "Spend_Range_Min_AUD": "spend_range_min_aud",\n  "Spend_Range_Max_AUD": "spend_range_max_aud",\n  "Spend_Range_Min_TWD": "spend_range_min_twd",\n  "Spend_Range_Max_TWD": "spend_range_max_twd",\n  "Spend_Range_Min_BRL": "spend_range_min_brl",\n  "Spend_Range_Max_BRL": "spend_range_max_brl",\n  "Spend_Range_Min_ARS": "spend_range_min_ars",\n  "Spend_Range_Max_ARS": "spend_range_max_ars",\n  "Spend_Range_Min_ZAR": "spend_range_min_zar",\n  "Spend_Range_Max_ZAR": "spend_range_max_zar",\n  "Spend_Range_Min_CLP": "spend_range_min_clp",\n  "Spend_Range_Max_CLP": "spend_range_max_clp",\n  "Age_Targeting": "age_targeting",\n  "Gender_Targeting": "gender_targeting",\n  "Geo_Targeting_Included": "geo_targeting_included",\n  "Geo_Targeting_Excluded": "geo_targeting_excluded",\n  "First_Served_Timestamp": "first_served_timestamp",\n  "Last_Served_Timestamp": "last_served_timestamp"\n}',
+            "CSV_HEADERS": '[\n  "ad_id",\n  "ad_url",\n  "ad_type",\n  "regions",\n  "advertiser_id",\n  "advertiser_name",\n  "ad_campaigns_list",\n  "date_range_start",\n  "date_range_end",\n  "num_of_days",\n  "impressions",\n  "spend_usd",\n  "first_served_timestamp",\n  "last_served_timestamp",\n  "age_targeting",\n  "gender_targeting",\n  "geo_targeting_included",\n  "geo_targeting_excluded",\n  "is_funded_by_google_ad_grants",\n  "spend_range_min_usd",\n  "spend_range_max_usd",\n  "spend_range_min_eur",\n  "spend_range_max_eur",\n  "spend_range_min_inr",\n  "spend_range_max_inr",\n  "spend_range_min_bgn",\n  "spend_range_max_bgn",\n  "spend_range_min_hrk",\n  "spend_range_max_hrk",\n  "spend_range_min_czk",\n  "spend_range_max_czk",\n  "spend_range_min_dkk",\n  "spend_range_max_dkk",\n  "spend_range_min_huf",\n  "spend_range_max_huf",\n  "spend_range_min_pln",\n  "spend_range_max_pln",\n  "spend_range_min_ron",\n  "spend_range_max_ron",\n  "spend_range_min_sek",\n  "spend_range_max_sek",\n  "spend_range_min_gbp",\n  "spend_range_max_gbp",\n  "spend_range_min_nzd",\n  "spend_range_max_nzd",\n  "spend_range_min_ils",\n  "spend_range_max_ils",\n  "spend_range_min_aud",\n  "spend_range_max_aud",\n  "spend_range_min_twd",\n  "spend_range_max_twd",\n  "spend_range_min_brl",\n  "spend_range_max_brl",\n  "spend_range_min_ars",\n  "spend_range_max_ars",\n  "spend_range_min_zar",\n  "spend_range_max_zar",\n  "spend_range_min_clp",\n  "spend_range_max_clp",\n  "spend_range_min_mxn",\n  "spend_range_max_mxn"\n]',
+            "RENAME_MAPPINGS": '{\n  "Ad_ID": "ad_id",\n  "Ad_URL": "ad_url",\n  "Ad_Type": "ad_type",\n  "Regions": "regions",\n  "Advertiser_ID": "advertiser_id",\n  "Advertiser_Name": "advertiser_name",\n  "Ad_Campaigns_List": "ad_campaigns_list",\n  "Date_Range_Start": "date_range_start",\n  "Date_Range_End": "date_range_end",\n  "Num_of_Days": "num_of_days",\n  "Impressions": "impressions",\n  "Spend_USD": "spend_usd",\n  "Spend_Range_Min_USD": "spend_range_min_usd",\n  "Spend_Range_Max_USD": "spend_range_max_usd",\n  "Spend_Range_Min_EUR": "spend_range_min_eur",\n  "Spend_Range_Max_EUR": "spend_range_max_eur",\n  "Spend_Range_Min_INR": "spend_range_min_inr",\n  "Spend_Range_Max_INR": "spend_range_max_inr",\n  "Spend_Range_Min_BGN": "spend_range_min_bgn",\n  "Spend_Range_Max_BGN": "spend_range_max_bgn",\n  "Spend_Range_Min_HRK": "spend_range_min_hrk",\n  "Spend_Range_Max_HRK": "spend_range_max_hrk",\n  "Spend_Range_Min_CZK": "spend_range_min_czk",\n  "Spend_Range_Max_CZK": "spend_range_max_czk",\n  "Spend_Range_Min_DKK": "spend_range_min_dkk",\n  "Spend_Range_Max_DKK": "spend_range_max_dkk",\n  "Spend_Range_Min_HUF": "spend_range_min_huf",\n  "Spend_Range_Max_HUF": "spend_range_max_huf",\n  "Spend_Range_Min_PLN": "spend_range_min_pln",\n  "Spend_Range_Max_PLN": "spend_range_max_pln",\n  "Spend_Range_Min_RON": "spend_range_min_ron",\n  "Spend_Range_Max_RON": "spend_range_max_ron",\n  "Spend_Range_Min_SEK": "spend_range_min_sek",\n  "Spend_Range_Max_SEK": "spend_range_max_sek",\n  "Spend_Range_Min_GBP": "spend_range_min_gbp",\n  "Spend_Range_Max_GBP": "spend_range_max_gbp",\n  "Spend_Range_Min_NZD": "spend_range_min_nzd",\n  "Spend_Range_Max_NZD": "spend_range_max_nzd",\n  "Spend_Range_Min_ILS": "spend_range_min_ils",\n  "Spend_Range_Max_ILS": "spend_range_max_ils",\n  "Spend_Range_Min_AUD": "spend_range_min_aud",\n  "Spend_Range_Max_AUD": "spend_range_max_aud",\n  "Spend_Range_Min_TWD": "spend_range_min_twd",\n  "Spend_Range_Max_TWD": "spend_range_max_twd",\n  "Spend_Range_Min_BRL": "spend_range_min_brl",\n  "Spend_Range_Max_BRL": "spend_range_max_brl",\n  "Spend_Range_Min_ARS": "spend_range_min_ars",\n  "Spend_Range_Max_ARS": "spend_range_max_ars",\n  "Spend_Range_Min_ZAR": "spend_range_min_zar",\n  "Spend_Range_Max_ZAR": "spend_range_max_zar",\n  "Spend_Range_Min_CLP": "spend_range_min_clp",\n  "Spend_Range_Max_CLP": "spend_range_max_clp",\n  "Age_Targeting": "age_targeting",\n  "Gender_Targeting": "gender_targeting",\n  "Geo_Targeting_Included": "geo_targeting_included",\n  "Geo_Targeting_Excluded": "geo_targeting_excluded",\n  "First_Served_Timestamp": "first_served_timestamp",\n  "Last_Served_Timestamp": "last_served_timestamp",\n  "Spend_Range_Min_MXN": "spend_range_min_mxn",\n  "Spend_Range_Max_MXN": "spend_range_max_mxn",\n  "Is_Funded_By_Google_Ad_Grants": "is_funded_by_google_ad_grants"\n}',
         },
         container_resources={
             "memory": {"request": "16G"},
@@ -956,6 +956,12 @@
                 "description": "Geographic locations excluded in the ad's targeting.",
                 "mode": "nullable",
             },
+            {
+                "name": "is_funded_by_google_ad_grants",
+                "type": "bool",
+                "description": "Indicates whether the ad is funded by an in-kind donation through Google Ad Grants. Google ad Grants donates Search advertising to eligible organizations. Google does not endorse these ad.",
+                "mode": "nullable",
+            },
             {
                 "name": "spend_range_min_usd",
                 "type": "integer",
@@ -1184,6 +1190,18 @@
                 "description": "Upper bound of the amount in CLP spent by the advertiser on the election ad.",
                 "mode": "nullable",
             },
+            {
+                "name": "spend_range_min_mxn",
+                "type": "integer",
+                "description": "Lower bound of the amount in MXN spent by the advertiser on the election ad.",
+                "mode": "nullable",
+            },
+            {
+                "name": "spend_range_max_mxn",
+                "type": "integer",
+                "description": "Upper bound of the amount in MXN spent by the advertiser on the election ad.",
+                "mode": "nullable",
+            },
         ],
     )