From e03810e012515d392326a30dd960fa27d4cf68d6 Mon Sep 17 00:00:00 2001 From: Christina Gosnell <cgosnell@catalyst.coop> Date: Fri, 20 Dec 2024 11:43:36 -0500 Subject: [PATCH 1/2] only check row nums for years that are in the dataset_settings --- src/pudl/transform/ferc714.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/pudl/transform/ferc714.py b/src/pudl/transform/ferc714.py index e188d18f10..067ea00250 100644 --- a/src/pudl/transform/ferc714.py +++ b/src/pudl/transform/ferc714.py @@ -1279,22 +1279,25 @@ class Ferc714CheckSpec: def make_check(spec: Ferc714CheckSpec) -> AssetChecksDefinition: """Turn the Ferc714CheckSpec into an actual Dagster asset check.""" - @asset_check(asset=spec.asset, blocking=True) - def _check(df): + @asset_check( + asset=spec.asset, required_resource_keys={"dataset_settings"}, blocking=True + ) + def _row_num_check(context, df): errors = [] - for year, expected_rows in spec.num_rows_by_report_year.items(): + for year in context.resources.dataset_settings.ferc714.years: + expected_rows = spec.num_rows_by_report_year[year] if (num_rows := len(df.loc[df.report_year == year])) != expected_rows: errors.append( f"Expected {expected_rows} for report year {year}, found {num_rows}" ) - logger.info(errors) + logger.warning(errors) if errors: return AssetCheckResult(passed=False, metadata={"errors": errors}) return AssetCheckResult(passed=True) - return _check + return _row_num_check _checks = [make_check(spec) for spec in check_specs] From d45005b16cbb55af0dcb9cfa6e5cdf4883f43d82 Mon Sep 17 00:00:00 2001 From: Christina Gosnell <cgosnell@catalyst.coop> Date: Fri, 20 Dec 2024 11:56:02 -0500 Subject: [PATCH 2/2] make the warning only warn once --- src/pudl/transform/ferc714.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pudl/transform/ferc714.py b/src/pudl/transform/ferc714.py index 067ea00250..8c2cc278e7 100644 --- a/src/pudl/transform/ferc714.py +++ b/src/pudl/transform/ferc714.py @@ -1276,7 +1276,7 @@ class Ferc714CheckSpec: ] -def make_check(spec: Ferc714CheckSpec) -> AssetChecksDefinition: +def make_row_num_check(spec: Ferc714CheckSpec) -> AssetChecksDefinition: """Turn the Ferc714CheckSpec into an actual Dagster asset check.""" @asset_check( @@ -1290,7 +1290,7 @@ def _row_num_check(context, df): errors.append( f"Expected {expected_rows} for report year {year}, found {num_rows}" ) - logger.warning(errors) + logger.warning(errors) if errors: return AssetCheckResult(passed=False, metadata={"errors": errors}) @@ -1300,4 +1300,4 @@ def _row_num_check(context, df): return _row_num_check -_checks = [make_check(spec) for spec in check_specs] +_checks = [make_row_num_check(spec) for spec in check_specs]