diff --git a/requirements.txt b/requirements.txt index 3db8a985..ce31599c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ # uv pip compile pyproject.toml --resolver=backtracking --all-extras -o requirements.txt annotated-types==0.7.0 # via pydantic -attrs==24.2.0 +attrs==24.3.0 # via # frictionless # jsonlines @@ -10,19 +10,19 @@ attrs==24.2.0 # referencing cachetools==5.5.0 # via google-auth -certifi==2024.8.30 +certifi==2024.12.14 # via requests cfgv==3.4.0 # via pre-commit chardet==5.2.0 # via frictionless -charset-normalizer==3.4.0 +charset-normalizer==3.4.1 # via requests ckanapi==4.8 # via hdx-python-api -click==8.1.7 +click==8.1.8 # via typer -coverage==7.6.9 +coverage==7.6.10 # via pytest-cov defopt==6.4.0 # via hdx-python-api @@ -44,7 +44,7 @@ filelock==3.16.1 # via virtualenv frictionless==5.18.0 # via hdx-python-utilities -google-auth==2.36.0 +google-auth==2.37.0 # via # google-auth-oauthlib # gspread @@ -77,7 +77,7 @@ hdx-python-utilities==3.7.4 # hdx-python-scraper humanize==4.11.0 # via frictionless -identify==2.6.3 +identify==2.6.5 # via pre-commit idna==3.10 # via @@ -85,13 +85,13 @@ idna==3.10 # requests ijson==3.3.0 # via hdx-python-utilities -inflect==7.4.0 +inflect==7.5.0 # via quantulum3 iniconfig==2.0.0 # via pytest isodate==0.7.2 # via frictionless -jinja2==3.1.4 +jinja2==3.1.5 # via frictionless jsonlines==4.0.0 # via hdx-python-utilities @@ -124,7 +124,7 @@ more-itertools==10.5.0 # via inflect nodeenv==1.9.1 # via pre-commit -num2words==0.5.13 +num2words==0.5.14 # via quantulum3 oauthlib==3.2.2 # via requests-oauthlib @@ -156,11 +156,11 @@ pyasn1==0.6.1 # rsa pyasn1-modules==0.4.1 # via google-auth -pydantic==2.10.3 +pydantic==2.10.4 # via frictionless -pydantic-core==2.27.1 +pydantic-core==2.27.2 # via pydantic -pygments==2.18.0 +pygments==2.19.0 # via rich pyphonetics==0.5.3 # via hdx-python-country @@ -224,11 +224,11 @@ rpds-py==0.22.3 # referencing rsa==4.9 # via google-auth -ruamel-yaml==0.18.6 +ruamel-yaml==0.18.9 # via hdx-python-utilities ruamel-yaml-clib==0.2.12 # via ruamel-yaml -setuptools==75.6.0 +setuptools==75.7.0 # via ckanapi shellingham==1.5.4 # via typer @@ -236,7 +236,7 @@ simpleeval==1.0.3 # via frictionless simplejson==3.19.3 # via ckanapi -six==1.16.0 +six==1.17.0 # via # ckanapi # pockets @@ -278,13 +278,13 @@ unidecode==1.3.8 # via # libhxl # pyphonetics -urllib3==2.2.3 +urllib3==2.3.0 # via # libhxl # requests validators==0.34.0 # via frictionless -virtualenv==20.28.0 +virtualenv==20.28.1 # via pre-commit wheel==0.45.1 # via libhxl diff --git a/src/hapi/pipelines/database/idps.py b/src/hapi/pipelines/database/idps.py index 977a14cc..85b5307b 100644 --- a/src/hapi/pipelines/database/idps.py +++ b/src/hapi/pipelines/database/idps.py @@ -70,6 +70,17 @@ def populate(self) -> None: date_reported = row[hxl_tags.index("#date+reported")] reporting_round = row[hxl_tags.index("#round+code")] operation = row[hxl_tags.index("#operation+name")] + # Ignore rows with no reporting date since it is part of + # the primary key of DBIDPs + if date_reported is None: + text = ( + f"No reportingDate for admin code {admin_code}, assessment type {assessment_type}, " + f"reporting round {reporting_round}, operation {operation}" + ) + self._error_manager.add_message( + "IDPs", dataset_name, text + ) + continue duplicate_row_check = ( admin2_ref, assessment_type,