Use get_code_from_name in HDX Python Utilities (#214)

Use hdx_error_manager from HDX Python API
Update requirements

.config/pre-commit-config.yaml

@@ -8,7 +8,7 @@ repos:
       - id: end-of-file-fixer
       - id: check-ast
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.7.0
+    rev: v0.8.6
     hooks:
       # Run the linter.
       - id: ruff
@@ -17,7 +17,7 @@ repos:
       - id: ruff-format
         args: [--config, .config/ruff.toml]
   - repo: https://github.com/astral-sh/uv-pre-commit
-    rev: 0.4.24
+    rev: 0.5.15
     hooks:
       # Run the pip compile
       - id: pip-compile

pyproject.toml

@@ -35,11 +35,11 @@ requires-python = ">=3.8"
 
 dependencies = [
     "hapi-schema>= 0.9.6",
-    "hdx-python-api>= 6.3.5",
-    "hdx-python-country>= 3.8.5",
+    "hdx-python-api>= 6.3.6",
+    "hdx-python-country>= 3.8.6",
     "hdx-python-database[postgresql]>= 1.3.4",
-    "hdx-python-scraper>= 2.5.0",
-    "hdx-python-utilities>= 3.7.4",
+    "hdx-python-scraper>= 2.5.1",
+    "hdx-python-utilities>= 3.8.0",
     "libhxl",
     "sqlalchemy"
 ]
@@ -97,7 +97,7 @@ run = """
        """
 
 [tool.hatch.envs.hatch-static-analysis]
-dependencies = ["ruff==0.7.0"]
+dependencies = ["ruff==0.8.6"]
 
 [tool.hatch.envs.hatch-static-analysis.scripts]
 format-check = ["ruff format --config .config/ruff.toml --check --diff {args:.}",]

requirements.txt

@@ -56,20 +56,20 @@ gspread==6.1.4
     # via hdx-python-scraper
 hapi-schema==0.9.6
     # via hapi-pipelines (pyproject.toml)
-hdx-python-api==6.3.5
+hdx-python-api==6.3.6
     # via
     #   hapi-pipelines (pyproject.toml)
     #   hdx-python-scraper
-hdx-python-country==3.8.5
+hdx-python-country==3.8.6
     # via
     #   hapi-pipelines (pyproject.toml)
     #   hdx-python-api
     #   hdx-python-scraper
 hdx-python-database==1.3.4
     # via hapi-pipelines (pyproject.toml)
-hdx-python-scraper==2.5.0
+hdx-python-scraper==2.5.1
     # via hapi-pipelines (pyproject.toml)
-hdx-python-utilities==3.7.4
+hdx-python-utilities==3.8.0
     # via
     #   hapi-pipelines (pyproject.toml)
     #   hdx-python-api
@@ -156,14 +156,14 @@ pyasn1==0.6.1
     #   rsa
 pyasn1-modules==0.4.1
     # via google-auth
-pydantic==2.10.4
+pydantic==2.10.5
     # via frictionless
 pydantic-core==2.27.2
     # via pydantic
-pygments==2.19.0
+pygments==2.19.1
     # via rich
 pyphonetics==0.5.3
-    # via hdx-python-country
+    # via hdx-python-utilities
 pytest==8.3.4
     # via
     #   hapi-pipelines (pyproject.toml)
@@ -224,11 +224,11 @@ rpds-py==0.22.3
     #   referencing
 rsa==4.9
     # via google-auth
-ruamel-yaml==0.18.9
+ruamel-yaml==0.18.10
     # via hdx-python-utilities
 ruamel-yaml-clib==0.2.12
     # via ruamel-yaml
-setuptools==75.7.0
+setuptools==75.8.0
     # via ckanapi
 shellingham==1.5.4
     # via typer
@@ -244,7 +244,7 @@ six==1.17.0
     #   sphinxcontrib-napoleon
 sphinxcontrib-napoleon==0.7
     # via defopt
-sqlalchemy==2.0.36
+sqlalchemy==2.0.37
     # via
     #   hapi-pipelines (pyproject.toml)
     #   hapi-schema

src/hapi/pipelines/app/__main__.py

@@ -8,6 +8,7 @@
 
 from hapi_schema.views import prepare_hapi_views
 from hdx.api.configuration import Configuration
+from hdx.api.utilities.hdx_error_handler import HDXErrorHandler
 from hdx.database import Database
 from hdx.database.dburi import (
     get_params_from_connection_uri,
@@ -18,7 +19,6 @@
 from hdx.utilities.dateparse import now_utc
 from hdx.utilities.dictandlist import args_to_dict
 from hdx.utilities.easy_logging import setup_logging
-from hdx.utilities.errors_onexit import ErrorsOnExit
 from hdx.utilities.path import temp_dir
 from hdx.utilities.typehint import ListTuple
 
@@ -142,7 +142,7 @@ def main(
         params["prepare_fn"] = prepare_hapi_views
     logger.info(f"> Database parameters: {params}")
     configuration = Configuration.read()
-    with ErrorsOnExit() as errors_on_exit:
+    with HDXErrorHandler(should_exit_on_error=False) as error_handler:
         with temp_dir() as temp_folder:
             with Database(**params) as database:
                 session = database.get_session()
@@ -165,7 +165,7 @@ def main(
                     today,
                     themes_to_run,
                     scrapers_to_run,
-                    errors_on_exit,
+                    error_handler,
                 )
                 pipelines.run()
                 pipelines.output()

src/hapi/pipelines/app/pipelines.py

@@ -3,11 +3,11 @@
 from typing import Dict, Optional
 
 from hdx.api.configuration import Configuration
+from hdx.api.utilities.hdx_error_handler import HDXErrorHandler
 from hdx.location.adminlevel import AdminLevel
 from hdx.scraper.framework.runner import Runner
 from hdx.scraper.framework.utilities.reader import Read
 from hdx.scraper.framework.utilities.sources import Sources
-from hdx.utilities.errors_onexit import ErrorsOnExit
 from hdx.utilities.typehint import ListTuple
 from sqlalchemy.orm import Session
 
@@ -31,7 +31,6 @@
 from hapi.pipelines.database.sector import Sector
 from hapi.pipelines.database.wfp_commodity import WFPCommodity
 from hapi.pipelines.database.wfp_market import WFPMarket
-from hapi.pipelines.utilities.error_handling import ErrorManager
 
 logger = logging.getLogger(__name__)
 
@@ -44,7 +43,7 @@ def __init__(
         today: datetime,
         themes_to_run: Optional[Dict] = None,
         scrapers_to_run: Optional[ListTuple[str]] = None,
-        errors_on_exit: Optional[ErrorsOnExit] = None,
+        error_handler: Optional[HDXErrorHandler] = None,
         use_live: bool = True,
         countries_to_run: Optional[ListTuple[str]] = None,
     ):
@@ -58,7 +57,7 @@ def __init__(
             countries=countries_to_run,
         )
         self.countries = self.locations.hapi_countries
-        self.error_manager = ErrorManager()
+        self.error_handler = error_handler
         reader = Read.get_reader("hdx")
         libhxl_dataset = AdminLevel.get_libhxl_dataset(
             retriever=reader
@@ -109,7 +108,7 @@ def __init__(
         self.runner = Runner(
             self.countries,
             today=today,
-            errors_on_exit=errors_on_exit,
+            error_handler=error_handler,
             scrapers_to_run=scrapers_to_run,
         )
         self.configurable_scrapers = {}
@@ -204,7 +203,7 @@ def output_population(self):
                 metadata=self.metadata,
                 admins=self.admins,
                 configuration=self.configuration,
-                error_manager=self.error_manager,
+                error_handler=self.error_handler,
             )
             population.populate()
 
@@ -227,7 +226,7 @@ def output_operational_presence(self):
                 sector=self.sector,
                 results=results,
                 config=self.configuration,
-                error_manager=self.error_manager,
+                error_handler=self.error_handler,
             )
             operational_presence.populate()
 
@@ -241,7 +240,7 @@ def output_food_security(self):
                 admintwo=self.admintwo,
                 countryiso3s=self.countries,
                 configuration=self.configuration,
-                error_manager=self.error_manager,
+                error_handler=self.error_handler,
             )
             food_security.populate()
 
@@ -256,7 +255,7 @@ def output_humanitarian_needs(self):
                 admins=self.admins,
                 sector=self.sector,
                 configuration=self.configuration,
-                error_manager=self.error_manager,
+                error_handler=self.error_handler,
             )
             humanitarian_needs.populate()
 
@@ -299,7 +298,7 @@ def output_idps(self):
                 metadata=self.metadata,
                 admins=self.admins,
                 results=results,
-                error_manager=self.error_manager,
+                error_handler=self.error_handler,
             )
             idps.populate()
 
@@ -311,7 +310,7 @@ def output_funding(self):
                 countryiso3s=self.countries,
                 locations=self.locations,
                 configuration=self.configuration,
-                error_manager=self.error_manager,
+                error_handler=self.error_handler,
             )
             funding.populate()
 
@@ -322,7 +321,7 @@ def output_poverty_rate(self):
                 metadata=self.metadata,
                 admins=self.admins,
                 configuration=self.configuration,
-                error_manager=self.error_manager,
+                error_handler=self.error_handler,
             )
             poverty_rate.populate()
 
@@ -337,7 +336,7 @@ def output_conflict_event(self):
                 admins=self.admins,
                 results=results,
                 configuration=self.configuration,
-                error_manager=self.error_manager,
+                error_handler=self.error_handler,
             )
             conflict_event.populate()
 
@@ -356,7 +355,7 @@ def output_food_prices(self):
                 adminone=self.adminone,
                 admintwo=self.admintwo,
                 configuration=self.configuration,
-                error_manager=self.error_manager,
+                error_handler=self.error_handler,
             )
             wfp_market.populate()
             food_price = FoodPrice(
@@ -367,7 +366,7 @@ def output_food_prices(self):
                 currency=self.currency,
                 commodity=wfp_commodity,
                 market=wfp_market,
-                error_manager=self.error_manager,
+                error_handler=self.error_handler,
             )
             food_price.populate()
 
@@ -395,4 +394,4 @@ def debug(self, folder: str) -> None:
         self.org.output_org_map(folder)
 
     def output_errors(self, err_to_hdx: bool) -> None:
-        self.error_manager.output_errors(err_to_hdx)
+        self.error_handler.output_errors(err_to_hdx)

src/hapi/pipelines/database/admins.py

@@ -9,12 +9,12 @@
 from hapi_schema.db_admin2 import DBAdmin2
 from hapi_schema.db_location import DBLocation
 from hdx.api.configuration import Configuration
+from hdx.api.utilities.hdx_error_handler import HDXErrorHandler
 from hdx.utilities.dateparse import parse_date
 from hxl.filters import AbstractStreamingFilter
 from sqlalchemy import select
 from sqlalchemy.orm import Session
 
-from ..utilities.error_handling import ErrorManager
 from .base_uploader import BaseUploader
 from .locations import Locations
 
@@ -164,15 +164,15 @@ def get_admin1_ref(
         admin_code: str,
         dataset_name: str,
         pipeline: str,
-        error_manager: ErrorManager,
+        error_handler: HDXErrorHandler,
     ) -> Optional[int]:
         admin1_code = get_admin1_code_based_on_level(
             admin_code=admin_code, admin_level=admin_level
         )
         ref = self.admin1_data.get(admin1_code)
         if ref is None:
             # TODO: resolve pipeline name
-            error_manager.add_missing_value_message(
+            error_handler.add_missing_value_message(
                 pipeline, dataset_name, "admin 1 code", admin1_code
             )
         return ref
@@ -183,15 +183,15 @@ def get_admin2_ref(
         admin_code: str,
         dataset_name: str,
         pipeline: str,
-        error_manager: ErrorManager,
+        error_handler: HDXErrorHandler,
     ) -> Optional[int]:
         admin2_code = get_admin2_code_based_on_level(
             admin_code=admin_code, admin_level=admin_level
         )
         ref = self.admin2_data.get(admin2_code)
         if ref is None:
             # TODO: resolve pipeline name
-            error_manager.add_missing_value_message(
+            error_handler.add_missing_value_message(
                 pipeline, dataset_name, "admin 2 code", admin2_code
             )
         return ref

src/hapi/pipelines/database/conflict_event.py

@@ -6,11 +6,11 @@
 from hapi_schema.db_conflict_event import DBConflictEvent
 from hapi_schema.utils.enums import EventType
 from hdx.api.configuration import Configuration
+from hdx.api.utilities.hdx_error_handler import HDXErrorHandler
 from hdx.utilities.dateparse import parse_date_range
 from sqlalchemy.orm import Session
 
 from ..utilities.batch_populate import batch_populate
-from ..utilities.error_handling import ErrorManager
 from ..utilities.provider_admin_names import get_provider_name
 from . import admins
 from .base_uploader import BaseUploader
@@ -27,14 +27,14 @@ def __init__(
         admins: admins.Admins,
         results: Dict,
         configuration: Configuration,
-        error_manager: ErrorManager,
+        error_handler: HDXErrorHandler,
     ):
         super().__init__(session)
         self._metadata = metadata
         self._admins = admins
         self._results = results
         self._configuration = configuration
-        self._error_manager = error_manager
+        self._error_handler = error_handler
 
     def populate(self) -> None:
         logger.info("Populating conflict event table")
@@ -117,13 +117,13 @@ def populate(self) -> None:
                             conflict_event_rows.append(conflict_event_row)
 
             if number_duplicates > 0:
-                self._error_manager.add_message(
+                self._error_handler.add_message(
                     "ConflictEvent",
                     dataset_name,
                     f"{number_duplicates} duplicate rows",
                 )
             if len(conflict_event_rows) == 0:
-                self._error_manager.add_message(
+                self._error_handler.add_message(
                     "ConflictEvent", dataset_name, "no rows found"
                 )
                 continue
@@ -133,7 +133,7 @@ def populate(self) -> None:
             "conflict_event_error_messages", {}
         ).items():
             dataset, resource_name = identifier.split("|")
-            self._error_manager.add_message(
+            self._error_handler.add_message(
                 "ConfictEvent",
                 dataset,
                 message,