Use age range YAML configuration

OCHA-DAP · Dec 15, 2023 · bf4eab9 · bf4eab9
1 parent 751c1a1
commit bf4eab9
Show file tree

Hide file tree

Showing 6 changed files with 38 additions and 17 deletions.
diff --git a/requirements.txt b/requirements.txt
@@ -29,7 +29,7 @@ click==8.1.7
     # via typer
 colorama==0.4.6
     # via typer
-coverage[toml]==7.3.2
+coverage[toml]==7.3.3
     # via pytest-cov
 cryptography==41.0.7
     # via pyopenssl
@@ -65,7 +65,7 @@ gspread==5.12.2
     # via hdx-python-scraper
 hapi-schema==0.4.0
     # via hapi-pipelines (pyproject.toml)
-hdx-python-api==6.1.4
+hdx-python-api==6.2.0
     # via hdx-python-scraper
 hdx-python-country==3.6.3
     # via
@@ -76,7 +76,7 @@ hdx-python-database[postgresql]==1.2.9
     # via hapi-pipelines (pyproject.toml)
 hdx-python-scraper==2.3.0
     # via hapi-pipelines (pyproject.toml)
-hdx-python-utilities==3.6.2
+hdx-python-utilities==3.6.3
     # via
     #   hdx-python-api
     #   hdx-python-country
@@ -149,9 +149,9 @@ pockets==0.9.1
     # via sphinxcontrib-napoleon
 pre-commit==3.6.0
     # via hapi-pipelines (pyproject.toml)
-psycopg[binary]==3.1.14
+psycopg[binary]==3.1.15
     # via hdx-python-database
-psycopg-binary==3.1.14
+psycopg-binary==3.1.15
     # via psycopg
 pyasn1==0.5.1
     # via

diff --git a/src/hapi/pipelines/app/pipelines.py b/src/hapi/pipelines/app/pipelines.py
@@ -69,8 +69,9 @@ def __init__(
             session=session,
             gender_descriptions=configuration["gender_descriptions"],
         )
-        self.age_range = AgeRange(session=session)
-
+        self.age_range = AgeRange(
+            session=session, age_range_codes=configuration["age_range_codes"]
+        )
         self.ipc_phase = IpcPhase(
             session=session,
             ipc_phase_names=configuration["ipc_phase_names"],
@@ -140,6 +141,7 @@ def output(self):
         self.sector.populate()
         self.org.populate()
         self.gender.populate()
+        self.age_range.populate()
         self.ipc_phase.populate()
         self.ipc_type.populate()
 

diff --git a/src/hapi/pipelines/configs/core.yaml b/src/hapi/pipelines/configs/core.yaml
@@ -23,6 +23,25 @@ gender_descriptions:
   m: "male"
   x: "non-binary"
 
+age_range_codes:
+  - "0-4"
+  - "10-14"
+  - "15-19"
+  - "20-24"
+  - "25-29"
+  - "30-34"
+  - "35-39"
+  - "40-44"
+  - "45-49"
+  - "5-9"
+  - "50-54"
+  - "55-59"
+  - "60-64"
+  - "65-69"
+  - "70-74"
+  - "75-79"
+  - "80+"
+
 sector:
   dataset: "global-coordination-groups-beta"
   resource: "Global Coordination Groups (Beta) CSV"

diff --git a/src/hapi/pipelines/database/age_range.py b/src/hapi/pipelines/database/age_range.py
@@ -1,4 +1,5 @@
 import logging
+from typing import List
 
 from hapi_schema.db_age_range import DBAgeRange
 from sqlalchemy.orm import Session
@@ -9,16 +10,17 @@
 
 
 class AgeRange(BaseUploader):
-    def __init__(self, session: Session):
+    def __init__(self, session: Session, age_range_codes: List[str]):
         super().__init__(session)
-        self.data = []
+        self.data = age_range_codes
 
     def populate(self):
-        raise NotImplementedError()
+        logger.info("Populating age ranges table")
+        for age_range_code in self.data:
+            self.populate_single(age_range_code)
+        self._session.commit()
 
     def populate_single(self, age_range_code: str):
-        logger.info(f"Adding age range code {age_range_code}")
-        # Check and exit if already exists
         ages = age_range_code.split("-")
         if len(ages) == 2:
             # Format: 0-5
@@ -31,5 +33,3 @@ def populate_single(self, age_range_code: str):
             code=age_range_code, age_min=age_min, age_max=age_max
         )
         self._session.add(age_range_row)
-        self._session.commit()
-        self.data.append(age_range_code)
diff --git a/src/hapi/pipelines/database/operational_presence.py b/src/hapi/pipelines/database/operational_presence.py
@@ -144,9 +144,9 @@ def populate(self):
                         rows.append(row)
                         operational_presence_row = DBOperationalPresence(
                             resource_ref=resource_ref,
+                            admin2_ref=admin2_ref,
                             org_ref=org_ref,
                             sector_code=sector_code,
-                            admin2_ref=admin2_ref,
                             reference_period_start=reference_period_start,
                             reference_period_end=reference_period_end,
                             # TODO: Add to scraper (HAPI-199)

diff --git a/src/hapi/pipelines/database/population.py b/src/hapi/pipelines/database/population.py
@@ -66,8 +66,8 @@ def populate(self):
                         age_range_code is not None
                         and age_range_code not in self._age_range.data
                     ):
-                        self._age_range.populate_single(
-                            age_range_code=age_range_code
+                        raise ValueError(
+                            f"Age range code {age_range_code} not in table"
                         )
                     for admin_code, value in values.items():
                         admin2_code = admins.get_admin2_code_based_on_level(