From 27f490bac6621d6cd577fd608ff8742b3b405c17 Mon Sep 17 00:00:00 2001 From: Ian Hopkinson Date: Wed, 9 Oct 2024 14:15:24 +0100 Subject: [PATCH 01/12] HDX-10213 Humanitarian Needs category column integration - all in one --- .gitignore | 2 ++ .vscode/launch.json | 5 +--- .../db/dao/humanitarian_needs_view_dao.py | 22 ++++---------- hdx_hapi/db/models/views/all_views.py | 8 +---- hdx_hapi/endpoints/get_affected_people.py | 15 ++-------- .../endpoints/models/humanitarian_needs.py | 29 ++++--------------- hdx_hapi/services/humanitarian_needs_logic.py | 12 ++------ tests/sample_data/humanitarian_needs.sql | 20 ++++++------- tests/test_endpoints/endpoint_data.py | 13 ++------- .../test_humanitarian_needs_endpoint.py | 19 ++++-------- 10 files changed, 37 insertions(+), 108 deletions(-) diff --git a/.gitignore b/.gitignore index 8d4aff37..af2bdb4f 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,5 @@ venv/ coverage.xml *.pg_restore + +src/* \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index feff2fe1..0ff20b13 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -11,10 +11,7 @@ "program": "main.py", "console": "integratedTerminal", "justMyCode": true, - "env":{ - // "HAPI_DB_NAME": "hapi", - "HAPI_USE_VAT": "false" - } + "env":{"HAPI_USE_VAT": "true"} } ], diff --git a/hdx_hapi/db/dao/humanitarian_needs_view_dao.py b/hdx_hapi/db/dao/humanitarian_needs_view_dao.py index 4dbbbcbe..586d7619 100644 --- a/hdx_hapi/db/dao/humanitarian_needs_view_dao.py +++ b/hdx_hapi/db/dao/humanitarian_needs_view_dao.py @@ -10,7 +10,7 @@ apply_reference_period_filter, ) from hdx_hapi.endpoints.util.util import PaginationParams, ReferencePeriodParameters -from hapi_schema.utils.enums import DisabledMarker, Gender, PopulationGroup, PopulationStatus +from hapi_schema.utils.enums import PopulationStatus async def humanitarian_needs_view_list( @@ -18,11 +18,8 @@ async def humanitarian_needs_view_list( ref_period_parameters: ReferencePeriodParameters, db: AsyncSession, admin2_ref: Optional[int] = None, - gender: Optional[Gender] = None, - age_range: Optional[str] = None, - disabled_marker: Optional[DisabledMarker] = None, + category: Optional[str] = None, sector_code: Optional[str] = None, - population_group: Optional[PopulationGroup] = None, population_status: Optional[PopulationStatus] = None, population_min: Optional[int] = None, population_max: Optional[int] = None, @@ -42,16 +39,10 @@ async def humanitarian_needs_view_list( ): query = select(HumanitarianNeedsView) - if gender: - query = query.where(HumanitarianNeedsView.gender == gender) - if age_range: - query = query.where(HumanitarianNeedsView.age_range == age_range) - if disabled_marker: - query = query.where(HumanitarianNeedsView.disabled_marker == disabled_marker) + if category: + query = query.where(HumanitarianNeedsView.category.icontains(category)) if sector_code: query = query.where(HumanitarianNeedsView.sector_code.icontains(sector_code)) - if population_group: - query = query.where(HumanitarianNeedsView.population_group == population_group) if population_status: query = query.where(HumanitarianNeedsView.population_status == population_status) @@ -85,12 +76,9 @@ async def humanitarian_needs_view_list( query = apply_pagination(query, pagination_parameters) query = query.order_by( HumanitarianNeedsView.admin2_ref, - HumanitarianNeedsView.gender, - HumanitarianNeedsView.age_range, + HumanitarianNeedsView.category, HumanitarianNeedsView.sector_code, - HumanitarianNeedsView.population_group, HumanitarianNeedsView.population_status, - HumanitarianNeedsView.disabled_marker, HumanitarianNeedsView.reference_period_start, ) diff --git a/hdx_hapi/db/models/views/all_views.py b/hdx_hapi/db/models/views/all_views.py index 930b97b6..a26a4cc4 100644 --- a/hdx_hapi/db/models/views/all_views.py +++ b/hdx_hapi/db/models/views/all_views.py @@ -38,7 +38,6 @@ from hapi_schema.utils.enums import ( CommodityCategory, - DisabledMarker, EventType, IPCPhase, IPCType, @@ -283,14 +282,9 @@ class HumanitarianNeedsView(Base): __table__ = humanitarian_needs_view resource_hdx_id: Mapped[str] = column_property(humanitarian_needs_view.c.resource_hdx_id) admin2_ref: Mapped[int] = column_property(humanitarian_needs_view.c.admin2_ref) - gender: Mapped[Gender] = column_property(humanitarian_needs_view.c.gender) - age_range: Mapped[str] = column_property(humanitarian_needs_view.c.age_range) - min_age: Mapped[int] = column_property(humanitarian_needs_view.c.min_age) - max_age: Mapped[int] = column_property(humanitarian_needs_view.c.max_age) sector_code: Mapped[str] = column_property(humanitarian_needs_view.c.sector_code) - population_group: Mapped[PopulationGroup] = column_property(humanitarian_needs_view.c.population_group) + category: Mapped[str] = column_property(humanitarian_needs_view.c.category) population_status: Mapped[PopulationStatus] = column_property(humanitarian_needs_view.c.population_status) - disabled_marker: Mapped[DisabledMarker] = column_property(humanitarian_needs_view.c.disabled_marker) population: Mapped[int] = column_property(humanitarian_needs_view.c.population) reference_period_start: Mapped[datetime.datetime] = column_property( humanitarian_needs_view.c.reference_period_start diff --git a/hdx_hapi/endpoints/get_affected_people.py b/hdx_hapi/endpoints/get_affected_people.py index b9b61457..e178cd51 100644 --- a/hdx_hapi/endpoints/get_affected_people.py +++ b/hdx_hapi/endpoints/get_affected_people.py @@ -6,7 +6,6 @@ from hdx_hapi.config.config import get_config from hdx_hapi.config.doc_snippets import ( - DOC_DISABLED_MARKER, DOC_GENDER, DOC_AGE_RANGE, DOC_POPULATION_GROUP, @@ -36,7 +35,7 @@ from hdx_hapi.services.humanitarian_needs_logic import get_humanitarian_needs_srv from hdx_hapi.services.refugees_logic import get_refugees_srv from hdx_hapi.services.sql_alchemy_session import get_db -from hapi_schema.utils.enums import DisabledMarker, Gender, PopulationGroup, PopulationStatus +from hapi_schema.utils.enums import Gender, PopulationGroup, PopulationStatus from hdx_hapi.endpoints.util.util import ( CommonEndpointParams, OutputFormat, @@ -145,13 +144,8 @@ async def get_humanitarian_needs( # ref_period_parameters: Annotated[ReferencePeriodParameters, Depends(reference_period_parameters)], common_parameters: Annotated[CommonEndpointParams, Depends(common_endpoint_parameters)], db: AsyncSession = Depends(get_db), - gender: Annotated[Optional[Gender], Query(max_length=3, description=f'{DOC_GENDER}')] = None, - age_range: Annotated[Optional[str], Query(max_length=32, description=f'{DOC_AGE_RANGE}')] = None, - disabled_marker: Annotated[Optional[DisabledMarker], Query(description=f'{DOC_DISABLED_MARKER}')] = None, + category: Annotated[Optional[str], Query(max_length=128, description='')] = None, sector_code: Annotated[Optional[str], Query(max_length=32, description=f'{DOC_SECTOR_CODE}')] = None, - population_group: Annotated[ - Optional[PopulationGroup], Query(max_length=32, description=f'{DOC_POPULATION_GROUP}') - ] = None, population_status: Annotated[ Optional[PopulationStatus], Query(max_length=32, description=f'{DOC_POPULATION_STATUS}') ] = None, @@ -192,11 +186,8 @@ async def get_humanitarian_needs( ref_period_parameters=ref_period_parameters, db=db, admin2_ref=admin2_ref, - gender=gender, - age_range=age_range, - disabled_marker=disabled_marker, + category=category, sector_code=sector_code, - population_group=population_group, population_status=population_status, population_min=population_min, population_max=population_max, diff --git a/hdx_hapi/endpoints/models/humanitarian_needs.py b/hdx_hapi/endpoints/models/humanitarian_needs.py index f88d37cc..eab1cad0 100644 --- a/hdx_hapi/endpoints/models/humanitarian_needs.py +++ b/hdx_hapi/endpoints/models/humanitarian_needs.py @@ -3,40 +3,18 @@ from typing import Optional from hdx_hapi.config.doc_snippets import ( - DOC_DISABLED_MARKER, - DOC_GENDER, - DOC_POPULATION_GROUP, DOC_POPULATION_STATUS, DOC_HDX_RESOURCE_ID, - DOC_AGE_RANGE, DOC_REFERENCE_PERIOD_START, DOC_REFERENCE_PERIOD_END, truncate_query_description, ) from hdx_hapi.endpoints.models.base import HapiBaseModel, HapiModelWithAdmins -from hapi_schema.utils.enums import Gender, PopulationGroup, PopulationStatus, DisabledMarker +from hapi_schema.utils.enums import PopulationStatus class HumanitarianNeedsResponse(HapiBaseModel, HapiModelWithAdmins): resource_hdx_id: str = Field(max_length=36, description=truncate_query_description(DOC_HDX_RESOURCE_ID)) - gender: Gender = Field(description=truncate_query_description(DOC_GENDER)) - age_range: str = Field(max_length=32, description=truncate_query_description(DOC_AGE_RANGE)) - min_age: Optional[int] = Field( - ge=0, - description=( - 'The minimum age from `age_range`, set to `null` if `age_range` is "all" and ' - 'there is no age disaggregation' - ), - ) - max_age: Optional[int] = Field( - ge=0, - description=( - 'The maximum age from `age_range`, set to `null` if `age_range` is "all" and ' - 'there is no age disaggregation, or if there is no upper limit to the ' - 'age range' - ), - ) - disabled_marker: DisabledMarker = Field(description=truncate_query_description(DOC_DISABLED_MARKER)) sector_code: str = Field( max_length=32, description=( @@ -45,7 +23,10 @@ class HumanitarianNeedsResponse(HapiBaseModel, HapiModelWithAdmins): 'dataset, or created for HDX HAPI' ), ) - population_group: PopulationGroup = Field(description=truncate_query_description(DOC_POPULATION_GROUP)) + category: str = Field( + max_length=128, + description='A category combining gender, age range, disability marker and population group information', + ) population_status: PopulationStatus = Field(description=truncate_query_description(DOC_POPULATION_STATUS)) population: int = Field(ge=0, description='The number of people') reference_period_start: datetime.datetime = Field(description=DOC_REFERENCE_PERIOD_START) diff --git a/hdx_hapi/services/humanitarian_needs_logic.py b/hdx_hapi/services/humanitarian_needs_logic.py index 5d287942..c90cd6f9 100644 --- a/hdx_hapi/services/humanitarian_needs_logic.py +++ b/hdx_hapi/services/humanitarian_needs_logic.py @@ -6,7 +6,7 @@ from hdx_hapi.db.dao.humanitarian_needs_view_dao import humanitarian_needs_view_list from hdx_hapi.services.admin_level_logic import compute_unspecified_values from hdx_hapi.endpoints.util.util import AdminLevel, PaginationParams, ReferencePeriodParameters -from hapi_schema.utils.enums import DisabledMarker, Gender, PopulationGroup, PopulationStatus +from hapi_schema.utils.enums import PopulationStatus async def get_humanitarian_needs_srv( @@ -14,11 +14,8 @@ async def get_humanitarian_needs_srv( ref_period_parameters: ReferencePeriodParameters, db: AsyncSession, admin2_ref: Optional[int] = None, - gender: Optional[Gender] = None, - age_range: Optional[str] = None, - disabled_marker: Optional[DisabledMarker] = None, + category: Optional[str] = None, sector_code: Optional[str] = None, - population_group: Optional[PopulationGroup] = None, population_status: Optional[PopulationStatus] = None, population_min: Optional[int] = None, population_max: Optional[int] = None, @@ -44,11 +41,8 @@ async def get_humanitarian_needs_srv( ref_period_parameters=ref_period_parameters, db=db, admin2_ref=admin2_ref, - gender=gender, - age_range=age_range, - disabled_marker=disabled_marker, + category=category, sector_code=sector_code, - population_group=population_group, population_status=population_status, population_min=population_min, population_max=population_max, diff --git a/tests/sample_data/humanitarian_needs.sql b/tests/sample_data/humanitarian_needs.sql index 9c2e3cca..e533776b 100644 --- a/tests/sample_data/humanitarian_needs.sql +++ b/tests/sample_data/humanitarian_needs.sql @@ -1,16 +1,16 @@ -- dummy data INSERT INTO public.humanitarian_needs -(resource_hdx_id, admin2_ref, gender, age_range, min_age, max_age, sector_code, population_group, population_status, disabled_marker, population, reference_period_start, reference_period_end) -VALUES ('17acb541-9431-409a-80a8-50eda7e8ebab',2, 'all', 'ALL',0,99,'EDU','REF','AFF','y',500000,'2023-01-01 00:00:00', '2025-01-01 00:00:00'); +(resource_hdx_id, admin2_ref, category, sector_code, population_status, population, reference_period_start, reference_period_end) +VALUES ('17acb541-9431-409a-80a8-50eda7e8ebab',2, 'All - Disabled - all - AFF','EDU','AFF',500000,'2023-01-01 00:00:00', '2025-01-01 00:00:00'); INSERT INTO public.humanitarian_needs -(resource_hdx_id, admin2_ref, gender, age_range, min_age, max_age, sector_code, population_group, population_status, disabled_marker, population, reference_period_start, reference_period_end) -VALUES ('17acb541-9431-409a-80a8-50eda7e8ebab', 4, 'f', '0-17',0,17,'SHL','IDP','INN','n',1500000,'2023-01-01 00:00:00', '2025-01-01 00:00:00'); +(resource_hdx_id, admin2_ref, category, sector_code, population_status, population, reference_period_start, reference_period_end) +VALUES ('17acb541-9431-409a-80a8-50eda7e8ebab',4,'Female - child - INN','SHL','INN',1500000,'2023-01-01 00:00:00', '2025-01-01 00:00:00'); INSERT INTO public.humanitarian_needs -(resource_hdx_id, admin2_ref, gender, age_range, min_age, max_age, sector_code, population_group, population_status, disabled_marker, population, reference_period_start, reference_period_end) -VALUES ('17acb541-9431-409a-80a8-50eda7e8ebab', 6, 'm', '65+',65,100,'WSH','all','all','all',100000,'2023-01-01 00:00:00', '2025-01-01 00:00:00'); +(resource_hdx_id, admin2_ref, category, sector_code, population_status, population, reference_period_start, reference_period_end) +VALUES ('17acb541-9431-409a-80a8-50eda7e8ebab', 6,'Male - elderly - all','WSH','all',100000,'2023-01-01 00:00:00', '2025-01-01 00:00:00'); INSERT INTO public.humanitarian_needs -(resource_hdx_id, admin2_ref, gender, age_range, min_age, max_age, sector_code, population_group, population_status, disabled_marker, population, reference_period_start, reference_period_end) -VALUES ('17acb541-9431-409a-80a8-50eda7e8ebab', 4, 'f', '18-64',18,64,'SHL','IDP','INN','n',1500000,'2023-01-01 00:00:00', '2025-01-01 00:00:00'); +(resource_hdx_id, admin2_ref, category, sector_code, population_status, population, reference_period_start, reference_period_end) +VALUES ('17acb541-9431-409a-80a8-50eda7e8ebab', 4, 'Female - adult - INN','SHL','INN',1500000,'2023-01-01 00:00:00', '2025-01-01 00:00:00'); INSERT INTO public.humanitarian_needs -(resource_hdx_id, admin2_ref, gender, age_range, min_age, max_age, sector_code, population_group, population_status, disabled_marker, population, reference_period_start, reference_period_end) -VALUES ('17acb541-9431-409a-80a8-50eda7e8ebab', 6, 'm', '18-64',18,64,'WSH','all','all','all',100000,'2023-01-01 00:00:00', '2025-01-01 00:00:00'); +(resource_hdx_id, admin2_ref, category, sector_code, population_status, population, reference_period_start, reference_period_end) +VALUES ('17acb541-9431-409a-80a8-50eda7e8ebab', 6,'Male - adult - all','WSH','all',100000,'2023-01-01 00:00:00', '2025-01-01 00:00:00'); diff --git a/tests/test_endpoints/endpoint_data.py b/tests/test_endpoints/endpoint_data.py index e469bc6c..2cf694b7 100644 --- a/tests/test_endpoints/endpoint_data.py +++ b/tests/test_endpoints/endpoint_data.py @@ -2,7 +2,6 @@ from hapi_schema.utils.enums import ( CommodityCategory, - DisabledMarker, EventType, Gender, PopulationGroup, @@ -456,11 +455,8 @@ '/api/v1/affected-people/humanitarian-needs': { 'query_parameters': { 'admin2_ref': 2, - 'gender': Gender.ALL.value, - 'age_range': 'ALL', - 'disabled_marker': DisabledMarker.YES.value, 'sector_code': 'EDU', - 'population_group': PopulationGroup.REFUGEES.value, + 'category': 'All - Disabled - all - AFF', 'population_status': PopulationStatus.AFFECTED.value, # 'reference_period_start_min': '2020-01-01T00:00:00', # 'reference_period_start_max': '2026-01-01T00:00:00', @@ -479,13 +475,8 @@ 'expected_fields': [ 'resource_hdx_id', 'admin2_ref', - 'gender', - 'age_range', - 'min_age', - 'max_age', - 'disabled_marker', + 'category', 'sector_code', - 'population_group', 'population_status', 'population', 'reference_period_start', diff --git a/tests/test_endpoints/test_humanitarian_needs_endpoint.py b/tests/test_endpoints/test_humanitarian_needs_endpoint.py index 239b7e9e..8c1f2ee5 100644 --- a/tests/test_endpoints/test_humanitarian_needs_endpoint.py +++ b/tests/test_endpoints/test_humanitarian_needs_endpoint.py @@ -3,6 +3,7 @@ from httpx import AsyncClient from hdx_hapi.endpoints.models.humanitarian_needs import HumanitarianNeedsResponse +from hapi_schema.utils.enums import PopulationStatus from main import app from tests.test_endpoints.endpoint_data import endpoint_data @@ -67,14 +68,9 @@ async def test_get_humanitarian_needs_adm_fields(event_loop, refresh_db): humanitarian_needs_view_adm_specified = HumanitarianNeedsResponse( resource_hdx_id='17acb541-9431-409a-80a8-50eda7e8ebab', - gender='f', - age_range='1-2', - min_age=1, - max_age=2, - disabled_marker='y', + category='', sector_code='EDU', - population_group='REF', - population_status='AFF', + population_status=PopulationStatus.AFFECTED, population=500000, reference_period_start='2023-01-01 00:00:00', reference_period_end='2023-03-31 23:59:59', @@ -111,14 +107,9 @@ async def test_get_humanitarian_needs_adm_fields(event_loop, refresh_db): humanitarian_needs_view_adm_unspecified = HumanitarianNeedsResponse( resource_hdx_id='17acb541-9431-409a-80a8-50eda7e8ebab', - gender='f', - age_range='1-2', - min_age=1, - max_age=2, - disabled_marker='y', sector_code='EDU', - population_group='REF', - population_status='AFF', + category='', + population_status=PopulationStatus.AFFECTED, population=500000, reference_period_start='2023-01-01 00:00:00', reference_period_end='2023-03-31 23:59:59', From b29adfec29a3c03e5fe646baf1faf56413dd6e16 Mon Sep 17 00:00:00 2001 From: Ian Hopkinson Date: Wed, 9 Oct 2024 14:42:02 +0100 Subject: [PATCH 02/12] HDX-10213 Minor updates --- .gitignore | 1 - .vscode/launch.json | 2 +- hdx_hapi/endpoints/get_affected_people.py | 8 +++++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index af2bdb4f..cc348ef7 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,3 @@ coverage.xml *.pg_restore -src/* \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index 0ff20b13..2834055d 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -11,7 +11,7 @@ "program": "main.py", "console": "integratedTerminal", "justMyCode": true, - "env":{"HAPI_USE_VAT": "true"} + "env":{"HAPI_USE_VAT": "false"} } ], diff --git a/hdx_hapi/endpoints/get_affected_people.py b/hdx_hapi/endpoints/get_affected_people.py index e178cd51..8faa49a0 100644 --- a/hdx_hapi/endpoints/get_affected_people.py +++ b/hdx_hapi/endpoints/get_affected_people.py @@ -144,7 +144,13 @@ async def get_humanitarian_needs( # ref_period_parameters: Annotated[ReferencePeriodParameters, Depends(reference_period_parameters)], common_parameters: Annotated[CommonEndpointParams, Depends(common_endpoint_parameters)], db: AsyncSession = Depends(get_db), - category: Annotated[Optional[str], Query(max_length=128, description='')] = None, + category: Annotated[ + Optional[str], + Query( + max_length=128, + description='A category combining gender, age range, disability marker and population group information', + ), + ] = None, sector_code: Annotated[Optional[str], Query(max_length=32, description=f'{DOC_SECTOR_CODE}')] = None, population_status: Annotated[ Optional[PopulationStatus], Query(max_length=32, description=f'{DOC_POPULATION_STATUS}') From 2ea4b756897af778b8b489f482038b379ac486c0 Mon Sep 17 00:00:00 2001 From: Ian Hopkinson Date: Tue, 22 Oct 2024 06:40:11 +0100 Subject: [PATCH 03/12] HDX-10213 Update schema to 0.9.2 for humanitarian needs --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 187ea2a2..3c25acc9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,4 @@ ua-parser==0.18.0 alembic~=1.12.00 psycopg2~=2.9.7 --e git+https://github.com/OCHA-DAP/hapi-sqlalchemy-schema@v0.9.0#egg=hapi-schema \ No newline at end of file +-e git+https://github.com/OCHA-DAP/hapi-sqlalchemy-schema@v0.9.2#egg=hapi-schema \ No newline at end of file From a4b6a6ec9c867d79c78332c82178015a72c912ab Mon Sep 17 00:00:00 2001 From: Ian Hopkinson Date: Tue, 22 Oct 2024 06:44:51 +0100 Subject: [PATCH 04/12] HDX-10213 Change psycog2 to psycog2-binary in requirements --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3c25acc9..7dfbd83e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,6 @@ mixpanel==4.10.1 ua-parser==0.18.0 alembic~=1.12.00 -psycopg2~=2.9.7 +psycopg2-binary~=2.9.7 -e git+https://github.com/OCHA-DAP/hapi-sqlalchemy-schema@v0.9.2#egg=hapi-schema \ No newline at end of file From b2a14010c137b155a758512ba52a3594bbef021e Mon Sep 17 00:00:00 2001 From: Ian Hopkinson Date: Mon, 28 Oct 2024 13:08:53 +0000 Subject: [PATCH 05/12] HDX-10213 Update humanitarian needs sql fixture to include provider names --- tests/sample_data/humanitarian_needs.sql | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/sample_data/humanitarian_needs.sql b/tests/sample_data/humanitarian_needs.sql index e533776b..0c635c7b 100644 --- a/tests/sample_data/humanitarian_needs.sql +++ b/tests/sample_data/humanitarian_needs.sql @@ -1,16 +1,16 @@ -- dummy data INSERT INTO public.humanitarian_needs -(resource_hdx_id, admin2_ref, category, sector_code, population_status, population, reference_period_start, reference_period_end) -VALUES ('17acb541-9431-409a-80a8-50eda7e8ebab',2, 'All - Disabled - all - AFF','EDU','AFF',500000,'2023-01-01 00:00:00', '2025-01-01 00:00:00'); +(resource_hdx_id, admin2_ref, provider_admin1_name, provider_admin2_name, category, sector_code, population_status, population, reference_period_start, reference_period_end) +VALUES ('17acb541-9431-409a-80a8-50eda7e8ebab',2, 'provider admin1 name 2','provider admin2 name 2','All - Disabled - all - AFF','EDU','AFF',500000,'2023-01-01 00:00:00', '2025-01-01 00:00:00'); INSERT INTO public.humanitarian_needs -(resource_hdx_id, admin2_ref, category, sector_code, population_status, population, reference_period_start, reference_period_end) -VALUES ('17acb541-9431-409a-80a8-50eda7e8ebab',4,'Female - child - INN','SHL','INN',1500000,'2023-01-01 00:00:00', '2025-01-01 00:00:00'); +(resource_hdx_id, admin2_ref, provider_admin1_name, provider_admin2_name, category, sector_code, population_status, population, reference_period_start, reference_period_end) +VALUES ('17acb541-9431-409a-80a8-50eda7e8ebab',4,'provider admin1 name 4','provider admin2 name 4','Female - child - INN','SHL','INN',1500000,'2023-01-01 00:00:00', '2025-01-01 00:00:00'); INSERT INTO public.humanitarian_needs -(resource_hdx_id, admin2_ref, category, sector_code, population_status, population, reference_period_start, reference_period_end) -VALUES ('17acb541-9431-409a-80a8-50eda7e8ebab', 6,'Male - elderly - all','WSH','all',100000,'2023-01-01 00:00:00', '2025-01-01 00:00:00'); +(resource_hdx_id, admin2_ref, provider_admin1_name, provider_admin2_name, category, sector_code, population_status, population, reference_period_start, reference_period_end) +VALUES ('17acb541-9431-409a-80a8-50eda7e8ebab', 6,'provider admin1 name 6','provider admin2 name 6','Male - elderly - all','WSH','all',100000,'2023-01-01 00:00:00', '2025-01-01 00:00:00'); INSERT INTO public.humanitarian_needs -(resource_hdx_id, admin2_ref, category, sector_code, population_status, population, reference_period_start, reference_period_end) -VALUES ('17acb541-9431-409a-80a8-50eda7e8ebab', 4, 'Female - adult - INN','SHL','INN',1500000,'2023-01-01 00:00:00', '2025-01-01 00:00:00'); +(resource_hdx_id, admin2_ref, provider_admin1_name, provider_admin2_name, category, sector_code, population_status, population, reference_period_start, reference_period_end) +VALUES ('17acb541-9431-409a-80a8-50eda7e8ebab', 4,'provider admin1 name 4','provider admin2 name 4', 'Female - adult - INN','SHL','INN',1500000,'2023-01-01 00:00:00', '2025-01-01 00:00:00'); INSERT INTO public.humanitarian_needs -(resource_hdx_id, admin2_ref, category, sector_code, population_status, population, reference_period_start, reference_period_end) -VALUES ('17acb541-9431-409a-80a8-50eda7e8ebab', 6,'Male - adult - all','WSH','all',100000,'2023-01-01 00:00:00', '2025-01-01 00:00:00'); +(resource_hdx_id, admin2_ref, provider_admin1_name, provider_admin2_name, category, sector_code, population_status, population, reference_period_start, reference_period_end) +VALUES ('17acb541-9431-409a-80a8-50eda7e8ebab', 6,'provider admin1 name 6','provider admin2 name 6','Male - adult - all','WSH','all',100000,'2023-01-01 00:00:00', '2025-01-01 00:00:00'); From 1e8f347eb67a42ac960bf2c45bb767c8b4660381 Mon Sep 17 00:00:00 2001 From: Ian Hopkinson Date: Mon, 28 Oct 2024 13:12:48 +0000 Subject: [PATCH 06/12] HDX-10213 Revert psycog2 requirement and update schema version to 0.9.3 --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7dfbd83e..d4649f0e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,6 @@ mixpanel==4.10.1 ua-parser==0.18.0 alembic~=1.12.00 -psycopg2-binary~=2.9.7 +psycopg2~=2.9.7 --e git+https://github.com/OCHA-DAP/hapi-sqlalchemy-schema@v0.9.2#egg=hapi-schema \ No newline at end of file +-e git+https://github.com/OCHA-DAP/hapi-sqlalchemy-schema@v0.9.3#egg=hapi-schema \ No newline at end of file From 8ae5389f31bd5ca4697f76215107e3563c3ce5ab Mon Sep 17 00:00:00 2001 From: Ian Hopkinson Date: Mon, 28 Oct 2024 13:23:23 +0000 Subject: [PATCH 07/12] HDX-10213 Update idps sql test fixture to accommodate new not nullable constraint on reference_period_end --- tests/sample_data/idps.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/sample_data/idps.sql b/tests/sample_data/idps.sql index cf73ba34..07adb10e 100644 --- a/tests/sample_data/idps.sql +++ b/tests/sample_data/idps.sql @@ -1,3 +1,3 @@ INSERT INTO idps (resource_hdx_id, admin2_ref, provider_admin1_name, provider_admin2_name, assessment_type, reporting_round, operation, population, reference_period_start, reference_period_end) VALUES -('17acb541-9431-409a-80a8-50eda7e8ebab', 1, 'Provider admin1 name 1','Provider admin2 name 2','BA', 1, 'operation', 50, '2023-01-01 00:00:00', NULL); \ No newline at end of file +('17acb541-9431-409a-80a8-50eda7e8ebab', 1, 'Provider admin1 name 1','Provider admin2 name 2','BA', 1, 'operation', 50, '2023-01-01 00:00:00', '2025-01-01 00:00:00'); \ No newline at end of file From cc82c8289cbd077742b3a2d28037fc9ac6cf6411 Mon Sep 17 00:00:00 2001 From: David Megginson Date: Tue, 29 Oct 2024 10:42:59 -0400 Subject: [PATCH 08/12] Doc updates for humanitarian-needs HDXDSYS-1087 --- docs/data_usage_guides/affected_people.md | 16 +++++++++++----- .../humanitarian_needs_details.yaml | 3 --- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/docs/data_usage_guides/affected_people.md b/docs/data_usage_guides/affected_people.md index acad0336..283984b6 100644 --- a/docs/data_usage_guides/affected_people.md +++ b/docs/data_usage_guides/affected_people.md @@ -135,11 +135,17 @@ crises. It includes an estimate of the number of people by sector who require assistance, often referred to as People in Need (PIN), which is derived using the [Joint Intersectoral Analysis Framework (JIAF)](https://www.jiaf.info/). -While the HNO data is -[directly available on HDX](https://data.humdata.org/dataset/?dataseries_name=Humanitarian+Needs+Overview), -it comes from different OCHA offices and is currently not standardised. Thus, -HDX HAPI obtains the PIN numbers from the -[HPC Tools API](https://api.hpc.tools/docs/v1/)-based datasets on HDX. +While the HNO data is [directly available on +HDX](https://data.humdata.org/dataset/?dataseries_name=Humanitarian+Needs+Overview), +it comes from different OCHA offices and is currently not +standardised. Thus, HDX HAPI obtains the PIN numbers from the [HPC +Tools API](https://api.hpc.tools/docs/v1/)-based datasets on HDX. +This results in some loss of standardisation — unlike the original +data, HPC Tools does not have consistent Sex and Age Disaggregation +(SADD), but instead includes a single, freeform `category` field — but +using it allows HAPI to include data for more countries and update +that data more quickly. As HPC Tools improves its SADD support, we +will update HAPI to take advantage of it. ### Summary diff --git a/docs/data_usage_guides/subcategory_details/humanitarian_needs_details.yaml b/docs/data_usage_guides/subcategory_details/humanitarian_needs_details.yaml index 84ecac1d..0b9e0460 100644 --- a/docs/data_usage_guides/subcategory_details/humanitarian_needs_details.yaml +++ b/docs/data_usage_guides/subcategory_details/humanitarian_needs_details.yaml @@ -9,7 +9,4 @@ Details: Annually - Attribute: Related tables Details: > - [ `DisabledMarker` ](enums.md#disabled-marker), - [`Gender`](enums.md#gender), - [`PopulationGroup`](enums.md#population-group), [`PopulationStatus`](enums.md#population-status) From 75072bc1420e1e00825b88e570eb4f24f968364a Mon Sep 17 00:00:00 2001 From: David Megginson Date: Tue, 29 Oct 2024 11:17:40 -0400 Subject: [PATCH 09/12] Add IDPs and Returnees to list of endpoints on the home page. --- docs/index.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/index.md b/docs/index.md index c32e75c3..c920c717 100644 --- a/docs/index.md +++ b/docs/index.md @@ -27,8 +27,10 @@ HDX HAPI is organised around a set of key humanitarian data subcategories like * #### Affected People -- [Refugees & Persons of Concern](https://hapi.humdata.org/docs#/Affected%20people/get_refugees_api_v1_affected_people_refugees_get) -- [Humanitarian Needs](https://hapi.humdata.org/docs#/Affected%20people/get_humanitarian_needs_api_v1_affected_people_humanitarian_needs_get) +- [IDPs](https://hapi.humdata.org/docs#/Affected%20People/get_idps_api_v1_affected_people_idps_get) +- [Refugees & Persons of Concern](https://hapi.humdata.org/docs#/Affected%20People/get_refugees_api_v1_affected_people_refugees_get) +- [Returnees](https://hapi.humdata.org/docs#/Affected%20People/get_returnees_api_v1_affected_people_returnees_get) +- [Humanitarian Needs](https://hapi.humdata.org/docs#/Affected%20People/get_humanitarian_needs_api_v1_affected_people_humanitarian_needs_get) #### Coordination & Context From 613cbd6e20a821bb527c30afa72958d605cc1ebb Mon Sep 17 00:00:00 2001 From: David Megginson Date: Tue, 29 Oct 2024 11:26:21 -0400 Subject: [PATCH 10/12] Remove the static data-coverage table and replace with a link out to Erica's data-availability viz. HDXDSYS-1087 --- docs/data.md | 448 --------------------------------------------------- mkdocs.yml | 2 +- 2 files changed, 1 insertion(+), 449 deletions(-) delete mode 100644 docs/data.md diff --git a/docs/data.md b/docs/data.md deleted file mode 100644 index 1386476d..00000000 --- a/docs/data.md +++ /dev/null @@ -1,448 +0,0 @@ - -# Data Coverage - ---- - -The initial aim of HDX HAPI is to cover all countries that have a -humanitarian response plan, and all data sub-categories from -[HDX data grids](https://data.humdata.org/dashboards/overview-of-data-grids). -In the table below we detail the sub-category coverage that we have achieved -at present, and to which administrative level: national (admin 0), -admin 1, or admin 2. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Refugees & Persons Of ConcernHumanitarian NeedsWho is Doing What Where - Operational PresenceFundingConflict EventsNational RiskFood SecurityFood PricesBaseline PopulationPoverty Rate
Afghanistanadmin 0admin 0, 1, 2admin 2admin 0admin 2admin 0admin 2admin 0, 1admin 0
Burkina Fasoadmin 0admin 0admin 0admin 0admin 2admin 0admin 2admin 2admin 0, 1, 2admin 0
Cameroonadmin 0admin 0, 1admin 2admin 0admin 2admin 0admin 2admin 2admin 0, 1admin 0
Central African Republicadmin 0admin 0, 2admin 2admin 0admin 2admin 0admin 1, 2admin 2admin 0
Chadadmin 0admin 0admin 1admin 0admin 2admin 0admin 1, 2admin 2admin 0, 1, 2admin 0
Colombiaadmin 0admin 0, 1admin 2admin 0admin 2admin 0admin 2admin 0, 1, 2admin 0
Democratic Republic of the Congoadmin 0admin 0admin 2admin 0admin 2admin 0admin 2admin 0, 1, 2admin 0
El Salvadoradmin 0admin 0admin 0admin 0admin 0admin 2admin 0, 1, 2admin 0
Ethiopiaadmin 0admin 0admin 0admin 0admin 2admin 0admin 2admin 0, 1, 2admin 0
Guatemalaadmin 0admin 0, 2admin 2admin 0admin 0admin 0admin 2admin 0, 1, 2admin 0
Haitiadmin 0admin 0, 2admin 2admin 0admin 2admin 0admin 2admin 0, 1, 2admin 0
Hondurasadmin 0admin 0, 2admin 2admin 0admin 0admin 0admin 2admin 0, 1, 2admin 0
Maliadmin 0admin 0, 1, 2admin 2admin 0admin 2admin 0admin 2admin 2admin 0, 1, 2admin 0
Mozambiqueadmin 0admin 0, 2admin 2admin 0admin 2admin 0admin 2admin 0, 1, 2admin 0
Myanmaradmin 0admin 0, 1admin 0admin 2admin 0admin 2admin 0, 1, 2admin 0
Nigeradmin 0admin 0, 2admin 0admin 0admin 2admin 0admin 1, 2admin 2admin 0, 1, 2admin 0
Nigeriaadmin 0admin 0, 2admin 2admin 0admin 2admin 0admin 1, 2admin 2admin 0, 1, 2admin 0
State of Palestineadmin 0admin 0admin 0admin 2admin 0admin 2admin 0, 1admin 0
Somaliaadmin 0admin 2admin 0admin 2admin 0admin 2admin 0, 1, 2
South Sudanadmin 0admin 0, 2admin 0admin 0admin 2admin 0admin 2admin 0, 1, 2
Sudanadmin 0admin 0, 2admin 0admin 0admin 2admin 0admin 2admin 0, 1admin 0
Syrian Arab Republicadmin 0admin 0admin 0admin 2admin 0admin 2
Ukraineadmin 0admin 0admin 0admin 2admin 0admin 2admin 0, 1admin 0
Venezuela (Bolivarian Republic of)admin 0admin 0, 2admin 2admin 0admin 2admin 0admin 0, 1, 2
Yemenadmin 0admin 0, 2admin 2admin 0admin 2admin 0admin 2admin 0
diff --git a/mkdocs.yml b/mkdocs.yml index 7540ee3c..1e8ed174 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -2,7 +2,7 @@ site_name: HDX HAPI - The Humanitarian API nav: - Home: index.md - Getting Started: getting-started.md - - Data Coverage: data.md + - Data Availability: https://ocha-dap.github.io/viz-hapi-availability/ - Code Examples: examples.md - Data Usage Guide: - Affected People: data_usage_guides/affected_people.md From fcfaef6e637f8686aae616c45aabc6643ebc46a7 Mon Sep 17 00:00:00 2001 From: David Megginson Date: Tue, 29 Oct 2024 11:42:13 -0400 Subject: [PATCH 11/12] More tweaks to update humanitarian needs docs, including new source dataset HDXDSYS-1087 --- docs/data_usage_guides/affected_people.md | 7 ++--- .../humanitarian_needs_parameters.yaml | 27 ------------------- .../humanitarian_needs_details.yaml | 2 +- 3 files changed, 3 insertions(+), 33 deletions(-) diff --git a/docs/data_usage_guides/affected_people.md b/docs/data_usage_guides/affected_people.md index 283984b6..a40e29b9 100644 --- a/docs/data_usage_guides/affected_people.md +++ b/docs/data_usage_guides/affected_people.md @@ -140,7 +140,7 @@ HDX](https://data.humdata.org/dataset/?dataseries_name=Humanitarian+Needs+Overvi it comes from different OCHA offices and is currently not standardised. Thus, HDX HAPI obtains the PIN numbers from the [HPC Tools API](https://api.hpc.tools/docs/v1/)-based datasets on HDX. -This results in some loss of standardisation — unlike the original +This results in some loss of detail — unlike the original data, HPC Tools does not have consistent Sex and Age Disaggregation (SADD), but instead includes a single, freeform `category` field — but using it allows HAPI to include data for more countries and update @@ -168,7 +168,6 @@ For available query parameters, please see the "all", "INN", "TGT", "AFF", and "REA", respectively * Sector values of “ALL” have been converted to “intersectoral”, as these represent the intersectoral PIN and **not** the disaggregated population -* Gender and disabled values of “a” have been converted to “all” * The methodology in Yemen leads to negative population values in some admin 2 level areas. Where negative values appear they have been omitted from the API. * The reference period is obtained from the HDX dataset @@ -178,6 +177,4 @@ For available query parameters, please see the * The PIN should **not** be summed across sectors or population statuses, as the same people can be present across multiple groups * For the number of people affected across all - sectors, please use the PIN value where sector=intersectoral. -* An “all” value in the `gender`, `age_range`, `disable_marker`, and - `population_group` columns indicates no disaggregation + sectors, please use the PIN value where sector=Intersectoral. diff --git a/docs/data_usage_guides/endpoint_parameters/humanitarian_needs_parameters.yaml b/docs/data_usage_guides/endpoint_parameters/humanitarian_needs_parameters.yaml index 0cd9e45b..8d2947d2 100644 --- a/docs/data_usage_guides/endpoint_parameters/humanitarian_needs_parameters.yaml +++ b/docs/data_usage_guides/endpoint_parameters/humanitarian_needs_parameters.yaml @@ -1,30 +1,6 @@ - Parameter: '`resource_hdx_id`' Description: Unique resource UUID on HDX Source: '[`Resource`](metadata.md#resource)' -- Parameter: '`gender`' - Description: > - The gender of the population group in question, set to "all" if there - is no disaggregation - Source: '[`Gender`](enums.md#gender)' -- Parameter: '`age_range`' - Description: > - The age range of the population group in question, set to "all" if there - is no disaggregation - Source: -- Parameter: '`min_age`' - Description: > - The minimum age from `age_range`, set to `null` if `age_range` is "all" and - there is no age disaggregation - Source: -- Parameter: '`max_age`' - Description: > - The maximum age from `age_range`, set to `null` if `age_range` is "all" and - there is no age disaggregation, or if there is no upper limit to the - age range - Source: -- Parameter: '`disabled_marker`' - Description: Marker used to indicate disaggregation by disability - Source: '[`Disabled Marker`](enums.md#disabled-marker)' - Parameter: '`sector_code`' Description: > The sector code, derived either from the @@ -34,9 +10,6 @@ - Parameter: '`sector_name`' Description: The name of the sector Source: '[`Sector`](metadata.md#sector)' -- Parameter: '`population_group`' - Description: The population group, such as IDP or refugee - Source: '[`Population Group`](enums.md#population-group)' - Parameter: '`population_status`' Description: The population status, such as in-need or targeted Source: '[`Population Status`](enums.md#population-status)' diff --git a/docs/data_usage_guides/subcategory_details/humanitarian_needs_details.yaml b/docs/data_usage_guides/subcategory_details/humanitarian_needs_details.yaml index 0b9e0460..f74ecdca 100644 --- a/docs/data_usage_guides/subcategory_details/humanitarian_needs_details.yaml +++ b/docs/data_usage_guides/subcategory_details/humanitarian_needs_details.yaml @@ -2,7 +2,7 @@ Details: OCHA HPC - Attribute: Dataset Details: > - [Humanitarian Needs Overview tag](https://data.humdata.org/dataset/?vocab_Topics=humanitarian+needs+overview-hno) + [Global Humanitarian Programme Cycle, Humanitarian Needs](https://data.humdata.org/dataset/global-hpc-hno) - Attribute: Time series Details: Past datasets are available in the HPC API, but are sparse prior to 2023 - Attribute: Update frequency From 210ad1d2bc187dbe324d746ac2067c7965bf0086 Mon Sep 17 00:00:00 2001 From: alexandru-m-g Date: Sun, 10 Nov 2024 23:37:45 +0000 Subject: [PATCH 12/12] updating versions to 0.8.0 and 0.9.3 --- hdx_hapi/endpoints/util/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hdx_hapi/endpoints/util/version.py b/hdx_hapi/endpoints/util/version.py index deef4a34..087089b3 100644 --- a/hdx_hapi/endpoints/util/version.py +++ b/hdx_hapi/endpoints/util/version.py @@ -1,2 +1,2 @@ -api_version = '0.7.3' -hapi_sqlalchemy_schema_version = '0.9.0' +api_version = '0.8.0' +hapi_sqlalchemy_schema_version = '0.9.3'