Simplify pipelines

CHANGELOG.md

@@ -4,6 +4,31 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [0.10.44] = 2025-02-26
+
+### Changed
+
+- Update to latest schema
+
+## [0.10.43] = 2025-02-25
+
+### Changed
+
+- Refugees and returnees read from HAPI datasets
+
+## [0.10.42] = 2025-02-24
+
+### Changed
+
+- Poverty rate reads from HAPI dataset
+- Fix hapi-schema version
+
+## [0.10.41] = 2025-02-20
+
+### Changed
+
+- Conflict event, IDPs, and Population read from HAPI dataset
+
 ## [0.10.40] = 2025-02-20
 
 ### Changed

pyproject.toml

@@ -34,7 +34,7 @@ classifiers = [
 requires-python = ">=3.8"
 
 dependencies = [
-    "hapi-schema>= 0.9.8",
+    "hapi-schema>= 0.9.10",
     "hdx-python-api>= 6.3.8",
     "hdx-python-country>= 3.8.8",
     "hdx-python-database[postgresql]>= 1.3.5",

requirements.txt

@@ -8,7 +8,7 @@ attrs==25.1.0
     #   jsonlines
     #   jsonschema
     #   referencing
-cachetools==5.5.1
+cachetools==5.5.2
     # via google-auth
 certifi==2025.1.31
     # via requests
@@ -54,7 +54,7 @@ greenlet==3.1.1
     # via sqlalchemy
 gspread==6.1.4
     # via hdx-python-scraper
-hapi-schema==0.9.8
+hapi-schema==0.9.10
     # via hapi-pipelines (pyproject.toml)
 hdx-python-api==6.3.8
     # via
@@ -77,7 +77,7 @@ hdx-python-utilities==3.8.4
     #   hdx-python-scraper
 humanize==4.12.1
     # via frictionless
-identify==2.6.7
+identify==2.6.8
     # via pre-commit
 idna==3.10
     # via
@@ -146,9 +146,9 @@ pockets==0.9.1
     # via sphinxcontrib-napoleon
 pre-commit==4.1.0
     # via hapi-pipelines (pyproject.toml)
-psycopg==3.2.4
+psycopg==3.2.5
     # via hdx-python-database
-psycopg-binary==3.2.4
+psycopg-binary==3.2.5
     # via psycopg
 pyasn1==0.6.1
     # via
@@ -218,7 +218,7 @@ rfc3986==2.0.0
     # via frictionless
 rich==13.9.4
     # via typer
-rpds-py==0.22.3
+rpds-py==0.23.1
     # via
     #   jsonschema
     #   referencing
@@ -228,7 +228,7 @@ ruamel-yaml==0.18.10
     # via hdx-python-utilities
 ruamel-yaml-clib==0.2.12
     # via ruamel-yaml
-setuptools==75.8.0
+setuptools==75.8.1
     # via ckanapi
 shellingham==1.5.4
     # via typer

src/hapi/pipelines/app/__main__.py

@@ -199,12 +199,9 @@ def main(
     if ehx is None:
         ehx = getenv("ERR_TO_HDX")
     project_configs = [
-        "conflict_event.yaml",
         "core.yaml",
         "food_security.yaml",
-        "idps.yaml",
         "national_risk.yaml",
-        "refugees_and_returnees.yaml",
         "wfp.yaml",
     ]
     project_config_dict = load_yamls(project_configs)

src/hapi/pipelines/app/pipelines.py

@@ -27,7 +27,8 @@
 from hapi.pipelines.database.org_type import OrgType
 from hapi.pipelines.database.population import Population
 from hapi.pipelines.database.poverty_rate import PovertyRate
-from hapi.pipelines.database.refugees_and_returnees import RefugeesAndReturnees
+from hapi.pipelines.database.refugees import Refugees
+from hapi.pipelines.database.returnees import Returnees
 from hapi.pipelines.database.sector import Sector
 from hapi.pipelines.database.wfp_commodity import WFPCommodity
 from hapi.pipelines.database.wfp_market import WFPMarket
@@ -170,18 +171,6 @@ def _create_configurable_scrapers(
             )
 
         _create_configurable_scrapers("national_risk", "national")
-        _create_configurable_scrapers("refugees_and_returnees", "national")
-        _create_configurable_scrapers("idps", "national")
-        _create_configurable_scrapers(
-            "idps", "adminone", adminlevel=self._adminone
-        )
-        _create_configurable_scrapers(
-            "idps", "admintwo", adminlevel=self._admintwo
-        )
-        _create_configurable_scrapers("conflict_event", "national")
-        _create_configurable_scrapers(
-            "conflict_event", "admintwo", adminlevel=self._admintwo
-        )
 
     def run(self):
         self._runner.run()
@@ -191,6 +180,7 @@ def output_population(self):
             population = Population(
                 session=self._session,
                 metadata=self._metadata,
+                locations=self._locations,
                 admins=self._admins,
                 configuration=self._configuration,
                 error_handler=self._error_handler,
@@ -211,6 +201,7 @@ def output_operational_presence(self):
             operational_presence = OperationalPresence(
                 session=self._session,
                 metadata=self._metadata,
+                locations=self._locations,
                 admins=self._admins,
                 configuration=self._configuration,
                 error_handler=self._error_handler,
@@ -239,6 +230,7 @@ def output_humanitarian_needs(self):
             humanitarian_needs = HumanitarianNeeds(
                 session=self._session,
                 metadata=self._metadata,
+                locations=self._locations,
                 admins=self._admins,
                 configuration=self._configuration,
                 error_handler=self._error_handler,
@@ -258,32 +250,38 @@ def output_national_risk(self):
             )
             national_risk.populate()
 
-    def output_refugees_and_returnees(self):
-        if (
-            not self._themes_to_run
-            or "refugees_and_returnees" in self._themes_to_run
-        ):
-            results = self._runner.get_hapi_results(
-                self._configurable_scrapers["refugees_and_returnees"]
+    def output_refugees(self):
+        if not self._themes_to_run or "refugees" in self._themes_to_run:
+            refugees = Refugees(
+                session=self._session,
+                metadata=self._metadata,
+                locations=self._locations,
+                admins=self._admins,
+                configuration=self._configuration,
+                error_handler=self._error_handler,
             )
-            refugees_and_returnees = RefugeesAndReturnees(
+            refugees.populate()
+
+    def output_returnees(self):
+        if not self._themes_to_run or "returnees" in self._themes_to_run:
+            returnees = Returnees(
                 session=self._session,
                 metadata=self._metadata,
                 locations=self._locations,
-                results=results,
+                admins=self._admins,
+                configuration=self._configuration,
+                error_handler=self._error_handler,
             )
-            refugees_and_returnees.populate()
+            returnees.populate()
 
     def output_idps(self):
         if not self._themes_to_run or "idps" in self._themes_to_run:
-            results = self._runner.get_hapi_results(
-                self._configurable_scrapers["idps"]
-            )
             idps = IDPs(
                 session=self._session,
                 metadata=self._metadata,
+                locations=self._locations,
                 admins=self._admins,
-                results=results,
+                configuration=self._configuration,
                 error_handler=self._error_handler,
             )
             idps.populate()
@@ -305,6 +303,7 @@ def output_poverty_rate(self):
             poverty_rate = PovertyRate(
                 session=self._session,
                 metadata=self._metadata,
+                locations=self._locations,
                 admins=self._admins,
                 configuration=self._configuration,
                 error_handler=self._error_handler,
@@ -313,14 +312,11 @@ def output_poverty_rate(self):
 
     def output_conflict_event(self):
         if not self._themes_to_run or "conflict_event" in self._themes_to_run:
-            results = self._runner.get_hapi_results(
-                self._configurable_scrapers["conflict_event"]
-            )
             conflict_event = ConflictEvent(
                 session=self._session,
                 metadata=self._metadata,
+                locations=self._locations,
                 admins=self._admins,
-                results=results,
                 configuration=self._configuration,
                 error_handler=self._error_handler,
             )
@@ -368,7 +364,8 @@ def output(self):
         self.output_food_security()
         self.output_humanitarian_needs()
         self.output_national_risk()
-        self.output_refugees_and_returnees()
+        self.output_refugees()
+        self.output_returnees()
         self.output_idps()
         self.output_funding()
         self.output_poverty_rate()