HAPI-333 Allow dates to be specified in scraper config

.config/pre-commit-config.yaml

@@ -2,23 +2,22 @@ default_language_version:
     python: python3.11
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: check-ast
-  - repo: https://github.com/psf/black
-    rev: 23.3.0
-    hooks:
-      - id: black
-        args: [--config, .config/black.toml]
-  - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: v0.0.267
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.1.14
     hooks:
+      # Run the linter.
       - id: ruff
-        args: [--config, .config/ruff.toml, --fix, --exit-non-zero-on-fix]
+        args: [--config, .config/ruff.toml, --fix]
+      # Run the formatter.
+      - id: ruff-format
+        args: [--config, .config/ruff.toml]
   - repo: https://github.com/jazzband/pip-tools
-    rev: 6.13.0
+    rev: 7.3.0
     hooks:
       - id: pip-compile
         name: pip-compile requirements.txt

pyproject.toml

@@ -34,10 +34,10 @@ classifiers = [
 requires-python = ">=3.8"
 
 dependencies = [
-    "hapi-schema>= 0.5.0",
-    "hdx-python-country>= 3.6.3",
+    "hapi-schema>= 0.6.0",
+    "hdx-python-country>= 3.6.4",
     "hdx-python-database[postgresql]>= 1.2.9",
-    "hdx-python-scraper>= 2.3.2",
+    "hdx-python-scraper>= 2.3.3",
     "libhxl",
     "sqlalchemy"
 ]
@@ -92,22 +92,14 @@ test = """
        """
 
 [[tool.hatch.envs.test.matrix]]
-python = ["3.11"]
+python = ["3.12"]
 
 [tool.hatch.envs.lint]
 detached = true
-dependencies = [
-  "black",
-  "ruff",
-]
+dependencies = ["ruff"]
 
 [tool.hatch.envs.lint.scripts]
 style = [
-  "ruff --config .config/ruff.toml {args:.}",
-  "black --config .config/black.toml --check --diff {args:.}",
-]
-# Not used for anything at the moment
-fmt = [
-  "black --config .config/black.toml {args:.}",
-  "ruff --config .config/ruff.toml --fix {args:.}",
+  "ruff check --config .config/ruff.toml --diff {args:.}",
+  "ruff format --config .config/ruff.toml --diff {args:.}",
 ]

requirements.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
-#    pip-compile --all-extras --output-file=requirements.txt --resolver=backtracking pyproject.toml
+#    pip-compile --all-extras --output-file=requirements.txt pyproject.toml
 #
 annotated-types==0.6.0
     # via pydantic
@@ -29,9 +29,11 @@ click==8.1.7
     # via typer
 colorama==0.4.6
     # via typer
-coverage[toml]==7.4.0
-    # via pytest-cov
-cryptography==42.0.0
+coverage[toml]==7.4.1
+    # via
+    #   coverage
+    #   pytest-cov
+cryptography==42.0.1
     # via pyopenssl
 defopt==6.4.0
     # via hdx-python-api
@@ -51,17 +53,17 @@ et-xmlfile==1.1.0
     # via openpyxl
 filelock==3.13.1
     # via virtualenv
-frictionless==5.16.0
+frictionless==5.16.1
     # via hdx-python-utilities
-google-auth==2.26.2
+google-auth==2.27.0
     # via
     #   google-auth-oauthlib
     #   gspread
 google-auth-oauthlib==1.2.0
     # via gspread
 greenlet==3.0.3
     # via sqlalchemy
-gspread==5.12.4
+gspread==6.0.0
     # via hdx-python-scraper
 hapi-schema==0.6.0
     # via hapi-pipelines (pyproject.toml)
@@ -73,8 +75,10 @@ hdx-python-country==3.6.4
     #   hdx-python-api
     #   hdx-python-scraper
 hdx-python-database[postgresql]==1.2.9
-    # via hapi-pipelines (pyproject.toml)
-hdx-python-scraper==2.3.2
+    # via
+    #   hapi-pipelines (pyproject.toml)
+    #   hdx-python-database
+hdx-python-scraper==2.3.3
     # via hapi-pipelines (pyproject.toml)
 hdx-python-utilities==3.6.4
     # via
@@ -139,7 +143,7 @@ petl==1.7.14
     # via frictionless
 platformdirs==4.1.0
     # via virtualenv
-pluggy==1.3.0
+pluggy==1.4.0
     # via pytest
 ply==3.11
     # via
@@ -163,11 +167,11 @@ pyasn1-modules==0.3.0
     # via google-auth
 pycparser==2.21
     # via cffi
-pydantic==2.5.3
+pydantic==2.6.0
     # via
     #   frictionless
     #   inflect
-pydantic-core==2.14.6
+pydantic-core==2.16.1
     # via pydantic
 pygments==2.17.2
     # via rich
@@ -179,7 +183,7 @@ pyphonetics==0.5.3
     # via hdx-python-country
 pyrsistent==0.20.0
     # via jsonschema
-pytest==7.4.4
+pytest==8.0.0
     # via
     #   hapi-pipelines (pyproject.toml)
     #   pytest-cov
@@ -192,7 +196,7 @@ python-dateutil==2.8.2
     #   libhxl
 python-io-wrapper==0.3.1
     # via libhxl
-python-slugify==8.0.1
+python-slugify==8.0.2
     # via
     #   ckanapi
     #   frictionless
@@ -215,7 +219,7 @@ requests==2.31.0
     #   libhxl
     #   requests-file
     #   requests-oauthlib
-requests-file==1.5.1
+requests-file==2.0.0
     # via hdx-python-utilities
 requests-oauthlib==1.3.1
     # via google-auth-oauthlib
@@ -239,7 +243,6 @@ six==1.16.0
     #   isodate
     #   pockets
     #   python-dateutil
-    #   requests-file
     #   sphinxcontrib-napoleon
 sphinxcontrib-napoleon==0.7
     # via defopt
@@ -248,6 +251,8 @@ sqlalchemy==2.0.25
     #   hapi-pipelines (pyproject.toml)
     #   hapi-schema
     #   hdx-python-database
+strenum==0.4.15
+    # via gspread
 stringcase==1.2.0
     # via frictionless
 structlog==24.1.0
@@ -259,7 +264,9 @@ tabulate==0.9.0
 text-unidecode==1.3
     # via python-slugify
 typer[all]==0.9.0
-    # via frictionless
+    # via
+    #   frictionless
+    #   typer
 typing-extensions==4.9.0
     # via
     #   frictionless

src/hapi/pipelines/database/admins.py

@@ -71,7 +71,7 @@ def _update_admin_table(
         for i, row in enumerate(admin_filter):
             code = row.get("#adm+code")
             name = row.get("#adm+name")
-            reference_period_start = parse_date(row.get("#date+start"))
+            time_period_start = parse_date(row.get("#date+start"))
             parent = row.get("#adm+code+parent")
             parent_ref = parent_dict.get(parent)
             if not parent_ref:
@@ -92,14 +92,14 @@ def _update_admin_table(
                     location_ref=parent_ref,
                     code=code,
                     name=name,
-                    reference_period_start=reference_period_start,
+                    reference_period_start=time_period_start,
                 )
             elif desired_admin_level == "2":
                 admin_row = DBAdmin2(
                     admin1_ref=parent_ref,
                     code=code,
                     name=name,
-                    reference_period_start=reference_period_start,
+                    reference_period_start=time_period_start,
                 )
             self._session.add(admin_row)
             if i % self._limit == 0:
@@ -108,7 +108,7 @@ def _update_admin_table(
 
     def _add_admin1_connector_rows(self):
         for location_code, location_ref in self._locations.data.items():
-            reference_period_start = (
+            time_period_start = (
                 self._session.query(DBLocation)
                 .filter(DBLocation.id == location_ref)
                 .one()
@@ -121,14 +121,14 @@ def _add_admin1_connector_rows(self):
                 ),
                 name="UNSPECIFIED",
                 is_unspecified=True,
-                reference_period_start=reference_period_start,
+                reference_period_start=time_period_start,
             )
             self._session.add(admin_row)
         self._session.commit()
 
     def _add_admin2_connector_rows(self):
         for admin1_code, admin1_ref in self.admin1_data.items():
-            reference_period_start = (
+            time_period_start = (
                 self._session.query(DBAdmin1)
                 .filter(DBAdmin1.id == admin1_ref)
                 .one()
@@ -141,7 +141,7 @@ def _add_admin2_connector_rows(self):
                 ),
                 name="UNSPECIFIED",
                 is_unspecified=True,
-                reference_period_start=reference_period_start,
+                reference_period_start=time_period_start,
             )
             self._session.add(admin_row)
         self._session.commit()

src/hapi/pipelines/database/food_security.py

@@ -73,9 +73,9 @@ def populate(self):
                             ][irow]
                         )
                         (
-                            reference_period_start,
-                            reference_period_end,
-                        ) = _get_reference_period(
+                            time_period_start,
+                            time_period_end,
+                        ) = _get_time_period(
                             month_range=values[reference_period_months_column][
                                 admin_code
                             ][irow],
@@ -101,8 +101,8 @@ def populate(self):
                                 admin2_ref=admin2_ref,
                                 ipc_phase_code=ipc_phase_code,
                                 ipc_type_code=ipc_type_code,
-                                reference_period_start=reference_period_start,
-                                reference_period_end=reference_period_end,
+                                reference_period_start=time_period_start,
+                                reference_period_end=time_period_end,
                                 population_in_phase=population_in_phase,
                                 population_fraction_in_phase=population_in_phase
                                 / population_total
@@ -128,11 +128,11 @@ def _get_ipc_type_code_from_data(ipc_type_from_data: str) -> str:
         ) from e
 
 
-def _get_reference_period(month_range: str, year: str) -> (datetime, datetime):
-    reference_period_start = parse_date_range(
+def _get_time_period(month_range: str, year: str) -> (datetime, datetime):
+    time_period_start = parse_date_range(
         f"{year} {month_range.split('-')[0]}"
     )[0]
-    reference_period_end = parse_date_range(
-        f"{year} {month_range.split('-')[1]}"
-    )[1]
-    return reference_period_start, reference_period_end
+    time_period_end = parse_date_range(f"{year} {month_range.split('-')[1]}")[
+        1
+    ]
+    return time_period_start, time_period_end

src/hapi/pipelines/database/humanitarian_needs.py

@@ -57,8 +57,8 @@ def match_column(col, pattern_to_code):
             return None
 
         for dataset in self._results.values():
-            reference_period_start = dataset["reference_period"]["startdate"]
-            reference_period_end = dataset["reference_period"]["enddate"]
+            time_period_start = dataset["time_period"]["start"]
+            time_period_end = dataset["time_period"]["end"]
 
             for admin_level, admin_results in dataset["results"].items():
                 resource_id = admin_results["hapi_resource_metadata"]["hdx_id"]
@@ -115,8 +115,8 @@ def match_column(col, pattern_to_code):
                             age_range_code=age_range_code,
                             disabled_marker=disabled_marker,
                             population=value,
-                            reference_period_start=reference_period_start,
-                            reference_period_end=reference_period_end,
+                            reference_period_start=time_period_start,
+                            reference_period_end=time_period_end,
                             # TODO: For v2+, add to scraper (HAPI-199)
                             source_data="not yet implemented",
                         )

src/hapi/pipelines/database/national_risk.py

@@ -32,12 +32,10 @@ def populate(self):
             datasetinfo = self._metadata.runner.scrapers[
                 "national_risk_national"
             ].datasetinfo
-            reference_period_start = datasetinfo["source_date"][
-                "default_date"
-            ]["start"]
-            reference_period_end = datasetinfo["source_date"]["default_date"][
-                "end"
+            time_period_start = datasetinfo["source_date"]["default_date"][
+                "start"
             ]
+            time_period_end = datasetinfo["source_date"]["default_date"]["end"]
             for admin_level, admin_results in dataset["results"].items():
                 resource_id = admin_results["hapi_resource_metadata"]["hdx_id"]
                 hxl_tags = admin_results["headers"][1]
@@ -73,8 +71,8 @@ def populate(self):
                         meta_avg_recentness_years=values[
                             "#meta+recentness+avg"
                         ].get(location),
-                        reference_period_start=reference_period_start,
-                        reference_period_end=reference_period_end,
+                        reference_period_start=time_period_start,
+                        reference_period_end=time_period_end,
                         # TODO: For v2+, add to scraper (HAPI-199)
                         source_data="not yet implemented",
                     )

src/hapi/pipelines/database/operational_presence.py

@@ -41,8 +41,8 @@ def populate(self):
         rows = []
         number_duplicates = 0
         for dataset in self._results.values():
-            reference_period_start = dataset["reference_period"]["startdate"]
-            reference_period_end = dataset["reference_period"]["enddate"]
+            time_period_start = dataset["time_period"]["start"]
+            time_period_end = dataset["time_period"]["end"]
             for admin_level, admin_results in dataset["results"].items():
                 resource_id = admin_results["hapi_resource_metadata"]["hdx_id"]
                 hxl_tags = admin_results["headers"][1]
@@ -104,9 +104,7 @@ def populate(self):
                                 acronym=org_acronym,
                                 org_name=org_name,
                                 org_type=org_type_code,
-                                reference_period_start=parse_date(
-                                    "2023-11-21"
-                                ),
+                                time_period_start=parse_date("2023-11-21"),
                             )
                         sector = values[sector_index][admin_code][i]
                         sector_code = self._sector.get_sector_code(sector)
@@ -134,8 +132,8 @@ def populate(self):
                             admin2_ref=admin2_ref,
                             org_ref=org_ref,
                             sector_code=sector_code,
-                            reference_period_start=reference_period_start,
-                            reference_period_end=reference_period_end,
+                            reference_period_start=time_period_start,
+                            reference_period_end=time_period_end,
                             # TODO: Add to scraper (HAPI-199)
                             source_data="not yet implemented",
                         )