HDXDSYS-1125 Fix test failures due to admin retriever change (#179)

* Fix tests

* Latest HDX Python Country

* Add changelog

.config/pre-commit-config.yaml

@@ -8,7 +8,7 @@ repos:
       - id: end-of-file-fixer
       - id: check-ast
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.5.5
+    rev: v0.6.9
     hooks:
       # Run the linter.
       - id: ruff
@@ -17,7 +17,7 @@ repos:
       - id: ruff-format
         args: [--config, .config/ruff.toml]
   - repo: https://github.com/astral-sh/uv-pre-commit
-    rev: 0.2.31
+    rev: 0.4.18
     hooks:
       # Run the pip compile
       - id: pip-compile

CHANGELOG.md

@@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [0.10.4] - 2024-10-07
+
+### Fixed
+
+- Fix broken tests
+
 ## [0.10.3] - 2024-09-24
 
 ### Changed

pyproject.toml

@@ -36,10 +36,10 @@ requires-python = ">=3.8"
 dependencies = [
     "hapi-schema>=0.8.17",
     "hdx-python-api>= 6.3.4",
-    "hdx-python-country>= 3.7.8",
+    "hdx-python-country>= 3.8.1",
     "hdx-python-database[postgresql]>= 1.3.1",
     "hdx-python-scraper>= 2.5.0",
-    "hdx-python-utilities>= 3.7.3",
+    "hdx-python-utilities>= 3.7.4",
     "libhxl",
     "sqlalchemy"
 ]

requirements.txt

@@ -28,7 +28,7 @@ defopt==6.4.0
     # via hdx-python-api
 distlib==0.3.8
     # via virtualenv
-dnspython==2.6.1
+dnspython==2.7.0
     # via email-validator
 docopt==0.6.2
     # via
@@ -42,7 +42,7 @@ et-xmlfile==1.1.0
     # via openpyxl
 filelock==3.16.1
     # via virtualenv
-frictionless==5.17.1
+frictionless==5.18.0
     # via hdx-python-utilities
 google-auth==2.35.0
     # via
@@ -52,15 +52,15 @@ google-auth-oauthlib==1.2.1
     # via gspread
 greenlet==3.1.1
     # via sqlalchemy
-gspread==6.1.2
+gspread==6.1.3
     # via hdx-python-scraper
 hapi-schema==0.8.17
     # via hapi-pipelines (pyproject.toml)
 hdx-python-api==6.3.4
     # via
     #   hapi-pipelines (pyproject.toml)
     #   hdx-python-scraper
-hdx-python-country==3.7.8
+hdx-python-country==3.8.1
     # via
     #   hapi-pipelines (pyproject.toml)
     #   hdx-python-api
@@ -69,13 +69,13 @@ hdx-python-database==1.3.3
     # via hapi-pipelines (pyproject.toml)
 hdx-python-scraper==2.5.0
     # via hapi-pipelines (pyproject.toml)
-hdx-python-utilities==3.7.3
+hdx-python-utilities==3.7.4
     # via
     #   hapi-pipelines (pyproject.toml)
     #   hdx-python-api
     #   hdx-python-country
     #   hdx-python-scraper
-humanize==4.10.0
+humanize==4.11.0
     # via frictionless
 identify==2.6.1
     # via pre-commit
@@ -110,7 +110,7 @@ libhxl==5.2.1
     #   hdx-python-country
 loguru==0.7.2
     # via hdx-python-utilities
-makefun==1.15.4
+makefun==1.15.6
     # via hdx-python-api
 markdown-it-py==3.0.0
     # via rich
@@ -144,11 +144,11 @@ ply==3.11
     #   libhxl
 pockets==0.9.1
     # via sphinxcontrib-napoleon
-pre-commit==3.8.0
+pre-commit==4.0.0
     # via hapi-pipelines (pyproject.toml)
-psycopg==3.2.2
+psycopg==3.2.3
     # via hdx-python-database
-psycopg-binary==3.2.2
+psycopg-binary==3.2.3
     # via psycopg
 pyasn1==0.6.1
     # via
@@ -216,7 +216,7 @@ requests-oauthlib==2.0.0
     # via google-auth-oauthlib
 rfc3986==2.0.0
     # via frictionless
-rich==13.8.1
+rich==13.9.2
     # via typer
 rpds-py==0.20.0
     # via
@@ -232,7 +232,7 @@ setuptools==75.1.0
     # via ckanapi
 shellingham==1.5.4
     # via typer
-simpleeval==0.9.13
+simpleeval==1.0.0
     # via frictionless
 simplejson==3.19.3
     # via ckanapi
@@ -283,7 +283,7 @@ urllib3==2.2.3
     #   requests
 validators==0.34.0
     # via frictionless
-virtualenv==20.26.5
+virtualenv==20.26.6
     # via pre-commit
 wheel==0.44.0
     # via libhxl

src/hapi/pipelines/app/pipelines.py

@@ -5,6 +5,7 @@
 from hdx.api.configuration import Configuration
 from hdx.location.adminlevel import AdminLevel
 from hdx.scraper.framework.runner import Runner
+from hdx.scraper.framework.utilities.reader import Read
 from hdx.scraper.framework.utilities.sources import Sources
 from hdx.utilities.errors_onexit import ErrorsOnExit
 from hdx.utilities.typehint import ListTuple
@@ -54,7 +55,13 @@ def __init__(
             use_live=use_live,
         )
         self.countries = configuration["HAPI_countries"]
-        libhxl_dataset = AdminLevel.get_libhxl_dataset().cache()
+        reader = Read.get_reader("hdx")
+        libhxl_dataset = AdminLevel.get_libhxl_dataset(
+            retriever=reader
+        ).cache()
+        libhxl_format_dataset = AdminLevel.get_libhxl_dataset(
+            url=AdminLevel.formats_url, retriever=reader
+        ).cache()
         self.admins = Admins(
             configuration, session, self.locations, libhxl_dataset
         )
@@ -63,9 +70,13 @@ def __init__(
         admin2_config = configuration["admin2"]
         self.admintwo = AdminLevel(admin_config=admin2_config, admin_level=2)
         self.adminone.setup_from_libhxl_dataset(libhxl_dataset, self.countries)
-        self.adminone.load_pcode_formats()
+        self.adminone.load_pcode_formats_from_libhxl_dataset(
+            libhxl_format_dataset
+        )
         self.admintwo.setup_from_libhxl_dataset(libhxl_dataset, self.countries)
-        self.admintwo.load_pcode_formats()
+        self.admintwo.load_pcode_formats_from_libhxl_dataset(
+            libhxl_format_dataset
+        )
         self.admintwo.set_parent_admins_from_adminlevels([self.adminone])
         logger.info("Admin one name mappings:")
         self.adminone.output_admin_name_mappings()

tests/fixtures/input/download-global-pcode-lengths.csv

@@ -0,0 +1,158 @@
+Location,Country Length,Admin 1 Length,Admin 2 Length,Admin 3 Length,Admin 4 Length,Admin 5 Length
+#country+code,#country+len,#adm1+len,#adm2+len,#adm3+len,#adm4+len,#adm5+len
+AFG,2,2,2,,,
+AGO,2,2,3,3,,
+ALB,2,2,2,2,,
+ARE,2,2,,,,
+ARG,2,3,3,,,
+ARM,2,2,1,3,,
+ATG,2,2,,,,
+AZE,2,8,,,,
+BDI,3,3,3,,,
+BEN,2,2,2,,,
+BES,3,1,,,,
+BFA,2,2,2,2,,
+BGD,2,2,2,2,2,
+BGR,2,3,3,,,
+BLR,2,3,3,,,
+BLZ,2,2,,,,
+BMU,2,2,1,,,
+BOL,2,2,2,2,,
+BRA,2,2,5,,,
+BRB,2,2,,,,
+BTN,2,3,2,,,
+BWA,2,2,2,2,,
+CAF,2,2,1,1,2,
+CHL,2,2,1,2,,
+CHN,2,3,3,,,
+CIV,2,2,2,2,,
+CMR,2,3,3,3,,
+COD,2,2,2,,,
+COG,2,2,2,,,
+COL,2,2,3,,,
+COM,2,1,1,1,,
+CPV,2,2,2,,,
+CRI,2,1,2,2,,
+CUB,2,2,2,,,
+CUW,2,2,,,,
+CYM,2,2,,,,
+DJI,2,2,2,,,
+DMA,2,2,,,,
+DOM,2,2,2,2,2,
+DZA,2,3,3,,,
+ECU,2,2,2,2,,
+EGY,2,2,2,2,,
+ERI,2,1,2,,,
+ESH,2,2,,,,
+ETH,2,2,2,2,,
+FJI,2,1,2,2,,
+FSM,2,1,2,,,
+GAB,3,3,3,,,
+GEO,2,2,2,,,
+GHA,2,2,2,,,
+GIN,2,3,3,2,,
+GLP,2,2,2,,,
+GMB,2,2,2,2,,
+GNB,2,2,2,,,
+GNQ,2,3,3,,,
+GRD,2,2,,,,
+GTM,2,2,2,,,
+GUF,2,1,3,,,
+GUY,2,2,2,,,
+HND,2,2,2,,,
+HTI,2,2,2,3,,
+HUN,2,3,3,,,
+IDN,2,2,2,3,,
+IRN,2,3,3,,,
+IRQ,2,3,3,3,,
+JAM,2,2,2,2,,
+KAZ,3,3,3,,,
+KEN,2,3,3,,,
+KGZ,2,11,0,0,,
+KHM,2,2,2,2,,
+KIR,2,1,2,,,
+KNA,2,2,,,,
+KWT,2,2,,,,
+LAO,2,2,2,,,
+LBN,2,1,1,1,,
+LBR,2,2,2,,,
+LBY,2,2,2,,,
+LCA,2,2,9,,,
+LKA,2,1,1,2,3,
+LSO,2,1,2,,,
+MAR,2,3,3,7,1,
+MDA,2,3,,,,
+MDG,2,2,7|3,-1|3,3,
+MDV,2,3,3,3,,
+MEX,2,2,3,,,
+MHL,2,2,2,,,
+MLI,2,2,2,2,,
+MMR,3,3,4,-1,3,3
+MNG,2,2,2,,,
+MOZ,2,2,2,2,,
+MRT,2,2,1,2,,
+MSR,2,2,,,,
+MTQ,2,2,2,,,
+MUS,2,2,,,,
+MWI,2,1,2,2,,
+MYS,2,2,2,,,
+NAM,2,2,2,,,
+NER,2,3,3,3,,
+NGA,2,3,3,3,,
+NIC,2,2,2,,,
+NPL,2,2,2,3,,
+OMN,2,2,2,,,
+PAK,2,1,2,2,,
+PAN,2,2,2,2,,
+PER,2,2,2,2,,
+PHL,2,2,3,2,3,
+PNG,2,2,2,2,,
+POL,2,3,3,,,
+PRI,2,2,,,,
+PRK,2,2,2,,,
+PRY,2,2,2,,,
+PSE,2,2,2,,,
+QAT,3,3,3,3,,
+ROU,2,3,3,,,
+RUS,2,3,3,,,
+RWA,2,1,1,2,2,
+SAU,2,2,,,,
+SDN,2,2,3,,,
+SEN,2,2,2,2,,
+SLB,2,2,4,4,,
+SLE,2,2,2,2,2,
+SLV,2,2,3,3,,
+SOM,2,2,2,,,
+SSD,2,2,2,2,,
+STP,2,2,2,,,
+SUR,2,2,2,,,
+SVK,2,3,3,,,
+SWZ,2,1,2,,,
+SXM,2,1,,,,
+SYC,2,1,1,4,,
+SYR,2,2,2,2,,
+TCA,2,1,2,,,
+TCD,2,2,2,2,,
+TGO,2,2,2,2,,
+THA,2,2,2,2,,
+TJK,0,7,0,,,
+TLS,2,2,2,2,,
+TON,2,1,1,2,,
+TTO,2,2,,,,
+TUN,2,1,1,2,2,
+TUR,3,3,3,3|9,3,
+TZA,2,2,2,3|4,,
+UGA,2,1,3,2,2,
+UKR,2,2,2,3,3,
+URY,2,2,3,,,
+UZB,2,2,3,,,
+VCT,2,1,2,,,
+VEN,2,2,2,2,,
+VGB,2,2,,,,
+VIR,2,3,5,,,
+VNM,2,3,2,,,
+VUT,2,2,3,,,
+YEM,2,2,2,2,,
+ZAF,2,1,2,1,3,
+ZMB,2,3,3,3,3,
+ZWE,2,2,2,2,,