HDXDSYS-1783 Investigate HAPI Pipelines failure (#226)

* TD -> TCD in p-codes

* Update test

pyproject.toml

@@ -35,10 +35,10 @@ requires-python = ">=3.8"
 
 dependencies = [
     "hapi-schema>= 0.9.8",
-    "hdx-python-api>= 6.3.7",
+    "hdx-python-api>= 6.3.8",
     "hdx-python-country>= 3.8.8",
     "hdx-python-database[postgresql]>= 1.3.5",
-    "hdx-python-scraper>= 2.5.8",
+    "hdx-python-scraper>= 2.6.2",
     "hdx-python-utilities>= 3.8.3",
     "libhxl",
     "sqlalchemy"

requirements.txt

@@ -56,7 +56,7 @@ gspread==6.1.4
     # via hdx-python-scraper
 hapi-schema==0.9.8
     # via hapi-pipelines (pyproject.toml)
-hdx-python-api==6.3.7
+hdx-python-api==6.3.8
     # via
     #   hapi-pipelines (pyproject.toml)
     #   hdx-python-scraper
@@ -67,15 +67,15 @@ hdx-python-country==3.8.8
     #   hdx-python-scraper
 hdx-python-database==1.3.5
     # via hapi-pipelines (pyproject.toml)
-hdx-python-scraper==2.5.9
+hdx-python-scraper==2.6.2
     # via hapi-pipelines (pyproject.toml)
 hdx-python-utilities==3.8.3
     # via
     #   hapi-pipelines (pyproject.toml)
     #   hdx-python-api
     #   hdx-python-country
     #   hdx-python-scraper
-humanize==4.11.0
+humanize==4.12.0
     # via frictionless
 identify==2.6.7
     # via pre-commit
@@ -234,7 +234,7 @@ shellingham==1.5.4
     # via typer
 simpleeval==1.0.3
     # via frictionless
-simplejson==3.19.3
+simplejson==3.20.1
     # via ckanapi
 six==1.17.0
     # via
@@ -261,7 +261,7 @@ tenacity==9.0.0
     # via hdx-python-country
 text-unidecode==1.3
     # via python-slugify
-typeguard==4.4.1
+typeguard==4.4.2
     # via inflect
 typer==0.15.1
     # via frictionless

src/hapi/pipelines/configs/core.yaml

@@ -72,7 +72,7 @@ admin1:
     "SOM|Juba Hoose": "SO28"
     "SOM|Shabelle Dhexe": "SO21"
     "SOM|Shabelle Hoose": "SO23"
-    "TCD|Mayo Kebbi Ouest": "TD12"
+    "TCD|Mayo Kebbi Ouest": "TCD12"
     "UKR|Avtonomna Respublika Krym": "UA01"
     "UKR|Kyivska": "UA80"
     "YEM|Amanat Al Asimah": "YE13"
@@ -156,9 +156,9 @@ admin2:
     "SD12|Al Gedaref Rural": "SD12080"
     "SD15|Wad Madani El Kobra": "SD15030"
     "SD18|El Salam": "SD18086"
-    "TD11|La kabbia": "TD1102"
-    "TD17|Iriba": "TD1703"
-    "TD22|Tibest-Ouest": "TD2202"
+    "TCD11|La kabbia": "TCD1102"
+    "TCD17|Iriba": "TCD1703"
+    "TCD22|Tibest-Ouest": "TCD2202"
     "TG05|Kpendial- Ouest": "TG0513"
     "TZ12|Mbozi": "TZ2606"  # TODO: this unit is not getting picked up because of the mismatched admin1
     "UA12|Apostolivskyi": "UA1206"

tests/fixtures/input/download-global-pcode-lengths.csv

@@ -1,160 +1,158 @@
-Location,Country Length,Admin 1 Length,Admin 2 Length,Admin 3 Length,Admin 4 Length,Admin 5 Length
+Location,Country Length,Admin 1 Length,Admin 2 Length,Admin 3 Length,Admin 4 Length,Admin 5 Length
 #country+code,#country+len,#adm1+len,#adm2+len,#adm3+len,#adm4+len,#adm5+len
 AFG,2,2,2,,,
-AGO,2,2,3,3,,
 ALB,2,2,2,2,,
-ARE,2,2,,,,
+DZA,2,3,3,,,
+AGO,2,2,3,3,,
 ARG,2,3,3,,,
 ARM,2,2,1,3,,
-ATG,2,2,,,,
 AZE,2,8,,,,
-BDI,3,3,3,,,
-BEN,2,2,2,,,
-BES,3,1,,,,
-BFA,2,2,2,2,,
 BGD,2,2,2,2,2,
-BGR,2,3,3,,,
+BRB,2,2,,,,
 BLR,2,3,3,,,
 BLZ,2,2,,,,
+BEN,2,2,2,,,
 BMU,2,2,1,,,
-BOL,2,2,2,2,,
-BRA,2,2,5,,,
-BRB,2,2,,,,
 BTN,2,3,2,,,
+BOL,2,2,2,2,,
+BES,3,1,,,,
 BWA,2,2,2,2,,
+BRA,2,2,5,,,
+BGR,2,3,3,,,
+BFA,2,2,2,2,,
+BDI,3,3,3,,,
+KHM,2,2,2,2,,
+CMR,2,3,3,3,,
+CPV,2,2,2,,,
+CYM,2,2,,,,
 CAF,2,2,1,1,,
+TCD,3,2,2,,,
 CHL,2,2,1,2,,
 CHN,2,3,3,,,
-CIV,2,2,2,2,,
-CMR,2,3,3,3,,
-COD,2,2,2,,,
-COG,2,2,2,,,
 COL,2,2,3,,,
 COM,2,1,1,1,,
-CPV,2,2,2,,,
+COG,2,2,2,,,
+COD,2,2,2,,,
 CRI,2,1,2,2,,
+CIV,2,2,2,2,,
 CUB,2,2,2,,,
 CUW,2,2,,,,
-CYM,2,2,,,,
 DJI,2,2,2,,,
 DMA,2,2,,,,
 DOM,2,2,2,2,2,
-DZA,2,3,3,,,
-ECU,2,2|4|9,3|8,2|0|5,,
+ECU,2,2,3,2,,
 EGY,2,2,2,2,,
+SLV,2,2,3,3,,
+GNQ,2,3,3,,,
 ERI,2,1,2,,,
-ESH,2,2,,,,
 ETH,2,2,2,2,,
 FJI,2,1,2,2,,
-FSM,2,1,2,,,
+GUF,2,1,3,,,
 GAB,3,3,3,,,
+GMB,2,2,2,2,,
 GEO,2,2,2,,,
 GHA,2,2,2,,,
-GIN,2,3,3,2,,
-GLP,2,2,2,,,
-GMB,2,2,2,2,,
-GNB,2,2,2,,,
-GNQ,2,3,3,,,
 GRC,2,2,2,,,
 GRD,2,2,,,,
+GLP,2,2,2,,,
 GTM,2,2,2,,,
-GUF,2,1,3,,,
+GIN,2,3,3,2,,
+GNB,2,2,2,,,
 GUY,2,2,2,,,
-HND,2,2,2,,,
 HTI,2,2,2,3,,
+HND,2,2,2,,,
 HUN,2,3,3,,,
 IDN,2,2,2,3,,
 IRN,2,3,3,,,
 IRQ,2,3,3,3,,
 JAM,2,2,2,2,,
 KAZ,3,3,3,,,
 KEN,2,3,3,,,
-KGZ,2,11,0,0,,
-KHM,2,2,2,2,,
 KIR,2,1,2,,,
-KNA,2,2,,,,
+PRK,2,2,2,,,
 KWT,2,2,,,,
+KGZ,2,11,0,0,,
 LAO,2,2,2,,,
 LBN,2,1,1,,,
+LSO,2,1,2,,,
 LBR,2,2,2,,,
 LBY,2,2,2,,,
-LCA,2,2,9,,,
-LKA,2,1,1,2,3,
-LSO,2,1,2,,,
-MAR,2,3,3,7,1,
-MDA,2,3,,,,
-MDG,2,2,7|3,-1|3,3,
-MDV,2,3,3,,,
-MEX,2,2,3,,,
-MHL,2,2,2,,,
 MKD,2,3,,,,
+MDG,2,2,3|7,3|-1,3,
+MWI,2,1,2,2,,
+MYS,2,2,2,,,
+MDV,2,3,3,,,
 MLI,2,2,2,2,,
-MMR,3,3,4,-1,,
-MNG,2,2,2,,,
-MOZ,2,2,2,2,,
-MRT,2,2,1,2,,
-MSR,2,2,,,,
+MHL,2,2,2,,,
 MTQ,2,2,2,,,
+MRT,2,2,1,2,,
 MUS,2,2,,,,
-MWI,2,1,2,2,,
-MYS,2,2,2,,,
+MEX,2,2,3,,,
+FSM,2,1,2,,,
+MDA,2,3,,,,
+MNG,2,2,2,,,
+MSR,2,2,,,,
+MAR,2,3,3,7,1,
+MOZ,2,2,2,2,,
+MMR,3,3,4,-1,,
 NAM,2,2,2,,,
+NPL,2,2,2,3,,
+NIC,2,2,2,,,
 NER,2,3,3,3,,
 NGA,2,3,3,3,,
-NIC,2,2,2,,,
-NPL,2,2,2,3,,
 OMN,2,2,2,,,
 PAK,2,1,2,2,,
+PSE,2,2,2,,,
 PAN,2,2,2,2,,
+PNG,2,2,2,2,,
+PRY,2,2,2,,,
 PER,2,2,2,2,,
 PHL,2,2,3,2,3,
-PNG,2,2,2,2,,
 POL,2,3,3,,,
-PRI,2,2,,,,
-PRK,2,2,2,,,
-PRY,2,2,2,,,
-PSE,2,2,2,,,
 QAT,3,3,3,3,,
 ROU,2,3,3,,,
 RUS,2,3,3,,,
 RWA,2,1,1,2,2,
+KNA,2,2,,,,
+LCA,2,2,9,,,
+VCT,2,1,2,,,
+STP,2,2,2,,,
 SAU,2,2,,,,
-SDN,2,2,3,,,
 SEN,2,2,2,2,,
-SLB,2,2,4,4,,
+SYC,2,1,1,4,,
 SLE,2,2,2,2,2,
-SLV,2,2,3,3,,
+SXM,2,1,,,,
+SVK,2,3,3,,,
+SLB,2,2,4,4,,
 SOM,2,2,2|7,,,
+ZAF,2,1,2,1,3,
+LKA,2,1,1,2,3,
 SSD,2,2,2,2,,
-STP,2,2,2,,,
+SDN,2,2,3,,,
 SUR,2,2,2,,,
-SVK,2,3,3,,,
 SWZ,2,1,2,,,
-SXM,2,1,,,,
-SYC,2,1,1,4,,
 SYR,2,2,2,2,,
-TCA,2,1,2,,,
-TCD,2,2,2,2,,
-TGO,2,2,2,2,,
-THA,2,2,2,2,,
 TJK,0,7,0,,,
+TZA,2,2,2,4|3,,
+THA,2,2,2,2,,
 TLS,2,2,2,2,,
+TGO,2,2,2,2,,
 TON,2,1,1,2,,
 TTO,2,2,,,,
 TUN,2,1,1,2,2,
 TUR,3,3,3,,,
-TZA,2,2,2,3|4,,
+TCA,2,1,2,,,
 UGA,2,1,3,,2,
 UKR,2,2,2,3,3,
+ARE,2,2,,,,
 URY,2,2,3,,,
 UZB,2,2,3,,,
-VCT,2,1,2,,,
+VUT,2,2,3,,,
 VEN,2,2,2,2,,
-VGB,2,2,,,,
-VIR,2,3,5,,,
 VNM,2,3,2,,,
-VUT,2,2,3,,,
+VGB,2,2,,,,
+ESH,2,2,,,,
 YEM,2,2,2,2,,
-ZAF,2,1,2,1,3,
 ZMB,2,3,3,3,3,
 ZWE,2,2,2,2,,
+VIR,2,3,5,,,