Add method to remove countries from config

requirements.txt

@@ -61,7 +61,7 @@ google-auth-oauthlib==1.2.0
     # via gspread
 greenlet==3.0.3
     # via sqlalchemy
-gspread==5.12.4
+gspread==6.0.0
     # via hdx-python-scraper
 hapi-schema==0.6.0
     # via hapi-pipelines (pyproject.toml)
@@ -163,11 +163,11 @@ pyasn1-modules==0.3.0
     # via google-auth
 pycparser==2.21
     # via cffi
-pydantic==2.5.3
+pydantic==2.6.0
     # via
     #   frictionless
     #   inflect
-pydantic-core==2.14.6
+pydantic-core==2.16.1
     # via pydantic
 pygments==2.17.2
     # via rich
@@ -179,7 +179,7 @@ pyphonetics==0.5.3
     # via hdx-python-country
 pyrsistent==0.20.0
     # via jsonschema
-pytest==7.4.4
+pytest==8.0.0
     # via
     #   hapi-pipelines (pyproject.toml)
     #   pytest-cov
@@ -215,7 +215,7 @@ requests==2.31.0
     #   libhxl
     #   requests-file
     #   requests-oauthlib
-requests-file==1.5.1
+requests-file==2.0.0
     # via hdx-python-utilities
 requests-oauthlib==1.3.1
     # via google-auth-oauthlib
@@ -239,7 +239,6 @@ six==1.16.0
     #   isodate
     #   pockets
     #   python-dateutil
-    #   requests-file
     #   sphinxcontrib-napoleon
 sphinxcontrib-napoleon==0.7
     # via defopt
@@ -248,6 +247,8 @@ sqlalchemy==2.0.25
     #   hapi-pipelines (pyproject.toml)
     #   hapi-schema
     #   hdx-python-database
+strenum==0.4.15
+    # via gspread
 stringcase==1.2.0
     # via frictionless
 structlog==24.1.0

src/hapi/pipelines/utilities/process_config_defaults.py

@@ -15,6 +15,26 @@ def add_defaults(config: Dict) -> Dict:
     return config
 
 
+def subset_scraper_countries(config: Dict, countries_to_remove: List) -> Dict:
+    countries_to_remove = [c.lower() for c in countries_to_remove]
+    for key in config:
+        scraper_name = key.rsplit("_")
+        if scraper_name[-1] not in ["national", "adminone", "admintwo"]:
+            continue
+        subkeys_to_delete = []
+        for subkey in config[key]:
+            subscraper_name = subkey.rsplit("_")
+            subscraper_country = list(
+                set(subscraper_name) & set(countries_to_remove)
+            )
+            if len(subscraper_country) == 0:
+                continue
+            subkeys_to_delete.append(subkey)
+        for subkey in subkeys_to_delete:
+            del config[key][subkey]
+    return config
+
+
 def _find_defaults(config: Dict) -> List:
     default_list = []
     for key in config:

tests/fixtures/input/population_bfa_bfa_admpop_adm1_2023_5yr.csv

@@ -0,0 +1,14 @@
+year,ISO3,ADM0_FR,ADM0_PCODE,ADM1_FR,ADM1_PCODE,F_TL,M_TL,T_TL,F_00_04,F_05_09,F_10_14,F_15_19,F_20_24,F_25_29,F_30_34,F_35_39,F_40_44,F_45_49,F_50_54,F_55_59,F_60_64,F_65_69,F_70_74,F_75_79,F_80Plus,M_00_04,M_05_09,M_10_14,M_15_19,M_20_24,M_25_29,M_30_34,M_35_39,M_40_44,M_45_49,M_50_54,M_55_59,M_60_64,M_65_69,M_70_74,M_75_79,M_80Plus,T_00_04,T_05_09,T_10_14,T_15_19,T_20_24,T_25_29,T_30_34,T_35_39,T_40_44,T_45_49,T_50_54,T_55_59,T_60_64,T_65_69,T_70_74,T_75_79,T_80Plus
+2023,BFA,Burkina Faso,BF,Boucle Du Mouhoun,BF46,1049130,1031594,2080724,183315,151820,134469,103443,91257,78681,67981,55749,45956,37323,29919,23287,16855,11849,7924,4840,4462,193853,165287,139967,118270,90799,69075,55789,44315,35594,29606,24707,20092,14899,11098,7774,5324,5145,377168,317107,274436,221713,182056,147756,123770,100064,81550,66929,54626,43379,31754,22947,15698,10164,9607
+2023,BFA,Burkina Faso,BF,Cascades,BF47,473492,446517,920009,83100,69045,61300,49840,42500,37458,32000,25233,19477,15033,12161,9445,6552,4580,2832,1646,1290,86147,72773,61058,49122,37954,29825,24718,20100,15809,12636,10278,8011,6282,4858,3126,2124,1696,169247,141818,122358,98962,80454,67283,56718,45333,35286,27669,22439,17456,12834,9438,5958,3770,2986
+2023,BFA,Burkina Faso,BF,Centre,BF13,1757045,1752983,3510028,234389,204347,198094,208129,203104,178925,146452,108334,77938,57024,42989,33740,23317,16308,10770,6619,6566,247747,214495,184561,172776,185482,174065,151670,119419,89680,65178,47200,35077,25110,16828,10667,6602,6426,482136,418842,382655,380905,388586,352990,298122,227753,167618,122202,90189,68817,48427,33136,21437,13221,12992
+2023,BFA,Burkina Faso,BF,Centre-Est,BF48,929201,819925,1749126,157514,134854,114863,96205,82831,71584,59876,48787,39499,32282,25241,19476,15074,11347,8387,5386,5995,166595,145818,123163,89172,60521,45781,36061,30678,25466,21860,18570,15198,12543,9583,7478,5201,6237,324109,280672,238026,185377,143352,117365,95937,79465,64965,54142,43811,34674,27617,20930,15865,10587,12232
+2023,BFA,Burkina Faso,BF,Centre-Nord,BF49,1101439,984803,2086242,197852,162970,143490,118246,98755,80867,67234,54858,45189,36022,28320,21966,15825,11307,8323,4786,5429,204879,169130,138513,105816,79131,61220,47052,37837,30201,24578,21730,18145,14204,11195,8598,5798,6776,402731,332100,282003,224062,177886,142087,114286,92695,75390,60600,50050,40111,30029,22502,16921,10584,12205
+2023,BFA,Burkina Faso,BF,Centre-Ouest,BF50,983228,845251,1828479,161911,135748,121776,102453,78631,67801,61654,54231,46063,38089,31098,24979,18825,14291,10789,6862,8027,171293,143301,118963,98800,72215,49056,36530,30585,26007,22529,18848,15481,12226,9760,7635,5811,6211,333204,279049,240739,201253,150846,116857,98184,84816,72070,60618,49946,40460,31051,24051,18424,12673,14238
+2023,BFA,Burkina Faso,BF,Centre-Sud,BF51,452088,405580,857668,70609,60088,54582,47050,34509,29667,28655,25670,21879,18855,16096,12883,9890,7839,5842,3664,4310,75888,65202,55448,46626,31036,23022,19580,16598,14077,12119,10694,9160,7344,6112,5013,3512,4149,146497,125290,110030,93676,65545,52689,48235,42268,35956,30974,26790,22043,17234,13951,10855,7176,8459
+2023,BFA,Burkina Faso,BF,Est,BF52,1111761,1072832,2184593,221547,178934,140748,120012,102033,86629,65227,52665,40237,30721,22538,16152,11756,8080,5963,3616,4903,230521,189648,152595,116761,88051,68924,51286,41641,32073,25632,21279,16472,12324,8674,6390,4505,6056,452068,368582,293343,236773,190084,155553,116513,94306,72310,56353,43817,32624,24080,16754,12353,8121,10959
+2023,BFA,Burkina Faso,BF,Hauts-Bassins,BF53,1272663,1237835,2510498,199835,177025,155595,142373,126960,107684,90210,70422,54619,42407,33147,25877,18075,12260,7698,4441,4035,204065,188268,161273,135813,119029,96689,78928,62857,49071,39207,30789,24047,17376,12216,8132,5294,4781,403900,365293,316868,278186,245989,204373,169138,133279,103690,81614,63936,49924,35451,24476,15830,9735,8816
+2023,BFA,Burkina Faso,BF,Nord,BF54,1005699,893752,1899451,179255,147456,128872,107780,86280,70603,59637,50420,42477,34027,27699,22660,16676,12264,8825,5392,5376,178559,148900,130585,98900,75079,56222,43314,33365,27370,22100,19870,16635,12677,9993,7977,5947,6259,357814,296356,259457,206680,161359,126825,102951,83785,69847,56127,47569,39295,29353,22257,16802,11339,11635
+2023,BFA,Burkina Faso,BF,Plateau-Central,BF55,572878,502071,1074949,96077,80002,71294,59760,46583,40080,35548,30924,25954,21631,17944,13888,10780,7968,6146,3784,4515,101473,84382,69676,55187,39141,29385,23267,19241,15885,13532,11866,10134,8391,6778,5433,3841,4459,197550,164384,140970,114947,85724,69465,58815,50165,41839,35163,29810,24022,19171,14746,11579,7625,8974
+2023,BFA,Burkina Faso,BF,Sahel,BF56,599100,601785,1200885,107819,89275,78940,67667,56309,47841,36722,29769,23562,18506,14049,10277,6949,4942,3103,1884,1486,112834,96048,75959,62696,54621,45860,34177,27371,21727,17565,14962,12016,8999,6666,4292,3157,2835,220653,185323,154899,130363,110930,93701,70899,57140,45289,36071,29011,22293,15948,11608,7395,5041,4321
+2023,BFA,Burkina Faso,BF,Sud-Ouest,BF57,500569,465694,966263,81775,68767,60089,49111,43433,37803,33338,27770,23835,19782,16115,12863,9663,6957,4402,2576,2290,86197,73895,63366,51305,41632,32550,26150,20347,16603,13518,10850,8891,7148,5095,3570,2468,2109,167972,142662,123455,100416,85065,70353,59488,48117,40438,33300,26965,21754,16811,12052,7972,5044,4399