Skip to content

Commit

Permalink
Merge pull request #63 from OCHA-DAP/admin_matching
Browse files Browse the repository at this point in the history
HAPI-282 Use new pcode length conversion
  • Loading branch information
mcarans authored Nov 7, 2023
2 parents ace7f28 + 46d9f9e commit 2625f15
Show file tree
Hide file tree
Showing 9 changed files with 63 additions and 50 deletions.
8 changes: 8 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file.

The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).

## [0.6.3] - 2023-11-07

### Added

- Use better pcode length conversion from HDX Python Country
- Add food security NGA
- When phase population is 0, set population_fraction_in_phase to 0.0

## [0.6.2] - 2023-11-06

### Added
Expand Down
4 changes: 2 additions & 2 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,9 @@ requires-python = ">=3.8"

dependencies = [
"hapi-schema>= 0.4.0",
"hdx-python-country>= 3.5.8",
"hdx-python-country>= 3.5.9",
"hdx-python-database[postgresql]>= 1.2.9",
"hdx-python-scraper>= 2.2.3",
"hdx-python-scraper>= 2.2.5",
"libhxl",
"sqlalchemy"
]
Expand Down
18 changes: 9 additions & 9 deletions requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ cfgv==3.4.0
# via pre-commit
chardet==5.2.0
# via frictionless
charset-normalizer==3.3.1
charset-normalizer==3.3.2
# via requests
ckanapi==4.7
# via hdx-python-api
Expand Down Expand Up @@ -49,11 +49,11 @@ email-validator==2.1.0.post1
# via hdx-python-api
et-xmlfile==1.1.0
# via openpyxl
filelock==3.13.0
filelock==3.13.1
# via virtualenv
frictionless==5.16.0
# via hdx-python-utilities
google-auth==2.23.3
google-auth==2.23.4
# via
# google-auth-oauthlib
# gspread
Expand All @@ -67,19 +67,19 @@ hapi-schema==0.4.0
# via hapi-pipelines (pyproject.toml)
hdx-python-api==6.1.3
# via hdx-python-scraper
hdx-python-country==3.5.8
hdx-python-country==3.5.9
# via
# hapi-pipelines (pyproject.toml)
# hdx-python-api
hdx-python-database[postgresql]==1.2.9
# via hapi-pipelines (pyproject.toml)
hdx-python-scraper==2.2.3
hdx-python-scraper==2.2.5
# via hapi-pipelines (pyproject.toml)
hdx-python-utilities==3.6.2
# via hdx-python-country
humanize==4.8.0
# via frictionless
identify==2.5.30
identify==2.5.31
# via pre-commit
idna==3.4
# via
Expand Down Expand Up @@ -220,7 +220,7 @@ rich==13.6.0
# via typer
rsa==4.9
# via google-auth
ruamel-yaml==0.18.2
ruamel-yaml==0.18.5
# via hdx-python-utilities
ruamel-yaml-clib==0.2.8
# via ruamel-yaml
Expand All @@ -238,7 +238,7 @@ six==1.16.0
# sphinxcontrib-napoleon
sphinxcontrib-napoleon==0.7
# via defopt
sqlalchemy==2.0.22
sqlalchemy==2.0.23
# via
# hapi-pipelines (pyproject.toml)
# hapi-schema
Expand Down Expand Up @@ -276,7 +276,7 @@ validators==0.22.0
# via frictionless
virtualenv==20.24.6
# via pre-commit
wheel==0.41.2
wheel==0.41.3
# via libhxl
xlrd==2.0.1
# via hdx-python-utilities
Expand Down
3 changes: 3 additions & 0 deletions src/hapi/pipelines/app/pipelines.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,10 @@ def __init__(
self.adminone = AdminLevel(admin_level=1)
self.admintwo = AdminLevel(admin_level=2)
self.adminone.setup_from_libhxl_dataset(libhxl_dataset, countries)
self.adminone.load_pcode_formats()
self.admintwo.setup_from_libhxl_dataset(libhxl_dataset, countries)
self.admintwo.load_pcode_formats()
self.admintwo.set_parent_admins_from_adminlevels([self.adminone])

self.org = Org(
session=session,
Expand Down
12 changes: 6 additions & 6 deletions src/hapi/pipelines/configs/food_security.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -107,10 +107,10 @@ food_security_adminone:
dataset: "cadre-harmonise"
format: "xlsx"
admin:
- ~
- "adm0_pcod3"
- "adm1_pcod2"
admin_exact: True
prefilter: "adm0_pcod3 in ['TCD'] and adm2_pcod2 is None"
admin_exact: False
prefilter: "adm0_pcod3 in ['NGA', 'TCD'] and adm2_pcod2 is None"
filter_cols:
- "adm0_pcod3"
- "adm2_pcod2"
Expand All @@ -120,9 +120,9 @@ food_security_admintwo:
dataset: "cadre-harmonise"
format: "xlsx"
admin:
- ~
- "adm0_pcod3"
- "adm2_pcod2"
admin_exact: True
prefilter: "adm0_pcod3 in ['BFA', 'MLI', 'TCD']"
admin_exact: False
prefilter: "adm0_pcod3 in ['BFA', 'MLI', 'NGA', 'TCD']"
filter_cols:
- "adm0_pcod3"
4 changes: 3 additions & 1 deletion src/hapi/pipelines/database/food_security.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,9 @@ def populate(self):
reference_period_end=reference_period_end,
population_in_phase=population_in_phase,
population_fraction_in_phase=population_in_phase
/ population_total,
/ population_total
if population_in_phase
else 0.0,
# TODO: For v2+, add to scraper (HAPI-199)
source_data="not yet implemented",
)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
HR.info ID,Preferred Term,Scope,Date Added (dd/mm/yyyy),Date updated (dd/mm/yyyy),RW API ID,RW Alt Term,Match type - RW term is:
#org +type +code +v_hrinfo,#org +type +preferred,#description +definition,#date +created,#date +updated,#org +type +code +v_reliefweb,#org +type +v_reliefweb,#meta +match
431,Academic / Research,"Includes universities, colleges, think tanks, private organizations focusing on research and analysis",16/03/2017,,270,Academic and Research Institution,Exact Match
433,Donor,"Organizations responsible for donating money, goods or other in-kind contributions. ",16/03/2017,,,,
434,Embassy,Includes embassies as a distinct category of government organisations - the official residence or offices of an ambassador.,16/03/2017,,271,Government,Broader
435,Government,"Includes national and local government agencies, including development, emergency, civil protection, etc. Embassies are a separate category.",16/03/2017,,271,Government,Broader
437,International NGO,"Includes international organizations that operate independently from any government, including civil society. UN and ICRC/IFRC are in separate categories.",16/03/2017,,274,Non-governmental Organization,Broader
438,International Organization,"Includes intergovernmental organizations, funds and programmes. UN and ICRC/IFRC are in separate categories.",16/03/2017,,272,International Organization,Broader
439,Media,"Includes local, national and international print, TV, online and radio media agencies",16/03/2017,,273,Media,Exact Match
440,Military,Includes National and International military and military-related organisations,16/03/2017,,,,
441,National NGO,"Includes national and subnational organizations that operate independently from any government, including civil society. Red Cross/Red Crescent Societies are in a separate category.",16/03/2017,,274,Non-governmental Organization,Broader
443,Other,Includes all organizations that are not applicable to other organization types,16/03/2017,,275,Other,Broader
444,Private sector,"Includes private sector organizations such as corporate charity arms or foundations, for-profit recruitment firms and consultancies",16/03/2017,,,,
445,Red Cross / Red Crescent,"Includes IFRC, ICRC and national Red Cross/Crescent societies",16/03/2017,,276,Red Cross/Red Crescent Movement,Exact Match
446,Religious,Includes faith-based organizations and organizations with affiliations to religious bodies,16/03/2017,,,,
HR.info ID,Preferred Term,Scope,Date Added (dd/mm/yyyy),Date updated (dd/mm/yyyy),RW API ID,RW Alt Term,Match type - RW term is:
#org +type +code +v_hrinfo,#org +type +preferred,#description +definition,#date +created,#date +updated,#org +type +code +v_reliefweb,#org +type +v_reliefweb,#meta +match
431,Academic / Research,"Includes universities, colleges, think tanks, private organizations focusing on research and analysis",16/03/2017,,270,Academic and Research Institution,Exact Match
433,Donor,"Organizations responsible for donating money, goods or other in-kind contributions. ",16/03/2017,,,,
434,Embassy,Includes embassies as a distinct category of government organisations - the official residence or offices of an ambassador.,16/03/2017,,271,Government,Broader
435,Government,"Includes national and local government agencies, including development, emergency, civil protection, etc. Embassies are a separate category.",16/03/2017,,271,Government,Broader
437,International NGO,"Includes international organizations that operate independently from any government, including civil society. UN and ICRC/IFRC are in separate categories.",16/03/2017,,274,Non-governmental Organization,Broader
438,International Organization,"Includes intergovernmental organizations, funds and programmes. UN and ICRC/IFRC are in separate categories.",16/03/2017,,272,International Organization,Broader
439,Media,"Includes local, national and international print, TV, online and radio media agencies",16/03/2017,,273,Media,Exact Match
440,Military,Includes National and International military and military-related organisations,16/03/2017,,,,
441,National NGO,"Includes national and subnational organizations that operate independently from any government, including civil society. Red Cross/Red Crescent Societies are in a separate category.",16/03/2017,,274,Non-governmental Organization,Broader
443,Other,Includes all organizations that are not applicable to other organization types,16/03/2017,,275,Other,Broader
444,Private sector,"Includes private sector organizations such as corporate charity arms or foundations, for-profit recruitment firms and consultancies",16/03/2017,,,,
445,Red Cross / Red Crescent,"Includes IFRC, ICRC and national Red Cross/Crescent societies",16/03/2017,,276,Red Cross/Red Crescent Movement,Exact Match
446,Religious,Includes faith-based organizations and organizations with affiliations to religious bodies,16/03/2017,,,,
447,United Nations,"Includes all UN agencies, funds and programmes",16/03/2017,,272,International Organization,Broader
Original file line number Diff line number Diff line change
@@ -1,17 +1,17 @@
HRinfo ID,Preferred Term,Preferred Term (fr),ACRONYM,Group Type,Homepage,Date Created (Post Date),Updated date,Notes
#sector +code +v_hrinfo_sector,#sector +name +preferred +i_en,#sector +name +preferred +i_fr,#sector +code +acronym,#sector +type,#meta +url,#date +created,#date +updated,#meta +comment
4,Emergency Shelter and NFI,Abris d'urgence et NFI,SHL,Cluster,https://www.sheltercluster.org/,03/16/2017,4/4/2018,"Added French name; shelter cluster uses ""NFI"" in French as well"
1,Camp Coordination / Management,Coordination et gestion des camps,CCM,Cluster,http://www.globalcccmcluster.org,03/16/2017,4/4/2018,Added French name
5406,Mine Action,Action contre les mines,PRO-MIN,Area of Responsibility (Sub-cluster),http://www.globalprotectioncluster.org/en/areas-of-responsibility/mine-action.html,03/16/2017,4/4/2018,Added French name
6,Food Security,Sécurité alimentaire,FSC,Cluster,http://foodsecuritycluster.net/,03/16/2017,4/4/2018,Added French name
11,Water Sanitation Hygiene,"Eau, assainissement et hygiène",WSH,Cluster,http://washcluster.net/,03/16/2017,4/4/2018,Added French name
8,Logistics,Logistique,LOG,Cluster,http://www.logcluster.org/,03/16/2017,4/4/2018,Added French name
5403,Child Protection,Protection de l'enfance,PRO-CPN,Area of Responsibility (Sub-cluster),http://cpwg.net/,03/16/2017,4/5/2018,Updated French name based feedback from OCHA-FIS
10,Protection,Protection,PRO,Cluster,http://www.globalprotectioncluster.org/,03/16/2017,4/4/2018,Added French name
3,Education,Education,EDU,Cluster,http://educationcluster.net/,03/16/2017,4/4/2018,Added French name
9,Nutrition,Nutrition,NUT,Cluster,http://nutritioncluster.net/,03/16/2017,4/4/2018,Added French name
7,Health,Santé,HEA,Cluster,http://www.who.int/health-cluster/en/,03/16/2017,4/4/2018,Added French name
2,Early Recovery,Relèvement rapide,ERY,Cluster,http://earlyrecovery.global/,03/16/2017,4/4/2018,Added French name
5,Emergency Telecommunications,Télécommunications d'urgence,TEL,Cluster,https://www.etcluster.org/,03/16/2017,4/4/2018,Added French name
5404,Gender Based Violence,Violences basées sur le genre,PRO-GBV,Area of Responsibility (Sub-cluster),http://gbvaor.net/,03/16/2017,4/5/2018,Updated French name based feedback from OCHA-FIS
HRinfo ID,Preferred Term,Preferred Term (fr),ACRONYM,Group Type,Homepage,Date Created (Post Date),Updated date,Notes
#sector +code +v_hrinfo_sector,#sector +name +preferred +i_en,#sector +name +preferred +i_fr,#sector +code +acronym,#sector +type,#meta +url,#date +created,#date +updated,#meta +comment
4,Emergency Shelter and NFI,Abris d'urgence et NFI,SHL,Cluster,https://www.sheltercluster.org/,03/16/2017,4/4/2018,"Added French name; shelter cluster uses ""NFI"" in French as well"
1,Camp Coordination / Management,Coordination et gestion des camps,CCM,Cluster,http://www.globalcccmcluster.org,03/16/2017,4/4/2018,Added French name
5406,Mine Action,Action contre les mines,PRO-MIN,Area of Responsibility (Sub-cluster),http://www.globalprotectioncluster.org/en/areas-of-responsibility/mine-action.html,03/16/2017,4/4/2018,Added French name
6,Food Security,Sécurité alimentaire,FSC,Cluster,http://foodsecuritycluster.net/,03/16/2017,4/4/2018,Added French name
11,Water Sanitation Hygiene,"Eau, assainissement et hygiène",WSH,Cluster,http://washcluster.net/,03/16/2017,4/4/2018,Added French name
8,Logistics,Logistique,LOG,Cluster,http://www.logcluster.org/,03/16/2017,4/4/2018,Added French name
5403,Child Protection,Protection de l'enfance,PRO-CPN,Area of Responsibility (Sub-cluster),http://cpwg.net/,03/16/2017,4/5/2018,Updated French name based feedback from OCHA-FIS
10,Protection,Protection,PRO,Cluster,http://www.globalprotectioncluster.org/,03/16/2017,4/4/2018,Added French name
3,Education,Education,EDU,Cluster,http://educationcluster.net/,03/16/2017,4/4/2018,Added French name
9,Nutrition,Nutrition,NUT,Cluster,http://nutritioncluster.net/,03/16/2017,4/4/2018,Added French name
7,Health,Santé,HEA,Cluster,http://www.who.int/health-cluster/en/,03/16/2017,4/4/2018,Added French name
2,Early Recovery,Relèvement rapide,ERY,Cluster,http://earlyrecovery.global/,03/16/2017,4/4/2018,Added French name
5,Emergency Telecommunications,Télécommunications d'urgence,TEL,Cluster,https://www.etcluster.org/,03/16/2017,4/4/2018,Added French name
5404,Gender Based Violence,Violences basées sur le genre,PRO-GBV,Area of Responsibility (Sub-cluster),http://gbvaor.net/,03/16/2017,4/5/2018,Updated French name based feedback from OCHA-FIS
5405,"Housing, Land and Property","Logement, terre, et biens",PRO-HLP,Area of Responsibility (Sub-cluster),http://www.globalprotectioncluster.org/en/areas-of-responsibility/housing-land-and-property.html,03/16/2017,4/4/2018,Added French name from 2012 Côte d'Ivoire workshop report
2 changes: 1 addition & 1 deletion tests/test_main.py
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ def test_pipelines(self, configuration, folder):
count = session.scalar(
select(func.count(DBFoodSecurity.id))
)
assert count == 43463
assert count == 103264

org_mapping = pipelines.org._org_lookup
assert org_mapping["Action against Hunger"] == {
Expand Down

0 comments on commit 2625f15

Please sign in to comment.