Add methodology notes

OCHA-DAP · Jan 23, 2025 · 182a61b · 182a61b
1 parent 5cbfb5d
commit 182a61b
Show file tree

Hide file tree

Showing 7 changed files with 51 additions and 33 deletions.
diff --git a/.config/pre-commit-config.yaml b/.config/pre-commit-config.yaml
@@ -8,7 +8,7 @@ repos:
       - id: end-of-file-fixer
       - id: check-ast
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.7.0
+    rev: v0.8.6
     hooks:
       # Run the linter.
       - id: ruff
@@ -17,7 +17,7 @@ repos:
       - id: ruff-format
         args: [--config, .config/ruff.toml]
   - repo: https://github.com/astral-sh/uv-pre-commit
-    rev: 0.4.24
+    rev: 0.5.15
     hooks:
       # Run the pip compile
       - id: pip-compile

diff --git a/pyproject.toml b/pyproject.toml
@@ -35,8 +35,8 @@ requires-python = ">=3.8"
 
 dependencies = [
     "hdx-python-api>= 6.3.5",
-    "hdx-python-country>= 3.8.5",
-    "hdx-python-utilities>= 3.7.4",
+    "hdx-python-country>= 3.8.7",
+    "hdx-python-utilities>= 3.8.2",
 ]
 dynamic = ["version"]
 
@@ -94,7 +94,7 @@ run = """
        """
 
 [tool.hatch.envs.hatch-static-analysis]
-dependencies = ["ruff==0.7.0"]
+dependencies = ["ruff==0.8.6"]
 
 [tool.hatch.envs.hatch-static-analysis.scripts]
 format-check = ["ruff format --config .config/ruff.toml --check --diff {args:.}",]

diff --git a/requirements.txt b/requirements.txt
@@ -2,25 +2,25 @@
 #    uv pip compile pyproject.toml --resolver=backtracking --all-extras -o requirements.txt
 annotated-types==0.7.0
     # via pydantic
-attrs==24.2.0
+attrs==24.3.0
     # via
     #   frictionless
     #   jsonlines
     #   jsonschema
     #   referencing
-certifi==2024.8.30
+certifi==2024.12.14
     # via requests
 cfgv==3.4.0
     # via pre-commit
 chardet==5.2.0
     # via frictionless
-charset-normalizer==3.4.0
+charset-normalizer==3.4.1
     # via requests
 ckanapi==4.8
     # via hdx-python-api
-click==8.1.7
+click==8.1.8
     # via typer
-coverage==7.6.8
+coverage==7.6.10
     # via pytest-cov
 defopt==6.4.0
     # via hdx-python-api
@@ -38,38 +38,38 @@ email-validator==2.2.0
     # via hdx-python-api
 et-xmlfile==2.0.0
     # via openpyxl
-filelock==3.16.1
+filelock==3.17.0
     # via virtualenv
 frictionless==5.18.0
     # via hdx-python-utilities
-hdx-python-api==6.3.5
+hdx-python-api==6.3.7
     # via hdx-scraper-ophi (pyproject.toml)
-hdx-python-country==3.8.5
+hdx-python-country==3.8.7
     # via
     #   hdx-scraper-ophi (pyproject.toml)
     #   hdx-python-api
-hdx-python-utilities==3.7.4
+hdx-python-utilities==3.8.2
     # via
     #   hdx-scraper-ophi (pyproject.toml)
     #   hdx-python-api
     #   hdx-python-country
 humanize==4.11.0
     # via frictionless
-identify==2.6.3
+identify==2.6.6
     # via pre-commit
 idna==3.10
     # via
     #   email-validator
     #   requests
 ijson==3.3.0
     # via hdx-python-utilities
-inflect==7.4.0
+inflect==7.5.0
     # via quantulum3
 iniconfig==2.0.0
     # via pytest
 isodate==0.7.2
     # via frictionless
-jinja2==3.1.4
+jinja2==3.1.5
     # via frictionless
 jsonlines==4.0.0
     # via hdx-python-utilities
@@ -85,7 +85,7 @@ libhxl==5.2.2
     # via
     #   hdx-python-api
     #   hdx-python-country
-loguru==0.7.2
+loguru==0.7.3
     # via hdx-python-utilities
 makefun==1.15.6
     # via hdx-python-api
@@ -97,11 +97,11 @@ markupsafe==3.0.2
     # via jinja2
 mdurl==0.1.2
     # via markdown-it-py
-more-itertools==10.5.0
+more-itertools==10.6.0
     # via inflect
 nodeenv==1.9.1
     # via pre-commit
-num2words==0.5.13
+num2words==0.5.14
     # via quantulum3
 openpyxl==3.1.5
     # via hdx-python-utilities
@@ -119,16 +119,16 @@ ply==3.11
     #   libhxl
 pockets==0.9.1
     # via sphinxcontrib-napoleon
-pre-commit==4.0.1
+pre-commit==4.1.0
     # via hdx-scraper-ophi (pyproject.toml)
-pydantic==2.10.3
+pydantic==2.10.5
     # via frictionless
-pydantic-core==2.27.1
+pydantic-core==2.27.2
     # via pydantic
-pygments==2.18.0
+pygments==2.19.1
     # via rich
 pyphonetics==0.5.3
-    # via hdx-python-country
+    # via hdx-python-utilities
 pytest==8.3.4
     # via
     #   hdx-scraper-ophi (pyproject.toml)
@@ -158,7 +158,7 @@ quantulum3==0.9.2
     # via hdx-python-api
 ratelimit==2.2.1
     # via hdx-python-utilities
-referencing==0.35.1
+referencing==0.36.1
     # via
     #   jsonschema
     #   jsonschema-specifications
@@ -179,11 +179,11 @@ rpds-py==0.22.3
     # via
     #   jsonschema
     #   referencing
-ruamel-yaml==0.18.6
+ruamel-yaml==0.18.10
     # via hdx-python-utilities
 ruamel-yaml-clib==0.2.12
     # via ruamel-yaml
-setuptools==75.6.0
+setuptools==75.8.0
     # via ckanapi
 shellingham==1.5.4
     # via typer
@@ -201,7 +201,7 @@ sphinxcontrib-napoleon==0.7
     # via defopt
 stringcase==1.2.0
     # via frictionless
-structlog==24.4.0
+structlog==25.1.0
     # via libhxl
 tableschema-to-template==0.0.13
     # via hdx-python-utilities
@@ -220,19 +220,20 @@ typing-extensions==4.12.2
     #   frictionless
     #   pydantic
     #   pydantic-core
+    #   referencing
     #   typeguard
     #   typer
 unidecode==1.3.8
     # via
     #   libhxl
     #   pyphonetics
-urllib3==2.2.3
+urllib3==2.3.0
     # via
     #   libhxl
     #   requests
 validators==0.34.0
     # via frictionless
-virtualenv==20.28.0
+virtualenv==20.29.1
     # via pre-commit
 wheel==0.45.1
     # via libhxl
@@ -242,7 +243,7 @@ xlrd3==1.1.0
     # via libhxl
 xlsx2csv==0.8.4
     # via hdx-python-utilities
-xlsxwriter==3.2.0
+xlsxwriter==3.2.1
     # via tableschema-to-template
 xlwt==1.3.0
     # via hdx-python-utilities
diff --git a/src/hdx/scraper/ophi/config/hdx_dataset_static.yaml b/src/hdx/scraper/ophi/config/hdx_dataset_static.yaml
@@ -3,7 +3,6 @@ notes: "The index provides the only comprehensive measure available for non-inco
 The resources subnational multidimensional poverty data from the [data tables](https://ophi.org.uk/multidimensional-poverty-index/data-tables-do-files/) published by the Oxford Poverty and Human Development Initiative (OPHI), University of Oxford. The global Multidimensional Poverty Index (MPI) measures multidimensional poverty in over 100 developing countries, using internationally comparable datasets and is updated annually. The measure captures the severe deprivations that each person faces at the same time using information from 10 indicators, which are grouped into three equally weighted dimensions: health, education, and living standards. The global MPI methodology is detailed in Alkire, Kanagaratnam & Suppa [(2023)](https://ophi.org.uk/sites/default/files/2024-03/OPHI_MPI_MN57_2023.pdf)"
 license_id: "other-pd-nr"
 methodology: "Other"
-methodology_other: "The global MPI is a leading policy tool that applies the multidimensional poverty methodology developed by Alkire and Foster [(2011)](https://www.sciencedirect.com/science/article/abs/pii/S0047272710001660?via%3Dihub). The global MPI is the product of incidence of poverty (H) and the average intensity of poverty (A)."
 dataset_source: "Alkire, S., Kanagaratnam, U., and Suppa, N. (2023). ‘The global Multidimensional Poverty Index (MPI) 2023 disaggregation results and methodological note’, OPHI MPI Methodological Note 56, Oxford Poverty and Human Development Initiative (OPHI), University of Oxford."
 caveats: ""
 private: False  # has to be True or False
diff --git a/src/hdx/scraper/ophi/config/project_configuration.yaml b/src/hdx/scraper/ophi/config/project_configuration.yaml
@@ -36,6 +36,12 @@ hxltags:
   "reference_period_start": "#date+start"
   "reference_period_end": "#date+end"
 
+methodology:
+  text: "The global MPI is a leading policy tool that applies the multidimensional poverty methodology developed by Alkire and Foster [(2011)](https://www.sciencedirect.com/science/article/abs/pii/S0047272710001660?via%3Dihub). The global MPI is the product of incidence of poverty (H) and the average intensity of poverty (A). More information on methodology can be found here: "
+  mpi_and_partial_indices: "https://ophi.org.uk/publications/MN-59"
+  trend_over_time: "https://ophi.org.uk/publications/MN-60"
+
+
 resource_descriptions:
   standardised_mpi: "This resource contains standardised MPI estimates by admin one unit and also shows the proportion of people who are MPI poor and experience deprivations in each of the indicators by admin one unit."
   standardised_trends: "This resource contains standardised MPI estimates and their changes over time by admin one unit and also shows the proportion of people who are MPI poor and experience deprivations in each of the indicators by admin one unit."

diff --git a/src/hdx/scraper/ophi/dataset_generator.py b/src/hdx/scraper/ophi/dataset_generator.py
@@ -144,6 +144,7 @@ def generate_dataset(
         name = self.get_name(countryname)
         dataset = self.generate_dataset_metadata(title, name)
         dataset.set_time_period(date_range["start"], date_range["end"])
+        methodology_info = self._configuration["methodology"]
         resource_descriptions = self._configuration["resource_descriptions"]
 
         resource_name = f"{countryname} MPI and Partial Indices"
@@ -169,6 +170,13 @@ def generate_dataset(
             logger.warning(f"{name} has no data!")
             return None
 
+        methodology_text = methodology_info["text"]
+        mpi_and_partial_indices_methodology = methodology_info[
+            "mpi_and_partial_indices"
+        ]
+        dataset["methodology_other"] = (
+            f"{methodology_text} {mpi_and_partial_indices_methodology}"
+        )
         if not standardised_trend_rows:
             return dataset
         resource_name = f"{countryname} MPI Trends Over Time"
@@ -190,6 +198,8 @@ def generate_dataset(
             folder,
             filename,
         )
+        trend_over_time_methodology = methodology_info["trend_over_time"]
+        dataset["methodology_other"] += f", {trend_over_time_methodology}"
         return dataset
 
     def generate_global_dataset(

diff --git a/tests/test_main.py b/tests/test_main.py
@@ -228,6 +228,7 @@ def test_main(
                     "data_update_frequency": "365",
                     "dataset_date": "[2001-01-01T00:00:00 TO 2023-12-31T23:59:59]",
                     "maintainer": "196196be-6037-4488-8b71-d786adf4c081",
+                    "methodology_other": "The global MPI is a leading policy tool that applies the multidimensional poverty methodology developed by Alkire and Foster [(2011)](https://www.sciencedirect.com/science/article/abs/pii/S0047272710001660?via%3Dihub). The global MPI is the product of incidence of poverty (H) and the average intensity of poverty (A). More information on methodology can be found here:  https://ophi.org.uk/publications/MN-59, https://ophi.org.uk/publications/MN-60",
                     "name": "global-mpi",
                     "owner_org": "00547685-9ded-4d69-9ca5-47d5278ead7c",
                     "subnational": "1",
@@ -349,6 +350,7 @@ def test_main(
                     "data_update_frequency": "365",
                     "dataset_date": "[2015-01-01T00:00:00 TO 2023-12-31T23:59:59]",
                     "maintainer": "196196be-6037-4488-8b71-d786adf4c081",
+                    "methodology_other": "The global MPI is a leading policy tool that applies the multidimensional poverty methodology developed by Alkire and Foster [(2011)](https://www.sciencedirect.com/science/article/abs/pii/S0047272710001660?via%3Dihub). The global MPI is the product of incidence of poverty (H) and the average intensity of poverty (A). More information on methodology can be found here:  https://ophi.org.uk/publications/MN-59, https://ophi.org.uk/publications/MN-60",
                     "name": "afghanistan-mpi",
                     "owner_org": "00547685-9ded-4d69-9ca5-47d5278ead7c",
                     "subnational": "1",