Add p-code check

OCHA-DAP · Feb 4, 2025 · 53ab7ec · 53ab7ec
1 parent b76f50c
commit 53ab7ec
Show file tree

Hide file tree

Showing 9 changed files with 40,249 additions and 6,580 deletions.
diff --git a/.config/ruff.toml b/.config/ruff.toml
@@ -1,4 +1,4 @@
-line-length = 79
+line-length = 90
 exclude = ["_version.py"]
 
 [lint]

diff --git a/src/hdx/scraper/dtm/__main__.py b/src/hdx/scraper/dtm/__main__.py
@@ -9,6 +9,7 @@
 from os.path import dirname, expanduser, join
 
 from hdx.api.configuration import Configuration
+from hdx.api.utilities.hdx_error_handler import HDXErrorHandler
 from hdx.facades.infer_arguments import facade
 from hdx.utilities.downloader import Download
 from hdx.utilities.path import (
@@ -29,79 +30,82 @@
 def main(
     save: bool = True,
     use_saved: bool = False,
+    err_to_hdx: bool = False,
 ) -> None:
     """Generate datasets and create them in HDX
 
     Args:
         save (bool): Save downloaded data. Defaults to True.
         use_saved (bool): Use saved data. Defaults to False.
+        err_to_hdx (bool): Whether to write any errors to HDX metadata. Defaults to False.
 
     Returns:
         None
     """
-    with wheretostart_tempdir_batch(folder=_USER_AGENT_LOOKUP) as info:
-        temp_dir = info["folder"]
-        with Download() as downloader:
-            configuration = Configuration.read()
-            retriever = Retrieve(
-                downloader=downloader,
-                fallback_dir=temp_dir,
-                saved_dir=_SAVED_DATA_DIR,
-                temp_dir=temp_dir,
-                save=save,
-                use_saved=use_saved,
-            )
-            dtm = Dtm(
-                configuration=configuration,
-                retriever=retriever,
-                temp_dir=temp_dir,
-            )
-
-            countries_list = dtm.get_countries()
-            operation_status = dtm.get_operation_status()
-            for countries in [
-                countries_list,
-                *[[x] for x in countries_list],
-            ]:
-                dataset = dtm.generate_dataset(
-                    countries=countries, operation_status=operation_status
+    with HDXErrorHandler(write_to_hdx=err_to_hdx) as error_handler:
+        with wheretostart_tempdir_batch(folder=_USER_AGENT_LOOKUP) as info:
+            temp_dir = info["folder"]
+            with Download() as downloader:
+                configuration = Configuration.read()
+                retriever = Retrieve(
+                    downloader=downloader,
+                    fallback_dir=temp_dir,
+                    saved_dir=_SAVED_DATA_DIR,
+                    temp_dir=temp_dir,
+                    save=save,
+                    use_saved=use_saved,
                 )
-                dataset.update_from_yaml(
-                    path=join(
-                        dirname(__file__), "config", "hdx_dataset_static.yaml"
-                    )
+                dtm = Dtm(
+                    configuration=configuration,
+                    retriever=retriever,
+                    temp_dir=temp_dir,
+                    error_handler=error_handler,
                 )
-                if len(countries) > 1:
-                    dataset.generate_quickcharts(
-                        resource=1,
-                        path=script_dir_plus_file(
-                            join("config", "hdx_resource_view_static.yaml"),
-                            main,
-                        ),
+
+                countries_list = dtm.get_countries()
+                operation_status = dtm.get_operation_status()
+                for countries in [
+                    countries_list,
+                    *[[x] for x in countries_list],
+                ]:
+                    dataset = dtm.generate_dataset(
+                        countries=countries, operation_status=operation_status
                     )
-                dataset.create_in_hdx(
-                    remove_additional_resources=True,
-                    match_resource_order=False,
-                    hxl_update=False,
-                    updated_by_script=_UPDATED_BY_SCRIPT,
-                    batch=info["batch"],
-                )
-                if len(countries) > 1:
-                    hapi_dataset = dtm.generate_hapi_dataset(dataset["name"])
-                    hapi_dataset.update_from_yaml(
-                        path=join(
-                            dirname(__file__),
-                            "config",
-                            "hdx_hapi_dataset_static.yaml",
-                        )
+                    dataset.update_from_yaml(
+                        path=join(dirname(__file__), "config", "hdx_dataset_static.yaml")
                     )
-                    hapi_dataset.create_in_hdx(
+                    if len(countries) > 1:
+                        dataset.generate_quickcharts(
+                            resource=1,
+                            path=script_dir_plus_file(
+                                join("config", "hdx_resource_view_static.yaml"),
+                                main,
+                            ),
+                        )
+                    dataset.create_in_hdx(
                         remove_additional_resources=True,
                         match_resource_order=False,
                         hxl_update=False,
                         updated_by_script=_UPDATED_BY_SCRIPT,
                         batch=info["batch"],
                     )
+                    if len(countries) > 1:
+                        hapi_dataset = dtm.generate_hapi_dataset(dataset["name"])
+                        hapi_dataset.update_from_yaml(
+                            path=join(
+                                dirname(__file__),
+                                "config",
+                                "hdx_hapi_dataset_static.yaml",
+                            )
+                        )
+                        hapi_dataset.create_in_hdx(
+                            remove_additional_resources=True,
+                            match_resource_order=False,
+                            hxl_update=False,
+                            updated_by_script=_UPDATED_BY_SCRIPT,
+                            batch=info["batch"],
+                        )
+    logger.info("HDX Scraper DTM pipeline completed!")
 
 
 if __name__ == "__main__":

diff --git a/src/hdx/scraper/dtm/config/hdx_hapi_dataset_static.yaml b/src/hdx/scraper/dtm/config/hdx_hapi_dataset_static.yaml
@@ -11,7 +11,7 @@ license_other: >
   as well as the following credit line: Source: “International Organization for
   Migration (IOM), Displacement Tracking Matrix (DTM)”.
 methodology: Registry
-caveats: None
+caveats: ""
 dataset_source: International Organization for Migration (IOM)
 package_creator: HDX Data Systems Team
 private: False

diff --git a/src/hdx/scraper/dtm/config/project_configuration.yaml b/src/hdx/scraper/dtm/config/project_configuration.yaml
@@ -53,10 +53,12 @@ hapi_hxl_tags:
   location_code: "#country+code"
   has_hrp: "#meta+has_hrp"
   in_gho: "#meta+in_gho"
-  admin1_name: "#adm1+name"
+  provider_admin1_name: "#adm1+name+provider"
+  provider_admin2_name: "#adm2+name+provider"
   admin1_code: "#adm1+code"
-  admin2_name: "#adm2+name"
+  admin1_name: "#adm1+name"
   admin2_code: "#adm2+code"
+  admin2_name: "#adm2+name"
   admin_level: "#adm+level"
   operation: "#operation+name"
   assessment_type: "#assessment+type"