Add GH Actions run script

Disable country dataset creation
OCHA-DAP · Nov 18, 2024 · e941129 · e941129
1 parent 819add0
commit e941129
Show file tree

Hide file tree

Showing 3 changed files with 65 additions and 19 deletions.
diff --git a/.github/workflows/run-python-script.yaml b/.github/workflows/run-python-script.yaml
@@ -0,0 +1,44 @@
+# This workflow will install Python dependencies and run the script
+
+name: Run script
+
+on:
+  workflow_dispatch: # add run button in github
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.x
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.x"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        pip install .
+    - name: Run script
+      env:
+        HDX_SITE: ${{ vars.HDX_SITE }}
+        HDX_KEY: ${{ secrets.HDX_BOT_SCRAPERS_API_TOKEN }}
+        PREPREFIX: ${{ vars.HDX_USER_AGENT_PREPREFIX }}
+        USER_AGENT: ${{ secrets.USER_AGENT }}
+        EXTRA_PARAMS: ${{ secrets.EXTRA_PARAMS }}
+      run: |
+        python -m hdx.scraper.ophi
+    - name: Send mail
+      if: failure()
+      uses: dawidd6/action-send-mail@v3
+      with:
+        server_address: ${{secrets.HDX_PIPELINE_EMAIL_SERVER}}
+        server_port: ${{secrets.HDX_PIPELINE_EMAIL_PORT}}
+        username: ${{secrets.HDX_PIPELINE_EMAIL_USERNAME}}
+        password: ${{secrets.HDX_PIPELINE_EMAIL_PASSWORD}}
+        subject: "FAILED: ${{github.repository}} run job"
+        body: GitHub Actions run job for ${{github.repository}} failed!
+        to: ${{vars.HDX_PIPELINE_EMAIL_LIST}}
+        from: ${{secrets.HDX_PIPELINE_EMAIL_FROM}}
+        content_type: text/html
diff --git a/requirements.txt b/requirements.txt
@@ -44,7 +44,7 @@ frictionless==5.18.0
     # via hdx-python-utilities
 hdx-python-api==6.3.5
     # via hdx-scraper-ophi (pyproject.toml)
-hdx-python-country==3.8.3
+hdx-python-country==3.8.4
     # via
     #   hdx-scraper-ophi (pyproject.toml)
     #   hdx-python-api
@@ -213,7 +213,7 @@ text-unidecode==1.3
     # via python-slugify
 typeguard==4.4.1
     # via inflect
-typer==0.13.0
+typer==0.13.1
     # via frictionless
 typing-extensions==4.12.2
     # via

diff --git a/src/hdx/scraper/ophi/__main__.py b/src/hdx/scraper/ophi/__main__.py
@@ -25,6 +25,7 @@
 lookup = "hdx-scraper-ophi"
 updated_by_script = "HDX Scraper: OPHI"
 
+create_country_datasets = False
 
 def main(
     save: bool = False,
@@ -97,24 +98,25 @@ def update_dataset(dataset):
             dataset.add_country_locations(list(standardised_countries.keys()))
             update_dataset(dataset)
 
-            for (
-                countryiso3,
-                standardised_country,
-            ) in standardised_countries.items():
-                countryname = Country.get_country_name_from_iso3(countryiso3)
-                standardised_country_trend = standardised_countries_trend.get(
-                    countryiso3
-                )
-                dataset = dataset_generator.generate_dataset(
-                    folder,
-                    standardised_country,
-                    standardised_country_trend,
+            if create_country_datasets:
+                for (
                     countryiso3,
-                    countryname,
-                    date_ranges[countryiso3],
-                )
-                dataset.add_country_location(countryiso3)
-                update_dataset(dataset)
+                    standardised_country,
+                ) in standardised_countries.items():
+                    countryname = Country.get_country_name_from_iso3(countryiso3)
+                    standardised_country_trend = standardised_countries_trend.get(
+                        countryiso3
+                    )
+                    dataset = dataset_generator.generate_dataset(
+                        folder,
+                        standardised_country,
+                        standardised_country_trend,
+                        countryiso3,
+                        countryname,
+                        date_ranges[countryiso3],
+                    )
+                    dataset.add_country_location(countryiso3)
+                    update_dataset(dataset)
 
     logger.info("HDX Scraper OPHI pipeline completed!")