diff --git a/.github/workflows/run-python-script.yaml b/.github/workflows/run-python-script.yaml new file mode 100644 index 0000000..b9ff694 --- /dev/null +++ b/.github/workflows/run-python-script.yaml @@ -0,0 +1,44 @@ +# This workflow will install Python dependencies and run the script + +name: Run script + +on: + workflow_dispatch: # add run button in github + +jobs: + run: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.x + uses: actions/setup-python@v5 + with: + python-version: "3.x" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + pip install . + - name: Run script + env: + HDX_SITE: ${{ vars.HDX_SITE }} + HDX_KEY: ${{ secrets.HDX_BOT_SCRAPERS_API_TOKEN }} + PREPREFIX: ${{ vars.HDX_USER_AGENT_PREPREFIX }} + USER_AGENT: ${{ secrets.USER_AGENT }} + EXTRA_PARAMS: ${{ secrets.EXTRA_PARAMS }} + run: | + python -m hdx.scraper.ophi + - name: Send mail + if: failure() + uses: dawidd6/action-send-mail@v3 + with: + server_address: ${{secrets.HDX_PIPELINE_EMAIL_SERVER}} + server_port: ${{secrets.HDX_PIPELINE_EMAIL_PORT}} + username: ${{secrets.HDX_PIPELINE_EMAIL_USERNAME}} + password: ${{secrets.HDX_PIPELINE_EMAIL_PASSWORD}} + subject: "FAILED: ${{github.repository}} run job" + body: GitHub Actions run job for ${{github.repository}} failed! + to: ${{vars.HDX_PIPELINE_EMAIL_LIST}} + from: ${{secrets.HDX_PIPELINE_EMAIL_FROM}} + content_type: text/html diff --git a/requirements.txt b/requirements.txt index 33ee4ca..8a05d49 100644 --- a/requirements.txt +++ b/requirements.txt @@ -44,7 +44,7 @@ frictionless==5.18.0 # via hdx-python-utilities hdx-python-api==6.3.5 # via hdx-scraper-ophi (pyproject.toml) -hdx-python-country==3.8.3 +hdx-python-country==3.8.4 # via # hdx-scraper-ophi (pyproject.toml) # hdx-python-api @@ -213,7 +213,7 @@ text-unidecode==1.3 # via python-slugify typeguard==4.4.1 # via inflect -typer==0.13.0 +typer==0.13.1 # via frictionless typing-extensions==4.12.2 # via diff --git a/src/hdx/scraper/ophi/__main__.py b/src/hdx/scraper/ophi/__main__.py index f7d5184..708f632 100755 --- a/src/hdx/scraper/ophi/__main__.py +++ b/src/hdx/scraper/ophi/__main__.py @@ -25,6 +25,7 @@ lookup = "hdx-scraper-ophi" updated_by_script = "HDX Scraper: OPHI" +create_country_datasets = False def main( save: bool = False, @@ -97,24 +98,25 @@ def update_dataset(dataset): dataset.add_country_locations(list(standardised_countries.keys())) update_dataset(dataset) - for ( - countryiso3, - standardised_country, - ) in standardised_countries.items(): - countryname = Country.get_country_name_from_iso3(countryiso3) - standardised_country_trend = standardised_countries_trend.get( - countryiso3 - ) - dataset = dataset_generator.generate_dataset( - folder, - standardised_country, - standardised_country_trend, + if create_country_datasets: + for ( countryiso3, - countryname, - date_ranges[countryiso3], - ) - dataset.add_country_location(countryiso3) - update_dataset(dataset) + standardised_country, + ) in standardised_countries.items(): + countryname = Country.get_country_name_from_iso3(countryiso3) + standardised_country_trend = standardised_countries_trend.get( + countryiso3 + ) + dataset = dataset_generator.generate_dataset( + folder, + standardised_country, + standardised_country_trend, + countryiso3, + countryname, + date_ranges[countryiso3], + ) + dataset.add_country_location(countryiso3) + update_dataset(dataset) logger.info("HDX Scraper OPHI pipeline completed!")