Run Pipelines On Schedule #7
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run Pipelines On Schedule | |
| on: | |
| # Only trigger workflow manually from GitHub UI | |
| schedule: | |
| - cron: '0 0 1 * *' | |
| workflow_dispatch: | |
| # Optional: Add input parameters that can be set when triggering the workflow | |
| inputs: | |
| script_name: | |
| description: 'Specific script to run (leave empty to run all)' | |
| required: false | |
| type: string | |
| logLevel: | |
| description: 'Log level' | |
| required: true | |
| default: 'warning' | |
| type: choice | |
| options: | |
| - info | |
| - warning | |
| - debug | |
| tags: | |
| description: 'Runs Data Pipelines on command' | |
| required: false | |
| type: boolean | |
| python_version: | |
| description: 'Python version to use' | |
| required: false | |
| default: '3.12.4' | |
| type: string | |
| env: | |
| CENSUS_API_KEY: ${{ secrets.CENSUS_API_KEY }} | |
| FRED_API_KEY: ${{ secrets.FRED_API_KEY }} | |
| MOTHERDUCK_TOKEN: ${{ secrets.MOTHERDUCK_TOKEN }} | |
| jobs: | |
| run-scripts: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v3 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ inputs.python_version }} | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| if [ -f requirements.txt ]; then pip install -r requirements.txt; fi | |
| - name: Run Python scripts | |
| continue-on-error: false | |
| run: | | |
| echo "Starting to run Python scripts (excluding utils.py)..." | |
| for script in python_scripts/*.py; do | |
| filename=$(basename "$script") | |
| if [ "$filename" != "utils.py" ]; then | |
| echo "----------------------------------------" | |
| echo "Running $script..." | |
| python "$script" || echo "Script $filename failed, continuing to next script..." | |
| echo "Finished running $script" | |
| else | |
| echo "Skipping utils.py" | |
| fi | |
| done | |
| echo "----------------------------------------" | |
| echo "All scripts completed" | |
| - name: Debug Environment Variables | |
| if: always() | |
| run: | | |
| echo "Python Version: ${{ inputs.python_version }}" | |
| echo "Log Level: ${{ env.LOG_LEVEL }}" | |
| echo "Has CENSUS_API_KEY: ${{ env.CENSUS_API_KEY != '' }}" | |
| echo "Has FRED_API_KEY: ${{ env.FRED_API_KEY != '' }}" |