Run Pipelines On Schedule #2
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Manually Run Pipelines | |
| on: | |
| # Only trigger workflow manually from GitHub UI | |
| schedule: | |
| - cron: '0 0 1 * *' | |
| workflow_dispatch: | |
| # Optional: Add input parameters that can be set when triggering the workflow | |
| inputs: | |
| script_name: | |
| description: 'Specific script to run (leave empty to run all)' | |
| required: false | |
| type: string | |
| logLevel: | |
| description: 'Log level' | |
| required: true | |
| default: 'warning' | |
| type: choice | |
| options: | |
| - info | |
| - warning | |
| - debug | |
| tags: | |
| description: 'Runs Data Pipelines on command' | |
| required: false | |
| type: boolean | |
| python_version: | |
| description: 'Python version to use' | |
| required: false | |
| default: '3.12.4' | |
| type: string | |
| jobs: | |
| run-scripts: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v3 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ inputs.python_version }} | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| if [ -f requirements.txt ]; then pip install -r requirements.txt; fi | |
| - name: Run Python scripts | |
| run: | | |
| echo "Starting to run Python scripts (excluding utils.py)..." | |
| for script in python_scripts/*.py; do | |
| filename=$(basename "$script") | |
| if [ "$filename" != "utils.py" ]; then | |
| echo "----------------------------------------" | |
| echo "Running $script..." | |
| python "$script" | |
| echo "Finished running $script" | |
| else | |
| echo "Skipping utils.py" | |
| fi | |
| done | |
| echo "----------------------------------------" | |
| echo "All scripts completed" | |
| - name: Report execution status | |
| if: always() | |
| run: | | |
| echo "Workflow execution completed" | |
| # List all scripts that were run | |
| echo "Scripts processed:" | |
| for script in python_scripts/*.py; do | |
| filename=$(basename "$script") | |
| if [ "$filename" != "utils.py" ]; then | |
| echo "- $filename" | |
| fi | |
| done |