Run Pipelines On Schedule #5
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Manually Run Pipelines | |
| on: | |
| # Only trigger workflow manually from GitHub UI | |
| schedule: | |
| - cron: '0 0 1 * *' | |
| workflow_dispatch: | |
| # Optional: Add input parameters that can be set when triggering the workflow | |
| inputs: | |
| script_name: | |
| description: 'Specific script to run (leave empty to run all)' | |
| required: false | |
| type: string | |
| logLevel: | |
| description: 'Log level' | |
| required: true | |
| default: 'warning' | |
| type: choice | |
| options: | |
| - info | |
| - warning | |
| - debug | |
| tags: | |
| description: 'Runs Data Pipelines on command' | |
| required: false | |
| type: boolean | |
| python_version: | |
| description: 'Python version to use' | |
| required: false | |
| default: '3.12.4' | |
| type: string | |
| env: | |
| CENSUS_API_KEY: ${{ secrets.CENSUS_API_KEY }} | |
| FRED_API_KEY: ${{ secrets.FRED_API_KEY }} | |
| jobs: | |
| run-scripts: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v3 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ inputs.python_version }} | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| if [ -f requirements.txt ]; then pip install -r requirements.txt; fi | |
| - name: Run Python scripts | |
| run: | | |
| echo "Starting to run Python scripts (excluding utils.py)..." | |
| # Initialize error log file | |
| echo "Script Execution Log - $(date)" > script_errors.log | |
| # Initialize success/failure counters | |
| successful_scripts=0 | |
| failed_scripts=0 | |
| for script in python_scripts/*.py; do | |
| filename=$(basename "$script") | |
| if [ "$filename" != "utils.py" ]; then | |
| echo "----------------------------------------" | |
| echo "Running $script..." | |
| # Run the script and capture its output and exit status | |
| if python "$script" 2>&1 | tee -a script_output.log; then | |
| echo "✅ Successfully completed: $filename" | |
| ((successful_scripts++)) | |
| else | |
| echo "❌ Failed: $filename" | |
| echo "$(date) - Error in $filename" >> script_errors.log | |
| cat script_output.log >> script_errors.log | |
| echo "----------------------------------------" >> script_errors.log | |
| ((failed_scripts++)) | |
| fi | |
| # Clear the output log for the next script | |
| > script_output.log | |
| echo "Finished running $script" | |
| else | |
| echo "Skipping utils.py" | |
| fi | |
| done | |
| echo "----------------------------------------" | |
| echo "Execution Summary:" | |
| echo "Successful scripts: $successful_scripts" | |
| echo "Failed scripts: $failed_scripts" | |
| # If any scripts failed, display the error log | |
| if [ $failed_scripts -gt 0 ]; then | |
| echo "Error Log:" | |
| cat script_errors.log | |
| fi | |
| - name: Upload error log | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: script-errors-${{ github.run_number }} | |
| path: script_errors.log | |
| retention-days: 7 | |
| - name: Report execution status | |
| if: always() | |
| run: | | |
| echo "Workflow execution completed" | |
| echo "Scripts processed:" | |
| for script in python_scripts/*.py; do | |
| filename=$(basename "$script") | |
| if [ "$filename" != "utils.py" ]; then | |
| echo "- $filename" | |
| fi | |
| done | |
| if [ -f script_errors.log ] && [ -s script_errors.log ]; then | |
| echo "⚠️ Some scripts failed during execution. See error log for details." | |
| exit 1 | |
| else | |
| echo "✅ All scripts completed successfully" | |
| fi | |
| - name: Debug Environment Variables | |
| if: always() | |
| run: | | |
| echo "Python Version: ${{ inputs.python_version }}" | |
| echo "Log Level: ${{ env.LOG_LEVEL }}" | |
| echo "Has CENSUS_API_KEY: ${{ env.CENSUS_API_KEY != '' }}" | |
| echo "Has FRED_API_KEY: ${{ env.FRED_API_KEY != '' }}" |