Skip to content

Commit c228f22

Browse files
egparedesegparedes
authored
build: update infrastructure to use the uv tool (#1813)
Modernize development infrastructure to use newer python tools. The `uv` (https://docs.astral.sh/uv/) tool is used as project management tool, `nox` (https://nox.thea.codes/) replaces `tox` as the project test runner, custom scripts and `cog` snippets are replaced by a central `dev-tasks.py` tool with multiple commands. Additionally, documentation and CI workflows are updated accordingly and some tests which were not properly executed properly are fixed. Other related tasks to enhance the development infrastructure which could be done after this PR is merged are being tracked in issue #1829
1 parent f732f08 commit c228f22

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

51 files changed

+4181
-1361
lines changed

‎.github/workflows/code-quality.yml

+11-7
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,17 @@ jobs:
1313
runs-on: ubuntu-latest
1414
steps:
1515
- uses: actions/checkout@v4
16+
1617
- name: Set up Python
1718
uses: actions/setup-python@v5
1819
with:
19-
python-version: "3.10"
20-
cache: 'pip'
21-
cache-dependency-path: |
22-
**/pyproject.toml
23-
**/constraints.txt
24-
**/requirements-dev.txt
25-
- uses: pre-commit/[email protected]
20+
python-version-file: ".python-version"
21+
22+
- name: Install uv
23+
uses: astral-sh/setup-uv@v5
24+
with:
25+
enable-cache: true
26+
cache-dependency-glob: "uv.lock"
27+
28+
- name: "Run pre-commit"
29+
uses: pre-commit/[email protected]

‎.github/workflows/daily-ci.yml

+54-72
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,8 @@ on:
55
- cron: '0 4 * * *'
66
workflow_dispatch:
77

8-
## COMMENTED OUT: only for testing CI action changes
8+
## COMMENTED OUT: only for testing CI action changes.
9+
## It only works for PRs to `main` branch from branches in the upstream gt4py repo.
910
# pull_request:
1011
# branches:
1112
# - main
@@ -15,106 +16,87 @@ jobs:
1516
daily-ci:
1617
strategy:
1718
matrix:
19+
# dependencies-strategy -> The strategy that `uv lock` should use to select
20+
# between the different compatible versions for a given package requirement
21+
# [arg: --resolution, env: UV_RESOLUTION=]
22+
dependencies-strategy: ["lowest-direct", "highest"]
23+
gt4py-module: ["cartesian", "eve", "next", "storage"]
24+
os: ["ubuntu-latest"] #, "macos-latest"]
1825
python-version: ["3.10", "3.11"]
19-
tox-module-factor: ["cartesian", "eve", "next", "storage"]
20-
os: ["ubuntu-latest"]
21-
requirements-file: ["requirements-dev.txt", "min-requirements-test.txt", "min-extra-requirements-test.txt"]
2226
fail-fast: false
2327

2428
runs-on: ${{ matrix.os }}
2529
steps:
2630
- uses: actions/checkout@v4
31+
2732
- name: Install C++ libraries
2833
if: ${{ matrix.os == 'macos-latest' }}
2934
shell: bash
30-
run: |
31-
brew install boost
35+
run: brew install boost
36+
3237
- name: Install C++ libraries
3338
if: ${{ matrix.os == 'ubuntu-latest' }}
3439
shell: bash
35-
run: |
36-
sudo apt install libboost-dev
37-
- name: Set up Python ${{ matrix.python-version }}
38-
uses: actions/setup-python@v5
40+
run: sudo apt install libboost-dev
41+
42+
- name: Install uv and set the python version
43+
uses: astral-sh/setup-uv@v5
3944
with:
45+
enable-cache: true
46+
cache-dependency-glob: "uv.lock"
4047
python-version: ${{ matrix.python-version }}
41-
cache: 'pip'
42-
cache-dependency-path: |
43-
**/pyproject.toml
44-
**/constraints.txt
45-
**/requirements-dev.txt
46-
- name: Install tox
47-
run: |
48-
python -m pip install -c ./constraints.txt pip setuptools wheel tox
49-
python -m pip list
50-
- name: Update requirements
51-
run: |
52-
pyversion=${{ matrix.python-version }}
53-
pyversion_no_dot=${pyversion//./}
54-
tox run -e requirements-py${pyversion_no_dot}
55-
# TODO(egparedes): add notification for dependencies updates
56-
# - name: Check for updated requirements
57-
# id: update-requirements
58-
# continue-on-error: true
59-
# if: ${{ matrix.python-version == '3.8' && matrix.tox-module-factor == 'cartesian' }}
60-
# shell: bash
61-
# run: |
62-
# if diff -q constraints.txt CURRENT-constraints.txt; then
63-
# echo "REQS_DIFF=''" >> $GITHUB_OUTPUT
64-
# else
65-
# diff --changed-group-format='%<' --unchanged-group-format='' constraints.txt CURRENT-constraints.txt | tr '\n' ' ' > constraints.txt.diff
66-
# echo "REQS_DIFF='$(cat constraints.txt.diff)'" >> $GITHUB_OUTPUT
67-
# fi
68-
# echo "REQS_DIFF_TEST="FOOOOOOOO" >> $GITHUB_OUTPUT
69-
# - name: Notify updated requirements (if any)
70-
# if: ${{ steps.update-requirements.outputs.REQS_DIFF }}
71-
# env:
72-
# SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
73-
# uses: slackapi/[email protected]
74-
# with:
75-
# channel-id: ${{ vars.SLACK_BOT_CHANNEL }}
76-
# payload: |
77-
# {
78-
# "text": "TEXT",
79-
# "blocks": [
80-
# {
81-
# "type": "section",
82-
# "text": {
83-
# "type": "plain_text",
84-
# "text": "@channel: AA/${{ steps.update-requirements.outputs.REQS_DIFF }}/BB/ ${{ steps.update-requirements.outputs.REQS_DIFF_TEST }} /CC"
85-
# }
86-
# },
87-
# {
88-
# "type": "section",
89-
# "text": {
90-
# "type": "mrkdwn",
91-
# "text": "@channel: AA/${{ steps.update-requirements.outputs.REQS_DIFF }}/BB/ ${{ steps.update-requirements.outputs.REQS_DIFF_TEST }} /CC"
92-
# }
93-
# }
94-
# ]
95-
# }
96-
- name: Run tests
48+
49+
- name: Run CPU tests for '${{ matrix.gt4py-module }}' with '${{ matrix.dependencies-strategy }}' resolution strategy
9750
env:
9851
NUM_PROCESSES: auto
99-
ENV_REQUIREMENTS_FILE: ${{ matrix.requirements-file }}
100-
run: |
101-
tox run --skip-missing-interpreters -m test-${{ matrix.tox-module-factor }}-cpu
52+
UV_RESOLUTION: ${{ matrix.dependencies-strategy }}
53+
run: uv run nox -s 'test_${{ matrix.gt4py-module }}-${{ matrix.python-version }}' -t 'cpu'
54+
10255
- name: Notify slack
10356
if: ${{ failure() }}
10457
env:
10558
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
10659
uses: slackapi/[email protected]
10760
with:
108-
channel-id: ${{ vars.SLACK_BOT_CHANNEL }}
61+
channel-id: ${{ vars.SLACK_BOT_CHANNEL }} # Use SLACK_BOT_CHANNEL_TEST for testing
62+
payload: |
63+
{
64+
"text": "Failed tests for ${{ github.workflow }} (dependencies-strategy=${{ matrix.dependencies-strategy }}, python=${{ matrix.python-version }}, component=${{ matrix.gt4py-module }}) [https://github.com/GridTools/gt4py/actions/runs/${{ github.run_id }}].",
65+
"blocks": [
66+
{
67+
"type": "section",
68+
"text": {
69+
"type": "mrkdwn",
70+
"text": "Failed tests: <https://github.com/GridTools/gt4py/actions/runs/${{ github.run_id }}|${{ github.workflow }}: **${{ matrix.gt4py-module }} (CPU)** for **Python-${{ matrix.python-version }}** with **'${{ matrix.dependencies-strategy }}' uv resolution strategy**.>"
71+
}
72+
}
73+
]
74+
}
75+
76+
weekly-reminder:
77+
runs-on: ubuntu-latest
78+
steps:
79+
- id: get_day_of_the_week
80+
name: Get day of the week
81+
run: echo "day_of_week=$(date +'%u')" >> $GITHUB_OUTPUT
82+
83+
- name: Weekly notification
84+
if: ${{ env.DAY_OF_WEEK == 1 }}
85+
env:
86+
DAY_OF_WEEK: ${{ steps.get_day_of_the_week.outputs.day_of_week }}
87+
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
88+
uses: slackapi/[email protected]
89+
with:
90+
channel-id: ${{ vars.SLACK_BOT_CHANNEL }} # Use SLACK_BOT_CHANNEL_TEST for testing
10991
payload: |
11092
{
111-
"text": "${{ github.workflow }}: `test-${{ matrix.tox-module-factor }}-cpu (python${{ matrix.python-version }})`>: *Failed tests!*",
93+
"text": "Weekly reminder to check the latest runs of the GT4Py Daily CI workflow at the GitHub Actions dashboard [https://github.com/GridTools/gt4py/actions/workflows/daily-ci.yml].",
11294
"blocks": [
11395
{
11496
"type": "section",
11597
"text": {
11698
"type": "mrkdwn",
117-
"text": "<https://github.com/GridTools/gt4py/actions/runs/${{ github.run_id }}|${{ github.workflow }}: `test-${{ matrix.tox-module-factor }}-cpu (python${{ matrix.python-version }})`>: *Failed tests!*"
99+
"text": "Weekly reminder to check the latest runs of the <https://github.com/GridTools/gt4py/actions/workflows/daily-ci.yml|GT4Py Daily CI> workflow at the GitHub Actions dashboard."
118100
}
119101
}
120102
]

‎.github/workflows/test-cartesian-fallback.yml

+3-3
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,11 @@ on:
1313

1414
jobs:
1515
test-cartesian:
16-
runs-on: ubuntu-latest
1716
strategy:
1817
matrix:
18+
codegen-factor: [internal, dace]
19+
os: ["ubuntu-latest"]
1920
python-version: ["3.10", "3.11"]
20-
tox-factor: [internal, dace]
21-
21+
runs-on: ${{ matrix.os }}
2222
steps:
2323
- run: 'echo "No build required"'

‎.github/workflows/test-cartesian.yml

+22-21
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ on:
77
pull_request:
88
branches:
99
- main
10-
paths-ignore: # Skip if only gt4py.next and irrelevant doc files have been updated
10+
paths-ignore: # Skip when only gt4py.next or doc files have been updated
1111
- "src/gt4py/next/**"
1212
- "tests/next_tests/**"
1313
- "examples/**"
@@ -20,35 +20,36 @@ concurrency:
2020

2121
jobs:
2222
test-cartesian:
23-
runs-on: ubuntu-latest
2423
strategy:
2524
matrix:
25+
codegen-factor: [internal, dace]
26+
os: ["ubuntu-latest"]
2627
python-version: ["3.10", "3.11"]
27-
tox-factor: [internal, dace]
28+
fail-fast: false
29+
30+
runs-on: ${{ matrix.os }}
2831
steps:
2932
- uses: actions/checkout@v4
33+
3034
- name: Install C++ libraries
35+
if: ${{ matrix.os == 'macos-latest' }}
3136
shell: bash
32-
run: |
33-
sudo apt install libboost-dev
34-
- name: Set up Python ${{ matrix.python-version }}
35-
uses: actions/setup-python@v5
37+
run: brew install boost
38+
39+
- name: Install C++ libraries
40+
if: ${{ matrix.os == 'ubuntu-latest' }}
41+
shell: bash
42+
run: sudo apt install libboost-dev
43+
44+
- name: Install uv and set the python version
45+
uses: astral-sh/setup-uv@v5
3646
with:
47+
enable-cache: true
48+
cache-dependency-glob: "uv.lock"
3749
python-version: ${{ matrix.python-version }}
38-
cache: 'pip'
39-
cache-dependency-path: |
40-
**/pyproject.toml
41-
**/constraints.txt
42-
**/requirements-dev.txt
43-
- name: Install python dependencies
44-
run: |
45-
python -m pip install -c ./constraints.txt pip setuptools wheel
46-
python -m pip install -r ./requirements-dev.txt
47-
- name: Test with tox
50+
51+
- name: Run CPU 'cartesian' tests with nox
4852
env:
4953
NUM_PROCESSES: auto
5054
shell: bash
51-
run: |
52-
pyversion=${{ matrix.python-version }}
53-
pyversion_no_dot=${pyversion//./}
54-
tox run -e cartesian-py${pyversion_no_dot}-${{ matrix.tox-factor }}-cpu
55+
run: uv run nox -s 'test_cartesian-${{ matrix.python-version }}(${{ matrix.codegen-factor }}, cpu)'

‎.github/workflows/test-eve-fallback.yml

+6-4
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,26 @@
11
name: "Fallback: Test Eve"
22

33
on:
4+
push:
5+
branches:
6+
- main
47
pull_request:
58
branches:
69
- main
710
paths-ignore: # Inverse of corresponding workflow
811
- "src/gt4py/eve/**"
912
- "tests/eve_tests/**"
10-
- "workflows/**"
11-
- "*.cfg"
12-
- "*.ini"
13+
- ".github/workflows/**"
14+
- "*.lock"
1315
- "*.toml"
1416
- "*.yml"
1517

1618
jobs:
1719
test-eve:
1820
strategy:
1921
matrix:
20-
python-version: ["3.10", "3.11"]
2122
os: ["ubuntu-latest"]
23+
python-version: ["3.10", "3.11"]
2224

2325
runs-on: ${{ matrix.os }}
2426
steps:

‎.github/workflows/test-eve.yml

+11-40
Original file line numberDiff line numberDiff line change
@@ -10,61 +10,32 @@ on:
1010
paths: # Run when gt4py.eve files (or package settings) are changed
1111
- "src/gt4py/eve/**"
1212
- "tests/eve_tests/**"
13-
- "workflows/**"
14-
- "*.cfg"
15-
- "*.ini"
13+
- ".github/workflows/**"
14+
- "*.lock"
1615
- "*.toml"
1716
- "*.yml"
1817

1918
jobs:
2019
test-eve:
2120
strategy:
2221
matrix:
23-
python-version: ["3.10", "3.11"]
2422
os: ["ubuntu-latest"]
23+
python-version: ["3.10", "3.11"]
2524
fail-fast: false
2625

2726
runs-on: ${{ matrix.os }}
2827
steps:
2928
- uses: actions/checkout@v4
30-
- name: Set up Python ${{ matrix.python-version }}
31-
uses: actions/setup-python@v5
29+
30+
- name: Install uv and set the python version
31+
uses: astral-sh/setup-uv@v5
3232
with:
33+
enable-cache: true
34+
cache-dependency-glob: "uv.lock"
3335
python-version: ${{ matrix.python-version }}
34-
cache: 'pip'
35-
cache-dependency-path: |
36-
**/pyproject.toml
37-
**/constraints.txt
38-
**/requirements-dev.txt
39-
- name: Install python dependencies
40-
run: |
41-
python -m pip install -c ./constraints.txt pip setuptools wheel
42-
python -m pip install -r ./requirements-dev.txt
43-
- name: Run tox tests
36+
37+
- name: Run 'eve' tests with nox
4438
env:
4539
NUM_PROCESSES: auto
4640
shell: bash
47-
run: |
48-
pyversion=${{ matrix.python-version }}
49-
pyversion_no_dot=${pyversion//./}
50-
tox run -e eve-py${pyversion_no_dot}
51-
# mv coverage.json coverage-py${{ matrix.python-version }}-${{ matrix.os }}.json
52-
# - name: Upload coverage.json artifact
53-
# uses: actions/upload-artifact@v4
54-
# with:
55-
# name: coverage-py${{ matrix.python-version }}-${{ matrix.os }}
56-
# path: coverage-py${{ matrix.python-version }}-${{ matrix.os }}.json
57-
# - name: Gather info
58-
# run: |
59-
# echo ${{ github.ref_type }} >> info.txt
60-
# echo ${{ github.ref }} >> info.txt
61-
# echo ${{ github.sha }} >> info.txt
62-
# echo ${{ github.event.number }} >> info.txt
63-
# echo ${{ github.event.pull_request.head.ref }} >> info.txt
64-
# echo ${{ github.event.pull_request.head.sha }} >> info.txt
65-
# echo ${{ github.run_id }} >> info.txt
66-
# - name: Upload info artifact
67-
# uses: actions/upload-artifact@v4
68-
# with:
69-
# name: info-py${{ matrix.python-version }}-${{ matrix.os }}
70-
# path: info.txt
41+
run: uv run nox -s test_eve-${{ matrix.python-version }}

0 commit comments

Comments
 (0)