Skip to content

Commit

Permalink
use uv in tox (#19871)
Browse files Browse the repository at this point in the history 
Use uv for env setup in tox to improve performance. 

## How I Tested These Changes

bk
  • Loading branch information
@alangenfeld
alangenfeld authored Feb 23, 2024
1 parent 03c93ac commit 059e3cd
Show file tree
Hide file tree
Showing 94 changed files with 206 additions and 106 deletions.
2 changes: 2 additions & 0 deletions .buildkite/dagster-buildkite/dagster_buildkite/steps/dagster_ui.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ def build_dagster_ui_components_steps() -> List[CommandStep]:
CommandStepBuilder(":typescript: dagster-ui-components")
.run(
"cd js_modules/dagster-ui/packages/ui-components",
"pip install -U uv",
f"tox -vv -e {AvailablePythonVersion.to_tox_factor(AvailablePythonVersion.get_default())}",
)
.on_test_image(AvailablePythonVersion.get_default())
Expand Down Expand Up @@ -57,6 +58,7 @@ def build_dagster_ui_core_steps() -> List[CommandStep]:
CommandStepBuilder(":typescript: dagster-ui-core")
.run(
"cd js_modules/dagster-ui",
"pip install -U uv",
f"tox -vv -e {AvailablePythonVersion.to_tox_factor(AvailablePythonVersion.get_default())}",
)
.on_test_image(AvailablePythonVersion.get_default())
Expand Down
1 change: 1 addition & 0 deletions .buildkite/dagster-buildkite/dagster_buildkite/steps/docs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ def build_docs_steps() -> List[BuildkiteStep]:
CommandStepBuilder("docs apidoc build")
.run(
"cd docs",
"pip install -U uv",
"make apidoc-build",
# "echo '--- Checking git diff (ignoring whitespace) after docs build...'",
# "git diff --ignore-all-space --stat",
Expand Down
1 change: 1 addition & 0 deletions .buildkite/dagster-buildkite/dagster_buildkite/steps/tox.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ def build_tox_step(
commands = [
*(extra_commands_pre or []),
f"cd {root_dir}",
"pip install -U uv",
f"echo -e {shlex.quote(buildkite_section_header)}",
tox_command,
*(extra_commands_post or []),
Expand Down
4 changes: 2 additions & 2 deletions docs/tox.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@ skipsdist = True
[testenv]
download = True
passenv = CI_* COVERALLS_REPO_TOKEN AWS_SECRET_ACCESS_KEY AWS_ACCESS_KEY_ID BUILDKITE*
usedevelop = False
allowlist_externals =
make
install_command = python -m pip install {opts} {packages} --config-settings editable_mode=compat
uv
install_command = uv pip install {opts} {packages}

[testenv:sphinx]
deps =
Expand Down
3 changes: 2 additions & 1 deletion examples/assets_dbt_python/tox.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ skipsdist = true
[testenv]
download = true
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
install_command = python -m pip install {opts} {packages} --config-settings editable_mode=compat
install_command = uv pip install {opts} {packages}
; note: "source" does not work at this time due to dagster-cloud source access
deps =
source: -e ../../python_modules/dagster[test]
Expand All @@ -19,6 +19,7 @@ deps =
-e .
allowlist_externals =
/bin/bash
uv
commands =
source: /bin/bash -c '! pip list --exclude-editable | grep -e dagster'
pytest -c ../../pyproject.toml -vv
3 changes: 2 additions & 1 deletion examples/assets_dynamic_partitions/tox.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ skipsdist = true
[testenv]
download = true
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
install_command = python -m pip install {opts} {packages} --config-settings editable_mode=compat
install_command = uv pip install {opts} {packages}
deps =
-e ../../python_modules/dagster[test]
-e ../../python_modules/dagster-pipes
Expand All @@ -13,6 +13,7 @@ deps =
-e .
allowlist_externals =
/bin/bash
uv
commands =
!windows: /bin/bash -c '! pip list --exclude-editable | grep -e dagster'
pytest -c ../../pyproject.toml -vv
3 changes: 2 additions & 1 deletion examples/assets_modern_data_stack/tox.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ skipsdist = true
[testenv]
download = True
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
install_command = python -m pip install {opts} {packages} --config-settings editable_mode=compat
install_command = uv pip install {opts} {packages}
; note: "source" does not work at this time due to dagster-cloud source access
deps =
source: -e ../../python_modules/dagster[test]
Expand All @@ -20,6 +20,7 @@ deps =
-e .
allowlist_externals =
/bin/bash
uv
commands =
source: /bin/bash -c '! pip list --exclude-editable | grep -e dagster'
pytest -c ../../pyproject.toml -vv
4 changes: 2 additions & 2 deletions examples/assets_pandas_pyspark/tox.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ skipsdist = true
[testenv]
download = true
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
install_command = python -m pip install {opts} {packages} --config-settings editable_mode=compat
install_command = uv pip install {opts} {packages}
deps =
-e ../../python_modules/dagster[test]
-e ../../python_modules/dagster-pipes
Expand All @@ -13,9 +13,9 @@ deps =
-e ../../python_modules/libraries/dagster-spark
-e ../../python_modules/libraries/dagster-pyspark
-e .[test]
-e .
allowlist_externals =
/bin/bash
uv
commands =
!windows: /bin/bash -c '! pip list --exclude-editable | grep -e dagster'
pytest -c ../../pyproject.toml -vv
4 changes: 2 additions & 2 deletions examples/assets_pandas_type_metadata/tox.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,17 @@ skipsdist = true
[testenv]
download = True
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
install_command = python -m pip install {opts} {packages} --config-settings editable_mode=compat
install_command = uv pip install {opts} {packages}
deps =
-e ../../python_modules/dagster[test]
-e ../../python_modules/dagster-pipes
-e ../../python_modules/dagster-webserver
-e ../../python_modules/dagster-graphql
-e ../../python_modules/dagster[test]
-e ../../python_modules/libraries/dagster-pandera/
-e .
allowlist_externals =
/bin/bash
uv
commands =
!windows: /bin/bash -c '! pip list --exclude-editable | grep -e dagster'
pytest -c ../../pyproject.toml -vv
3 changes: 2 additions & 1 deletion examples/assets_smoke_test/tox.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ skipsdist = true
[testenv]
download = true
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
install_command = python -m pip install {opts} {packages} --config-settings editable_mode=compat
install_command = uv pip install {opts} {packages}
deps =
-e ../../python_modules/dagster[test]
-e ../../python_modules/dagster-pipes
Expand All @@ -17,6 +17,7 @@ deps =
-e .
allowlist_externals =
/bin/bash
uv
commands =
!windows: /bin/bash -c '! pip list --exclude-editable | grep -e dagster'
pytest -vv
3 changes: 2 additions & 1 deletion examples/deploy_docker/tox.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,13 @@ skipsdist = True
[testenv]
download = True
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE* DEPLOY_DOCKER_WEBSERVER_HOST
install_command = python -m pip install {opts} {packages} --config-settings editable_mode=compat
install_command = uv pip install {opts} {packages}
deps =
-e ../../python_modules/dagster[test]
-e ../../python_modules/dagster-pipes
allowlist_externals =
/bin/bash
uv
commands =
!windows: /bin/bash -c '! pip list --exclude-editable | grep -e dagster'
pytest -c ../../pyproject.toml -vv {posargs}
3 changes: 2 additions & 1 deletion examples/deploy_ecs/tox.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,14 @@ skipsdist = True
[testenv]
download = True
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE* DEPLOY_DOCKER_WEBSERVER_HOST
install_command = python -m pip install {opts} {packages} --config-settings editable_mode=compat
install_command = uv pip install {opts} {packages}
deps =
-e ../../python_modules/dagster[test]
-e ../../python_modules/dagster-pipes
-e ../../python_modules/dagster-test
allowlist_externals =
/bin/bash
uv
commands =
!windows: /bin/bash -c '! pip list --exclude-editable | grep -e dagster'
pytest -s -vv {posargs}
3 changes: 2 additions & 1 deletion examples/deploy_k8s/tox.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ skipsdist = True
[testenv]
download = True
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
install_command = python -m pip install {opts} {packages} --config-settings editable_mode=compat
install_command = uv pip install {opts} {packages}
deps =
-e ../../python_modules/dagster[test]
-e ../../python_modules/dagster-pipes
Expand All @@ -15,6 +15,7 @@ deps =
-e ../../python_modules/libraries/dagster-celery-k8s
allowlist_externals =
/bin/bash
uv
commands =
!windows: /bin/bash -c '! pip list --exclude-editable | grep -e dagster'
pytest -c ../../pyproject.toml -vv {posargs}
5 changes: 3 additions & 2 deletions examples/development_to_production/tox.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,18 @@ skipsdist = True
[testenv]
download = True
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
install_command = python -m pip install {opts} {packages} --config-settings editable_mode=compat
install_command = uv pip install {opts} {packages}
deps =
-e ../../python_modules/dagster[test]
-e ../../python_modules/dagster-pipes
-e .[test]
-e ../../python_modules/dagster-webserver
-e ../../python_modules/dagster-graphql
-e ../../python_modules/libraries/dagster-snowflake/
-e ../../python_modules/libraries/dagster-snowflake-pandas/
-e .[test]
allowlist_externals =
/bin/bash
uv
commands =
!windows: /bin/bash -c '! pip list --exclude-editable | grep -e dagster'
pytest -c ../../pyproject.toml -vv {posargs}
2 changes: 2 additions & 0 deletions examples/docs_snippets/tox.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ skipsdist = true
[testenv]
download = True
passenv = CI_* COVERALLS_REPO_TOKEN POSTGRES_TEST_DB_HOST BUILDKITE*
; dagstermill test errors prevent using uv, require compat mode install
; install_command = uv pip install {opts} {packages}
install_command = python -m pip install {opts} {packages} --config-settings editable_mode=compat
deps =
-e ../../python_modules/dagster[test]
Expand Down
3 changes: 2 additions & 1 deletion examples/experimental/assets_yaml_dsl/tox.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ skipsdist = true
[testenv]
download = True
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
install_command = python -m pip install {opts} {packages} --config-settings editable_mode=compat
install_command = uv pip install {opts} {packages}
deps =
-e ../../../python_modules/dagster[test]
-e ../../../python_modules/dagster-pipes
Expand All @@ -13,6 +13,7 @@ deps =
-e .
allowlist_externals =
/bin/bash
uv
commands =
!windows: /bin/bash -c '! pip list --exclude-editable | grep -e dagster'
pytest -c ../../../pyproject.toml -vv
3 changes: 2 additions & 1 deletion examples/feature_graph_backed_assets/tox.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ skipsdist = true
[testenv]
download = True
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
install_command = python -m pip install {opts} {packages} --config-settings editable_mode=compat
install_command = uv pip install {opts} {packages}
deps =
-e ../../python_modules/dagster[test]
-e ../../python_modules/dagster-pipes
Expand All @@ -13,6 +13,7 @@ deps =
-e .
allowlist_externals =
/bin/bash
uv
commands =
!windows: /bin/bash -c '! pip list --exclude-editable | grep -e dagster'
pytest -c ../../pyproject.toml -vv
4 changes: 3 additions & 1 deletion examples/project_fully_featured/tox.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@ skipsdist = true
[testenv]
download = True
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE* SNOWFLAKE_ACCOUNT SNOWFLAKE_USER SNOWFLAKE_PASSWORD
install_command = python -m pip install {opts} {packages} --config-settings editable_mode=compat
; version resolution using uv
; install_command = uv pip install {opts} {packages}
deps =
-e ../../python_modules/dagster[test]
-e ../../python_modules/dagster-pipes
Expand All @@ -20,6 +21,7 @@ deps =
-e .
allowlist_externals =
/bin/bash
uv
commands =
!windows: /bin/bash -c '! pip list --exclude-editable | grep -e dagster'
pytest -c ../../pyproject.toml -vv
5 changes: 4 additions & 1 deletion examples/quickstart_aws/tox.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ skipsdist = true
[testenv]
download = true
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
install_command = python -m pip install {opts} {packages} --config-settings editable_mode=compat
; version resolution issues prevent using uv
; install_command = uv pip install {opts} {packages}

; note: "source" does not work at this time due to dagster-cloud source access
deps =
source: -e ../../python_modules/dagster[test]
Expand All @@ -15,6 +17,7 @@ deps =
-e .
allowlist_externals =
/bin/bash
uv
commands =
source: /bin/bash -c '! pip list --exclude-editable | grep -e dagster'
pytest -c ../../pyproject.toml -vv
3 changes: 2 additions & 1 deletion examples/quickstart_etl/tox.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ skipsdist = true
[testenv]
download = true
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
install_command = python -m pip install {opts} {packages} --config-settings editable_mode=compat
install_command = uv pip install {opts} {packages}
; note: "source" does not work at this time due to dagster-cloud source access
deps =
source: -e ../../python_modules/dagster[test]
Expand All @@ -14,6 +14,7 @@ deps =
-e .
allowlist_externals =
/bin/bash
uv
commands =
source: /bin/bash -c '! pip list --exclude-editable | grep -e dagster'
pytest -c ../../pyproject.toml -vv
3 changes: 2 additions & 1 deletion examples/quickstart_gcp/tox.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ skipsdist = true
[testenv]
download = true
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
install_command = python -m pip install {opts} {packages} --config-settings editable_mode=compat
install_command = uv pip install {opts} {packages}
; note: "source" does not work at this time due to dagster-cloud source access
deps =
source: -e ../../python_modules/dagster[test]
Expand All @@ -17,6 +17,7 @@ deps =
-e .
allowlist_externals =
/bin/bash
uv
commands =
source: /bin/bash -c '! pip list --exclude-editable | grep -e dagster'
pytest -c ../../pyproject.toml -vv
3 changes: 2 additions & 1 deletion examples/quickstart_snowflake/tox.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ skipsdist = true
[testenv]
download = true
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
install_command = python -m pip install {opts} {packages} --config-settings editable_mode=compat
install_command = uv pip install {opts} {packages}
; note: "source" does not work at this time due to dagster-cloud source access
deps =
source: -e ../../python_modules/dagster[test]
Expand All @@ -18,6 +18,7 @@ deps =
-e .
allowlist_externals =
/bin/bash
uv
commands =
source: /bin/bash -c '! pip list --exclude-editable | grep -e dagster'
pytest -c ../../pyproject.toml -vv
5 changes: 4 additions & 1 deletion examples/with_airflow/tox.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ skipsdist = true
[testenv]
download = True
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
install_command = python -m pip install {opts} {packages} --config-settings editable_mode=compat
; uv has trouble with resolve
; https://buildkite.com/dagster/dagster-dagster/builds/76502#018dd221-e24c-40c2-a459-693bdb456f8f
; install_command = uv pip install {opts} {packages}
deps =
-e ../../python_modules/dagster[test]
-e ../../python_modules/dagster-pipes
Expand All @@ -14,6 +16,7 @@ deps =
-e .
allowlist_externals =
/bin/bash
uv
commands =
!windows: /bin/bash -c '! pip list --exclude-editable | grep -e dagster'
airflow db init
Expand Down
3 changes: 2 additions & 1 deletion examples/with_great_expectations/tox.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ skipsdist = true
[testenv]
download = True
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
install_command = python -m pip install {opts} {packages} --config-settings editable_mode=compat
install_command = uv pip install {opts} {packages}
deps =
-e ../../python_modules/dagster[test]
-e ../../python_modules/dagster-pipes
Expand All @@ -15,6 +15,7 @@ deps =
-e .
allowlist_externals =
/bin/bash
uv
commands =
!windows: /bin/bash -c '! pip list --exclude-editable | grep -e dagster'
pytest -c ../../pyproject.toml -vv
Loading

2 comments on commit 059e3cd

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Deploy preview for dagit-storybook ready!

✅ Preview
https://dagit-storybook-6muqndic7-elementl.vercel.app

Built with commit 059e3cd.
This pull request is being automatically deployed with vercel-action

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Deploy preview for dagster-docs ready!

✅ Preview
https://dagster-docs-m8ctcudl4-elementl.vercel.app
https://master.dagster.dagster-docs.io

Built with commit 059e3cd.
This pull request is being automatically deployed with vercel-action

Please sign in to comment.