diff --git a/docs/content/api/modules.json.gz b/docs/content/api/modules.json.gz index 9b25ec58bab53..20a84ccb2c8c8 100644 Binary files a/docs/content/api/modules.json.gz and b/docs/content/api/modules.json.gz differ diff --git a/docs/content/api/searchindex.json.gz b/docs/content/api/searchindex.json.gz index 71d0947f4455c..238bda0393437 100644 Binary files a/docs/content/api/searchindex.json.gz and b/docs/content/api/searchindex.json.gz differ diff --git a/docs/content/api/sections.json.gz b/docs/content/api/sections.json.gz index 1532334fb5d0d..2986d8c5a2a33 100644 Binary files a/docs/content/api/sections.json.gz and b/docs/content/api/sections.json.gz differ diff --git a/pyright/alt-1/requirements-pinned.txt b/pyright/alt-1/requirements-pinned.txt index 1d8dc11d8fcc5..15b0ad535cf52 100644 --- a/pyright/alt-1/requirements-pinned.txt +++ b/pyright/alt-1/requirements-pinned.txt @@ -9,7 +9,6 @@ alembic==1.13.2 aniso8601==9.0.1 annotated-types==0.7.0 anyio==4.4.0 -appnope==0.1.4 argon2-cffi==23.1.0 argon2-cffi-bindings==21.2.0 arrow==1.3.0 @@ -25,9 +24,9 @@ backports-tarfile==1.2.0 beautifulsoup4==4.12.3 bleach==6.1.0 boto3==1.35.16 -boto3-stubs-lite==1.35.16 +boto3-stubs-lite==1.35.17 botocore==1.35.16 -botocore-stubs==1.35.16 +botocore-stubs==1.35.17 buildkite-test-collector==0.1.9 cachetools==5.5.0 caio==0.9.17 @@ -91,8 +90,8 @@ filelock==3.16.0 fonttools==4.53.1 fqdn==1.5.1 frozenlist==1.4.1 -fsspec==2024.3.1 -gcsfs==2024.3.1 +fsspec==2024.3.0 +gcsfs==0.8.0 google-api-core==2.19.2 google-api-python-client==2.145.0 google-auth==2.34.0 @@ -108,6 +107,7 @@ gql==3.5.0 graphene==3.3 graphql-core==3.2.4 graphql-relay==3.2.0 +greenlet==3.1.0 grpcio==1.66.1 grpcio-health-checking==1.62.3 grpcio-status==1.62.3 @@ -131,6 +131,7 @@ jaraco-classes==3.4.0 jaraco-context==6.0.1 jaraco-functools==4.0.2 jedi==0.19.1 +jeepney==0.8.0 jinja2==3.1.4 jmespath==1.0.1 joblib==1.4.2 @@ -251,18 +252,19 @@ rfc3986-validator==0.1.1 rich==13.8.1 rpds-py==0.20.0 rsa==4.9 -s3fs==2024.3.1 +s3fs==2024.3.0 s3transfer==0.10.2 -scikit-learn==1.5.1 +scikit-learn==1.5.2 scipy==1.14.1 seaborn==0.13.2 +secretstorage==3.3.3 send2trash==1.8.3 setuptools==74.1.2 shellingham==1.5.4 six==1.16.0 slack-sdk==3.32.0 sniffio==1.3.1 -snowflake-connector-python==3.12.1 +snowflake-connector-python==3.12.2 snowflake-sqlalchemy==1.5.1 sortedcontainers==2.4.0 soupsieve==2.6 @@ -314,10 +316,11 @@ types-tzlocal==5.1.0.1 typing-extensions==4.12.2 typing-inspect==0.9.0 tzdata==2024.1 -universal-pathlib==0.2.2 +ujson==5.10.0 +universal-pathlib==0.2.5 uri-template==1.3.0 uritemplate==4.1.1 -urllib3==2.2.2 +urllib3==2.2.3 uvicorn==0.30.6 uvloop==0.20.0 virtualenv==20.26.4 diff --git a/pyright/alt-1/requirements.txt b/pyright/alt-1/requirements.txt index 225c3d34764bc..56e5dca335050 100644 --- a/pyright/alt-1/requirements.txt +++ b/pyright/alt-1/requirements.txt @@ -34,7 +34,7 @@ -e python_modules/dagster-pipes -e python_modules/dagster-webserver -e python_modules/dagster-graphql - -e python_modules/libraries/dagster-aws/ + -e python_modules/libraries/dagster-aws[stubs] -e python_modules/libraries/dagster-dbt/ -e python_modules/libraries/dagster-duckdb-pandas/ -e python_modules/libraries/dagster-duckdb/ @@ -49,4 +49,3 @@ -e python_modules/libraries/dagster-snowflake/ -e python_modules/libraries/dagster-snowflake-pandas/ -e python_modules/libraries/dagster-snowflake-pyspark/ - \ No newline at end of file diff --git a/pyright/master/requirements-pinned.txt b/pyright/master/requirements-pinned.txt index 2fa017aad5606..ea4b0131cb490 100644 --- a/pyright/master/requirements-pinned.txt +++ b/pyright/master/requirements-pinned.txt @@ -25,7 +25,6 @@ apache-airflow-providers-sqlite==3.8.2 apeye==1.4.1 apeye-core==1.1.5 apispec==6.6.1 -appnope==0.1.4 argcomplete==3.5.0 argon2-cffi==23.1.0 argon2-cffi-bindings==21.2.0 @@ -58,10 +57,10 @@ bitmath==1.3.3.1 bleach==6.1.0 blinker==1.8.2 bokeh==3.5.2 -boto3==1.35.16 -boto3-stubs-lite==1.35.16 -botocore==1.35.16 -botocore-stubs==1.35.16 +boto3==1.35.17 +boto3-stubs-lite==1.35.17 +botocore==1.35.17 +botocore-stubs==1.35.17 buildkite-test-collector==0.1.9 cachecontrol==0.14.0 cached-property==1.5.2 @@ -256,6 +255,7 @@ graphql-core==3.2.4 graphql-relay==3.2.0 graphviz==0.20.3 great-expectations==0.17.11 +greenlet==3.1.0 grpcio==1.66.1 grpcio-health-checking==1.62.3 grpcio-status==1.62.3 @@ -321,7 +321,7 @@ langchain-community==0.2.9 langchain-core==0.2.39 langchain-openai==0.1.14 langchain-text-splitters==0.2.4 -langsmith==0.1.117 +langsmith==0.1.118 lazy-object-proxy==1.10.0 leather==0.4.0 limits==3.13.0 @@ -375,6 +375,18 @@ noteable-origami==1.1.5 notebook==7.2.2 notebook-shim==0.2.4 numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.6.68 +nvidia-nvtx-cu12==12.1.105 oauth2client==4.1.3 oauthlib==3.2.2 objgraph==3.6.1 @@ -483,7 +495,7 @@ querystring-parser==1.2.4 rapidfuzz==3.9.7 readme-renderer==44.0 referencing==0.35.1 -regex==2024.7.24 +regex==2024.9.11 requests==2.32.3 requests-file==2.1.0 requests-mock==1.12.1 @@ -500,7 +512,7 @@ rsa==4.9 ruamel-yaml==0.17.17 ruff==0.6.4 s3transfer==0.10.2 -scikit-learn==1.5.1 +scikit-learn==1.5.2 scipy==1.14.1 scrapbook==0.5.0 sdf-cli==0.3.23 @@ -519,11 +531,11 @@ skein==0.8.2 skl2onnx==1.17.0 slack-sdk==3.32.0 sling==1.2.18 -sling-mac-arm64==1.2.18 +sling-linux-amd64==1.2.18 smmap==5.0.1 sniffio==1.3.1 snowballstemmer==2.2.0 -snowflake-connector-python==3.12.1 +snowflake-connector-python==3.12.2 snowflake-sqlalchemy==1.6.1 sortedcontainers==2.4.0 soupsieve==2.6 @@ -575,6 +587,7 @@ tqdm==4.66.5 traitlets==5.14.3 trio==0.26.2 trio-websocket==0.11.1 +triton==3.0.0 -e examples/experimental/dagster-airlift/examples/tutorial-example -e examples/tutorial_notebook_assets twilio==9.3.0 @@ -621,7 +634,7 @@ uvicorn==0.30.6 uvloop==0.20.0 vine==5.1.0 virtualenv==20.25.0 -wandb==0.17.9 +wandb==0.18.0 watchdog==5.0.2 watchfiles==0.24.0 wcwidth==0.2.13 diff --git a/pyright/master/requirements.txt b/pyright/master/requirements.txt index 5a776a4a569df..50d705ad23229 100644 --- a/pyright/master/requirements.txt +++ b/pyright/master/requirements.txt @@ -37,7 +37,7 @@ -e python_modules/dagster-webserver -e python_modules/libraries/dagster-airbyte/ -e python_modules/libraries/dagster-airflow[test_airflow_2] --e python_modules/libraries/dagster-aws[test] +-e python_modules/libraries/dagster-aws[stubs,test] -e python_modules/libraries/dagster-azure/ -e python_modules/libraries/dagster-celery/ -e python_modules/libraries/dagster-celery-docker/ diff --git a/python_modules/libraries/dagster-aws/MANIFEST.in b/python_modules/libraries/dagster-aws/MANIFEST.in index 175697c9b999c..635e368f3c118 100644 --- a/python_modules/libraries/dagster-aws/MANIFEST.in +++ b/python_modules/libraries/dagster-aws/MANIFEST.in @@ -4,3 +4,5 @@ recursive-include dagster_aws *.txt recursive-include dagster_aws *.template include LICENSE include dagster_aws/py.typed + +exclude ruff.toml diff --git a/python_modules/libraries/dagster-aws/dagster_aws/pipes/clients/glue.py b/python_modules/libraries/dagster-aws/dagster_aws/pipes/clients/glue.py index ac1945af324cc..ebd00aba3fa84 100644 --- a/python_modules/libraries/dagster-aws/dagster_aws/pipes/clients/glue.py +++ b/python_modules/libraries/dagster-aws/dagster_aws/pipes/clients/glue.py @@ -15,13 +15,13 @@ PipesMessageReader, ) from dagster._core.pipes.utils import open_pipes_session -from mypy_boto3_glue.type_defs import StartJobRunRequestRequestTypeDef from dagster_aws.pipes.context_injectors import PipesS3ContextInjector from dagster_aws.pipes.message_readers import PipesCloudWatchMessageReader if TYPE_CHECKING: from mypy_boto3_glue.client import GlueClient + from mypy_boto3_glue.type_defs import StartJobRunRequestRequestTypeDef RUN_PARAMS_BREAKING_VERSION = "1.9.0" @@ -87,7 +87,7 @@ def run( self, *, context: OpExecutionContext, - start_job_run_params: Optional[StartJobRunRequestRequestTypeDef] = None, + start_job_run_params: Optional["StartJobRunRequestRequestTypeDef"] = None, extras: Optional[Dict[str, Any]] = None, job_name: Optional[str] = None, arguments: Optional[Mapping[str, Any]] = None, @@ -137,7 +137,7 @@ def run( "JobName": job_name, } - params = cast(StartJobRunRequestRequestTypeDef, params) + params = cast("StartJobRunRequestRequestTypeDef", params) params["Arguments"] = arguments diff --git a/python_modules/libraries/dagster-aws/dagster_aws_tests/athena_tests/test_resources.py b/python_modules/libraries/dagster-aws/dagster_aws_tests/athena_tests/test_resources.py index 1761f5d442a89..457eb2207554a 100644 --- a/python_modules/libraries/dagster-aws/dagster_aws_tests/athena_tests/test_resources.py +++ b/python_modules/libraries/dagster-aws/dagster_aws_tests/athena_tests/test_resources.py @@ -1,5 +1,7 @@ import boto3 import pytest +from moto import mock_athena + from dagster_aws.athena.resources import ( AthenaError, AthenaTimeout, @@ -7,7 +9,6 @@ ResourceWithAthenaConfig, fake_athena_resource, ) -from moto import mock_athena class TestAthenaClientResource(ResourceWithAthenaConfig): diff --git a/python_modules/libraries/dagster-aws/dagster_aws_tests/cloudwatch_tests/test_loggers.py b/python_modules/libraries/dagster-aws/dagster_aws_tests/cloudwatch_tests/test_loggers.py index bfc854102b01e..af0ef2b2e440d 100644 --- a/python_modules/libraries/dagster-aws/dagster_aws_tests/cloudwatch_tests/test_loggers.py +++ b/python_modules/libraries/dagster-aws/dagster_aws_tests/cloudwatch_tests/test_loggers.py @@ -3,9 +3,10 @@ import boto3 import pytest from dagster import job, op -from dagster_aws.cloudwatch import cloudwatch_logger from moto import mock_logs +from dagster_aws.cloudwatch import cloudwatch_logger + @op def hello_op(context): diff --git a/python_modules/libraries/dagster-aws/dagster_aws_tests/ecr_tests/test_resources.py b/python_modules/libraries/dagster-aws/dagster_aws_tests/ecr_tests/test_resources.py index 51807bc49b922..9c58f6361ae83 100644 --- a/python_modules/libraries/dagster-aws/dagster_aws_tests/ecr_tests/test_resources.py +++ b/python_modules/libraries/dagster-aws/dagster_aws_tests/ecr_tests/test_resources.py @@ -1,5 +1,6 @@ from dagster import op from dagster._utils.test import wrap_op_in_graph_and_execute + from dagster_aws.ecr import FakeECRPublicResource, fake_ecr_public_resource diff --git a/python_modules/libraries/dagster-aws/dagster_aws_tests/ecs_tests/launcher_tests/test_container_context.py b/python_modules/libraries/dagster-aws/dagster_aws_tests/ecs_tests/launcher_tests/test_container_context.py index 8b0596ecc6e73..ab0523deaa5fe 100644 --- a/python_modules/libraries/dagster-aws/dagster_aws_tests/ecs_tests/launcher_tests/test_container_context.py +++ b/python_modules/libraries/dagster-aws/dagster_aws_tests/ecs_tests/launcher_tests/test_container_context.py @@ -1,6 +1,7 @@ import pytest from dagster._core.errors import DagsterInvalidConfigError from dagster._core.test_utils import environ + from dagster_aws.ecs.container_context import EcsContainerContext diff --git a/python_modules/libraries/dagster-aws/dagster_aws_tests/ecs_tests/launcher_tests/test_launching.py b/python_modules/libraries/dagster-aws/dagster_aws_tests/ecs_tests/launcher_tests/test_launching.py index 75f1ce8de2898..5bb5af32585a9 100644 --- a/python_modules/libraries/dagster-aws/dagster_aws_tests/ecs_tests/launcher_tests/test_launching.py +++ b/python_modules/libraries/dagster-aws/dagster_aws_tests/ecs_tests/launcher_tests/test_launching.py @@ -3,7 +3,6 @@ import time from concurrent.futures import ThreadPoolExecutor -import dagster_aws import pytest from botocore.exceptions import ClientError from dagster._check import CheckError @@ -14,6 +13,8 @@ from dagster._core.origin import JobPythonOrigin, RepositoryPythonOrigin from dagster._core.storage.dagster_run import DagsterRunStatus from dagster._core.storage.tags import RUN_WORKER_ID_TAG + +import dagster_aws from dagster_aws.ecs import EcsEventualConsistencyTimeout from dagster_aws.ecs.launcher import ( DEFAULT_LINUX_RESOURCES, diff --git a/python_modules/libraries/dagster-aws/dagster_aws_tests/ecs_tests/launcher_tests/test_utils.py b/python_modules/libraries/dagster-aws/dagster_aws_tests/ecs_tests/launcher_tests/test_utils.py index eb3f80fca7b97..84ca5f7cd622d 100644 --- a/python_modules/libraries/dagster-aws/dagster_aws_tests/ecs_tests/launcher_tests/test_utils.py +++ b/python_modules/libraries/dagster-aws/dagster_aws_tests/ecs_tests/launcher_tests/test_utils.py @@ -3,6 +3,7 @@ RemoteJobOrigin, RemoteRepositoryOrigin, ) + from dagster_aws.ecs.utils import get_task_definition_family, sanitize_family diff --git a/python_modules/libraries/dagster-aws/dagster_aws_tests/emr_tests/test_configs_spark.py b/python_modules/libraries/dagster-aws/dagster_aws_tests/emr_tests/test_configs_spark.py index 4552e0cbfc040..fefeeb4f8ac81 100644 --- a/python_modules/libraries/dagster-aws/dagster_aws_tests/emr_tests/test_configs_spark.py +++ b/python_modules/libraries/dagster-aws/dagster_aws_tests/emr_tests/test_configs_spark.py @@ -1,9 +1,10 @@ # from dagster_aws.emr.configs_spark import spark_config as dagster_aws_spark_config # from dagster_spark.configs_spark import spark_config as dagster_spark_spark_config -import dagster_aws.emr.configs_spark as aws_configs_spark import dagster_spark.configs_spark as spark_configs_spark +import dagster_aws.emr.configs_spark as aws_configs_spark + def test_spark_configs_same_as_in_dagster_spark(): aws_contents = open(aws_configs_spark.__file__, encoding="utf8").read() diff --git a/python_modules/libraries/dagster-aws/dagster_aws_tests/emr_tests/test_emr.py b/python_modules/libraries/dagster-aws/dagster_aws_tests/emr_tests/test_emr.py index 958e22e7b75c7..25e11b0770af4 100644 --- a/python_modules/libraries/dagster-aws/dagster_aws_tests/emr_tests/test_emr.py +++ b/python_modules/libraries/dagster-aws/dagster_aws_tests/emr_tests/test_emr.py @@ -7,9 +7,10 @@ import pytest from dagster._utils.test import create_test_pipeline_execution_context +from moto import mock_emr + from dagster_aws.emr import EmrClusterState, EmrError, EmrJobRunner from dagster_aws.utils.mrjob.utils import _boto3_now -from moto import mock_emr REGION = "us-west-1" diff --git a/python_modules/libraries/dagster-aws/dagster_aws_tests/emr_tests/test_emr_step_main.py b/python_modules/libraries/dagster-aws/dagster_aws_tests/emr_tests/test_emr_step_main.py index 3003b9d9fe2a6..18c5583b5b1d5 100644 --- a/python_modules/libraries/dagster-aws/dagster_aws_tests/emr_tests/test_emr_step_main.py +++ b/python_modules/libraries/dagster-aws/dagster_aws_tests/emr_tests/test_emr_step_main.py @@ -3,6 +3,7 @@ from dagster import DagsterEvent from dagster._core.events import EngineEventData + from dagster_aws.emr.emr_step_main import DONE, event_writing_loop diff --git a/python_modules/libraries/dagster-aws/dagster_aws_tests/emr_tests/test_pyspark.py b/python_modules/libraries/dagster-aws/dagster_aws_tests/emr_tests/test_pyspark.py index ee7d5a5110bd0..d0506939fe5c0 100644 --- a/python_modules/libraries/dagster-aws/dagster_aws_tests/emr_tests/test_pyspark.py +++ b/python_modules/libraries/dagster-aws/dagster_aws_tests/emr_tests/test_pyspark.py @@ -16,14 +16,15 @@ from dagster._core.test_utils import instance_for_test from dagster._utils.merger import deep_merge_dicts from dagster._utils.test import create_test_pipeline_execution_context -from dagster_aws.emr import EmrError, EmrJobRunner -from dagster_aws.emr.pyspark_step_launcher import EmrPySparkStepLauncher, emr_pyspark_step_launcher -from dagster_aws.s3 import s3_resource from dagster_pyspark import DataFrame, pyspark_resource from moto import mock_emr from pyspark.sql import Row from pyspark.sql.types import IntegerType, StringType, StructField, StructType +from dagster_aws.emr import EmrError, EmrJobRunner +from dagster_aws.emr.pyspark_step_launcher import EmrPySparkStepLauncher, emr_pyspark_step_launcher +from dagster_aws.s3 import s3_resource + S3_BUCKET = "dagster-scratch-80542c2" diff --git a/python_modules/libraries/dagster-aws/dagster_aws_tests/emr_tests/test_pyspark_step_launcher.py b/python_modules/libraries/dagster-aws/dagster_aws_tests/emr_tests/test_pyspark_step_launcher.py index 455ed2e940a86..92645dcade79a 100644 --- a/python_modules/libraries/dagster-aws/dagster_aws_tests/emr_tests/test_pyspark_step_launcher.py +++ b/python_modules/libraries/dagster-aws/dagster_aws_tests/emr_tests/test_pyspark_step_launcher.py @@ -4,6 +4,7 @@ import pytest from dagster import DagsterEvent, EventLogEntry, build_init_resource_context from dagster._core.execution.plan.objects import StepSuccessData + from dagster_aws.emr.pyspark_step_launcher import EmrPySparkStepLauncher, emr_pyspark_step_launcher EVENTS = [ diff --git a/python_modules/libraries/dagster-aws/dagster_aws_tests/pipes_tests/test_pipes.py b/python_modules/libraries/dagster-aws/dagster_aws_tests/pipes_tests/test_pipes.py index 134c7378cbac7..1b8067d312c60 100644 --- a/python_modules/libraries/dagster-aws/dagster_aws_tests/pipes_tests/test_pipes.py +++ b/python_modules/libraries/dagster-aws/dagster_aws_tests/pipes_tests/test_pipes.py @@ -10,7 +10,7 @@ import time from contextlib import contextmanager from tempfile import NamedTemporaryFile -from typing import Any, Callable, Iterator, Literal +from typing import TYPE_CHECKING, Any, Callable, Iterator, Literal import boto3 import pytest @@ -27,6 +27,8 @@ from dagster._core.pipes.subprocess import PipesSubprocessClient from dagster._core.pipes.utils import PipesEnvContextInjector from dagster._core.storage.asset_check_execution_record import AssetCheckExecutionRecordStatus +from moto.server import ThreadedMotoServer # type: ignore # (pyright bug) + from dagster_aws.pipes import ( PipesCloudWatchMessageReader, PipesECSClient, @@ -36,9 +38,6 @@ PipesS3ContextInjector, PipesS3MessageReader, ) -from moto.server import ThreadedMotoServer # type: ignore # (pyright bug) -from mypy_boto3_ecs import ECSClient - from dagster_aws_tests.pipes_tests.fake_ecs import LocalECSMockClient from dagster_aws_tests.pipes_tests.fake_glue import LocalGlueMockClient from dagster_aws_tests.pipes_tests.fake_lambda import ( @@ -47,6 +46,9 @@ LambdaFunctions, ) +if TYPE_CHECKING: + from mypy_boto3_ecs import ECSClient + _PYTHON_EXECUTABLE = shutil.which("python") or "python" @@ -630,7 +632,7 @@ def materialize_asset(env, return_dict): @pytest.fixture -def ecs_client(moto_server, external_s3_glue_script, s3_client) -> ECSClient: +def ecs_client(moto_server, external_s3_glue_script, s3_client) -> "ECSClient": return boto3.client("ecs", region_name="us-east-1", endpoint_url=_MOTO_SERVER_URL) diff --git a/python_modules/libraries/dagster-aws/dagster_aws_tests/redshift_tests/test_resources.py b/python_modules/libraries/dagster-aws/dagster_aws_tests/redshift_tests/test_resources.py index c2202dbacff35..3f8b6b3d46af6 100644 --- a/python_modules/libraries/dagster-aws/dagster_aws_tests/redshift_tests/test_resources.py +++ b/python_modules/libraries/dagster-aws/dagster_aws_tests/redshift_tests/test_resources.py @@ -7,6 +7,7 @@ import pytest from dagster._core.definitions.decorators import op from dagster._utils.test import wrap_op_in_graph_and_execute + from dagster_aws.redshift import ( FakeRedshiftClient, FakeRedshiftClientResource, diff --git a/python_modules/libraries/dagster-aws/dagster_aws_tests/s3_tests/test_compute_log_manager.py b/python_modules/libraries/dagster-aws/dagster_aws_tests/s3_tests/test_compute_log_manager.py index 835092d7c715c..b2514804caa32 100644 --- a/python_modules/libraries/dagster-aws/dagster_aws_tests/s3_tests/test_compute_log_manager.py +++ b/python_modules/libraries/dagster-aws/dagster_aws_tests/s3_tests/test_compute_log_manager.py @@ -15,6 +15,7 @@ from dagster._core.storage.runs import SqliteRunStorage from dagster._core.test_utils import ensure_dagster_tests_import, environ, instance_for_test from dagster._time import get_current_datetime + from dagster_aws.s3 import S3ComputeLogManager ensure_dagster_tests_import() diff --git a/python_modules/libraries/dagster-aws/dagster_aws_tests/s3_tests/test_file_handle_to_s3.py b/python_modules/libraries/dagster-aws/dagster_aws_tests/s3_tests/test_file_handle_to_s3.py index 598cb93c64b64..ff453edd34dc0 100644 --- a/python_modules/libraries/dagster-aws/dagster_aws_tests/s3_tests/test_file_handle_to_s3.py +++ b/python_modules/libraries/dagster-aws/dagster_aws_tests/s3_tests/test_file_handle_to_s3.py @@ -3,6 +3,7 @@ import pytest from dagster import DagsterEventType, job, op from dagster._core.definitions.metadata import MetadataValue + from dagster_aws.s3 import ( S3FileHandle, S3FileManagerResource, diff --git a/python_modules/libraries/dagster-aws/dagster_aws_tests/s3_tests/test_io_manager.py b/python_modules/libraries/dagster-aws/dagster_aws_tests/s3_tests/test_io_manager.py index bb8183656164c..c057ddaf6f053 100644 --- a/python_modules/libraries/dagster-aws/dagster_aws_tests/s3_tests/test_io_manager.py +++ b/python_modules/libraries/dagster-aws/dagster_aws_tests/s3_tests/test_io_manager.py @@ -22,6 +22,7 @@ from dagster._core.definitions.definitions_class import Definitions from dagster._core.definitions.source_asset import SourceAsset from dagster._core.definitions.unresolved_asset_job_definition import define_asset_job + from dagster_aws.s3.io_manager import S3PickleIOManager, s3_pickle_io_manager from dagster_aws.s3.utils import construct_s3_client diff --git a/python_modules/libraries/dagster-aws/dagster_aws_tests/s3_tests/test_s3_file_manager.py b/python_modules/libraries/dagster-aws/dagster_aws_tests/s3_tests/test_s3_file_manager.py index 13dd84244eb6e..7830c275db5bb 100644 --- a/python_modules/libraries/dagster-aws/dagster_aws_tests/s3_tests/test_s3_file_manager.py +++ b/python_modules/libraries/dagster-aws/dagster_aws_tests/s3_tests/test_s3_file_manager.py @@ -5,6 +5,7 @@ import pytest from botocore import exceptions from dagster import DagsterResourceFunctionError, In, Out, build_op_context, configured, job, op + from dagster_aws.s3 import ( S3FileHandle, S3FileManager, diff --git a/python_modules/libraries/dagster-aws/dagster_aws_tests/s3_tests/test_s3_sensor.py b/python_modules/libraries/dagster-aws/dagster_aws_tests/s3_tests/test_s3_sensor.py index 0584c1fd5e7b3..9b52d22c29d30 100644 --- a/python_modules/libraries/dagster-aws/dagster_aws_tests/s3_tests/test_s3_sensor.py +++ b/python_modules/libraries/dagster-aws/dagster_aws_tests/s3_tests/test_s3_sensor.py @@ -1,5 +1,6 @@ import boto3 import moto + from dagster_aws.s3.sensor import get_s3_keys diff --git a/python_modules/libraries/dagster-aws/dagster_aws_tests/secretsmanager_tests/test_secrets.py b/python_modules/libraries/dagster-aws/dagster_aws_tests/secretsmanager_tests/test_secrets.py index ff6743430bd94..12790ae99e468 100644 --- a/python_modules/libraries/dagster-aws/dagster_aws_tests/secretsmanager_tests/test_secrets.py +++ b/python_modules/libraries/dagster-aws/dagster_aws_tests/secretsmanager_tests/test_secrets.py @@ -3,6 +3,7 @@ from dagster._core.execution.context.init import build_init_resource_context from dagster._core.test_utils import environ + from dagster_aws.secretsmanager import ( SecretsManagerSecretsResource, get_secrets_from_arns, diff --git a/python_modules/libraries/dagster-aws/dagster_aws_tests/ssm_tests/test_resources.py b/python_modules/libraries/dagster-aws/dagster_aws_tests/ssm_tests/test_resources.py index 34d010ed6ed0c..1b4f1d1162b07 100644 --- a/python_modules/libraries/dagster-aws/dagster_aws_tests/ssm_tests/test_resources.py +++ b/python_modules/libraries/dagster-aws/dagster_aws_tests/ssm_tests/test_resources.py @@ -2,6 +2,7 @@ from dagster import asset, build_init_resource_context, materialize from dagster._core.test_utils import environ + from dagster_aws.ssm import ParameterStoreResource, ParameterStoreTag, parameter_store_resource diff --git a/python_modules/libraries/dagster-aws/dagster_aws_tests/test_version.py b/python_modules/libraries/dagster-aws/dagster_aws_tests/test_version.py index 59dc3f695233a..6cfe61b67f0ae 100644 --- a/python_modules/libraries/dagster-aws/dagster_aws_tests/test_version.py +++ b/python_modules/libraries/dagster-aws/dagster_aws_tests/test_version.py @@ -3,6 +3,7 @@ import boto3 import pytest from botocore.exceptions import ClientError + from dagster_aws.version import __version__ diff --git a/python_modules/libraries/dagster-aws/dagster_aws_tests/utils_tests/mrjob_tests/test_utils.py b/python_modules/libraries/dagster-aws/dagster_aws_tests/utils_tests/mrjob_tests/test_utils.py index 584b79511914f..9e98766248b49 100644 --- a/python_modules/libraries/dagster-aws/dagster_aws_tests/utils_tests/mrjob_tests/test_utils.py +++ b/python_modules/libraries/dagster-aws/dagster_aws_tests/utils_tests/mrjob_tests/test_utils.py @@ -5,6 +5,7 @@ import botocore from dagster._vendored.dateutil.tz import tzutc + from dagster_aws.utils.mrjob.utils import ( _boto3_now, _client_error_code, diff --git a/python_modules/libraries/dagster-aws/ruff.toml b/python_modules/libraries/dagster-aws/ruff.toml new file mode 100644 index 0000000000000..6fd266058fb41 --- /dev/null +++ b/python_modules/libraries/dagster-aws/ruff.toml @@ -0,0 +1,13 @@ +extend = "../../../pyproject.toml" + +[lint] +extend-select = [ + # ban specific top-level imports + "TID253", +] + +[lint.flake8-tidy-imports] +banned-module-level-imports = [ + "mypy_boto3_ecs", + "mypy_boto3_glue", +] diff --git a/python_modules/libraries/dagster-aws/setup.py b/python_modules/libraries/dagster-aws/setup.py index 1808a13fcd820..78d03ffba1b9d 100644 --- a/python_modules/libraries/dagster-aws/setup.py +++ b/python_modules/libraries/dagster-aws/setup.py @@ -37,7 +37,6 @@ def get_version() -> str: python_requires=">=3.8,<3.13", install_requires=[ "boto3", - "boto3-stubs-lite[ecs,glue]", f"dagster{pin}", "packaging", "requests", @@ -45,6 +44,9 @@ def get_version() -> str: extras_require={ "redshift": ["psycopg2-binary"], "pyspark": ["dagster-pyspark"], + "stubs": [ + "boto3-stubs-lite[ecs,glue]", + ], "test": [ "botocore!=1.32.1", "moto[s3,server,glue]>=2.2.8,<5.0", diff --git a/scripts/install_dev_python_modules.py b/scripts/install_dev_python_modules.py index 7c70ab48f1720..af860e611b140 100644 --- a/scripts/install_dev_python_modules.py +++ b/scripts/install_dev_python_modules.py @@ -58,7 +58,7 @@ def main( "python_modules/automation", "python_modules/libraries/dagster-managed-elements", "python_modules/libraries/dagster-airbyte", - "python_modules/libraries/dagster-aws[test]", + "python_modules/libraries/dagster-aws[stubs,test]", "python_modules/libraries/dagster-celery", "python_modules/libraries/dagster-celery-docker", "python_modules/libraries/dagster-dask[yarn,pbs,kube]",