[dagster-aws] extend main ruff config + move boto3-stubs to stubs e…

…xtras (#24153)

## Summary & Motivation

1. We currently have `boto3-stubs-lite` in `dagster-aws` runtime
dependencies. This might be causing type-checkers to fail for our users
if they have "bad" `boto3` code (which previously wasn't checked because
no stubs package was installed). This PR moves `boto3-stubs-lite` to a
new `stubs` extras of `dagster-aws`, ensuring it won't be installed by
default. It's now only imported under `if TYPE_CHECKING` blocks.
Packages like `mypy_boto3_glue` were added to banned top-level modules
(via ruff rule).

2. Additionally, `dagster-aws`'s ruff config now extends the root
config. This mostly fixed import ordering, previously `dagster_aws`
imports weren't coming after all other packages.

## How I Tested These Changes

## Changelog [New | Bug | Docs]

> NOCHANGELOG

pyright/alt-1/requirements-pinned.txt

@@ -9,7 +9,6 @@ alembic==1.13.2
 aniso8601==9.0.1
 annotated-types==0.7.0
 anyio==4.4.0
-appnope==0.1.4
 argon2-cffi==23.1.0
 argon2-cffi-bindings==21.2.0
 arrow==1.3.0
@@ -25,9 +24,9 @@ backports-tarfile==1.2.0
 beautifulsoup4==4.12.3
 bleach==6.1.0
 boto3==1.35.16
-boto3-stubs-lite==1.35.16
+boto3-stubs-lite==1.35.17
 botocore==1.35.16
-botocore-stubs==1.35.16
+botocore-stubs==1.35.17
 buildkite-test-collector==0.1.9
 cachetools==5.5.0
 caio==0.9.17
@@ -91,8 +90,8 @@ filelock==3.16.0
 fonttools==4.53.1
 fqdn==1.5.1
 frozenlist==1.4.1
-fsspec==2024.3.1
-gcsfs==2024.3.1
+fsspec==2024.3.0
+gcsfs==0.8.0
 google-api-core==2.19.2
 google-api-python-client==2.145.0
 google-auth==2.34.0
@@ -108,6 +107,7 @@ gql==3.5.0
 graphene==3.3
 graphql-core==3.2.4
 graphql-relay==3.2.0
+greenlet==3.1.0
 grpcio==1.66.1
 grpcio-health-checking==1.62.3
 grpcio-status==1.62.3
@@ -131,6 +131,7 @@ jaraco-classes==3.4.0
 jaraco-context==6.0.1
 jaraco-functools==4.0.2
 jedi==0.19.1
+jeepney==0.8.0
 jinja2==3.1.4
 jmespath==1.0.1
 joblib==1.4.2
@@ -251,18 +252,19 @@ rfc3986-validator==0.1.1
 rich==13.8.1
 rpds-py==0.20.0
 rsa==4.9
-s3fs==2024.3.1
+s3fs==2024.3.0
 s3transfer==0.10.2
-scikit-learn==1.5.1
+scikit-learn==1.5.2
 scipy==1.14.1
 seaborn==0.13.2
+secretstorage==3.3.3
 send2trash==1.8.3
 setuptools==74.1.2
 shellingham==1.5.4
 six==1.16.0
 slack-sdk==3.32.0
 sniffio==1.3.1
-snowflake-connector-python==3.12.1
+snowflake-connector-python==3.12.2
 snowflake-sqlalchemy==1.5.1
 sortedcontainers==2.4.0
 soupsieve==2.6
@@ -314,10 +316,11 @@ types-tzlocal==5.1.0.1
 typing-extensions==4.12.2
 typing-inspect==0.9.0
 tzdata==2024.1
-universal-pathlib==0.2.2
+ujson==5.10.0
+universal-pathlib==0.2.5
 uri-template==1.3.0
 uritemplate==4.1.1
-urllib3==2.2.2
+urllib3==2.2.3
 uvicorn==0.30.6
 uvloop==0.20.0
 virtualenv==20.26.4

pyright/alt-1/requirements.txt

@@ -34,7 +34,7 @@
   -e python_modules/dagster-pipes
   -e python_modules/dagster-webserver
   -e python_modules/dagster-graphql
-  -e python_modules/libraries/dagster-aws/
+  -e python_modules/libraries/dagster-aws[stubs]
   -e python_modules/libraries/dagster-dbt/
   -e python_modules/libraries/dagster-duckdb-pandas/
     -e python_modules/libraries/dagster-duckdb/
@@ -49,4 +49,3 @@
   -e python_modules/libraries/dagster-snowflake/
     -e python_modules/libraries/dagster-snowflake-pandas/
     -e python_modules/libraries/dagster-snowflake-pyspark/
-

pyright/master/requirements-pinned.txt

@@ -25,7 +25,6 @@ apache-airflow-providers-sqlite==3.8.2
 apeye==1.4.1
 apeye-core==1.1.5
 apispec==6.6.1
-appnope==0.1.4
 argcomplete==3.5.0
 argon2-cffi==23.1.0
 argon2-cffi-bindings==21.2.0
@@ -58,10 +57,10 @@ bitmath==1.3.3.1
 bleach==6.1.0
 blinker==1.8.2
 bokeh==3.5.2
-boto3==1.35.16
-boto3-stubs-lite==1.35.16
-botocore==1.35.16
-botocore-stubs==1.35.16
+boto3==1.35.17
+boto3-stubs-lite==1.35.17
+botocore==1.35.17
+botocore-stubs==1.35.17
 buildkite-test-collector==0.1.9
 cachecontrol==0.14.0
 cached-property==1.5.2
@@ -256,6 +255,7 @@ graphql-core==3.2.4
 graphql-relay==3.2.0
 graphviz==0.20.3
 great-expectations==0.17.11
+greenlet==3.1.0
 grpcio==1.66.1
 grpcio-health-checking==1.62.3
 grpcio-status==1.62.3
@@ -321,7 +321,7 @@ langchain-community==0.2.9
 langchain-core==0.2.39
 langchain-openai==0.1.14
 langchain-text-splitters==0.2.4
-langsmith==0.1.117
+langsmith==0.1.118
 lazy-object-proxy==1.10.0
 leather==0.4.0
 limits==3.13.0
@@ -375,6 +375,18 @@ noteable-origami==1.1.5
 notebook==7.2.2
 notebook-shim==0.2.4
 numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.6.68
+nvidia-nvtx-cu12==12.1.105
 oauth2client==4.1.3
 oauthlib==3.2.2
 objgraph==3.6.1
@@ -483,7 +495,7 @@ querystring-parser==1.2.4
 rapidfuzz==3.9.7
 readme-renderer==44.0
 referencing==0.35.1
-regex==2024.7.24
+regex==2024.9.11
 requests==2.32.3
 requests-file==2.1.0
 requests-mock==1.12.1
@@ -500,7 +512,7 @@ rsa==4.9
 ruamel-yaml==0.17.17
 ruff==0.6.4
 s3transfer==0.10.2
-scikit-learn==1.5.1
+scikit-learn==1.5.2
 scipy==1.14.1
 scrapbook==0.5.0
 sdf-cli==0.3.23
@@ -519,11 +531,11 @@ skein==0.8.2
 skl2onnx==1.17.0
 slack-sdk==3.32.0
 sling==1.2.18
-sling-mac-arm64==1.2.18
+sling-linux-amd64==1.2.18
 smmap==5.0.1
 sniffio==1.3.1
 snowballstemmer==2.2.0
-snowflake-connector-python==3.12.1
+snowflake-connector-python==3.12.2
 snowflake-sqlalchemy==1.6.1
 sortedcontainers==2.4.0
 soupsieve==2.6
@@ -575,6 +587,7 @@ tqdm==4.66.5
 traitlets==5.14.3
 trio==0.26.2
 trio-websocket==0.11.1
+triton==3.0.0
 -e examples/experimental/dagster-airlift/examples/tutorial-example
 -e examples/tutorial_notebook_assets
 twilio==9.3.0
@@ -621,7 +634,7 @@ uvicorn==0.30.6
 uvloop==0.20.0
 vine==5.1.0
 virtualenv==20.25.0
-wandb==0.17.9
+wandb==0.18.0
 watchdog==5.0.2
 watchfiles==0.24.0
 wcwidth==0.2.13

pyright/master/requirements.txt

@@ -37,7 +37,7 @@
 -e python_modules/dagster-webserver
 -e python_modules/libraries/dagster-airbyte/
 -e python_modules/libraries/dagster-airflow[test_airflow_2]
--e python_modules/libraries/dagster-aws[test]
+-e python_modules/libraries/dagster-aws[stubs,test]
 -e python_modules/libraries/dagster-azure/
 -e python_modules/libraries/dagster-celery/
 -e python_modules/libraries/dagster-celery-docker/

python_modules/libraries/dagster-aws/MANIFEST.in

@@ -4,3 +4,5 @@ recursive-include dagster_aws *.txt
 recursive-include dagster_aws *.template
 include LICENSE
 include dagster_aws/py.typed
+
+exclude ruff.toml

python_modules/libraries/dagster-aws/dagster_aws/pipes/clients/glue.py

@@ -15,13 +15,13 @@
     PipesMessageReader,
 )
 from dagster._core.pipes.utils import open_pipes_session
-from mypy_boto3_glue.type_defs import StartJobRunRequestRequestTypeDef
 
 from dagster_aws.pipes.context_injectors import PipesS3ContextInjector
 from dagster_aws.pipes.message_readers import PipesCloudWatchMessageReader
 
 if TYPE_CHECKING:
     from mypy_boto3_glue.client import GlueClient
+    from mypy_boto3_glue.type_defs import StartJobRunRequestRequestTypeDef
 
 RUN_PARAMS_BREAKING_VERSION = "1.9.0"
 
@@ -87,7 +87,7 @@ def run(
         self,
         *,
         context: OpExecutionContext,
-        start_job_run_params: Optional[StartJobRunRequestRequestTypeDef] = None,
+        start_job_run_params: Optional["StartJobRunRequestRequestTypeDef"] = None,
         extras: Optional[Dict[str, Any]] = None,
         job_name: Optional[str] = None,
         arguments: Optional[Mapping[str, Any]] = None,
@@ -137,7 +137,7 @@ def run(
                     "JobName": job_name,
                 }
 
-        params = cast(StartJobRunRequestRequestTypeDef, params)
+        params = cast("StartJobRunRequestRequestTypeDef", params)
 
         params["Arguments"] = arguments
 

python_modules/libraries/dagster-aws/dagster_aws_tests/athena_tests/test_resources.py

@@ -1,13 +1,14 @@
 import boto3
 import pytest
+from moto import mock_athena
+
 from dagster_aws.athena.resources import (
     AthenaError,
     AthenaTimeout,
     FakeAthenaClient,
     ResourceWithAthenaConfig,
     fake_athena_resource,
 )
-from moto import mock_athena
 
 
 class TestAthenaClientResource(ResourceWithAthenaConfig):

python_modules/libraries/dagster-aws/dagster_aws_tests/cloudwatch_tests/test_loggers.py

@@ -3,9 +3,10 @@
 import boto3
 import pytest
 from dagster import job, op
-from dagster_aws.cloudwatch import cloudwatch_logger
 from moto import mock_logs
 
+from dagster_aws.cloudwatch import cloudwatch_logger
+
 
 @op
 def hello_op(context):

python_modules/libraries/dagster-aws/dagster_aws_tests/ecr_tests/test_resources.py

@@ -1,5 +1,6 @@
 from dagster import op
 from dagster._utils.test import wrap_op_in_graph_and_execute
+
 from dagster_aws.ecr import FakeECRPublicResource, fake_ecr_public_resource
 
 

python_modules/libraries/dagster-aws/dagster_aws_tests/ecs_tests/launcher_tests/test_container_context.py

@@ -1,6 +1,7 @@
 import pytest
 from dagster._core.errors import DagsterInvalidConfigError
 from dagster._core.test_utils import environ
+
 from dagster_aws.ecs.container_context import EcsContainerContext
 
 

python_modules/libraries/dagster-aws/dagster_aws_tests/ecs_tests/launcher_tests/test_launching.py

@@ -3,7 +3,6 @@
 import time
 from concurrent.futures import ThreadPoolExecutor
 
-import dagster_aws
 import pytest
 from botocore.exceptions import ClientError
 from dagster._check import CheckError
@@ -14,6 +13,8 @@
 from dagster._core.origin import JobPythonOrigin, RepositoryPythonOrigin
 from dagster._core.storage.dagster_run import DagsterRunStatus
 from dagster._core.storage.tags import RUN_WORKER_ID_TAG
+
+import dagster_aws
 from dagster_aws.ecs import EcsEventualConsistencyTimeout
 from dagster_aws.ecs.launcher import (
     DEFAULT_LINUX_RESOURCES,

python_modules/libraries/dagster-aws/dagster_aws_tests/ecs_tests/launcher_tests/test_utils.py

@@ -3,6 +3,7 @@
     RemoteJobOrigin,
     RemoteRepositoryOrigin,
 )
+
 from dagster_aws.ecs.utils import get_task_definition_family, sanitize_family
 
 

python_modules/libraries/dagster-aws/dagster_aws_tests/emr_tests/test_configs_spark.py

@@ -1,9 +1,10 @@
 # from dagster_aws.emr.configs_spark import spark_config as dagster_aws_spark_config
 # from dagster_spark.configs_spark import spark_config as dagster_spark_spark_config
 
-import dagster_aws.emr.configs_spark as aws_configs_spark
 import dagster_spark.configs_spark as spark_configs_spark
 
+import dagster_aws.emr.configs_spark as aws_configs_spark
+
 
 def test_spark_configs_same_as_in_dagster_spark():
     aws_contents = open(aws_configs_spark.__file__, encoding="utf8").read()

python_modules/libraries/dagster-aws/dagster_aws_tests/emr_tests/test_emr.py

@@ -7,9 +7,10 @@
 
 import pytest
 from dagster._utils.test import create_test_pipeline_execution_context
+from moto import mock_emr
+
 from dagster_aws.emr import EmrClusterState, EmrError, EmrJobRunner
 from dagster_aws.utils.mrjob.utils import _boto3_now
-from moto import mock_emr
 
 REGION = "us-west-1"
 

python_modules/libraries/dagster-aws/dagster_aws_tests/emr_tests/test_emr_step_main.py

@@ -3,6 +3,7 @@
 
 from dagster import DagsterEvent
 from dagster._core.events import EngineEventData
+
 from dagster_aws.emr.emr_step_main import DONE, event_writing_loop
 
 

python_modules/libraries/dagster-aws/dagster_aws_tests/emr_tests/test_pyspark.py

@@ -16,14 +16,15 @@
 from dagster._core.test_utils import instance_for_test
 from dagster._utils.merger import deep_merge_dicts
 from dagster._utils.test import create_test_pipeline_execution_context
-from dagster_aws.emr import EmrError, EmrJobRunner
-from dagster_aws.emr.pyspark_step_launcher import EmrPySparkStepLauncher, emr_pyspark_step_launcher
-from dagster_aws.s3 import s3_resource
 from dagster_pyspark import DataFrame, pyspark_resource
 from moto import mock_emr
 from pyspark.sql import Row
 from pyspark.sql.types import IntegerType, StringType, StructField, StructType
 
+from dagster_aws.emr import EmrError, EmrJobRunner
+from dagster_aws.emr.pyspark_step_launcher import EmrPySparkStepLauncher, emr_pyspark_step_launcher
+from dagster_aws.s3 import s3_resource
+
 S3_BUCKET = "dagster-scratch-80542c2"
 
 

python_modules/libraries/dagster-aws/dagster_aws_tests/emr_tests/test_pyspark_step_launcher.py

@@ -4,6 +4,7 @@
 import pytest
 from dagster import DagsterEvent, EventLogEntry, build_init_resource_context
 from dagster._core.execution.plan.objects import StepSuccessData
+
 from dagster_aws.emr.pyspark_step_launcher import EmrPySparkStepLauncher, emr_pyspark_step_launcher
 
 EVENTS = [