[dagster-aws] add PipesEMRServerlessClient (#24318)

## Summary & Motivation

resolve [DS-443](https://linear.app/dagster-labs/issue/DS-443/implement-pipes-emr-serverless)

Implemented message reading for CloudWatch logging configuration (using `CloudWatchMessageReader`). Mapping Spark driver stdout/stderr to Dagster's stdout/stderr. 

S3-backed logs are currently blocked by #24098

## How I Tested These Changes

1. Manually tested with real AWS infra (personal account)
2. Added a very simple test which checks that context env vars are set correctly. This is as far as we can get with `moto` since they don't have `start_job_run` implemented. 

I'm planning to add integration tests in CI with real AWS infra. 

## Changelog

Insert changelog entry or "NOCHANGELOG" here.

- [x] `NEW` _([dagster-aws] added `PipesEMRServerlessClient`, but it's untested yet so please don't include into release notes until we have tests)_
- [ ] `BUGFIX` _(fixed a bug)_
- [ ] `DOCS` _(added or updated documentation)_

pyright/alt-1/requirements-pinned.txt

@@ -171,7 +171,8 @@ multidict==6.1.0
 multimethod==1.10
 mypy==1.11.2
 mypy-boto3-ecs==1.35.2
-mypy-boto3-glue==1.35.3
+mypy-boto3-emr-serverless==1.35.0
+mypy-boto3-glue==1.35.18
 mypy-extensions==1.0.0
 mypy-protobuf==3.6.0
 nbclient==0.10.0
@@ -201,7 +202,7 @@ pillow==10.4.0
 pip==24.2
 platformdirs==4.3.2
 pluggy==1.5.0
-polars==1.7.0
+polars==1.7.1
 -e examples/project_fully_featured
 prometheus-client==0.20.0
 prompt-toolkit==3.0.47
@@ -269,7 +270,7 @@ snowflake-sqlalchemy==1.5.1
 sortedcontainers==2.4.0
 soupsieve==2.6
 sqlalchemy==1.4.54
-sqlglot==25.20.1
+sqlglot==25.21.0
 sqlglotrs==0.2.12
 sqlparse==0.5.1
 stack-data==0.6.3

pyright/master/requirements-pinned.txt

@@ -57,9 +57,9 @@ bitmath==1.3.3.1
 bleach==6.1.0
 blinker==1.8.2
 bokeh==3.5.2
-boto3==1.35.17
+boto3==1.35.18
 boto3-stubs-lite==1.35.17
-botocore==1.35.17
+botocore==1.35.18
 botocore-stubs==1.35.17
 buildkite-test-collector==0.1.9
 cachecontrol==0.14.0
@@ -322,7 +322,7 @@ langchain-community==0.2.9
 langchain-core==0.2.39
 langchain-openai==0.1.14
 langchain-text-splitters==0.2.4
-langsmith==0.1.118
+langsmith==0.1.119
 lazy-object-proxy==1.10.0
 leather==0.4.0
 limits==3.13.0
@@ -360,7 +360,8 @@ msgpack==1.1.0
 multidict==6.1.0
 multimethod==1.10
 mypy-boto3-ecs==1.35.2
-mypy-boto3-glue==1.35.3
+mypy-boto3-emr-serverless==1.35.0
+mypy-boto3-glue==1.35.18
 mypy-extensions==1.0.0
 mypy-protobuf==3.6.0
 mysql-connector-python==9.0.0
@@ -394,7 +395,7 @@ objgraph==3.6.1
 onnx==1.16.2
 onnxconverter-common==1.13.0
 onnxruntime==1.19.2
-openai==1.44.1
+openai==1.45.0
 openapi-schema-validator==0.6.2
 openapi-spec-validator==0.7.1
 opentelemetry-api==1.27.0
@@ -432,10 +433,10 @@ pillow==10.4.0
 pip==24.2
 pkginfo==1.11.1
 platformdirs==4.3.2
-plotly==5.24.0
+plotly==5.24.1
 pluggy==1.5.0
 ply==3.11
-polars==1.7.0
+polars==1.7.1
 portalocker==2.10.1
 prison==0.2.1
 progressbar2==4.5.0
@@ -541,7 +542,7 @@ snowflake-sqlalchemy==1.6.1
 sortedcontainers==2.4.0
 soupsieve==2.6
 sphinx==8.0.2
-sphinx-autodoc-typehints==2.4.0
+sphinx-autodoc-typehints==2.4.1
 sphinx-jinja2-compat==0.3.0
 sphinx-prompt==1.9.0
 sphinx-tabs==3.4.5
@@ -555,7 +556,7 @@ sphinxcontrib-serializinghtml==2.0.0
 sqlalchemy==1.4.54
 sqlalchemy-jsonfield==1.0.2
 sqlalchemy-utils==0.41.2
-sqlglot==25.20.1
+sqlglot==25.21.0
 sqlglotrs==0.2.12
 sqlparse==0.5.1
 sshpubkeys==3.3.1

python_modules/libraries/dagster-aws/dagster_aws/pipes/__init__.py

@@ -1,4 +1,9 @@
-from dagster_aws.pipes.clients import PipesECSClient, PipesGlueClient, PipesLambdaClient
+from dagster_aws.pipes.clients import (
+    PipesECSClient,
+    PipesEMRServerlessClient,
+    PipesGlueClient,
+    PipesLambdaClient,
+)
 from dagster_aws.pipes.context_injectors import (
     PipesLambdaEventContextInjector,
     PipesS3ContextInjector,
@@ -18,4 +23,5 @@
     "PipesS3MessageReader",
     "PipesLambdaLogsMessageReader",
     "PipesCloudWatchMessageReader",
+    "PipesEMRServerlessClient",
 ]

python_modules/libraries/dagster-aws/dagster_aws/pipes/clients/__init__.py

@@ -1,5 +1,6 @@
 from dagster_aws.pipes.clients.ecs import PipesECSClient
+from dagster_aws.pipes.clients.emr_serverless import PipesEMRServerlessClient
 from dagster_aws.pipes.clients.glue import PipesGlueClient
 from dagster_aws.pipes.clients.lambda_ import PipesLambdaClient
 
-__all__ = ["PipesGlueClient", "PipesLambdaClient", "PipesECSClient"]
+__all__ = ["PipesGlueClient", "PipesLambdaClient", "PipesECSClient", "PipesEMRServerlessClient"]