Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[dagster-aws] delete dummy Pipes classes #23441

Merged
merged 1 commit into from
Aug 8, 2024
Merged

[dagster-aws] delete dummy Pipes classes #23441

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file modified docs/content/api/modules.json.gz
View file
Open in desktop
Binary file not shown.
Binary file modified docs/content/api/searchindex.json.gz
View file
Open in desktop
Binary file not shown.
Binary file modified docs/content/api/sections.json.gz
View file
Open in desktop
Binary file not shown.
5 changes: 2 additions & 3 deletions docs/content/concepts/dagster-pipes/aws-glue.mdx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,8 +107,7 @@ Next, add the `PipesGlueClient` resource to your project's <PyObject object="Def

```python file=/guides/dagster/dagster_pipes/glue/dagster_code.py startafter=start_definitions_marker endbefore=end_definitions_marker
from dagster import Definitions # noqa
from dagster_aws.pipes import PipesGlueContextInjector, PipesS3MessageReader

from dagster_aws.pipes import PipesS3ContextInjector, PipesS3MessageReader

bucket = os.environ["DAGSTER_GLUE_S3_CONTEXT_BUCKET"]

Expand All @@ -118,7 +117,7 @@ defs = Definitions(
resources={
"pipes_glue_client": PipesGlueClient(
client=boto3.client("glue"),
context_injector=PipesGlueContextInjector(
context_injector=PipesS3ContextInjector(
client=boto3.client("s3"),
bucket=bucket,
),
Expand Down
5 changes: 2 additions & 3 deletions examples/docs_snippets/docs_snippets/guides/dagster/dagster_pipes/glue/dagster_code.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,7 @@ def glue_pipes_asset(
# start_definitions_marker

from dagster import Definitions # noqa
from dagster_aws.pipes import PipesGlueContextInjector, PipesS3MessageReader

from dagster_aws.pipes import PipesS3ContextInjector, PipesS3MessageReader

bucket = os.environ["DAGSTER_GLUE_S3_CONTEXT_BUCKET"]

Expand All @@ -35,7 +34,7 @@ def glue_pipes_asset(
resources={
"pipes_glue_client": PipesGlueClient(
client=boto3.client("glue"),
context_injector=PipesGlueContextInjector(
context_injector=PipesS3ContextInjector(
client=boto3.client("s3"),
bucket=bucket,
),
Expand Down
15 changes: 3 additions & 12 deletions python_modules/libraries/dagster-aws/dagster_aws/pipes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -334,24 +334,15 @@ def run(
return PipesClientCompletedInvocation(session)


class PipesGlueContextInjector(PipesS3ContextInjector):
def no_messages_debug_text(self) -> str:
return "Attempted to inject context via Glue job Arguments"


class PipesGlueLogsMessageReader(PipesCloudWatchMessageReader):
pass


@experimental
class PipesGlueClient(PipesClient, TreatAsResourceParam):
"""A pipes client for invoking AWS Glue jobs.

Args:
context_injector (Optional[PipesContextInjector]): A context injector to use to inject
context into the Glue job, for example, :py:class:`PipesGlueContextInjector`.
context into the Glue job, for example, :py:class:`PipesS3ContextInjector`.
message_reader (Optional[PipesMessageReader]): A message reader to use to read messages
from the glue job run. Defaults to :py:class:`PipesGlueLogsMessageReader`.
from the glue job run. Defaults to :py:class:`PipesCloudWatchsMessageReader`.
client (Optional[boto3.client]): The boto Glue client used to launch the Glue job
"""

Expand All @@ -363,7 +354,7 @@ def __init__(
):
self._client = client or boto3.client("glue")
self._context_injector = context_injector
self._message_reader = message_reader or PipesGlueLogsMessageReader()
self._message_reader = message_reader or PipesCloudWatchMessageReader()

@classmethod
def _is_dagster_maintained(cls) -> bool:
Expand Down