diff --git a/docs/content/api/modules.json.gz b/docs/content/api/modules.json.gz index dec32b7540357..14f7400a10ace 100644 Binary files a/docs/content/api/modules.json.gz and b/docs/content/api/modules.json.gz differ diff --git a/docs/content/api/searchindex.json.gz b/docs/content/api/searchindex.json.gz index e18a84a9a4a0f..31892b8358ddc 100644 Binary files a/docs/content/api/searchindex.json.gz and b/docs/content/api/searchindex.json.gz differ diff --git a/docs/content/api/sections.json.gz b/docs/content/api/sections.json.gz index a42af61969192..44cc54b90d718 100644 Binary files a/docs/content/api/sections.json.gz and b/docs/content/api/sections.json.gz differ diff --git a/docs/next/public/objects.inv b/docs/next/public/objects.inv index 9177a72132c81..c007be86def04 100644 Binary files a/docs/next/public/objects.inv and b/docs/next/public/objects.inv differ diff --git a/docs/sphinx/sections/api/apidocs/internals.rst b/docs/sphinx/sections/api/apidocs/internals.rst index 8e2ed341a734c..1234e8a2ab6d1 100644 --- a/docs/sphinx/sections/api/apidocs/internals.rst +++ b/docs/sphinx/sections/api/apidocs/internals.rst @@ -214,8 +214,8 @@ Exception handling ---- -Step Launchers (Experimental) ------------------------------ +Step Launchers (Superseded) +--------------------------- .. currentmodule:: dagster diff --git a/python_modules/dagster/dagster/_core/definitions/step_launcher.py b/python_modules/dagster/dagster/_core/definitions/step_launcher.py index 65be9687bbff1..4978b47dfe339 100644 --- a/python_modules/dagster/dagster/_core/definitions/step_launcher.py +++ b/python_modules/dagster/dagster/_core/definitions/step_launcher.py @@ -2,6 +2,7 @@ from typing import TYPE_CHECKING, Iterator, Mapping, NamedTuple, Optional import dagster._check as check +from dagster._annotations import superseded from dagster._core.definitions.reconstruct import ReconstructableJob from dagster._core.execution.retries import RetryMode from dagster._core.storage.dagster_run import DagsterRun @@ -56,6 +57,13 @@ def __new__( ) +_step_launcher_supersession = superseded( + subject="StepLauncher", + additional_warn_text="Consider using Dagster Pipes instead. Learn more here: https://docs.dagster.io/concepts/dagster-pipes", +) + + +@_step_launcher_supersession class StepLauncher(ABC): """A StepLauncher is responsible for executing steps, either in-process or in an external process.""" diff --git a/python_modules/libraries/dagster-aws/dagster_aws/emr/pyspark_step_launcher.py b/python_modules/libraries/dagster-aws/dagster_aws/emr/pyspark_step_launcher.py index 532589b79d7e1..76e325ef973f6 100644 --- a/python_modules/libraries/dagster-aws/dagster_aws/emr/pyspark_step_launcher.py +++ b/python_modules/libraries/dagster-aws/dagster_aws/emr/pyspark_step_launcher.py @@ -13,7 +13,7 @@ resource, ) from dagster._core.definitions.resource_definition import dagster_maintained_resource -from dagster._core.definitions.step_launcher import StepLauncher +from dagster._core.definitions.step_launcher import StepLauncher, _step_launcher_supersession from dagster._core.errors import DagsterInvariantViolationError, raise_execution_interrupts from dagster._core.execution.plan.external_step import ( PICKLED_EVENTS_FILE_NAME, @@ -147,6 +147,7 @@ ), } ) +@_step_launcher_supersession def emr_pyspark_step_launcher(context): # Resolve legacy arguments if context.resource_config.get("local_job_package_path") and context.resource_config.get( @@ -216,6 +217,7 @@ def emr_pyspark_step_launcher(context): ) +@_step_launcher_supersession class EmrPySparkStepLauncher(StepLauncher): def __init__( self, diff --git a/python_modules/libraries/dagster-databricks/dagster_databricks/databricks_pyspark_step_launcher.py b/python_modules/libraries/dagster-databricks/dagster_databricks/databricks_pyspark_step_launcher.py index 67612de0ddb3a..9caf9a54f3737 100644 --- a/python_modules/libraries/dagster-databricks/dagster_databricks/databricks_pyspark_step_launcher.py +++ b/python_modules/libraries/dagster-databricks/dagster_databricks/databricks_pyspark_step_launcher.py @@ -19,7 +19,11 @@ ) from dagster._core.definitions.metadata import MetadataValue, RawMetadataValue from dagster._core.definitions.resource_definition import dagster_maintained_resource -from dagster._core.definitions.step_launcher import StepLauncher, StepRunRef +from dagster._core.definitions.step_launcher import ( + StepLauncher, + StepRunRef, + _step_launcher_supersession, +) from dagster._core.errors import DagsterInvariantViolationError, raise_execution_interrupts from dagster._core.events import DagsterEvent, DagsterEventType, EngineEventData from dagster._core.events.log import EventLogEntry @@ -169,6 +173,7 @@ ), } ) +@_step_launcher_supersession def databricks_pyspark_step_launcher( context: InitResourceContext, ) -> "DatabricksPySparkStepLauncher": @@ -188,6 +193,7 @@ def databricks_pyspark_step_launcher( return DatabricksPySparkStepLauncher(**context.resource_config) +@_step_launcher_supersession class DatabricksPySparkStepLauncher(StepLauncher): def __init__( self,