diff --git a/examples/docs_snippets/docs_snippets_tests/guides_tests/asset_tutorial_tests/test_cereal.py b/examples/docs_snippets/docs_snippets_tests/guides_tests/asset_tutorial_tests/test_cereal.py index 9c2bb1a39a817..d693b73d05388 100644 --- a/examples/docs_snippets/docs_snippets_tests/guides_tests/asset_tutorial_tests/test_cereal.py +++ b/examples/docs_snippets/docs_snippets_tests/guides_tests/asset_tutorial_tests/test_cereal.py @@ -1,5 +1,7 @@ -from dagster._core.definitions.load_assets_from_modules import load_assets_from_modules from dagster._core.definitions.materialize import materialize +from dagster._core.definitions.module_loaders.load_assets_from_modules import ( + load_assets_from_modules, +) from docs_snippets.guides.dagster.asset_tutorial import cereal from docs_snippets.intro_tutorial.test_util import patch_cereal_requests diff --git a/examples/docs_snippets/docs_snippets_tests/guides_tests/asset_tutorial_tests/test_serial_asset_graph.py b/examples/docs_snippets/docs_snippets_tests/guides_tests/asset_tutorial_tests/test_serial_asset_graph.py index 70a5d6cbefdb9..80f95ee6e920e 100644 --- a/examples/docs_snippets/docs_snippets_tests/guides_tests/asset_tutorial_tests/test_serial_asset_graph.py +++ b/examples/docs_snippets/docs_snippets_tests/guides_tests/asset_tutorial_tests/test_serial_asset_graph.py @@ -1,5 +1,7 @@ -from dagster._core.definitions.load_assets_from_modules import load_assets_from_modules from dagster._core.definitions.materialize import materialize +from dagster._core.definitions.module_loaders.load_assets_from_modules import ( + load_assets_from_modules, +) from docs_snippets.guides.dagster.asset_tutorial import serial_asset_graph from docs_snippets.intro_tutorial.test_util import patch_cereal_requests diff --git a/python_modules/dagster/dagster/__init__.py b/python_modules/dagster/dagster/__init__.py index 8c5ee3805c460..441135c0c3217 100644 --- a/python_modules/dagster/dagster/__init__.py +++ b/python_modules/dagster/dagster/__init__.py @@ -250,18 +250,6 @@ InputMapping as InputMapping, ) from dagster._core.definitions.job_definition import JobDefinition as JobDefinition -from dagster._core.definitions.load_asset_checks_from_modules import ( - load_asset_checks_from_current_module as load_asset_checks_from_current_module, - load_asset_checks_from_modules as load_asset_checks_from_modules, - load_asset_checks_from_package_module as load_asset_checks_from_package_module, - load_asset_checks_from_package_name as load_asset_checks_from_package_name, -) -from dagster._core.definitions.load_assets_from_modules import ( - load_assets_from_current_module as load_assets_from_current_module, - load_assets_from_modules as load_assets_from_modules, - load_assets_from_package_module as load_assets_from_package_module, - load_assets_from_package_name as load_assets_from_package_name, -) from dagster._core.definitions.logger_definition import ( LoggerDefinition as LoggerDefinition, build_init_logger_context as build_init_logger_context, @@ -309,6 +297,18 @@ TableRecord as TableRecord, TableSchema as TableSchema, ) +from dagster._core.definitions.module_loaders.load_asset_checks_from_modules import ( + load_asset_checks_from_current_module as load_asset_checks_from_current_module, + load_asset_checks_from_modules as load_asset_checks_from_modules, + load_asset_checks_from_package_module as load_asset_checks_from_package_module, + load_asset_checks_from_package_name as load_asset_checks_from_package_name, +) +from dagster._core.definitions.module_loaders.load_assets_from_modules import ( + load_assets_from_current_module as load_assets_from_current_module, + load_assets_from_modules as load_assets_from_modules, + load_assets_from_package_module as load_assets_from_package_module, + load_assets_from_package_name as load_assets_from_package_name, +) from dagster._core.definitions.multi_asset_sensor_definition import ( MultiAssetSensorDefinition as MultiAssetSensorDefinition, MultiAssetSensorEvaluationContext as MultiAssetSensorEvaluationContext, diff --git a/python_modules/dagster/dagster/_core/code_pointer.py b/python_modules/dagster/dagster/_core/code_pointer.py index b79293722ea84..915ecf29b7b60 100644 --- a/python_modules/dagster/dagster/_core/code_pointer.py +++ b/python_modules/dagster/dagster/_core/code_pointer.py @@ -184,7 +184,9 @@ def describe(self) -> str: def _load_target_from_module(module: ModuleType, fn_name: str, error_suffix: str) -> object: - from dagster._core.definitions.load_assets_from_modules import load_assets_from_modules + from dagster._core.definitions.module_loaders.load_assets_from_modules import ( + load_assets_from_modules, + ) from dagster._core.workspace.autodiscovery import LOAD_ALL_ASSETS if fn_name == LOAD_ALL_ASSETS: diff --git a/python_modules/dagster/dagster/_core/definitions/__init__.py b/python_modules/dagster/dagster/_core/definitions/__init__.py index e8ad5b2d6d2a5..75fb66abfa513 100644 --- a/python_modules/dagster/dagster/_core/definitions/__init__.py +++ b/python_modules/dagster/dagster/_core/definitions/__init__.py @@ -148,16 +148,16 @@ ) from dagster._core.definitions.graph_definition import GraphDefinition as GraphDefinition from dagster._core.definitions.job_definition import JobDefinition as JobDefinition -from dagster._core.definitions.load_assets_from_modules import ( +from dagster._core.definitions.materialize import ( + materialize as materialize, + materialize_to_memory as materialize_to_memory, +) +from dagster._core.definitions.module_loaders.load_assets_from_modules import ( load_assets_from_current_module as load_assets_from_current_module, load_assets_from_modules as load_assets_from_modules, load_assets_from_package_module as load_assets_from_package_module, load_assets_from_package_name as load_assets_from_package_name, ) -from dagster._core.definitions.materialize import ( - materialize as materialize, - materialize_to_memory as materialize_to_memory, -) from dagster._core.definitions.op_definition import OpDefinition as OpDefinition from dagster._core.definitions.partition import ( DynamicPartitionsDefinition as DynamicPartitionsDefinition, diff --git a/python_modules/dagster/dagster_tests/asset_defs_tests/asset_package/asset_subpackage/__init__.py b/python_modules/dagster/dagster/_core/definitions/module_loaders/__init__.py similarity index 100% rename from python_modules/dagster/dagster_tests/asset_defs_tests/asset_package/asset_subpackage/__init__.py rename to python_modules/dagster/dagster/_core/definitions/module_loaders/__init__.py diff --git a/python_modules/dagster/dagster/_core/definitions/load_asset_checks_from_modules.py b/python_modules/dagster/dagster/_core/definitions/module_loaders/load_asset_checks_from_modules.py similarity index 98% rename from python_modules/dagster/dagster/_core/definitions/load_asset_checks_from_modules.py rename to python_modules/dagster/dagster/_core/definitions/module_loaders/load_asset_checks_from_modules.py index a587066410f00..56d76aa254922 100644 --- a/python_modules/dagster/dagster/_core/definitions/load_asset_checks_from_modules.py +++ b/python_modules/dagster/dagster/_core/definitions/module_loaders/load_asset_checks_from_modules.py @@ -9,7 +9,7 @@ CoercibleToAssetKeyPrefix, check_opt_coercible_to_asset_key_prefix_param, ) -from dagster._core.definitions.load_assets_from_modules import ( +from dagster._core.definitions.module_loaders.load_assets_from_modules import ( LoadedAssetsList, find_modules_in_package, ) diff --git a/python_modules/dagster/dagster/_core/definitions/load_assets_from_modules.py b/python_modules/dagster/dagster/_core/definitions/module_loaders/load_assets_from_modules.py similarity index 100% rename from python_modules/dagster/dagster/_core/definitions/load_assets_from_modules.py rename to python_modules/dagster/dagster/_core/definitions/module_loaders/load_assets_from_modules.py diff --git a/python_modules/dagster/dagster/_core/workspace/autodiscovery.py b/python_modules/dagster/dagster/_core/workspace/autodiscovery.py index fc9142f40ecca..63733e9d45f98 100644 --- a/python_modules/dagster/dagster/_core/workspace/autodiscovery.py +++ b/python_modules/dagster/dagster/_core/workspace/autodiscovery.py @@ -5,7 +5,9 @@ from dagster import DagsterInvariantViolationError, GraphDefinition, RepositoryDefinition from dagster._core.code_pointer import load_python_file, load_python_module from dagster._core.definitions.definitions_class import Definitions -from dagster._core.definitions.load_assets_from_modules import load_assets_from_modules +from dagster._core.definitions.module_loaders.load_assets_from_modules import ( + load_assets_from_modules, +) LOAD_ALL_ASSETS = "<>" diff --git a/python_modules/dagster/dagster_tests/asset_defs_tests/test_asset_job.py b/python_modules/dagster/dagster_tests/asset_defs_tests/test_asset_job.py index 0e96216bd1477..99d8733f7e7f4 100644 --- a/python_modules/dagster/dagster_tests/asset_defs_tests/test_asset_job.py +++ b/python_modules/dagster/dagster_tests/asset_defs_tests/test_asset_job.py @@ -2363,10 +2363,10 @@ def test_asset_group_build_subset_job(job_selection, expected_assets, use_multi, for prefix in reversed(prefixes or []): all_assets = [ assets_def.with_attributes( - input_asset_key_replacements={ - k: k.with_prefix(prefix) for k in assets_def.keys_by_input_name.values() + asset_key_replacements={ + k: k.with_prefix(prefix) + for k in set(assets_def.keys_by_input_name.values()) | set(assets_def.keys) }, - output_asset_key_replacements={k: k.with_prefix(prefix) for k in assets_def.keys}, ) for assets_def in all_assets ] diff --git a/python_modules/dagster/dagster_tests/definitions_tests/asset_check_tests/__init__.py b/python_modules/dagster/dagster_tests/definitions_tests/module_loader_tests/__init__.py similarity index 100% rename from python_modules/dagster/dagster_tests/definitions_tests/asset_check_tests/__init__.py rename to python_modules/dagster/dagster_tests/definitions_tests/module_loader_tests/__init__.py diff --git a/python_modules/dagster/dagster_tests/asset_defs_tests/asset_package/__init__.py b/python_modules/dagster/dagster_tests/definitions_tests/module_loader_tests/asset_package/__init__.py similarity index 100% rename from python_modules/dagster/dagster_tests/asset_defs_tests/asset_package/__init__.py rename to python_modules/dagster/dagster_tests/definitions_tests/module_loader_tests/asset_package/__init__.py diff --git a/python_modules/dagster/dagster_tests/definitions_tests/module_loader_tests/asset_package/asset_subpackage/__init__.py b/python_modules/dagster/dagster_tests/definitions_tests/module_loader_tests/asset_package/asset_subpackage/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/python_modules/dagster/dagster_tests/asset_defs_tests/asset_package/asset_subpackage/another_module_with_assets.py b/python_modules/dagster/dagster_tests/definitions_tests/module_loader_tests/asset_package/asset_subpackage/another_module_with_assets.py similarity index 100% rename from python_modules/dagster/dagster_tests/asset_defs_tests/asset_package/asset_subpackage/another_module_with_assets.py rename to python_modules/dagster/dagster_tests/definitions_tests/module_loader_tests/asset_package/asset_subpackage/another_module_with_assets.py diff --git a/python_modules/dagster/dagster_tests/asset_defs_tests/asset_package/asset_subpackage/asset_subsubpackage/__init__.py b/python_modules/dagster/dagster_tests/definitions_tests/module_loader_tests/asset_package/asset_subpackage/asset_subsubpackage/__init__.py similarity index 100% rename from python_modules/dagster/dagster_tests/asset_defs_tests/asset_package/asset_subpackage/asset_subsubpackage/__init__.py rename to python_modules/dagster/dagster_tests/definitions_tests/module_loader_tests/asset_package/asset_subpackage/asset_subsubpackage/__init__.py diff --git a/python_modules/dagster/dagster_tests/asset_defs_tests/asset_package/module_with_assets.py b/python_modules/dagster/dagster_tests/definitions_tests/module_loader_tests/asset_package/module_with_assets.py similarity index 88% rename from python_modules/dagster/dagster_tests/asset_defs_tests/asset_package/module_with_assets.py rename to python_modules/dagster/dagster_tests/definitions_tests/module_loader_tests/asset_package/module_with_assets.py index eaa8819f9fc57..ace7bfbf9c8af 100644 --- a/python_modules/dagster/dagster_tests/asset_defs_tests/asset_package/module_with_assets.py +++ b/python_modules/dagster/dagster_tests/definitions_tests/module_loader_tests/asset_package/module_with_assets.py @@ -7,7 +7,7 @@ ) # importing this makes it show up twice when we collect everything -from dagster_tests.asset_defs_tests.asset_package.asset_subpackage.another_module_with_assets import ( +from dagster_tests.definitions_tests.module_loader_tests.asset_package.asset_subpackage.another_module_with_assets import ( miles_davis, ) diff --git a/python_modules/dagster/dagster_tests/asset_defs_tests/asset_package_with_cacheable/__init__.py b/python_modules/dagster/dagster_tests/definitions_tests/module_loader_tests/asset_package_with_cacheable/__init__.py similarity index 100% rename from python_modules/dagster/dagster_tests/asset_defs_tests/asset_package_with_cacheable/__init__.py rename to python_modules/dagster/dagster_tests/definitions_tests/module_loader_tests/asset_package_with_cacheable/__init__.py diff --git a/python_modules/dagster/dagster_tests/definitions_tests/asset_check_tests/checks_module/__init__.py b/python_modules/dagster/dagster_tests/definitions_tests/module_loader_tests/checks_module/__init__.py similarity index 100% rename from python_modules/dagster/dagster_tests/definitions_tests/asset_check_tests/checks_module/__init__.py rename to python_modules/dagster/dagster_tests/definitions_tests/module_loader_tests/checks_module/__init__.py diff --git a/python_modules/dagster/dagster_tests/definitions_tests/asset_check_tests/checks_module/checks_submodule/__init__.py b/python_modules/dagster/dagster_tests/definitions_tests/module_loader_tests/checks_module/checks_submodule/__init__.py similarity index 100% rename from python_modules/dagster/dagster_tests/definitions_tests/asset_check_tests/checks_module/checks_submodule/__init__.py rename to python_modules/dagster/dagster_tests/definitions_tests/module_loader_tests/checks_module/checks_submodule/__init__.py diff --git a/python_modules/dagster/dagster_tests/definitions_tests/asset_check_tests/checks_module/checks_submodule_2/__init__.py b/python_modules/dagster/dagster_tests/definitions_tests/module_loader_tests/checks_module/checks_submodule_2/__init__.py similarity index 100% rename from python_modules/dagster/dagster_tests/definitions_tests/asset_check_tests/checks_module/checks_submodule_2/__init__.py rename to python_modules/dagster/dagster_tests/definitions_tests/module_loader_tests/checks_module/checks_submodule_2/__init__.py diff --git a/python_modules/dagster/dagster_tests/definitions_tests/asset_check_tests/test_load_from_modules.py b/python_modules/dagster/dagster_tests/definitions_tests/module_loader_tests/test_load_asset_checks_from_modules.py similarity index 92% rename from python_modules/dagster/dagster_tests/definitions_tests/asset_check_tests/test_load_from_modules.py rename to python_modules/dagster/dagster_tests/definitions_tests/module_loader_tests/test_load_asset_checks_from_modules.py index c299c488aae4a..9256adf4efadd 100644 --- a/python_modules/dagster/dagster_tests/definitions_tests/asset_check_tests/test_load_from_modules.py +++ b/python_modules/dagster/dagster_tests/definitions_tests/module_loader_tests/test_load_asset_checks_from_modules.py @@ -19,8 +19,8 @@ def test_load_asset_checks_from_modules(): - from dagster_tests.definitions_tests.asset_check_tests import checks_module - from dagster_tests.definitions_tests.asset_check_tests.checks_module import asset_check_1 + from dagster_tests.definitions_tests.module_loader_tests import checks_module + from dagster_tests.definitions_tests.module_loader_tests.checks_module import asset_check_1 checks = load_asset_checks_from_modules([checks_module]) assert len(checks) == 1 @@ -47,7 +47,7 @@ def test_load_asset_checks_from_modules(): def test_load_asset_checks_from_modules_prefix(): - from dagster_tests.definitions_tests.asset_check_tests import checks_module + from dagster_tests.definitions_tests.module_loader_tests import checks_module checks = load_asset_checks_from_modules([checks_module], asset_key_prefix="foo") assert len(checks) == 1 @@ -98,7 +98,7 @@ def test_load_asset_checks_from_current_module(): ], ) def test_load_asset_checks_from_package(load_fns): - from dagster_tests.definitions_tests.asset_check_tests import checks_module + from dagster_tests.definitions_tests.module_loader_tests import checks_module assets_load_fn, checks_load_fn = load_fns diff --git a/python_modules/dagster/dagster_tests/asset_defs_tests/test_assets_from_modules.py b/python_modules/dagster/dagster_tests/definitions_tests/module_loader_tests/test_load_assets_from_modules.py similarity index 89% rename from python_modules/dagster/dagster_tests/asset_defs_tests/test_assets_from_modules.py rename to python_modules/dagster/dagster_tests/definitions_tests/module_loader_tests/test_load_assets_from_modules.py index 97d9549e615da..4b479e4b372d8 100644 --- a/python_modules/dagster/dagster_tests/asset_defs_tests/test_assets_from_modules.py +++ b/python_modules/dagster/dagster_tests/definitions_tests/module_loader_tests/test_load_assets_from_modules.py @@ -88,7 +88,7 @@ def get_source_asset_with_key( def test_load_assets_from_package_name(): - from dagster_tests.asset_defs_tests import asset_package + from dagster_tests.definitions_tests.module_loader_tests import asset_package assets_defs = load_assets_from_package_name(asset_package.__name__) assert len(assets_defs) == 11 @@ -113,7 +113,7 @@ def test_load_assets_from_package_name(): def test_load_assets_from_package_module(): - from dagster_tests.asset_defs_tests import asset_package + from dagster_tests.definitions_tests.module_loader_tests import asset_package assets_1 = load_assets_from_package_module(asset_package) assert len(assets_1) == 11 @@ -138,8 +138,8 @@ def test_load_assets_from_package_module(): def test_load_assets_from_modules(monkeypatch): - from dagster_tests.asset_defs_tests import asset_package - from dagster_tests.asset_defs_tests.asset_package import module_with_assets + from dagster_tests.definitions_tests.module_loader_tests import asset_package + from dagster_tests.definitions_tests.module_loader_tests.asset_package import module_with_assets collection_1 = load_assets_from_modules([asset_package, module_with_assets]) @@ -204,8 +204,8 @@ def test_load_assets_from_current_module(): def test_load_assets_from_modules_with_group_name(): - from dagster_tests.asset_defs_tests import asset_package - from dagster_tests.asset_defs_tests.asset_package import module_with_assets + from dagster_tests.definitions_tests.module_loader_tests import asset_package + from dagster_tests.definitions_tests.module_loader_tests.asset_package import module_with_assets assets = load_assets_from_modules( [asset_package, module_with_assets], group_name="my_cool_group" @@ -226,8 +226,8 @@ def test_respect_existing_groups(): def test_load_assets_with_freshness_policy(): - from dagster_tests.asset_defs_tests import asset_package - from dagster_tests.asset_defs_tests.asset_package import module_with_assets + from dagster_tests.definitions_tests.module_loader_tests import asset_package + from dagster_tests.definitions_tests.module_loader_tests.asset_package import module_with_assets assets = load_assets_from_modules( [asset_package, module_with_assets], @@ -242,8 +242,8 @@ def test_load_assets_with_freshness_policy(): def test_load_assets_with_auto_materialize_policy(): - from dagster_tests.asset_defs_tests import asset_package - from dagster_tests.asset_defs_tests.asset_package import module_with_assets + from dagster_tests.definitions_tests.module_loader_tests import asset_package + from dagster_tests.definitions_tests.module_loader_tests.asset_package import module_with_assets assets = load_assets_from_modules( [asset_package, module_with_assets], auto_materialize_policy=AutoMaterializePolicy.eager() @@ -265,8 +265,8 @@ def test_load_assets_with_auto_materialize_policy(): ], ) def test_prefix(prefix): - from dagster_tests.asset_defs_tests import asset_package - from dagster_tests.asset_defs_tests.asset_package import module_with_assets + from dagster_tests.definitions_tests.module_loader_tests import asset_package + from dagster_tests.definitions_tests.module_loader_tests.asset_package import module_with_assets assets = load_assets_from_modules([asset_package, module_with_assets], key_prefix=prefix) assert_assets_have_prefix(prefix, assets) # pyright: ignore[reportArgumentType] @@ -276,7 +276,7 @@ def test_prefix(prefix): def _load_assets_from_module_with_assets(**kwargs): - from dagster_tests.asset_defs_tests.asset_package import module_with_assets + from dagster_tests.definitions_tests.module_loader_tests.asset_package import module_with_assets return load_assets_from_modules([module_with_assets], **kwargs) @@ -286,7 +286,7 @@ def _load_assets_from_module_with_assets(**kwargs): [ _load_assets_from_module_with_assets, lambda **kwargs: load_assets_from_package_name( - "dagster_tests.asset_defs_tests.asset_package", **kwargs + "dagster_tests.definitions_tests.module_loader_tests.asset_package", **kwargs ), ], ) @@ -339,7 +339,7 @@ def test_source_key_prefix(load_fn): ) def test_load_assets_cacheable(load_fn, prefix): """Tests the load-from-module and load-from-package-name functinos with cacheable assets.""" - from dagster_tests.asset_defs_tests import asset_package_with_cacheable + from dagster_tests.definitions_tests.module_loader_tests import asset_package_with_cacheable assets_defs = load_fn(asset_package_with_cacheable) assert len(assets_defs) == 3 diff --git a/python_modules/libraries/dagster-components/dagster_components/core/component.py b/python_modules/libraries/dagster-components/dagster_components/core/component.py index dc8ca5e0dab20..bd28674a1c94d 100644 --- a/python_modules/libraries/dagster-components/dagster_components/core/component.py +++ b/python_modules/libraries/dagster-components/dagster_components/core/component.py @@ -182,7 +182,9 @@ def __repr__(self) -> str: def get_registered_components_in_module(module: ModuleType) -> Iterable[Type[Component]]: - from dagster._core.definitions.load_assets_from_modules import find_subclasses_in_module + from dagster._core.definitions.module_loaders.load_assets_from_modules import ( + find_subclasses_in_module, + ) for component in find_subclasses_in_module(module, (Component,)): if is_registered_component(component): diff --git a/python_modules/libraries/dagster-dbt/dagster_dbt/cli/app.py b/python_modules/libraries/dagster-dbt/dagster_dbt/cli/app.py index cb439b79c5fd2..3a80099287c07 100644 --- a/python_modules/libraries/dagster-dbt/dagster_dbt/cli/app.py +++ b/python_modules/libraries/dagster-dbt/dagster_dbt/cli/app.py @@ -7,7 +7,9 @@ import yaml from dagster._cli.project import check_if_pypi_package_conflict_exists from dagster._core.code_pointer import load_python_file -from dagster._core.definitions.load_assets_from_modules import find_objects_in_module_of_types +from dagster._core.definitions.module_loaders.load_assets_from_modules import ( + find_objects_in_module_of_types, +) from jinja2 import Environment, FileSystemLoader from rich.console import Console from rich.syntax import Syntax