From cee5d3eb916d326333d42345906bd237bbbd2151 Mon Sep 17 00:00:00 2001 From: Sean Mackesey Date: Sun, 22 Dec 2024 00:07:43 -0500 Subject: [PATCH] [components] Move global options to subcommands (#26640) ## Summary & Motivation This changes the options scheme in `dagster-dg` so that options are no longer hierarchically processed and passed down the command tree, _except_ in the special case of `dg component generate` subcommands, where global options are defined on the containing group. Now, all leaves of the tree (except for `dg component generate` subcommands) have the same set of "global options" appended to their options list. These are visually separated in the help message. We correspondingly simplify the Usage message to have a single `[OPTIONS]` input spot. There are also now fewer apparent global options because two of the old ones, `--clear-cache` and `--rebuild-component-registry`, must be executed as standalone pseudo-commands (e.g. `dg --clear-cache`) rather than used as modifiers to other commands. Before (`dg component-type info --help`): ``` Usage: dg [OPTIONS] component-type info [OPTIONS] COMPONENT_TYPE Get detailed information on a registered Dagster component type. Options: --description --generate-params-schema --component-params-schema -h, --help Show this message and exit. Options (dg): --builtin-component-lib TEXT Specify a builitin component library to use. --verbose Enable verbose output for debugging. --disable-cache Disable caching of component registry data. --clear-cache Clear the cache before running the command. --rebuild-component-registry Recompute and cache the set of available component types for the current environment. Note that this also happens automatically whenever the cache is detected to be stale. --cache-dir PATH Specify a directory to use for the cache. ``` After (`dg component-type info --help`): ``` Usage: dg component-type info [OPTIONS] COMPONENT_TYPE Get detailed information on a registered Dagster component type. Options: --description --generate-params-schema --component-params-schema -h, --help Show this message and exit. Global options: --builtin-component-lib TEXT Specify a builitin component library to use. --verbose Enable verbose output for debugging. --disable-cache Disable the cache.. --cache-dir PATH Specify a directory to use for the cache. ``` --- Before (`dg generate component dagster_components.dbt_project --help`): ``` Usage: dg [OPTIONS] component generate dagster_components.dbt_project [OPTIONS] COMPONENT_NAME Options: --json-params TEXT JSON string of component parameters. --init BOOLEAN init --project-path TEXT project_path -h, --help Show this message and exit. Options (dg): --builtin-component-lib TEXT Specify a builitin component library to use. --verbose Enable verbose output for debugging. --disable-cache Disable caching of component registry data. --clear-cache Clear the cache before running the command. --rebuild-component-registry Recompute and cache the set of available component types for the current environment. Note that this also happens automatically whenever the cache is detected to be stale. --cache-dir PATH Specify a directory to use for the cache. ``` After (`dg generate component dagster_components.dbt_project --help`): ``` Usage: dg component generate [GLOBAL OPTIONS] dagster_components.dbt_project [OPTIONS] COMPONENT_NAME Options: --json-params TEXT JSON string of component parameters. --init BOOLEAN init --project-path TEXT project_path -h, --help Show this message and exit. Global options: --builtin-component-lib TEXT Specify a builitin component library to use. --verbose Enable verbose output for debugging. --disable-cache Disable the cache.. --cache-dir PATH Specify a directory to use for the cache. ``` ## How I Tested These Changes Modified unit tests, played with it on the command line. --- .../dagster-dg/dagster_dg/cli/__init__.py | 80 ++----- .../dagster_dg/cli/code_location.py | 19 +- .../dagster-dg/dagster_dg/cli/component.py | 90 ++++++- .../dagster_dg/cli/component_type.py | 19 +- .../dagster-dg/dagster_dg/cli/deployment.py | 8 +- .../dagster_dg/cli/global_options.py | 67 ++++++ .../libraries/dagster-dg/dagster_dg/config.py | 38 ++- .../dagster-dg/dagster_dg/context.py | 30 ++- .../dagster-dg/dagster_dg/generate.py | 2 +- .../libraries/dagster-dg/dagster_dg/utils.py | 88 +++---- .../cli_tests/test_code_location_commands.py | 49 ++-- .../cli_tests/test_custom_help_format.py | 222 +++++++----------- .../cli_tests/test_integrity.py | 14 ++ .../dagster-dg/dagster_dg_tests/test_cache.py | 17 +- .../dagster-dg/dagster_dg_tests/utils.py | 29 ++- 15 files changed, 439 insertions(+), 333 deletions(-) create mode 100644 python_modules/libraries/dagster-dg/dagster_dg/cli/global_options.py diff --git a/python_modules/libraries/dagster-dg/dagster_dg/cli/__init__.py b/python_modules/libraries/dagster-dg/dagster_dg/cli/__init__.py index 8302e5e659877..01a868c8c0bea 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg/cli/__init__.py +++ b/python_modules/libraries/dagster-dg/dagster_dg/cli/__init__.py @@ -3,12 +3,11 @@ import click -from dagster_dg.cache import DgCache from dagster_dg.cli.code_location import code_location_group from dagster_dg.cli.component import component_group from dagster_dg.cli.component_type import component_type_group from dagster_dg.cli.deployment import deployment_group -from dagster_dg.config import DgConfig, set_config_on_cli_context +from dagster_dg.cli.global_options import dg_global_options from dagster_dg.context import ( DgContext, ensure_uv_lock, @@ -20,44 +19,30 @@ from dagster_dg.utils import DgClickGroup from dagster_dg.version import __version__ +DG_CLI_MAX_OUTPUT_WIDTH = 120 -def create_dg_cli(): - commands = { - "code-location": code_location_group, - "deployment": deployment_group, - "component": component_group, - "component-type": component_type_group, - } - # Defaults are defined on the DgConfig object. +def create_dg_cli(): @click.group( - commands=commands, - context_settings={"max_content_width": 120, "help_option_names": ["-h", "--help"]}, + name="dg", + commands={ + "code-location": code_location_group, + "deployment": deployment_group, + "component": component_group, + "component-type": component_type_group, + }, + context_settings={ + "max_content_width": DG_CLI_MAX_OUTPUT_WIDTH, + "help_option_names": ["-h", "--help"], + }, invoke_without_command=True, cls=DgClickGroup, ) - @click.option( - "--builtin-component-lib", - type=str, - default=DgConfig.builtin_component_lib, - help="Specify a builitin component library to use.", - ) - @click.option( - "--verbose", - is_flag=True, - default=DgConfig.verbose, - help="Enable verbose output for debugging.", - ) - @click.option( - "--disable-cache", - is_flag=True, - default=DgConfig.disable_cache, - help="Disable caching of component registry data.", - ) + @dg_global_options @click.option( "--clear-cache", is_flag=True, - help="Clear the cache before running the command.", + help="Clear the cache.", default=False, ) @click.option( @@ -69,33 +54,15 @@ def create_dg_cli(): ), default=False, ) - @click.option( - "--cache-dir", - type=Path, - default=DgConfig.cache_dir, - help="Specify a directory to use for the cache.", - ) @click.version_option(__version__, "--version", "-v") @click.pass_context def group( context: click.Context, - builtin_component_lib: str, - verbose: bool, - disable_cache: bool, - cache_dir: Path, clear_cache: bool, rebuild_component_registry: bool, + **global_options: object, ): - """CLI tools for working with Dagster components.""" - context.ensure_object(dict) - config = DgConfig( - builtin_component_lib=builtin_component_lib, - verbose=verbose, - disable_cache=disable_cache, - cache_dir=cache_dir, - ) - set_config_on_cli_context(context, config) - + """CLI for working with Dagster components.""" if clear_cache and rebuild_component_registry: click.echo( click.style( @@ -104,10 +71,12 @@ def group( ) sys.exit(1) elif clear_cache: - DgCache.from_config(config).clear_all() + dg_context = DgContext.from_cli_global_options(global_options) + dg_context.cache.clear_all() if context.invoked_subcommand is None: context.exit(0) elif rebuild_component_registry: + dg_context = DgContext.from_cli_global_options(global_options) if context.invoked_subcommand is not None: click.echo( click.style( @@ -115,7 +84,7 @@ def group( ) ) sys.exit(1) - _rebuild_component_registry(context) + _rebuild_component_registry(dg_context) elif context.invoked_subcommand is None: click.echo(context.get_help()) context.exit(0) @@ -123,8 +92,7 @@ def group( return group -def _rebuild_component_registry(cli_context: click.Context): - dg_context = DgContext.from_cli_context(cli_context) +def _rebuild_component_registry(dg_context: DgContext): if not is_inside_code_location_directory(Path.cwd()): click.echo( click.style( @@ -132,7 +100,7 @@ def _rebuild_component_registry(cli_context: click.Context): ) ) sys.exit(1) - if not dg_context.cache: + if not dg_context.has_cache: click.echo( click.style("Cache is disabled. This command cannot be run without a cache.", fg="red") ) diff --git a/python_modules/libraries/dagster-dg/dagster_dg/cli/code_location.py b/python_modules/libraries/dagster-dg/dagster_dg/cli/code_location.py index bed03bd9f804c..e2d6e7e54f765 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg/cli/code_location.py +++ b/python_modules/libraries/dagster-dg/dagster_dg/cli/code_location.py @@ -5,6 +5,7 @@ import click +from dagster_dg.cli.global_options import dg_global_options from dagster_dg.context import DeploymentDirectoryContext, DgContext, is_inside_deployment_directory from dagster_dg.generate import generate_code_location from dagster_dg.utils import DgClickCommand, DgClickGroup @@ -40,9 +41,12 @@ def code_location_group(): default=False, help="Do not create a virtual environment for the code location.", ) -@click.pass_context +@dg_global_options def code_location_generate_command( - cli_context: click.Context, name: str, use_editable_dagster: Optional[str], skip_venv: bool + name: str, + use_editable_dagster: Optional[str], + skip_venv: bool, + **global_options: object, ) -> None: """Generate a Dagster code location file structure and a uv-managed virtual environment scoped to the code location. @@ -53,6 +57,7 @@ def code_location_generate_command( The code location file structure defines a Python package with some pre-existing internal structure: + \b ├── │ ├── __init__.py │ ├── components @@ -66,8 +71,8 @@ def code_location_generate_command( The `.components` directory holds components (which can be created with `dg generate component`). The `.lib` directory holds custom component types scoped to the code location (which can be created with `dg component-type generate`). - """ - dg_context = DgContext.from_cli_context(cli_context) + """ # noqa: D301 + dg_context = DgContext.from_cli_global_options(global_options) if is_inside_deployment_directory(Path.cwd()): context = DeploymentDirectoryContext.from_path(Path.cwd(), dg_context) if context.has_code_location(name): @@ -101,10 +106,10 @@ def code_location_generate_command( @code_location_group.command(name="list", cls=DgClickCommand) -@click.pass_context -def code_location_list_command(cli_context: click.Context) -> None: +@dg_global_options +def code_location_list_command(**global_options: object) -> None: """List code locations in the current deployment.""" - dg_context = DgContext.from_cli_context(cli_context) + dg_context = DgContext.from_cli_global_options(global_options) if not is_inside_deployment_directory(Path.cwd()): click.echo( click.style("This command must be run inside a Dagster deployment directory.", fg="red") diff --git a/python_modules/libraries/dagster-dg/dagster_dg/cli/component.py b/python_modules/libraries/dagster-dg/dagster_dg/cli/component.py index a39592c51fa84..b58fb707a1426 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg/cli/component.py +++ b/python_modules/libraries/dagster-dg/dagster_dg/cli/component.py @@ -1,11 +1,18 @@ import sys from pathlib import Path -from typing import Any, Mapping, Optional +from typing import Any, List, Mapping, Optional import click from click.core import ParameterSource +from dagster_dg.cli.global_options import dg_global_options from dagster_dg.component import RemoteComponentType +from dagster_dg.config import ( + DgConfig, + get_config_from_cli_context, + has_config_on_cli_context, + set_config_on_cli_context, +) from dagster_dg.context import ( CodeLocationDirectoryContext, DgContext, @@ -16,6 +23,7 @@ DgClickCommand, DgClickGroup, json_schema_property_to_click_option, + not_none, parse_json_option, ) @@ -48,14 +56,17 @@ def get_command(self, cli_context: click.Context, cmd_name: str) -> Optional[cli self._define_commands(cli_context) return super().get_command(cli_context, cmd_name) - def list_commands(self, cli_context): + def list_commands(self, cli_context: click.Context) -> List[str]: if not self._commands_defined: self._define_commands(cli_context) return super().list_commands(cli_context) def _define_commands(self, cli_context: click.Context) -> None: """Dynamically define a command for each registered component type.""" - app_context = DgContext.from_cli_context(cli_context) + if not has_config_on_cli_context(cli_context): + cli_context.invoke(not_none(self.callback), **cli_context.params) + config = get_config_from_cli_context(cli_context) + dg_context = DgContext.from_config(config) if not is_inside_code_location_directory(Path.cwd()): click.echo( @@ -65,21 +76,77 @@ def _define_commands(self, cli_context: click.Context) -> None: ) sys.exit(1) - context = CodeLocationDirectoryContext.from_path(Path.cwd(), app_context) + context = CodeLocationDirectoryContext.from_path(Path.cwd(), dg_context) for key, component_type in context.iter_component_types(): command = _create_component_generate_subcommand(key, component_type) self.add_command(command) -@component_group.group(name="generate", cls=ComponentGenerateGroup) -def component_generate_group() -> None: +class ComponentGenerateSubCommand(DgClickCommand): + def format_usage(self, context: click.Context, formatter: click.HelpFormatter) -> None: + if not isinstance(self, click.Command): + raise ValueError("This mixin is only intended for use with click.Command instances.") + arg_pieces = self.collect_usage_pieces(context) + command_parts = context.command_path.split(" ") + command_parts.insert(-1, "[GLOBAL OPTIONS]") + return formatter.write_usage(" ".join(command_parts), " ".join(arg_pieces)) + + def format_options(self, context: click.Context, formatter: click.HelpFormatter) -> None: + # This will not produce any global options since there are none defined on component + # generate subcommands. + super().format_options(context, formatter) + + # Get the global options off the parent group. + parent_context = not_none(context.parent) + parent_command = not_none(context.parent).command + if not isinstance(parent_command, DgClickGroup): + raise ValueError("Parent command must be a DgClickGroup.") + _, global_opts = parent_command.get_partitioned_opts(context) + + with formatter.section("Global options"): + records = [not_none(p.get_help_record(parent_context)) for p in global_opts] + formatter.write_dl(records) + + +# We have to override the usual Click processing of `--help` here. The issue is +# that click will process this option before processing anything else, but because we are +# dynamically generating subcommands based on the content of other options, the output of --help +# actually depends on these other options. So we opt out of Click's short-circuiting +# behavior of `--help` by setting `help_option_names=[]`, ensuring that we can process the other +# options first and generate the correct subcommands. We then add a custom `--help` option that +# gets invoked inside the callback. +@component_group.group( + name="generate", + cls=ComponentGenerateGroup, + invoke_without_command=True, + context_settings={"help_option_names": []}, +) +@click.option("-h", "--help", "help_", is_flag=True, help="Show this message and exit.") +@dg_global_options +@click.pass_context +def component_generate_group(context: click.Context, help_: bool, **global_options: object) -> None: """Generate a scaffold of a Dagster component.""" + # Click attempts to resolve subcommands BEFORE it invokes this callback. + # Therefore we need to manually invoke this callback during subcommand generation to make sure + # it runs first. It will be invoked again later by Click. We make it idempotent to deal with + # that. + if not has_config_on_cli_context(context): + set_config_on_cli_context(context, DgConfig.from_cli_global_options(global_options)) + if help_: + click.echo(context.get_help()) + context.exit(0) def _create_component_generate_subcommand( component_key: str, component_type: RemoteComponentType ) -> DgClickCommand: - @click.command(name=component_key, cls=DgClickCommand) + # We need to "reset" the help option names to the default ones because we inherit the parent + # value of context settings from the parent group, which has been customized. + @click.command( + name=component_key, + cls=ComponentGenerateSubCommand, + context_settings={"help_option_names": ["-h", "--help"]}, + ) @click.argument("component_name", type=str) @click.option( "--json-params", @@ -112,7 +179,8 @@ def generate_component_command( It is an error to pass both --json-params and key-value pairs as options. """ - dg_context = DgContext.from_cli_context(cli_context) + config = get_config_from_cli_context(cli_context) + dg_context = DgContext.from_config(config) if not is_inside_code_location_directory(Path.cwd()): click.echo( click.style( @@ -185,10 +253,10 @@ def generate_component_command( @component_group.command(name="list", cls=DgClickCommand) -@click.pass_context -def component_list_command(cli_context: click.Context) -> None: +@dg_global_options +def component_list_command(**global_options: object) -> None: """List Dagster component instances defined in the current code location.""" - dg_context = DgContext.from_cli_context(cli_context) + dg_context = DgContext.from_cli_global_options(global_options) if not is_inside_code_location_directory(Path.cwd()): click.echo( click.style( diff --git a/python_modules/libraries/dagster-dg/dagster_dg/cli/component_type.py b/python_modules/libraries/dagster-dg/dagster_dg/cli/component_type.py index b222d3f921e78..28f0dc2282654 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg/cli/component_type.py +++ b/python_modules/libraries/dagster-dg/dagster_dg/cli/component_type.py @@ -5,6 +5,7 @@ import click +from dagster_dg.cli.global_options import dg_global_options from dagster_dg.context import ( CodeLocationDirectoryContext, DgContext, @@ -26,14 +27,14 @@ def component_type_group(): @component_type_group.command(name="generate", cls=DgClickCommand) @click.argument("name", type=str) -@click.pass_context -def component_type_generate_command(cli_context: click.Context, name: str) -> None: +@dg_global_options +def component_type_generate_command(name: str, **global_options: object) -> None: """Generate a scaffold of a custom Dagster component type. This command must be run inside a Dagster code location directory. The component type scaffold will be generated in submodule `.lib.`. """ - dg_context = DgContext.from_cli_context(cli_context) + dg_context = DgContext.from_cli_global_options(global_options) if not is_inside_code_location_directory(Path.cwd()): click.echo( click.style( @@ -60,16 +61,16 @@ def component_type_generate_command(cli_context: click.Context, name: str) -> No @click.option("--description", is_flag=True, default=False) @click.option("--generate-params-schema", is_flag=True, default=False) @click.option("--component-params-schema", is_flag=True, default=False) -@click.pass_context +@dg_global_options def component_type_info_command( - cli_context: click.Context, component_type: str, description: bool, generate_params_schema: bool, component_params_schema: bool, + **global_options: object, ) -> None: """Get detailed information on a registered Dagster component type.""" - dg_context = DgContext.from_cli_context(cli_context) + dg_context = DgContext.from_cli_global_options(global_options) if not is_inside_code_location_directory(Path.cwd()): click.echo( click.style( @@ -136,10 +137,10 @@ def _serialize_json_schema(schema: Mapping[str, Any]) -> str: @component_type_group.command(name="list", cls=DgClickCommand) -@click.pass_context -def component_type_list(cli_context: click.Context) -> None: +@dg_global_options +def component_type_list(**global_options: object) -> None: """List registered Dagster components in the current code location environment.""" - dg_context = DgContext.from_cli_context(cli_context) + dg_context = DgContext.from_cli_global_options(global_options) if not is_inside_code_location_directory(Path.cwd()): click.echo( click.style( diff --git a/python_modules/libraries/dagster-dg/dagster_dg/cli/deployment.py b/python_modules/libraries/dagster-dg/dagster_dg/cli/deployment.py index c20e1f31a3cec..ef48ffb60b8f0 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg/cli/deployment.py +++ b/python_modules/libraries/dagster-dg/dagster_dg/cli/deployment.py @@ -4,6 +4,8 @@ import click +from dagster_dg.cli.global_options import dg_global_options +from dagster_dg.context import DgContext from dagster_dg.generate import generate_deployment from dagster_dg.utils import DgClickCommand, DgClickGroup @@ -19,13 +21,15 @@ def deployment_group(): @deployment_group.command(name="generate", cls=DgClickCommand) +@dg_global_options @click.argument("path", type=Path) -def deployment_generate_command(path: Path) -> None: +def deployment_generate_command(path: Path, **global_options: object) -> None: """Generate a Dagster deployment file structure. The deployment file structure includes a directory for code locations and configuration files for deploying to Dagster Plus. """ + dg_context = DgContext.from_cli_global_options(global_options) dir_abspath = os.path.abspath(path) if os.path.exists(dir_abspath): click.echo( @@ -33,4 +37,4 @@ def deployment_generate_command(path: Path) -> None: + "\nPlease delete the contents of this path or choose another location." ) sys.exit(1) - generate_deployment(path) + generate_deployment(path, dg_context) diff --git a/python_modules/libraries/dagster-dg/dagster_dg/cli/global_options.py b/python_modules/libraries/dagster-dg/dagster_dg/cli/global_options.py new file mode 100644 index 0000000000000..306b633f4b646 --- /dev/null +++ b/python_modules/libraries/dagster-dg/dagster_dg/cli/global_options.py @@ -0,0 +1,67 @@ +from pathlib import Path +from typing import Any, Callable, Optional, Sequence, TypeVar, Union + +import click + +from dagster_dg.config import DgConfig + +T_Command = TypeVar("T_Command", bound=Union[Callable[..., Any], click.Command]) + +# Defaults are defined on the DgConfig object. +GLOBAL_OPTIONS = { + option.name: option + for option in [ + click.Option( + ["--cache-dir"], + type=Path, + default=DgConfig.cache_dir, + help="Specify a directory to use for the cache.", + ), + click.Option( + ["--disable-cache"], + is_flag=True, + default=DgConfig.disable_cache, + help="Disable the cache..", + ), + click.Option( + ["--verbose"], + is_flag=True, + default=DgConfig.verbose, + help="Enable verbose output for debugging.", + ), + click.Option( + ["--builtin-component-lib"], + type=str, + default=DgConfig.builtin_component_lib, + help="Specify a builitin component library to use.", + ), + ] +} + + +def dg_global_options( + fn: Optional[T_Command] = None, *, names: Optional[Sequence[str]] = None +) -> Union[T_Command, Callable[[T_Command], T_Command]]: + if fn: + options = [GLOBAL_OPTIONS[name] for name in names or list(GLOBAL_OPTIONS.keys())] + if isinstance(fn, click.Command): + for option in options: + fn.params.append(option) + else: + # This is borrowed from click itself, it is how its decorators operate on both commands + # and regular functions. + if not hasattr(fn, "__click_params__"): + fn.__click_params__ = [] # type: ignore + for option in options: + fn.__click_params__.append(option) # type: ignore + + return fn + else: + return lambda fn: dg_global_options(fn, names=names) # type: ignore + + +def validate_global_opts(context: click.Context, **global_options: object) -> None: + for name, value in global_options.items(): + if name not in GLOBAL_OPTIONS: + raise click.UsageError(f"Unknown global option: {name}") + GLOBAL_OPTIONS[name].process_value(context, value) diff --git a/python_modules/libraries/dagster-dg/dagster_dg/config.py b/python_modules/libraries/dagster-dg/dagster_dg/config.py index d5a7abd9adaa4..abfb727cd332d 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg/config.py +++ b/python_modules/libraries/dagster-dg/dagster_dg/config.py @@ -1,12 +1,15 @@ import sys from dataclasses import dataclass from pathlib import Path +from typing import Mapping, Type, TypeVar import click from typing_extensions import Self from dagster_dg.error import DgError +T = TypeVar("T") + DEFAULT_BUILTIN_COMPONENT_LIB = "dagster_components" @@ -40,21 +43,44 @@ class DgConfig: builtin_component_lib: str = DEFAULT_BUILTIN_COMPONENT_LIB @classmethod - def from_cli_context(cls, cli_context: click.Context) -> Self: - if _CLI_CONTEXT_CONFIG_KEY not in cli_context.obj: - raise DgError( - "Attempted to extract DgConfig from CLI context but nothing stored under designated key `{_CLI_CONTEXT_CONFIG_KEY}`." - ) - return cli_context.obj[_CLI_CONTEXT_CONFIG_KEY] + def from_cli_global_options(cls, global_options: Mapping[str, object]) -> Self: + return cls( + disable_cache=_validate_global_option(global_options, "disable_cache", bool), + cache_dir=_validate_global_option(global_options, "cache_dir", Path), + verbose=_validate_global_option(global_options, "verbose", bool), + builtin_component_lib=_validate_global_option( + global_options, "builtin_component_lib", str + ), + ) @classmethod def default(cls) -> "DgConfig": return cls() +# This validation will generally already be done by click, but this internal validation routine +# provides insurance and satisfies the type checker. +def _validate_global_option( + global_options: Mapping[str, object], key: str, expected_type: Type[T] +) -> T: + value = global_options.get(key, getattr(DgConfig, key)) + if not isinstance(value, expected_type): + raise DgError(f"Global option {key} must be of type {expected_type}.") + return value + + _CLI_CONTEXT_CONFIG_KEY = "config" def set_config_on_cli_context(cli_context: click.Context, config: DgConfig) -> None: cli_context.ensure_object(dict) cli_context.obj[_CLI_CONTEXT_CONFIG_KEY] = config + + +def has_config_on_cli_context(cli_context: click.Context) -> bool: + return _CLI_CONTEXT_CONFIG_KEY in cli_context.ensure_object(dict) + + +def get_config_from_cli_context(cli_context: click.Context) -> DgConfig: + cli_context.ensure_object(dict) + return cli_context.obj[_CLI_CONTEXT_CONFIG_KEY] diff --git a/python_modules/libraries/dagster-dg/dagster_dg/context.py b/python_modules/libraries/dagster-dg/dagster_dg/context.py index 1a6bb50966ed1..c169a630d1b12 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg/context.py +++ b/python_modules/libraries/dagster-dg/dagster_dg/context.py @@ -4,9 +4,8 @@ import subprocess from dataclasses import dataclass from pathlib import Path -from typing import Final, Iterable, Optional, Tuple +from typing import Final, Iterable, Mapping, Optional, Tuple -import click import tomli from typing_extensions import Self @@ -80,21 +79,31 @@ def _is_code_location_root_directory(path: Path) -> bool: @dataclass class DgContext: config: DgConfig - cache: Optional[DgCache] = None + _cache: Optional[DgCache] = None @classmethod - def from_cli_context(cls, cli_context: click.Context) -> Self: - return cls.from_config(config=DgConfig.from_cli_context(cli_context)) + def from_cli_global_options(cls, global_options: Mapping[str, object]) -> Self: + return cls.from_config(config=DgConfig.from_cli_global_options(global_options)) @classmethod def from_config(cls, config: DgConfig) -> Self: cache = None if config.disable_cache else DgCache.from_config(config) - return cls(config=config, cache=cache) + return cls(config=config, _cache=cache) @classmethod def default(cls) -> Self: return cls.from_config(DgConfig.default()) + @property + def cache(self) -> DgCache: + if not self._cache: + raise DgError("Cache is disabled") + return self._cache + + @property + def has_cache(self) -> bool: + return self._cache is not None + @dataclass class DeploymentDirectoryContext: @@ -146,17 +155,16 @@ def ensure_uv_lock(root_path: Path) -> None: def fetch_component_registry(path: Path, dg_context: DgContext) -> RemoteComponentRegistry: root_path = resolve_code_location_root_directory(path) - cache = dg_context.cache - if cache: + if dg_context.has_cache: cache_key = make_cache_key(root_path, "component_registry_data") - raw_registry_data = cache.get(cache_key) if cache else None + raw_registry_data = dg_context.cache.get(cache_key) if dg_context.has_cache else None if not raw_registry_data: raw_registry_data = execute_code_location_command( root_path, ["list", "component-types"], dg_context ) - if cache: - cache.set(cache_key, raw_registry_data) + if dg_context.has_cache: + dg_context.cache.set(cache_key, raw_registry_data) registry_data = json.loads(raw_registry_data) return RemoteComponentRegistry.from_dict(registry_data) diff --git a/python_modules/libraries/dagster-dg/dagster_dg/generate.py b/python_modules/libraries/dagster-dg/dagster_dg/generate.py index a7f8721ad4c7f..bac2d7facfe4b 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg/generate.py +++ b/python_modules/libraries/dagster-dg/dagster_dg/generate.py @@ -18,7 +18,7 @@ # ######################## -def generate_deployment(path: Path) -> None: +def generate_deployment(path: Path, dg_context: DgContext) -> None: click.echo(f"Creating a Dagster deployment at {path}.") generate_subtree( diff --git a/python_modules/libraries/dagster-dg/dagster_dg/utils.py b/python_modules/libraries/dagster-dg/dagster_dg/utils.py index 8bcf29db6b277..8627ac083be0c 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg/utils.py +++ b/python_modules/libraries/dagster-dg/dagster_dg/utils.py @@ -242,12 +242,8 @@ def not_none(value: Optional[T]) -> T: # ######################## # Here we subclass click.Command and click.Group to customize the help output. We do this in order -# to show the options for each parent group in the help output of a subcommand. The form of the -# output can be seen in dagster_dg_tests.test_custom_help_format. - -# When rendering options for parent groups, exclude these options since they are not used when -# executing a subcommand. -_EXCLUDE_PARENT_OPTIONS = ["help", "version"] +# to visually separate global from command-specific options. The form of the output can be seen in +# dagster_dg_tests.test_custom_help_format. class DgClickHelpMixin: @@ -265,71 +261,43 @@ def format_help(self, context: click.Context, formatter: click.HelpFormatter): self.format_commands(context, formatter) self.format_options(context, formatter) - # Add section for each parent option group - for ctx, cmd in self._walk_parents(context): - cmd.format_options(ctx, formatter, as_parent=True) - - def format_options( - self, ctx: click.Context, formatter: HelpFormatter, as_parent: bool = False - ) -> None: - """Writes all the options into the formatter if they exist. + def get_partitioned_opts( + self, ctx: click.Context + ) -> Tuple[Sequence[click.Parameter], Sequence[click.Parameter]]: + from dagster_dg.cli.global_options import GLOBAL_OPTIONS - If `as_parent` is True, the header will include the command path and the `--help` option - will be excluded. - """ if not isinstance(self, click.Command): raise ValueError("This mixin is only intended for use with click.Command instances.") - params = [ - p - for p in self.get_params(ctx) - if p.name and not (as_parent and p.name in _EXCLUDE_PARENT_OPTIONS) - ] - opts = [rv for p in params if (rv := p.get_help_record(ctx)) is not None] - if as_parent: - opts = [opt for opt in opts if not opt[0].startswith("--help")] - if opts: - header = f"Options ({ctx.command_path})" if as_parent else "Options" - with formatter.section(header): - formatter.write_dl(opts) - - def format_usage(self, context: click.Context, formatter: HelpFormatter) -> None: - if not isinstance(self, click.Command): - raise ValueError("This mixin is only intended for use with click.Command instances.") - arg_pieces = self.collect_usage_pieces(context) - - path_parts: List[str] = [not_none(context.info_name)] - for ctx, cmd in self._walk_parents(context): - if cmd.has_visible_options_as_parent(ctx): - path_parts.append("[OPTIONS]") - path_parts.append(not_none(ctx.info_name)) - path_parts.reverse() - return formatter.write_usage(" ".join(path_parts), " ".join(arg_pieces)) - - def has_visible_options_as_parent(self, ctx: click.Context) -> bool: - """Returns True if the command has options that are not help-related.""" + # Filter out arguments + opts = [p for p in self.get_params(ctx) if p.get_help_record(ctx) is not None] + command_opts = [opt for opt in opts if opt.name not in GLOBAL_OPTIONS] + global_opts = [opt for opt in self.get_params(ctx) if opt.name in GLOBAL_OPTIONS] + return command_opts, global_opts + + def format_options(self, ctx: click.Context, formatter: HelpFormatter) -> None: + """Writes all the options into the formatter if they exist.""" if not isinstance(self, click.Command): raise ValueError("This mixin is only intended for use with click.Command instances.") - return any( - p for p in self.get_params(ctx) if (p.name and p.name not in _EXCLUDE_PARENT_OPTIONS) - ) - def _walk_parents( - self, ctx: click.Context - ) -> Iterator[Tuple[click.Context, "DgClickHelpMixin"]]: - while ctx.parent: - if not isinstance(ctx.parent.command, DgClickHelpMixin): - raise DgError("Parent command must be an instance of DgClickHelpMixin.") - yield ctx.parent, ctx.parent.command - ctx = ctx.parent + # Filter out arguments + command_opts, global_opts = self.get_partitioned_opts(ctx) + + if command_opts: + records = [not_none(p.get_help_record(ctx)) for p in command_opts] + with formatter.section("Options"): + formatter.write_dl(records) + + if global_opts: + with formatter.section("Global options"): + records = [not_none(p.get_help_record(ctx)) for p in global_opts] + formatter.write_dl(records) -class DgClickCommand(DgClickHelpMixin, click.Command): - pass +class DgClickCommand(DgClickHelpMixin, click.Command): ... -class DgClickGroup(DgClickHelpMixin, click.Group): - pass +class DgClickGroup(DgClickHelpMixin, click.Group): ... # ######################## diff --git a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_code_location_commands.py b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_code_location_commands.py index c144fc9357726..88e2d4b533ad2 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_code_location_commands.py +++ b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_code_location_commands.py @@ -17,15 +17,23 @@ # ##### GENERATE # ######################## +# At this time all of our tests are against an editable install of dagster-components. The reason +# for this is that this package should always be tested against the corresponding version of +# dagster-copmonents (i.e. from the same commit), and the only way to achieve this right now is +# using the editable install variant of `dg code-location generate`. +# +# Ideally we would have a way to still use the matching dagster-components without using the +# editable install variant, but this will require somehow configuring uv to ensure that it builds +# and returns the local version of the package. + + +def test_code_location_generate_inside_deployment_success(monkeypatch) -> None: + # Remove when we are able to test without editable install + dagster_git_repo_dir = discover_git_root(Path(__file__)) + monkeypatch.setenv("DAGSTER_GIT_REPO_DIR", str(dagster_git_repo_dir)) -def test_code_location_generate_inside_deployment_success() -> None: - # Don't use the test component lib because it is not present in published dagster-components, - # which this test is currently accessing since we are not doing an editable install. - with ( - ProxyRunner.test(use_test_component_lib=False) as runner, - isolated_example_deployment_foo(runner), - ): - result = runner.invoke("code-location", "generate", "bar") + with ProxyRunner.test() as runner, isolated_example_deployment_foo(runner): + result = runner.invoke("code-location", "generate", "bar", "--use-editable-dagster") assert_runner_result(result) assert Path("code_locations/bar").exists() assert Path("code_locations/bar/bar").exists() @@ -38,23 +46,26 @@ def test_code_location_generate_inside_deployment_success() -> None: assert Path("code_locations/bar/.venv").exists() assert Path("code_locations/bar/uv.lock").exists() - with open("code_locations/bar/pyproject.toml") as f: - toml = tomli.loads(f.read()) - - # No tool.uv.sources added without --use-editable-dagster - assert "uv" not in toml["tool"] + # Restore when we are able to test without editable install + # with open("code_locations/bar/pyproject.toml") as f: + # toml = tomli.loads(f.read()) + # + # # No tool.uv.sources added without --use-editable-dagster + # assert "uv" not in toml["tool"] # Check cache was populated with pushd("code_locations/bar"): - result = runner.invoke("--verbose", "component-type", "list") + result = runner.invoke("component-type", "list", "--verbose") assert "CACHE [hit]" in result.output -def test_code_location_generate_outside_deployment_success() -> None: - # Don't use the test component lib because it is not present in published dagster-components, - # which this test is currently accessing since we are not doing an editable install. - with ProxyRunner.test(use_test_component_lib=False) as runner, runner.isolated_filesystem(): - result = runner.invoke("code-location", "generate", "bar") +def test_code_location_generate_outside_deployment_success(monkeypatch) -> None: + # Remove when we are able to test without editable install + dagster_git_repo_dir = discover_git_root(Path(__file__)) + monkeypatch.setenv("DAGSTER_GIT_REPO_DIR", str(dagster_git_repo_dir)) + + with ProxyRunner.test() as runner, runner.isolated_filesystem(): + result = runner.invoke("code-location", "generate", "bar", "--use-editable-dagster") assert_runner_result(result) assert Path("bar").exists() assert Path("bar/bar").exists() diff --git a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_custom_help_format.py b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_custom_help_format.py index 7819f4935d5a0..e4e7c3ea5ce49 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_custom_help_format.py +++ b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_custom_help_format.py @@ -6,57 +6,54 @@ ensure_dagster_dg_tests_import() -from dagster_dg_tests.utils import assert_runner_result +from dagster_dg_tests.utils import ( + ProxyRunner, + assert_runner_result, + isolated_example_code_location_bar, +) # ######################## # ##### TEST CLI # ######################## - -@click.group(name="test", cls=DgClickGroup) -@click.option("--test-opt", type=str, default="test", help="Test option.") -def test_cli(test_opt): - """Test CLI group.""" - pass +# The names of our global options are special-cased, so use one of them here (--disable-cache) as +# a test case. -@test_cli.group(name="sub-test-1", cls=DgClickGroup) -@click.option("--sub-test-1-opt", type=str, default="sub-test-1", help="Sub-test 1 option.") -def sub_test_1(sub_test_1_opt): - """Sub-test 1 group.""" +@click.group(name="root", cls=DgClickGroup) +@click.option("--root-opt", type=str, default="root", help="Root option.") +@click.option("--disable-cache", type=str, default="test", help="Disable cache.") +def root(root_opt, disable_cache): + """Root group.""" pass -@sub_test_1.command(name="alpha", cls=DgClickCommand) -@click.option("--alpha-opt", type=str, default="alpha", help="Alpha option.") -def alpha(alpha_opt): - """Alpha command.""" +@root.group(name="sub-group", cls=DgClickGroup) +@click.option("--sub-group-opt", type=str, default="sub-group", help="Sub-group option.") +@click.option("--disable-cache", type=str, default="test", help="Disable cache.") +def sub_group(sub_group_opt, disable_cache): + """Sub-group.""" pass -@test_cli.group(name="sub-test-2", cls=DgClickGroup) -def sub_test_2(): - """Sub-test 2 group.""" +@sub_group.command(name="sub-group-command", cls=DgClickCommand) +@click.option( + "--sub-group-command-opt", + type=str, + default="sub_group_command", + help="Sub-group-command option.", +) +@click.option("--disable-cache", type=str, default="test", help="Disable cache.") +def sub_group_command(sub_group_command_opt, disable_cache): + """Sub-group-command.""" pass -@click.option("--beta-opt", type=str, default="alpha", help="Beta option.") -@sub_test_2.command(name="beta", cls=DgClickCommand) -def beta(beta_opt): - """Beta command.""" - pass - - -@click.option("--delta-opt", type=str, default="delta", help="Delta option.") -@test_cli.command(name="delta", cls=DgClickCommand) -def delta(delta_opt): - """Delta command.""" - pass - - -@test_cli.command(name="gamma", cls=DgClickCommand) -def gamma(gamma_opt): - """Gamma command.""" +@root.group(name="sub-command", cls=DgClickGroup) +@click.option("--sub-command-opt", type=str, default="sub-command", help="Sub-command option.") +@click.option("--disable-cache", type=str, default="test", help="Disable cache.") +def sub_command(sub_command_opt, disable_cache): + """Sub-command.""" pass @@ -65,158 +62,117 @@ def gamma(gamma_opt): # ######################## -def test_root_group_help_message(): +def test_root_help_message(): runner = CliRunner() - result = runner.invoke(test_cli, ["--help"]) + result = runner.invoke(root, ["--help"]) assert_runner_result(result) assert ( result.output.strip() == textwrap.dedent(""" - Usage: test [OPTIONS] COMMAND [ARGS]... + Usage: root [OPTIONS] COMMAND [ARGS]... - Test CLI group. + Root group. Commands: - delta Delta command. - gamma Gamma command. - sub-test-1 Sub-test 1 group. - sub-test-2 Sub-test 2 group. + sub-command Sub-command. + sub-group Sub-group. Options: - --test-opt TEXT Test option. + --root-opt TEXT Root option. --help Show this message and exit. - """).strip() - ) - - -def test_sub_group_with_option_help_message(): - runner = CliRunner() - result = runner.invoke(test_cli, ["sub-test-1", "--help"]) - assert_runner_result(result) - assert ( - result.output.strip() - == textwrap.dedent(""" - Usage: test [OPTIONS] sub-test-1 [OPTIONS] COMMAND [ARGS]... - - Sub-test 1 group. - - Commands: - alpha Alpha command. - - Options: - --sub-test-1-opt TEXT Sub-test 1 option. - --help Show this message and exit. - - Options (test): - --test-opt TEXT Test option. - """).strip() - ) - -def test_command_in_sub_group_with_option_help_message(): - runner = CliRunner() - result = runner.invoke(test_cli, ["sub-test-1", "alpha", "--help"]) - assert_runner_result(result) - assert ( - result.output.strip() - == textwrap.dedent(""" - Usage: test [OPTIONS] sub-test-1 [OPTIONS] alpha [OPTIONS] - - Alpha command. - - Options: - --alpha-opt TEXT Alpha option. - --help Show this message and exit. - - Options (test sub-test-1): - --sub-test-1-opt TEXT Sub-test 1 option. - - Options (test): - --test-opt TEXT Test option. + Global options: + --disable-cache TEXT Disable cache. """).strip() ) -def test_sub_group_with_no_option_help_message(): +def test_sub_group_with_option_help_message(): runner = CliRunner() - result = runner.invoke(test_cli, ["sub-test-2", "--help"]) + result = runner.invoke(root, ["sub-group", "--help"]) assert_runner_result(result) assert ( result.output.strip() == textwrap.dedent(""" - Usage: test [OPTIONS] sub-test-2 [OPTIONS] COMMAND [ARGS]... + Usage: root sub-group [OPTIONS] COMMAND [ARGS]... - Sub-test 2 group. + Sub-group. Commands: - beta Beta command. + sub-group-command Sub-group-command. Options: - --help Show this message and exit. + --sub-group-opt TEXT Sub-group option. + --help Show this message and exit. - Options (test): - --test-opt TEXT Test option. + Global options: + --disable-cache TEXT Disable cache. """).strip() ) -def test_command_in_sub_group_with_no_option_help_message(): +def test_sub_group_command_with_option_help_message(): runner = CliRunner() - result = runner.invoke(test_cli, ["sub-test-2", "beta", "--help"]) + result = runner.invoke(root, ["sub-group", "sub-group-command", "--help"]) assert_runner_result(result) assert ( result.output.strip() == textwrap.dedent(""" - Usage: test [OPTIONS] sub-test-2 beta [OPTIONS] + Usage: root sub-group sub-group-command [OPTIONS] - Beta command. + Sub-group-command. Options: - --beta-opt TEXT Beta option. - --help Show this message and exit. + --sub-group-command-opt TEXT Sub-group-command option. + --help Show this message and exit. - Options (test): - --test-opt TEXT Test option. + Global options: + --disable-cache TEXT Disable cache. """).strip() ) -def test_command_with_option_in_root_group_help_message(): +def test_sub_command_with_option_help_message(): runner = CliRunner() - result = runner.invoke(test_cli, ["delta", "--help"]) + result = runner.invoke(root, ["sub-command", "--help"]) assert_runner_result(result) assert ( result.output.strip() == textwrap.dedent(""" - Usage: test [OPTIONS] delta [OPTIONS] + Usage: root sub-command [OPTIONS] COMMAND [ARGS]... - Delta command. + Sub-command. Options: - --delta-opt TEXT Delta option. - --help Show this message and exit. + --sub-command-opt TEXT Sub-command option. + --help Show this message and exit. - Options (test): - --test-opt TEXT Test option. + Global options: + --disable-cache TEXT Disable cache. """).strip() ) -def test_command_with_no_option_in_root_group_help_message(): - runner = CliRunner() - result = runner.invoke(test_cli, ["gamma", "--help"]) - assert_runner_result(result) - assert ( - result.output.strip() - == textwrap.dedent(""" - Usage: test [OPTIONS] gamma [OPTIONS] - - Gamma command. - - Options: - --help Show this message and exit. - - Options (test): - --test-opt TEXT Test option. - """).strip() - ) +def test_dynamic_subcommand_help_message(): + with ProxyRunner.test() as runner, isolated_example_code_location_bar(runner): + result = runner.invoke( + "component", "generate", "dagster_components.test.simple_pipes_script_asset", "--help" + ) + assert ( + result.output.strip() + == textwrap.dedent(""" + Usage: dg component generate [GLOBAL OPTIONS] dagster_components.test.simple_pipes_script_asset [OPTIONS] COMPONENT_NAME + + Options: + --json-params TEXT JSON string of component parameters. + --asset-key TEXT asset_key + --filename TEXT filename + -h, --help Show this message and exit. + + Global options: + --builtin-component-lib TEXT Specify a builitin component library to use. + --verbose Enable verbose output for debugging. + --disable-cache Disable the cache.. + --cache-dir PATH Specify a directory to use for the cache. + """).strip() + ) diff --git a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_integrity.py b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_integrity.py index 5588c3dec1c10..5e7ce8f0b018c 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_integrity.py +++ b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_integrity.py @@ -14,3 +14,17 @@ def crawl(command): crawl(command) crawl(cli) + + +# Important that all nodes of the command tree inherit from one of our customized click +# Command/Group subclasses to ensure that the help formatting is consistent. +def test_all_commands_have_global_options(): + def crawl(command): + assert isinstance( + command, (DgClickGroup, DgClickCommand) + ), f"Group is not a DgClickGroup or DgClickCommand: {command}" + if isinstance(command, DgClickGroup): + for command in command.commands.values(): + crawl(command) + + crawl(cli) diff --git a/python_modules/libraries/dagster-dg/dagster_dg_tests/test_cache.py b/python_modules/libraries/dagster-dg/dagster_dg_tests/test_cache.py index e7448c0f7dfef..4424e53905d64 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg_tests/test_cache.py +++ b/python_modules/libraries/dagster-dg/dagster_dg_tests/test_cache.py @@ -2,8 +2,6 @@ from functools import partial from pathlib import Path -import pytest - from dagster_dg_tests.utils import ( ProxyRunner, assert_runner_result, @@ -69,22 +67,17 @@ def test_cache_no_invalidation_modified_pkg(): assert "CACHE [hit]" in result.output -@pytest.mark.parametrize("with_command", [True, False]) -def test_cache_clear(with_command: bool): +def test_clear_cache(): with ProxyRunner.test(verbose=True) as runner, example_code_location(runner): result = runner.invoke("component-type", "list") assert_runner_result(result) assert "CACHE [miss]" in result.output assert "CACHE [write]" in result.output - if with_command: - result = runner.invoke("--clear-cache", "component-type", "list") - assert "CACHE [clear-all]" in result.output - else: - result = runner.invoke("--clear-cache") - assert_runner_result(result) - assert "CACHE [clear-all]" in result.output - result = runner.invoke("component-type", "list") + result = runner.invoke("--clear-cache") + assert_runner_result(result) + assert "CACHE [clear-all]" in result.output + result = runner.invoke("component-type", "list") assert_runner_result(result) assert "CACHE [miss]" in result.output diff --git a/python_modules/libraries/dagster-dg/dagster_dg_tests/utils.py b/python_modules/libraries/dagster-dg/dagster_dg_tests/utils.py index 45dbdebbf3216..2c646175902c9 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg_tests/utils.py +++ b/python_modules/libraries/dagster-dg/dagster_dg_tests/utils.py @@ -7,7 +7,10 @@ from typing import Iterator, Optional, Sequence, Tuple, Type, Union from click.testing import CliRunner, Result -from dagster_dg.cli import cli as dg_cli +from dagster_dg.cli import ( + DG_CLI_MAX_OUTPUT_WIDTH, + cli as dg_cli, +) from dagster_dg.utils import discover_git_root, pushd from typing_extensions import Self @@ -56,7 +59,7 @@ def isolated_example_code_location_bar( @dataclass class ProxyRunner: original: CliRunner - prepend_args: Optional[Sequence[str]] = None + append_args: Optional[Sequence[str]] = None @classmethod @contextmanager @@ -64,7 +67,7 @@ def test( cls, use_test_component_lib: bool = True, verbose: bool = False, disable_cache: bool = False ) -> Iterator[Self]: with TemporaryDirectory() as cache_dir: - prepend_args = [ + append_opts = [ *( ["--builtin-component-lib", "dagster_components.test"] if use_test_component_lib @@ -75,11 +78,25 @@ def test( *(["--verbose"] if verbose else []), *(["--disable-cache"] if disable_cache else []), ] - yield cls(CliRunner(), prepend_args=prepend_args) + yield cls(CliRunner(), append_args=append_opts) def invoke(self, *args: str): - all_args = [*(self.prepend_args or []), *args] - return self.original.invoke(dg_cli, all_args) + # We need to find the right spot to inject global options. For the `dg component generate` + # command, we need to inject the global options before the final subcommand. For everything + # else they can be appended at the end of the options. + if args[:2] == ("component", "generate"): + index = 2 + elif "--help" in args: + index = args.index("--help") + elif "--" in args: + index = args.index("--") + else: + index = len(args) + all_args = [*args[:index], *(self.append_args or []), *args[index:]] + + # For some reason the context setting `max_content_width` is not respected when using the + # CliRunner, so we have to set it manually. + return self.original.invoke(dg_cli, all_args, terminal_width=DG_CLI_MAX_OUTPUT_WIDTH) @contextmanager def isolated_filesystem(self) -> Iterator[None]: