From 07a861bd4de1c6e95e6be212393ec65516780f63 Mon Sep 17 00:00:00 2001 From: Rene Thomsen <40211569+ReneTC@users.noreply.github.com> Date: Tue, 10 Oct 2023 10:40:41 +0200 Subject: [PATCH 1/5] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7f0ded3..c99ae9f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ packages = [ [tool.poetry.dependencies] python = ">=3.8,<3.12" -dagster = ">=1.3,<2" +dagster = ">=1.5.0,<2" dagster-shell = ">=0,<1" [tool.poetry.group.dev.dependencies] From 9f97b83f5d4d4c2bc3273efcdaaad695ed84564c Mon Sep 17 00:00:00 2001 From: Rene Thomsen <40211569+ReneTC@users.noreply.github.com> Date: Tue, 10 Oct 2023 14:08:51 +0200 Subject: [PATCH 2/5] Update utils.py --- dagster_meltano/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dagster_meltano/utils.py b/dagster_meltano/utils.py index 2c5edcc..7585c25 100644 --- a/dagster_meltano/utils.py +++ b/dagster_meltano/utils.py @@ -16,7 +16,7 @@ def generate_dagster_name(input_string) -> str: """ Generate a dagster safe name (^[A-Za-z0-9_]+$.) """ - return input_string.replace("-", "_").replace(" ", "_").replace(":", "_") + return input_string.replace("-", "_").replace(" ", "_").replace(":", "_").replace("=", "_") def generate_dbt_group_name(node_info: dict) -> str: From 282e0eed9ec9915f89524b1c74148e377632af70 Mon Sep 17 00:00:00 2001 From: Rene Thomsen <40211569+ReneTC@users.noreply.github.com> Date: Tue, 10 Oct 2023 14:11:03 +0200 Subject: [PATCH 3/5] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c99ae9f..7f0ded3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ packages = [ [tool.poetry.dependencies] python = ">=3.8,<3.12" -dagster = ">=1.5.0,<2" +dagster = ">=1.3,<2" dagster-shell = ">=0,<1" [tool.poetry.group.dev.dependencies] From 225b5dc33dc9f337028f1619435720d8d120153e Mon Sep 17 00:00:00 2001 From: Rene Thomsen <40211569+ReneTC@users.noreply.github.com> Date: Wed, 11 Oct 2023 10:11:39 +0200 Subject: [PATCH 4/5] Add posibilty of custom dagster op name Added the posibility of a custom dagster op name. If not set, it will default to the command generated name --- dagster_meltano/ops.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/dagster_meltano/ops.py b/dagster_meltano/ops.py index 2e47f85..8e36bdc 100644 --- a/dagster_meltano/ops.py +++ b/dagster_meltano/ops.py @@ -24,7 +24,7 @@ @lru_cache def meltano_command_op( command: str, - dagster_name: Optional[str] = None, + dagster_name: str, ) -> OpDefinition: """ Run `meltano ` using a Dagster op. @@ -34,13 +34,11 @@ def meltano_command_op( Args: command (str): The Meltano command to run. - dagster_name (Optional[str], optional): The Dagster name to use for the op. - Defaults to None. + dagster_name (str): The Dagster name to use for the op. Returns: OpDefinition: The Dagster op definition. """ - dagster_name = dagster_name or generate_dagster_name(command) ins = { "after": In(Nothing), "env": In( @@ -101,14 +99,23 @@ def dagster_op( @lru_cache def meltano_run_op( command: str, + dagster_name: Optional[str] = None, + ) -> OpDefinition: """ Run `meltano run ` using a Dagster op. This factory is cached to make sure the same commands can be reused in the same repository. + + Args: + command (str): The Meltano command to run. + dagster_name (Optional[str], optional): The Dagster name to use for the op. + Defaults to None. """ - dagster_name = generate_dagster_name(command) + # if not set, generate a dagster_name from the command + dagster_name = dagster_name or generate_dagster_name(command) + return meltano_command_op( command=f"run {command} --force", dagster_name=dagster_name ) From c7b7979b0d88fe8c6358cd1228ea521657bb2820 Mon Sep 17 00:00:00 2001 From: Rene Thomsen <40211569+ReneTC@users.noreply.github.com> Date: Fri, 13 Oct 2023 09:17:57 +0200 Subject: [PATCH 5/5] Update test_meltano_command.py --- tests/test_meltano_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_meltano_command.py b/tests/test_meltano_command.py index 184b025..8ddb4d5 100644 --- a/tests/test_meltano_command.py +++ b/tests/test_meltano_command.py @@ -9,7 +9,7 @@ @job(resource_defs={"meltano": meltano_resource}) def meltano_command_job(): - meltano_command_op("install extractor tap-smoke-test")() + meltano_command_op("install extractor tap-smoke-test", "custom_job_name")() def test_meltano_command():