Removed some Dagster Cloud options

quantile-development · Dec 21, 2022 · bd8ef2b · bd8ef2b
1 parent 70a02e3
commit bd8ef2b
Show file tree

Hide file tree

Showing 6 changed files with 48 additions and 100 deletions.
diff --git a/README.md b/README.md
@@ -12,7 +12,6 @@ This project is still a work in progress. Please create an issue if you find any
 - (todo) Load all Singer tap streams as Dagster assets.
 - (todo) Ops to perform all Meltano actions.
 - (todo) Extract Singer metrics from logs and store them using Dagster.
-- (todo) Add more integrations with Dagster Cloud.
 
 ## Installation
 

diff --git a/dagster_ext/extension.py b/dagster_ext/extension.py
@@ -89,43 +89,43 @@ def initialize(self, force: bool = False) -> None:
 
         repository_name = repository_name.replace("-", "_").replace(" ", "-")
 
-        setup_cloud = Confirm.ask(
-            "Do you want to setup Dagster Cloud Serverless?",
-            default=True,
-        )
+        # setup_cloud = Confirm.ask(
+        #     "Do you want to setup Dagster Cloud Serverless?",
+        #     default=True,
+        # )
 
-        if setup_cloud:
-            cloud_organization = Prompt.ask(
-                "What is your Dagster Cloud organization name?",
-                default=os.getenv("DAGSTER_CLOUD_ORGANIZATION"),
-            )
-            self.set_meltano_config(
-                description="Setting Dagster Cloud organization",
-                config_name="cloud_organization",
-                config_value=cloud_organization,
-            )
+        # if setup_cloud:
+        #     cloud_organization = Prompt.ask(
+        #         "What is your Dagster Cloud organization name?",
+        #         default=os.getenv("DAGSTER_CLOUD_ORGANIZATION"),
+        #     )
+        #     self.set_meltano_config(
+        #         description="Setting Dagster Cloud organization",
+        #         config_name="cloud_organization",
+        #         config_value=cloud_organization,
+        #     )
 
-            cloud_api_token = Prompt.ask(
-                "What is your Dagster Cloud api token?",
-                password=True,
-            )
-            self.set_meltano_config(
-                description="Setting Dagster Cloud organization",
-                config_name="cloud_api_token",
-                config_value=cloud_api_token,
-            )
+        #     cloud_api_token = Prompt.ask(
+        #         "What is your Dagster Cloud api token?",
+        #         password=True,
+        #     )
+        #     self.set_meltano_config(
+        #         description="Setting Dagster Cloud organization",
+        #         config_name="cloud_api_token",
+        #         config_value=cloud_api_token,
+        #     )
 
-            cloud_location_name = Prompt.ask(
-                "What is your Dagster Cloud location name?",
-                default=os.getenv("DAGSTER_CLOUD_LOCATION_NAME", "meltano"),
-            )
-            self.set_meltano_config(
-                description="Setting Dagster Cloud location name",
-                config_name="cloud_location_name",
-                config_value=cloud_location_name,
-            )
+        #     cloud_location_name = Prompt.ask(
+        #         "What is your Dagster Cloud location name?",
+        #         default=os.getenv("DAGSTER_CLOUD_LOCATION_NAME", "meltano"),
+        #     )
+        #     self.set_meltano_config(
+        #         description="Setting Dagster Cloud location name",
+        #         config_name="cloud_location_name",
+        #         config_value=cloud_location_name,
+        #     )
 
-            # TODO: Create a dockerignore here
+        # TODO: Create a dockerignore here
 
         # dbt_installed = Confirm.ask(
         #     "Do you have DBT installed?",
@@ -161,7 +161,7 @@ def initialize(self, force: bool = False) -> None:
 
         print("[green]Successfully initialized your Dagster project![/green]")
         print("[green]Start Dagit by running `meltano invoke dagster:start`[/green]")
-        print("[blue]Or deploy by running `meltano invoke dagster:deploy`[/blue]")
+        # print("[blue]Or deploy by running `meltano invoke dagster:deploy`[/blue]")
 
     def get_env_value(self, env_variable: str) -> str:
         try:

diff --git a/files_dagster_ext/dagster/{{ cookiecutter.project_name }}/deploy/Dockerfile b/files_dagster_ext/dagster/{{ cookiecutter.project_name }}/deploy/Dockerfile
diff --git a/project/meltano.yml b/project/meltano.yml
@@ -62,6 +62,11 @@ plugins:
           env: DAGSTER_CLOUD_LOCATION_NAME
           value: meltano
           label: The name of the Cloud Deployment location.
+        - description: The root folder of the Dagster project.
+          env: REPOSITORY_DIR
+          label: Repository Directory
+          name: repository_dir
+          value: $MELTANO_PROJECT_ROOT/orchestrate/dagster
       commands:
         initialize:
           args: initialize
@@ -92,12 +97,6 @@ plugins:
         DBT_POSTGRES_PASSWORD: postgres
         DBT_POSTGRES_SCHEMA: warehouse
         DAGSTER_CLOUD_ORGANIZATION: quantile
-
-      config:
-        cloud_organization: quantile
-        cloud_location_name: dagster-ext
-        cloud_env_variables:
-          - MELTANO_ENVIRONMENT
 jobs:
   - name: ingestion
     tasks:
@@ -117,6 +116,10 @@ schedules:
     interval: "@daily"
     job: tap-target
 
+  - name: schedule-weekly
+    interval: "@weekly"
+    job: tap-target
+
 environments:
   - name: dev
   - name: staging

diff --git a/project/orchestrate/dagster/deploy/Dockerfile b/project/orchestrate/dagster/deploy/Dockerfile
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dagster-ext"
-version = "0.0.1a9"
+version = "0.0.1a11"
 description = "`dagster-ext` is a Meltano utility extension."
 authors = ["Jules Huisman"]
 license = "Apache 2.0"
@@ -18,18 +18,16 @@ include = [
 python = "<3.11,>=3.8"
 click = "^8.1.3"
 typer = "^0.6.1"
-dagit = ">=1.0,<1.1"
-dagster = ">=1.0,<1.1"
-dagster-dbt = ">=0.16,<0.17"
-# "meltano.edk" = {git = "https://github.com/meltano/edk.git", rev="main"}
+dagit = ">=1.0"
+dagster = ">=1.0"
+dagster-dbt = ">=0.16"
 cookiecutter = "^2.1.1"
 rich = "^12.5.1"
 # Dependencies from Meltano EDK, remove when dependency fixed
 structlog = "^21.2.0"
 PyYAML = "^6.0.0"
 pydantic = "^1.9.0"
 devtools = "^0.9.0"
-dagster-cloud = "^1.0.15"
 
 [tool.poetry.dev-dependencies]
 black = "^22.3.0"
@@ -42,7 +40,7 @@ requires = ["poetry-core>=1.0.8"]
 build-backend = "poetry.core.masonry.api"
 
 [project.urls]
-Homepage = "https://github.com/JulesHuisman/dagster-ext"
+Homepage = "https://github.com/quantile-development/dagster-ext"
 
 [tool.poetry.scripts]
 dagster_extension = 'dagster_ext.main:app'