diff --git a/README.md b/README.md index 3003bfc..53031f1 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,6 @@ This project is still a work in progress. Please create an issue if you find any - (todo) Load all Singer tap streams as Dagster assets. - (todo) Ops to perform all Meltano actions. - (todo) Extract Singer metrics from logs and store them using Dagster. -- (todo) Add more integrations with Dagster Cloud. ## Installation diff --git a/dagster_ext/extension.py b/dagster_ext/extension.py index 592069a..8fb6e2e 100644 --- a/dagster_ext/extension.py +++ b/dagster_ext/extension.py @@ -89,43 +89,43 @@ def initialize(self, force: bool = False) -> None: repository_name = repository_name.replace("-", "_").replace(" ", "-") - setup_cloud = Confirm.ask( - "Do you want to setup Dagster Cloud Serverless?", - default=True, - ) + # setup_cloud = Confirm.ask( + # "Do you want to setup Dagster Cloud Serverless?", + # default=True, + # ) - if setup_cloud: - cloud_organization = Prompt.ask( - "What is your Dagster Cloud organization name?", - default=os.getenv("DAGSTER_CLOUD_ORGANIZATION"), - ) - self.set_meltano_config( - description="Setting Dagster Cloud organization", - config_name="cloud_organization", - config_value=cloud_organization, - ) + # if setup_cloud: + # cloud_organization = Prompt.ask( + # "What is your Dagster Cloud organization name?", + # default=os.getenv("DAGSTER_CLOUD_ORGANIZATION"), + # ) + # self.set_meltano_config( + # description="Setting Dagster Cloud organization", + # config_name="cloud_organization", + # config_value=cloud_organization, + # ) - cloud_api_token = Prompt.ask( - "What is your Dagster Cloud api token?", - password=True, - ) - self.set_meltano_config( - description="Setting Dagster Cloud organization", - config_name="cloud_api_token", - config_value=cloud_api_token, - ) + # cloud_api_token = Prompt.ask( + # "What is your Dagster Cloud api token?", + # password=True, + # ) + # self.set_meltano_config( + # description="Setting Dagster Cloud organization", + # config_name="cloud_api_token", + # config_value=cloud_api_token, + # ) - cloud_location_name = Prompt.ask( - "What is your Dagster Cloud location name?", - default=os.getenv("DAGSTER_CLOUD_LOCATION_NAME", "meltano"), - ) - self.set_meltano_config( - description="Setting Dagster Cloud location name", - config_name="cloud_location_name", - config_value=cloud_location_name, - ) + # cloud_location_name = Prompt.ask( + # "What is your Dagster Cloud location name?", + # default=os.getenv("DAGSTER_CLOUD_LOCATION_NAME", "meltano"), + # ) + # self.set_meltano_config( + # description="Setting Dagster Cloud location name", + # config_name="cloud_location_name", + # config_value=cloud_location_name, + # ) - # TODO: Create a dockerignore here + # TODO: Create a dockerignore here # dbt_installed = Confirm.ask( # "Do you have DBT installed?", @@ -161,7 +161,7 @@ def initialize(self, force: bool = False) -> None: print("[green]Successfully initialized your Dagster project![/green]") print("[green]Start Dagit by running `meltano invoke dagster:start`[/green]") - print("[blue]Or deploy by running `meltano invoke dagster:deploy`[/blue]") + # print("[blue]Or deploy by running `meltano invoke dagster:deploy`[/blue]") def get_env_value(self, env_variable: str) -> str: try: diff --git a/files_dagster_ext/dagster/{{ cookiecutter.project_name }}/deploy/Dockerfile b/files_dagster_ext/dagster/{{ cookiecutter.project_name }}/deploy/Dockerfile deleted file mode 100644 index ba51a07..0000000 --- a/files_dagster_ext/dagster/{{ cookiecutter.project_name }}/deploy/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -# Use python image as a base -FROM python:3.8 - -# Set the workdir to the final Dagster workdir -WORKDIR /opt/dagster/app - -# Set the meltano project root to the Dagster root -ENV MELTANO_PROJECT_ROOT /opt/dagster/app - -# Install pipx -RUN pip install pipx - -# Make sure the pipx install location is in the path -ENV PATH "$PATH:/root/.local/bin" - -# Install meltano -RUN pipx install meltano - -# Make sure we install the requirements -RUN pip install dagster-ext dagster-cloud - -# Copy the meltano project root to the working dir -COPY . /opt/dagster/app - -# Install all Meltano plugins -RUN meltano install \ No newline at end of file diff --git a/project/meltano.yml b/project/meltano.yml index 5741d77..8c65110 100644 --- a/project/meltano.yml +++ b/project/meltano.yml @@ -62,6 +62,11 @@ plugins: env: DAGSTER_CLOUD_LOCATION_NAME value: meltano label: The name of the Cloud Deployment location. + - description: The root folder of the Dagster project. + env: REPOSITORY_DIR + label: Repository Directory + name: repository_dir + value: $MELTANO_PROJECT_ROOT/orchestrate/dagster commands: initialize: args: initialize @@ -92,12 +97,6 @@ plugins: DBT_POSTGRES_PASSWORD: postgres DBT_POSTGRES_SCHEMA: warehouse DAGSTER_CLOUD_ORGANIZATION: quantile - - config: - cloud_organization: quantile - cloud_location_name: dagster-ext - cloud_env_variables: - - MELTANO_ENVIRONMENT jobs: - name: ingestion tasks: @@ -117,6 +116,10 @@ schedules: interval: "@daily" job: tap-target + - name: schedule-weekly + interval: "@weekly" + job: tap-target + environments: - name: dev - name: staging diff --git a/project/orchestrate/dagster/deploy/Dockerfile b/project/orchestrate/dagster/deploy/Dockerfile deleted file mode 100644 index dab40e0..0000000 --- a/project/orchestrate/dagster/deploy/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -# Use python image as a base -FROM python:3.8 - -# Set the workdir to the final Dagster workdir -WORKDIR /opt/dagster/app - -# Set the meltano project root to the Dagster root -ENV MELTANO_PROJECT_ROOT /opt/dagster/app - -# Install pipx -RUN pip install pipx - -# Make sure the pipx install location is in the path -ENV PATH "$PATH:/root/.local/bin" - -# Install meltano -RUN pipx install meltano - -# Make sure we install the requirements -RUN pip install dagster-ext dagster-cloud - -# Copy the meltano project root to the working dir -COPY . /opt/dagster/app - -# Install all Meltano plugins -# RUN meltano install \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index bcf4ff9..f2c1a7f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dagster-ext" -version = "0.0.1a9" +version = "0.0.1a11" description = "`dagster-ext` is a Meltano utility extension." authors = ["Jules Huisman"] license = "Apache 2.0" @@ -18,10 +18,9 @@ include = [ python = "<3.11,>=3.8" click = "^8.1.3" typer = "^0.6.1" -dagit = ">=1.0,<1.1" -dagster = ">=1.0,<1.1" -dagster-dbt = ">=0.16,<0.17" -# "meltano.edk" = {git = "https://github.com/meltano/edk.git", rev="main"} +dagit = ">=1.0" +dagster = ">=1.0" +dagster-dbt = ">=0.16" cookiecutter = "^2.1.1" rich = "^12.5.1" # Dependencies from Meltano EDK, remove when dependency fixed @@ -29,7 +28,6 @@ structlog = "^21.2.0" PyYAML = "^6.0.0" pydantic = "^1.9.0" devtools = "^0.9.0" -dagster-cloud = "^1.0.15" [tool.poetry.dev-dependencies] black = "^22.3.0" @@ -42,7 +40,7 @@ requires = ["poetry-core>=1.0.8"] build-backend = "poetry.core.masonry.api" [project.urls] -Homepage = "https://github.com/JulesHuisman/dagster-ext" +Homepage = "https://github.com/quantile-development/dagster-ext" [tool.poetry.scripts] dagster_extension = 'dagster_ext.main:app'