New dagster_meltano package (#15)

* added a meltano elt solid and unit tests

* fixed tests

* add meltano install step to ci/cd tests

* add navigation to tests setp

* Created initial version to consolidate the two solid variants

* Created a new dagster type for environment variables

* Delete solids_new.py

* Replaced the solids file with the new MeltanoEltSolid

* Updated the tests

* Updated the readme

* Removed the select from meltano.yaml

* cleaned up logging for errors

* Update devcontainer.json

* Ignored the build folder

* Made changes to overal structure

* Ported `dagster-meltano` code

* Added pip cache

* Added dbt to the meltano project

* Improved the extract load op

* Meltano job notebook

* Added notebook content

* Add ipykernel / delete copilot

* Added meltano jobs and schedules

* Removed old run_streams

* Correctly parsed logging

* Add metadata to tap runs

* Revert "Added meltano jobs and schedules"

This reverts commit 52a4f14.

* Revert "Add ipykernel / delete copilot"

This reverts commit 6602781.

* Revert "Added notebook content"

This reverts commit 664d3da.

* Revert "Meltano job notebook"

This reverts commit 767b24b.

* Initial working set-up of dagster repo

* Added a `request_duration` parser

* Notebook contents

* Add test jobs in meltano config

* Add dagster jobs to the meltano resource

* Add Job class that transforms meltano job to a dagster job

* Add load_jobs_from_meltano_project func

* Add auto recognition of meltano schedules

* Disabled tests

* Moved the schedule generation to the job file

* Added meltano as extra

* Locked versions

* Add dagit as requirement

* Removed dagit requirement

* Added package finder

* Moved to new `dagster_meltano`

* Deleted output files

* Added tests

* Ignore pytest_cache

* Added testing suite

* Install in Meltano project

* Added support for project_dir

Co-authored-by: Josh Lloyd <[email protected]>
Co-authored-by: jlloyd-widen <[email protected]>
Co-authored-by: Josh Lloyd <[email protected]>
Co-authored-by: BernardWez <[email protected]>

.devcontainer/devcontainer.json

@@ -1,53 +1,23 @@
-{
-    "name": "dagster-meltano",
-    "build": {
-        "dockerfile": "../Dockerfile"
-    },
-    "extensions": [
-        "ms-python.vscode-pylance",
-        "mechatroner.rainbow-csv",
-        "njpwerner.autodocstring",
-        "github.autopilot",
-        "waderyan.gitblame"
-    ],
-    "postCreateCommand": "cd meltano && meltano install",
-    "settings": {
-        "python.defaultInterpreterPath": "/usr/local/bin/python",
-        "python.linting.ignorePatterns": [
-            ".vscode/*.py",
-            "**/site-packages/**/*.py",
-            ".meltano"
-        ],
-        "files.exclude": {
-            "**/.git": true,
-            "**/.svn": true,
-            "**/.hg": true,
-            "**/CVS": true,
-            "**/.DS_Store": true,
-            "**/__pycache__": true,
-            ".pytest_cache": true
-        },
-        "python.testing.unittestEnabled": false,
-        "python.testing.pytestEnabled": true,
-        "editor.formatOnSave": true,
-        "python.formatting.provider": "black",
-        "python.formatting.blackArgs": [
-            "--line-length",
-            "100",
-            "-S",
-            "--fast",
-        ],
-        "python.linting.enabled": true,
-        "python.linting.pylintEnabled": true,
-        "python.linting.lintOnSave": true,
-        "python.sortImports.args": [
-            "--profile",
-            "black"
-        ],
-        "[python]": {
-            "editor.codeActionsOnSave": {
-                "source.organizeImports": true
-            }
-        }
-    }
-}
+{
+  "name": "dagster-meltano",
+  "image": "ghcr.io/quantile-development/vscode-python-base:3.8",
+  "features": {
+    "ghcr.io/devcontainers-contrib/features/black:1": {},
+    "ghcr.io/devcontainers-contrib/features/isort:1": {},
+    "ghcr.io/devcontainers-contrib/features/meltano:1": {
+      "version": "2.12.0"
+    },
+    "ghcr.io/devcontainers-contrib/features/poetry:1": {}
+  },
+  "extensions": [
+    "ms-python.vscode-pylance",
+    "njpwerner.autodocstring",
+    "zhuangtongfa.material-theme",
+    "bungcip.better-toml",
+    "redhat.vscode-yaml",
+    "waderyan.gitblame",
+    "esbenp.prettier-vscode"
+  ],
+  "postCreateCommand": "poetry config virtualenvs.in-project true && poetry install && git config --global --add safe.directory /workspaces/dagster-meltano",
+  "remoteUser": "root"
+}

.github/workflows/test.yaml

@@ -1,27 +1,37 @@
-name: test
-on: 
-  pull_request:
-    paths:
-      - 'dagster_meltano/**'
-      - 'tests/**'
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-
-    steps:
-      - name: Check out repository code
-        uses: actions/checkout@v2
-
-      - name: Setup Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: "3.9"
-
-      - name: Install dependencies
-        run: |
-          pip install ".[development]"
-      - name: Run test suite
-        run: |
-          pytest
+name: test
+on:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/[email protected]
+        with:
+          python-version: 3.8
+          architecture: x64
+          cache: "pip"
+          cache-dependency-path: "poetry.lock"
+
+      - name: Install Poetry
+        run: pipx install poetry
+
+      - name: Install Dependencies
+        run: poetry install
+
+      - name: Install Meltano
+        run: pipx install meltano==2.12.0
+
+      - name: Install Meltano plugins
+        run: |
+          cd tests/meltano_test_project
+          meltano install
+
+      - name: Run test suite
+        run: poetry run pytest

.gitignore

@@ -3,6 +3,6 @@
 dist/
 .idea
 .env
-.meltano
-.vscode
+**/.meltano/
+build/
 meltano/load/sample.jsonl

.pylintrc

@@ -1,32 +1,33 @@
-[MESSAGES CONTROL]
-
-# Enable the message, report, category or checker with the given id(s). You can
-# either give multiple identifier separated by comma (,) or put this option
-# multiple time.
-#enable=
-
-# Disable the message, report, category or checker with the given id(s). You
-# can either give multiple identifier separated by comma (,) or put this option
-# multiple time (only on the command line, not in the configuration file where
-# it should appear only once).#
-#
-# R - refactoring related checks
-# C - convention related checks
-# W0511 disable TODO warning
-# W1201, W1202 disable log format warning. False positives (I think)
-# W0707 disable raise-missing-from which we cant use because py2 back compat
-
-disable=C,R,duplicate-code,W0511,W1201,W1202,W0707,no-init
-
-# See: https://github.com/getsentry/responses/issues/74
-[TYPECHECK]
-ignored-classes=responses
-signature-mutators=solid,composite_solid,lambda_solid,configured,op,graph
-
-[MASTER]
-ignore=snapshots
-load-plugins=dagster.utils.linter
-
-# See: https://stackoverflow.com/questions/40163106/cannot-find-col-function-in-pyspark
-generated-members=pyspark.*
-ignored-modules=pyspark.sql.functions
+[MESSAGES CONTROL]
+
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time.
+#enable=
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifier separated by comma (,) or put this option
+# multiple time (only on the command line, not in the configuration file where
+# it should appear only once).#
+#
+# R - refactoring related checks
+# C - convention related checks
+# W0511 disable TODO warning
+# W1201, W1202 disable log format warning. False positives (I think)
+# W0707 disable raise-missing-from which we cant use because py2 back compat
+
+disable=C,R,duplicate-code,W0511,W1201,W1202,W0707,no-init
+
+# See: https://github.com/getsentry/responses/issues/74
+[TYPECHECK]
+ignored-classes=responses
+signature-mutators=solid,composite_solid,lambda_solid,configured,op,graph
+
+[MASTER]
+ignore=snapshots
+load-plugins=dagster.utils.linter
+init-hook='import sys; sys.path.append("/workspaces/dagster-meltano")'
+
+# See: https://stackoverflow.com/questions/40163106/cannot-find-col-function-in-pyspark
+generated-members=pyspark.*
+ignored-modules=pyspark.sql.functions

.vscode/settings.json

@@ -0,0 +1,17 @@
+{
+  "python.linting.pylintPath": "/workspaces/dagster-meltano/.venv/bin/pylint",
+  "files.exclude": {
+    "**/.git": true,
+    "**/.svn": true,
+    "**/.hg": true,
+    "**/CVS": true,
+    "**/.DS_Store": true,
+    "**/Thumbs.db": true,
+    "**/.venv": true,
+    "**/__pycache__": true,
+    "**/.pytest_cache": true
+  },
+  "python.testing.pytestArgs": ["tests"],
+  "python.testing.unittestEnabled": false,
+  "python.testing.pytestEnabled": true
+}

.vscode/tasks.json

@@ -0,0 +1,42 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+      {
+        "label": "meltano-install",
+        "type": "shell",
+        "command": "meltano",
+        "args": [
+          "install"
+        ],
+        "options": {
+          "cwd": "${workspaceFolder}/meltano"
+        },
+        "isBackground": true,
+        "presentation": {
+          "reveal": "always",
+          "panel": "shared",
+          "close": true
+        },
+        "problemMatcher": []
+      },
+      {
+        "label": "dagit-up",
+        "type": "shell",
+        "command": "meltano",
+        "args": [
+          "run",
+          "dagit:up"
+        ],
+        "options": {
+          "cwd": "${workspaceFolder}/meltano"
+        },
+        "isBackground": true,
+        "presentation": {
+          "reveal": "always",
+          "panel": "shared",
+          "close": true
+        },
+        "problemMatcher": []
+      }
+    ]
+  }