working mvp

.github/workflows/ci_workflow.yml

@@ -12,18 +12,18 @@ jobs:
     strategy:
       matrix:
         # Only lint using the primary version used for dev
-        python-version: [3.9]
+        python-version: ["3.10"]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install Poetry
-      uses: snok/install-poetry@v1
-      with:
-        version: 1.1.8
+      run: |
+        python -m pip install --upgrade pip
+        pip install poetry==1.2.*
     - name: Install dependencies
       run: |
         poetry install
@@ -36,20 +36,26 @@ jobs:
     runs-on: ubuntu-latest
     env:
       GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+      SF_USER: ${{secrets.SF_USER}}
+      SF_PASSWORD: ${{secrets.SF_PASSWORD}}
+      SF_ACCOUNT: ${{secrets.SF_ACCOUNT}}
+      SF_DATABASE: ${{secrets.SF_DATABASE}}
+      SF_WAREHOUSE: ${{secrets.SF_WAREHOUSE}}
+      SF_ROLE: ${{secrets.SF_ROLE}}
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9]
+        python-version: ["3.7", "3.8", "3.9", "3.10"]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install Poetry
-      uses: snok/install-poetry@v1
-      with:
-        version: 1.1.11
+      run: |
+        python -m pip install --upgrade pip
+        pip install poetry==1.2.*
     - name: Install dependencies
       run: |
         poetry install

.gitignore

@@ -2,9 +2,14 @@
 **/.secrets/*
 
 # Ignore meltano internal cache and sqlite systemdb
-
 .meltano/
 
+# IDEs
+.vscode
+
+#macOS
+.DS_Store
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

README.md

@@ -6,18 +6,14 @@ Built with the [Meltano Tap SDK](https://sdk.meltano.com) for Singer Taps.
 
 ## Installation
 
-- [ ] `Developer TODO:` Update the below as needed to correctly describe the install procedure. For instance, if you do not have a PyPi repo, or if you want users to directly install from your git repo, you can modify this step as appropriate.
-
 ```bash
-pipx install tap-snowflake
+pipx install git+https://github.com/MeltanoLabs/tap-snowflake.git
 ```
 
 ## Configuration
 
 ### Accepted Config Options
 
-- [ ] `Developer TODO:` Provide a list of config options accepted by the tap.
-
 A full list of supported settings and capabilities for this
 tap is available by running:
 
@@ -33,7 +29,47 @@ environment variable is set either in the terminal context or in the `.env` file
 
 ### Source Authentication and Authorization
 
-- [ ] `Developer TODO:` If your tap requires special access on the source system, or any special authentication requirements, provide those here.
+Standard `username` and `password` auth is supported.
+
+### Enabling Batch Messaging
+
+This tap is built using the Meltano SDK and therefore supports a `BATCH` [message type](https://sdk.meltano.com/en/latest/batch.html), in
+addition to the `RECORD` messages of the Singer spec. This can be enabled either by adding the following to your `config.json`:
+
+```json
+{
+  // ...
+  "batch_config": {
+    "encoding": {
+      "format": "jsonl",
+      "compression": "gzip"
+    },
+    "storage": {
+      "root": "file://tests/core/resources",
+      "prefix": "test-batch"
+    }
+  }
+}
+```
+
+or its equivalent to your `meltano.yml`
+
+```yaml
+config:
+  plugins:
+    extractors:
+      - name: tap-snowflake
+        config:
+          batch_config:
+            encoding:
+              format: jsonl
+              compression: gzip
+            storage:
+              root: "file://tests/core/resources"
+              prefix: test-batch
+```
+
+**Note:** This variant of `tap-snowflake` does not yet support the `INCREMENTAL` replication strategy in `BATCH` mode. Follow [here](https://github.com/meltano/sdk/issues/976#issuecomment-1257848119) for updates.
 
 ## Usage
 
@@ -49,8 +85,6 @@ tap-snowflake --config CONFIG --discover > ./catalog.json
 
 ## Developer Resources
 
-- [ ] `Developer TODO:` As a first step, scan the entire project for the text "`TODO:`" and complete any recommended steps, deleting the "TODO" references once completed.
-
 ### Initialize your Development Environment
 
 ```bash
@@ -61,7 +95,7 @@ poetry install
 ### Create and Run Tests
 
 Create tests within the `tap_snowflake/tests` subfolder and
-  then run:
+then run:
 
 ```bash
 poetry run pytest

meltano.yml

@@ -1,25 +1,28 @@
 version: 1
 send_anonymous_usage_stats: true
-project_id: "tap-snowflake"
+project_id: tap-snowflake
 plugins:
   extractors:
-  - name: "tap-snowflake"
-    namespace: "tap_snowflake"
+  - name: tap-snowflake
+    namespace: tap_snowflake
     pip_url: -e .
     capabilities:
     - state
     - catalog
     - discover
-    config:
-      start_date: '2010-01-01T00:00:00Z'
     settings:
-    # TODO: To configure using Meltano, declare settings and their types here:
-    - name: username
+    - name: user
     - name: password
       kind: password
-    - name: start_date
-      value: '2010-01-01T00:00:00Z'
+    - name: account
+      kind: password
+    - name: role
+    - name: warehouse
+    - name: database
+    - name: schema
   loaders:
   - name: target-jsonl
     variant: andyh1203
     pip_url: target-jsonl
+environments:
+- name: dev

mypy.ini

@@ -1,6 +1,7 @@
 [mypy]
 python_version = 3.9
 warn_unused_configs = True
+ignore_missing_imports = True
 
 [mypy-backoff.*]
 ignore_missing_imports = True