Merge branch 'main' of https://github.com/BuzzCutNorman/sdk into 2045…

…-validate-records-with-fastjsonschema
BuzzCutNorman · Jan 22, 2024 · a727cae · a727cae
2 parents 8adfd7c + 376421e
commit a727cae
Show file tree

Hide file tree

Showing 64 changed files with 590 additions and 983 deletions.
diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml
@@ -2,8 +2,6 @@ name: Bug Report
 description: File a bug report
 title: "bug: <title>"
 labels: ["kind/Bug", "valuestream/SDK"]
-assignees:
-  - meltano/engineering
 
 body:
   - type: markdown
@@ -31,12 +29,13 @@ body:
       label: Python Version
       description: Version of Python you are using
       options:
-        - "3.6 (deprecated)"
-        - "3.7"
+        - "3.6 (EOL)"
+        - "3.7 (EOL)"
         - "3.8"
         - "3.9"
         - "3.10"
         - "3.11"
+        - "3.12"
         - "NA"
     validations:
       required: true
@@ -76,6 +75,6 @@ body:
     attributes:
       label: Code
       description: Paste the failing code and/or traceback, if applicable
-      render: python
+      render: Python
     validations:
       required: false
diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml
@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/dependency-review-action@v3
+      - uses: actions/dependency-review-action@v4
         if: ${{ github.event_name == 'pull_request_target' }}
         with:
           fail-on-severity: high
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -47,7 +47,7 @@ jobs:
       matrix:
         session: [tests]
         os: ["ubuntu-latest", "macos-latest", "windows-latest"]
-        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
         sqlalchemy: ["2"]
         include:
         - { session: tests,   python-version: "3.11", os: "ubuntu-latest", sqlalchemy: "1" }
@@ -163,6 +163,8 @@ jobs:
     name: Coverage
     runs-on: ubuntu-latest
     needs: tests
+    env:
+      NOXSESSION: coverage
     steps:
     - uses: actions/checkout@v4
 
@@ -199,13 +201,16 @@ jobs:
         pipx inject nox nox-poetry
         nox --version
 
+    - run: nox --install-only
+
     - name: Combine coverage data and display human readable report
       run: |
-        nox --session=coverage
+        nox -r --no-install -- combine --debug=pathmap
+        nox -r --no-install -- report --show-missing --ignore-errors
 
     - name: Create coverage report
       run: |
-        nox --session=coverage -- xml
+        nox -r --no-install -- xml
 
     - uses: codecov/codecov-action@v3
       with:

diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,7 @@
+# Local Poetry configuration file
+
+poetry.toml
+
 # CI
 
 _changelog_fragment.md

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -45,7 +45,7 @@ repos:
   - id: check-readthedocs
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.1.13
+  rev: v0.1.14
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]

diff --git a/cookiecutter/mapper-template/cookiecutter.json b/cookiecutter/mapper-template/cookiecutter.json
@@ -7,13 +7,15 @@
   "variant": "None (Skip)",
   "include_ci_files": ["GitHub", "None (Skip)"],
   "license": ["Apache-2.0"],
+  "ide": ["VSCode", "None"],
   "__prompts__": {
     "name": "The name of the mapper, in CamelCase",
     "admin_name": "Provide your [bold yellow]full name[/]",
     "admin_email": "Provide your [bold yellow]email[/]",
     "mapper_id": "The ID of the tap, in kebab-case",
     "library_name": "The name of the library, in snake_case. This is how the library will be imported in Python.",
     "include_ci_files": "Whether to include CI files for a common CI services",
-    "license": "The license for the project"
+    "license": "The license for the project",
+    "ide": "Add configuration files for your preferred IDE"
   }
 }
diff --git a/cookiecutter/mapper-template/hooks/post_gen_project.py b/cookiecutter/mapper-template/hooks/post_gen_project.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+from pathlib import Path
+import shutil
+
+
+BASE_PATH = Path("{{cookiecutter.library_name}}")
+
+
+if __name__ == "__main__":
+    if "{{ cookiecutter.license }}" != "Apache-2.0":
+        Path("LICENSE").unlink()
+
+    if "{{ cookiecutter.include_ci_files }}" != "GitHub":
+        shutil.rmtree(Path(".github"))
+
+    if "{{ cookiecutter.ide }}" != "VSCode":
+        shutil.rmtree(".vscode")
diff --git a/...les == 'GitHub' %}dependabot.yml{%endif%} → ...cutter.mapper_id}}/.github/dependabot.yml b/...les == 'GitHub' %}dependabot.yml{%endif%} → ...cutter.mapper_id}}/.github/dependabot.yml
diff --git a/..._ci_files == 'GitHub' %}test.yml{%endif%} → ...er.mapper_id}}/.github/workflows/test.yml b/..._ci_files == 'GitHub' %}test.yml{%endif%} → ...er.mapper_id}}/.github/workflows/test.yml
diff --git a/...= cookiecutter.license %}LICENSE{%endif%} → ...mplate/{{cookiecutter.mapper_id}}/LICENSE b/...= cookiecutter.license %}LICENSE{%endif%} → ...mplate/{{cookiecutter.mapper_id}}/LICENSE
diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tests/conftest.py b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tests/conftest.py
diff --git a/...tter/mapper-template/{{cookiecutter.mapper_id}}/{{cookiecutter.library_name}}/__main__.py b/...tter/mapper-template/{{cookiecutter.mapper_id}}/{{cookiecutter.library_name}}/__main__.py
@@ -0,0 +1,7 @@
+"""{{ cookiecutter.name }} entry point."""
+
+from __future__ import annotations
+
+from {{ cookiecutter.library_name }}.mapper import {{ cookiecutter.name }}Mapper
+
+{{ cookiecutter.name }}Mapper.cli()
diff --git a/cookiecutter/tap-template/cookiecutter.json b/cookiecutter/tap-template/cookiecutter.json
@@ -16,6 +16,7 @@
   ],
   "include_ci_files": ["GitHub", "None"],
   "license": ["Apache-2.0", "None"],
+  "ide": ["VSCode", "None"],
   "__prompts__": {
     "source_name": "The name of the source, in CamelCase",
     "admin_name": "Provide your [bold yellow]full name[/]",
@@ -25,6 +26,7 @@
     "stream_type": "The type of stream the source provides",
     "auth_method": "The [bold red]authentication[/] method used by the source, for REST and GraphQL sources",
     "include_ci_files": "Whether to include CI files for a common CI services",
-    "license": "The license for the project"
+    "license": "The license for the project",
+    "ide": "Add configuration files for your preferred IDE"
   }
 }
diff --git a/cookiecutter/tap-template/hooks/post_gen_project.py b/cookiecutter/tap-template/hooks/post_gen_project.py
@@ -26,3 +26,6 @@
 
     if "{{ cookiecutter.include_ci_files }}" != "GitHub":
         shutil.rmtree(".github")
+
+    if "{{ cookiecutter.ide }}" != "VSCode":
+        shutil.rmtree(".vscode")
diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/tests/conftest.py b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/tests/conftest.py
diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/__main__.py b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/__main__.py
@@ -0,0 +1,7 @@
+"""{{ cookiecutter.source_name }} entry point."""
+
+from __future__ import annotations
+
+from {{ cookiecutter.library_name }}.tap import Tap{{ cookiecutter.source_name }}
+
+Tap{{ cookiecutter.source_name }}.cli()
diff --git a/cookiecutter/target-template/cookiecutter.json b/cookiecutter/target-template/cookiecutter.json
@@ -8,6 +8,7 @@
   "serialization_method": ["Per record", "Per batch", "SQL"],
   "include_ci_files": ["GitHub", "None (Skip)"],
   "license": ["Apache-2.0"],
+  "ide": ["VSCode", "None"],
   "__prompts__": {
     "name": "The name of the mapper, in CamelCase",
     "admin_name": "Provide your [bold yellow]full name[/]",
@@ -16,6 +17,7 @@
     "library_name": "The name of the library, in snake_case. This is how the library will be imported in Python.",
     "serialization_method": "The serialization method to use for loading data",
     "include_ci_files": "Whether to include CI files for a common CI services",
-    "license": "The license for the project"
+    "license": "The license for the project",
+    "ide": "Add configuration files for your preferred IDE"
   }
 }
diff --git a/cookiecutter/target-template/hooks/post_gen_project.py b/cookiecutter/target-template/hooks/post_gen_project.py
@@ -12,3 +12,6 @@
 
     if "{{ cookiecutter.include_ci_files }}" != "GitHub":
         shutil.rmtree(Path(".github"))
+
+    if "{{ cookiecutter.ide }}" != "VSCode":
+        shutil.rmtree(".vscode")
diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/tests/conftest.py b/cookiecutter/target-template/{{cookiecutter.target_id}}/tests/conftest.py
diff --git a/...tter/target-template/{{cookiecutter.target_id}}/{{cookiecutter.library_name}}/__main__.py b/...tter/target-template/{{cookiecutter.target_id}}/{{cookiecutter.library_name}}/__main__.py
@@ -0,0 +1,7 @@
+"""{{ cookiecutter.destination_name }} entry point."""
+
+from __future__ import annotations
+
+from {{ cookiecutter.library_name }}.target import Target{{ cookiecutter.destination_name }}
+
+Target{{ cookiecutter.destination_name }}.cli()
diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md
@@ -1,6 +1,6 @@
 # Contributing Guide
 
-_**Note:** The SDK currently works with Python versions 3.7 through 3.10.x. Python 3.6 is no longer supported._
+_**Note:** The SDK currently works with Python versions 3.8 through 3.12.x._
 
 Let's build together! Please see our [Contributor Guide](https://docs.meltano.com/contribute/)
 for more information on contributing to Meltano.

diff --git a/docs/classes/singer_sdk.authenticators.APIAuthenticatorBase.rst b/docs/classes/singer_sdk.authenticators.APIAuthenticatorBase.rst
@@ -5,4 +5,4 @@
 
 .. autoclass:: APIAuthenticatorBase
     :members:
-    :special-members: __init__, __call__
+    :special-members: __init__, __call__
diff --git a/docs/testing.md b/docs/testing.md
@@ -15,13 +15,6 @@ The Meltano SDK test framework consists of 4 main components:
 ## Example Usage
 
 If you created your Tap/Target using the provided cookiecutter templates, you will find the following snippets in `tests/test_core.py`.
-You will also find a `conftest.py` file containing configuration of the SDK as a `pytest` plugin.
-This is required for tests to collect correctly:
-
-```python
-# register the singer_sdk pytest plugin
-pytest_plugins = ("singer_sdk.testing.pytest_plugin",)
-```
 
 ### Testing Taps
 

diff --git a/e2e-tests/cookiecutters/mapper-base.json b/e2e-tests/cookiecutters/mapper-base.json
@@ -8,6 +8,7 @@
       "variant": "None (Skip)",
       "include_ci_files": "None (Skip)",
       "license": "Apache-2.0",
+      "ide": "VSCode",
       "_template": "../mapper-template/",
       "_output_dir": "."
     }

diff --git a/e2e-tests/cookiecutters/tap-graphql-jwt.json b/e2e-tests/cookiecutters/tap-graphql-jwt.json
@@ -10,6 +10,7 @@
       "auth_method": "JWT",
       "include_ci_files": "None (Skip)",
       "license": "Apache-2.0",
+      "ide": "VSCode",
       "_template": "../tap-template/",
       "_output_dir": "."
     }

diff --git a/e2e-tests/cookiecutters/tap-other-custom.json b/e2e-tests/cookiecutters/tap-other-custom.json
@@ -10,6 +10,7 @@
       "auth_method": "Custom or N/A",
       "include_ci_files": "None (Skip)",
       "license": "Apache-2.0",
+      "ide": "VSCode",
       "_template": "../tap-template/",
       "_output_dir": "."
     }

diff --git a/e2e-tests/cookiecutters/tap-rest-api_key-github.json b/e2e-tests/cookiecutters/tap-rest-api_key-github.json
@@ -10,6 +10,7 @@
       "auth_method": "API Key",
       "include_ci_files": "GitHub",
       "license": "Apache-2.0",
+      "ide": "VSCode",
       "_template": "../tap-template/",
       "_output_dir": "."
     }

diff --git a/e2e-tests/cookiecutters/tap-rest-basic_auth.json b/e2e-tests/cookiecutters/tap-rest-basic_auth.json
@@ -10,6 +10,7 @@
       "auth_method": "Basic Auth",
       "include_ci_files": "None (Skip)",
       "license": "Apache-2.0",
+      "ide": "VSCode",
       "_template": "../tap-template/",
       "_output_dir": "."
     }

diff --git a/e2e-tests/cookiecutters/tap-rest-bearer_token.json b/e2e-tests/cookiecutters/tap-rest-bearer_token.json
@@ -10,6 +10,7 @@
       "auth_method": "Bearer Token",
       "include_ci_files": "None (Skip)",
       "license": "Apache-2.0",
+      "ide": "VSCode",
       "_template": "../tap-template/",
       "_output_dir": "."
     }

diff --git a/e2e-tests/cookiecutters/tap-rest-custom.json b/e2e-tests/cookiecutters/tap-rest-custom.json
@@ -10,6 +10,7 @@
       "auth_method": "Custom or N/A",
       "include_ci_files": "None (Skip)",
       "license": "Apache-2.0",
+      "ide": "VSCode",
       "_template": "../tap-template/",
       "_output_dir": "."
     }

diff --git a/e2e-tests/cookiecutters/tap-rest-jwt.json b/e2e-tests/cookiecutters/tap-rest-jwt.json
@@ -10,6 +10,7 @@
       "auth_method": "JWT",
       "include_ci_files": "None (Skip)",
       "license": "Apache-2.0",
+      "ide": "VSCode",
       "_template": "../tap-template/",
       "_output_dir": "."
     }

diff --git a/e2e-tests/cookiecutters/tap-rest-oauth2.json b/e2e-tests/cookiecutters/tap-rest-oauth2.json
@@ -10,6 +10,7 @@
       "auth_method": "OAuth2",
       "include_ci_files": "None (Skip)",
       "license": "Apache-2.0",
+      "ide": "VSCode",
       "_template": "../tap-template/",
       "_output_dir": "."
     }

diff --git a/e2e-tests/cookiecutters/tap-sql-custom.json b/e2e-tests/cookiecutters/tap-sql-custom.json
@@ -10,6 +10,7 @@
       "auth_method": "Custom or N/A",
       "include_ci_files": "None (Skip)",
       "license": "Apache-2.0",
+      "ide": "VSCode",
       "_template": "../tap-template/",
       "_output_dir": "."
     }

diff --git a/e2e-tests/cookiecutters/target-per_record.json b/e2e-tests/cookiecutters/target-per_record.json
@@ -9,6 +9,7 @@
       "serialization_method": "Per record",
       "include_ci_files": "None (Skip)",
       "license": "Apache-2.0",
+      "ide": "VSCode",
       "_template": "./sdk/cookiecutter/target-template",
       "_output_dir": "."
     }

diff --git a/e2e-tests/cookiecutters/target-sql.json b/e2e-tests/cookiecutters/target-sql.json
@@ -9,6 +9,7 @@
       "serialization_method": "SQL",
       "include_ci_files": "None (Skip)",
       "license": "Apache-2.0",
+      "ide": "VSCode",
       "_template": "./sdk/cookiecutter/target-template",
       "_output_dir": "."
     }

diff --git a/noxfile.py b/noxfile.py
@@ -28,7 +28,7 @@
 COOKIECUTTER_REPLAY_FILES = list(Path("./e2e-tests/cookiecutters").glob("*.json"))
 
 package = "singer_sdk"
-python_versions = ["3.12", "3.11", "3.10", "3.9", "3.8", "3.7"]
+python_versions = ["3.12", "3.11", "3.10", "3.9", "3.8"]
 main_python_version = "3.11"
 locations = "singer_sdk", "tests", "noxfile.py", "docs/conf.py"
 nox.options.sessions = (
@@ -42,11 +42,12 @@
     "coverage[toml]",
     "duckdb",
     "duckdb-engine",
+    "pyarrow",
     "pytest",
     "pytest-benchmark",
     "pytest-durations",
     "pytest-snapshot",
-    "pyarrow",
+    "pytz",
     "requests-mock",
     "rfc3339-validator",
     "time-machine",