Merge branch 'main' into 2045-validate-records-with-fastjsonschema

BuzzCutNorman · Feb 9, 2024 · 9d4fb94 · 9d4fb94
2 parents f055237 + 89d6afc
commit 9d4fb94
Show file tree

Hide file tree

Showing 78 changed files with 1,129 additions and 729 deletions.
diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml
@@ -13,7 +13,7 @@ body:
     attributes:
       label: Singer SDK Version
       description: Version of the library you are using
-      placeholder: "0.34.1"
+      placeholder: "0.35.0"
     validations:
       required: true
   - type: checkboxes

diff --git a/.github/workflows/constraints.txt b/.github/workflows/constraints.txt
@@ -1,4 +1,4 @@
-pip==23.3.2
+pip==24.0
 poetry==1.7.1
 poetry-plugin-export==1.6.0
 poetry-dynamic-versioning==1.2.0

diff --git a/.github/workflows/cookiecutter-e2e.yml b/.github/workflows/cookiecutter-e2e.yml
@@ -30,7 +30,7 @@ jobs:
       fail-fast: true
       matrix:
         include:
-          - { python-version: "3.11",  os: "ubuntu-latest" }
+          - { python-version: "3.12",  os: "ubuntu-latest" }
 
     steps:
     - uses: actions/checkout@v4

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -51,9 +51,9 @@ jobs:
         python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
         sqlalchemy: ["2"]
         include:
-        - { session: tests,   python-version: "3.11", os: "ubuntu-latest", sqlalchemy: "1" }
-        - { session: doctest, python-version: "3.11", os: "ubuntu-latest", sqlalchemy: "2" }
-        - { session: mypy,    python-version: "3.11",  os: "ubuntu-latest", sqlalchemy: "2" }
+        - { session: tests,   python-version: "3.12", os: "ubuntu-latest", sqlalchemy: "1" }
+        - { session: doctest, python-version: "3.12", os: "ubuntu-latest", sqlalchemy: "2" }
+        - { session: mypy,    python-version: "3.12",  os: "ubuntu-latest", sqlalchemy: "2" }
 
     steps:
     - uses: actions/checkout@v4
@@ -109,7 +109,7 @@ jobs:
     runs-on: ubuntu-latest
     if: ${{ !github.event.pull_request.head.repo.fork }}
     env:
-      NOXPYTHON: "3.11"
+      NOXPYTHON: "3.12"
       NOXSESSION: tests
       SAMPLE_TAP_GITLAB_AUTH_TOKEN: ${{ secrets.SAMPLE_TAP_GITLAB_AUTH_TOKEN }}
       SAMPLE_TAP_GITLAB_GROUP_IDS: ${{ secrets.SAMPLE_TAP_GITLAB_GROUP_IDS }}
@@ -180,7 +180,7 @@ jobs:
 
     - uses: actions/setup-python@v5
       with:
-        python-version: '3.11'
+        python-version: '3.12'
         cache: 'pip'
         cache-dependency-path: 'poetry.lock'
 
@@ -213,7 +213,7 @@ jobs:
       run: |
         nox -r --no-install -- xml
 
-    - uses: codecov/codecov-action@v3
+    - uses: codecov/codecov-action@v4
       with:
         fail_ci_if_error: true
         token: ${{ secrets.CODECOV_TOKEN }}
diff --git a/.github/workflows/version_bump.yml b/.github/workflows/version_bump.yml
@@ -42,7 +42,7 @@ jobs:
 
     - uses: actions/setup-python@v5
       with:
-        python-version: "3.11"
+        python-version: "3.12"
         architecture: x64
 
     - name: Bump version
@@ -74,7 +74,7 @@ jobs:
         sudo chown -R $USER:$USER .git/objects
 
     - name: Create Pull Request
-      uses: peter-evans/create-pull-request@v5
+      uses: peter-evans/create-pull-request@v6
       id: create-pull-request
       with:
         token: ${{ secrets.MELTYBOT_GITHUB_AUTH_TOKEN }}

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -42,14 +42,14 @@ repos:
         )$
 
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.27.3
+  rev: 0.27.4
   hooks:
   - id: check-dependabot
   - id: check-github-workflows
   - id: check-readthedocs
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.1.14
+  rev: v0.2.0
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,45 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## v0.35.0 (2024-02-02)
+
+### ✨ New
+
+- [#2208](https://github.com/meltano/sdk/issues/2208) Allow users to disable schema validation in targets
+- [#2170](https://github.com/meltano/sdk/issues/2170) Generate fake data with stream maps -- _**Thanks @ReubenFrankel!**_
+- [#937](https://github.com/meltano/sdk/issues/937) Support validating configuration for any tap with a dynamic catalog
+- [#2144](https://github.com/meltano/sdk/issues/2144) Support fanning out parent record into multiple child contexts/syncs
+- [#1918](https://github.com/meltano/sdk/issues/1918) End RESTStream pagination if an empty page is received
+
+### 🐛 Fixes
+
+- [#2203](https://github.com/meltano/sdk/issues/2203) Fix serialization of arbitrary objects (e.g. `ObjectId` from mongoDB) during flattening -- _**Thanks @dgawlowsky!**_
+- [#2200](https://github.com/meltano/sdk/issues/2200) Quote column names in INSERT statement
+- [#2195](https://github.com/meltano/sdk/issues/2195) Include empty `schemas` directory in REST tap cookiecutter
+- [#2187](https://github.com/meltano/sdk/issues/2187) Replace use of deprecated `jsonschema._RefResolver` with recommended `referencing` library
+- [#2184](https://github.com/meltano/sdk/issues/2184) Reduce amount of unnecessary whitespace in Singer output
+- [#2183](https://github.com/meltano/sdk/issues/2183) Ensure `.vscode` directory is included when requested in cookiecutters and avoid failing if it does not exist
+- [#2180](https://github.com/meltano/sdk/issues/2180) Limit supported Python versions in `--about` output to existing ones
+- [#2108](https://github.com/meltano/sdk/issues/2108) Log sink name when an unhandled error occurs during setup
+- [#2158](https://github.com/meltano/sdk/issues/2158) Fix pytest plugin declaration so it can be used without requiring defining `pytest_plugins` in `conftest.py`
+- [#2105](https://github.com/meltano/sdk/issues/2105) Default handling of `ACTIVATE_VERSION` messages to soft deletes and add new `SQLConnector.delete_old_versions` method
+
+### ⚙️ Under the Hood
+
+- [#2189](https://github.com/meltano/sdk/issues/2189) Use `functools.lru_cache` instead of the stale `memoization` library (#1981)
+- [#2188](https://github.com/meltano/sdk/issues/2188) Remove unused `logger` parameter from private catalog helper functions
+- [#2143](https://github.com/meltano/sdk/issues/2143) Drop support for Python 3.7
+- [#2157](https://github.com/meltano/sdk/issues/2157) Remove `pytz` dependency and use `datetime.timezone.utc` instead of `pytz.UTC` where possible
+- [#2136](https://github.com/meltano/sdk/issues/2136) Create interface for schema validation in sinks, and implement it for `python-jsonschema` -- _**Thanks @BuzzCutNorman!**_
+- [#2130](https://github.com/meltano/sdk/issues/2130) Allow loading stream schemas from `importlib.resources.abc.Traversable` types
+
+### 📚 Documentation Improvements
+
+- [#2204](https://github.com/meltano/sdk/issues/2204) Document supported package extras
+- [#2186](https://github.com/meltano/sdk/issues/2186) Call out minimum recommended `cookiecutter` version
+- [#2168](https://github.com/meltano/sdk/issues/2168) Explain `Progress is not resumable if interrupted` in docs FAQ
+- [#2140](https://github.com/meltano/sdk/issues/2140) Update auth caching example to use `functools.cached_property`
+
 ## v0.34.1 (2023-12-19)
 
 ### 🐛 Fixes

diff --git a/cookiecutter/mapper-template/cookiecutter.json b/cookiecutter/mapper-template/cookiecutter.json
@@ -5,6 +5,7 @@
   "mapper_id": "mapper-{{ cookiecutter.name.lower() }}",
   "library_name": "{{ cookiecutter.mapper_id.replace('-', '_') }}",
   "variant": "None (Skip)",
+  "faker_extra": false,
   "include_ci_files": ["GitHub", "None (Skip)"],
   "license": ["Apache-2.0"],
   "ide": ["VSCode", "None"],
@@ -14,6 +15,7 @@
     "admin_email": "Provide your [bold yellow]email[/]",
     "mapper_id": "The ID of the tap, in kebab-case",
     "library_name": "The name of the library, in snake_case. This is how the library will be imported in Python.",
+    "faker_extra": "Add [bold orange1][link=https://faker.readthedocs.io/en/master/]Faker[/link][/] as an extra dependency to support generating fake data in stream maps?",
     "include_ci_files": "Whether to include CI files for a common CI services",
     "license": "The license for the project",
     "ide": "Add configuration files for your preferred IDE"

diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml
@@ -31,12 +31,12 @@ packages = [
 
 [tool.poetry.dependencies]
 python = ">=3.8"
-singer-sdk = { version="~=0.34.1" }
+singer-sdk = { version="~=0.35.0"{{ ', extras = ["faker"]' if cookiecutter.faker_extra }} }
 fs-s3fs = { version = "~=1.1.1", optional = true }
 
 [tool.poetry.group.dev.dependencies]
 pytest = ">=7.4.0"
-singer-sdk = { version="~=0.34.1", extras = ["testing"] }
+singer-sdk = { version="~=0.35.0", extras = ["testing"] }
 
 [tool.poetry.extras]
 s3 = ["fs-s3fs"]

diff --git a/cookiecutter/tap-template/cookiecutter.json b/cookiecutter/tap-template/cookiecutter.json
@@ -14,6 +14,7 @@
     "JWT",
     "Custom or N/A"
   ],
+  "faker_extra": false,
   "include_ci_files": ["GitHub", "None"],
   "license": ["Apache-2.0", "None"],
   "ide": ["VSCode", "None"],
@@ -25,6 +26,7 @@
     "library_name": "The name of the library, in snake_case. This is how the library will be imported in Python.",
     "stream_type": "The type of stream the source provides",
     "auth_method": "The [bold red]authentication[/] method used by the source, for REST and GraphQL sources",
+    "faker_extra": "Add [bold orange1][link=https://faker.readthedocs.io/en/master/]Faker[/link][/] as an extra dependency to support generating fake data in stream maps?",
     "include_ci_files": "Whether to include CI files for a common CI services",
     "license": "The license for the project",
     "ide": "Add configuration files for your preferred IDE"

diff --git a/cookiecutter/tap-template/hooks/post_gen_project.py b/cookiecutter/tap-template/hooks/post_gen_project.py
@@ -15,6 +15,9 @@
     for client_py in PACKAGE_PATH.rglob("*-client.py"):
         client_py.unlink()
 
+    if "{{ cookiecutter.stream_type }}" != "REST":
+        shutil.rmtree(PACKAGE_PATH.joinpath("schemas"), ignore_errors=True)
+
     if "{{ cookiecutter.auth_method }}" not in ("OAuth2", "JWT"):
         PACKAGE_PATH.joinpath("auth.py").unlink()
 

diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml
@@ -31,15 +31,15 @@ packages = [
 [tool.poetry.dependencies]
 python = ">=3.8"
 importlib-resources = { version = "==6.1.*", python = "<3.9" }
-singer-sdk = { version="~=0.34.1" }
+singer-sdk = { version="~=0.35.0"{{ ', extras = ["faker"]' if cookiecutter.faker_extra }} }
 fs-s3fs = { version = "~=1.1.1", optional = true }
 {%- if cookiecutter.stream_type in ["REST", "GraphQL"] %}
 requests = "~=2.31.0"
 {%- endif %}
 
 [tool.poetry.group.dev.dependencies]
 pytest = ">=7.4.0"
-singer-sdk = { version="~=0.34.1", extras = ["testing"] }
+singer-sdk = { version="~=0.35.0", extras = ["testing"] }
 
 [tool.poetry.extras]
 s3 = ["fs-s3fs"]

diff --git a/...er/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/schemas/__init__.py b/...er/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/schemas/__init__.py
@@ -0,0 +1 @@
+"""JSON schema files for the REST API."""
diff --git a/cookiecutter/target-template/cookiecutter.json b/cookiecutter/target-template/cookiecutter.json
@@ -6,6 +6,7 @@
   "library_name": "{{ cookiecutter.target_id.replace('-', '_') }}",
   "variant": "None (Skip)",
   "serialization_method": ["Per record", "Per batch", "SQL"],
+  "faker_extra": false,
   "include_ci_files": ["GitHub", "None (Skip)"],
   "license": ["Apache-2.0"],
   "ide": ["VSCode", "None"],
@@ -16,6 +17,7 @@
     "mapper_id": "The ID of the tap, in kebab-case",
     "library_name": "The name of the library, in snake_case. This is how the library will be imported in Python.",
     "serialization_method": "The serialization method to use for loading data",
+    "faker_extra": "Add [bold orange1][link=https://faker.readthedocs.io/en/master/]Faker[/link][/] as an extra dependency to support generating fake data in stream maps?",
     "include_ci_files": "Whether to include CI files for a common CI services",
     "license": "The license for the project",
     "ide": "Add configuration files for your preferred IDE"

diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml b/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml
@@ -30,15 +30,15 @@ packages = [
 
 [tool.poetry.dependencies]
 python = ">=3.8"
-singer-sdk = { version="~=0.34.1" }
+singer-sdk = { version="~=0.35.0"{{ ', extras = ["faker"]' if cookiecutter.faker_extra }} }
 fs-s3fs = { version = "~=1.1.1", optional = true }
 {%- if cookiecutter.serialization_method != "SQL" %}
 requests = "~=2.31.0"
 {%- endif %}
 
 [tool.poetry.dev-dependencies]
 pytest = ">=7.4.0"
-singer-sdk = { version="~=0.34.1", extras = ["testing"] }
+singer-sdk = { version="~=0.35.0", extras = ["testing"] }
 
 [tool.poetry.extras]
 s3 = ["fs-s3fs"]

diff --git a/docs/batch.md b/docs/batch.md
@@ -52,7 +52,7 @@ AWS S3
 
 ### `encoding`
 
-The `encoding` field is used to specify the format and compression of the batch files. Currently only `jsonl` and `gzip` are supported, respectively.
+The `encoding` field is used to specify the format and compression of the batch files. Currently `jsonl`, `gzip` and `parquet` are supported.
 
 ### `manifest`
 

diff --git a/docs/classes/singer_sdk.authenticators.APIAuthenticatorBase.rst b/docs/classes/singer_sdk.authenticators.APIAuthenticatorBase.rst
@@ -5,4 +5,4 @@
 
 .. autoclass:: APIAuthenticatorBase
     :members:
-    :special-members: __init__, __call__
+    :special-members: __init__, __call__
diff --git a/docs/conf.py b/docs/conf.py
@@ -26,7 +26,7 @@
 author = "Meltano Core Team and Contributors"
 
 # The full version, including alpha/beta/rc tags
-release = "0.34.1"
+release = "0.35.0"
 
 
 # -- General configuration ---------------------------------------------------
@@ -39,6 +39,7 @@
     "sphinx.ext.napoleon",
     "sphinx.ext.autosectionlabel",
     "sphinx.ext.autosummary",
+    "sphinx.ext.intersphinx",
     "sphinx_copybutton",
     "myst_parser",
     "sphinx_reredirects",
@@ -55,7 +56,7 @@
 exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 
 # Show typehints in the description, along with parameter descriptions
-autodoc_typehints = "signature"
+autodoc_typehints = "description"
 autodoc_class_signature = "separated"
 autodoc_member_order = "groupwise"
 
@@ -127,3 +128,10 @@
 redirects = {
     "porting.html": "guides/porting.html",
 }
+
+# -- Options for intersphinx -------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#configuration
+intersphinx_mapping = {
+    "requests": ("https://requests.readthedocs.io/en/latest/", None),
+    "python": ("https://docs.python.org/3/", None),
+}
diff --git a/docs/dev_guide.md b/docs/dev_guide.md
@@ -177,6 +177,15 @@ Some APIs instead return the records as values inside an object where each key i
   ]
   ```
 
+## Extra features
+
+The following [extra features](https://packaging.python.org/en/latest/specifications/dependency-specifiers/#extras) are available for the Singer SDK:
+
+- `faker` - Enables the use of [Faker](https://faker.readthedocs.io/en/master/) in [stream maps](stream_maps.md).
+- `s3` - Enables AWS S3 as a [BATCH storage](batch.md#the-batch-message).
+- `parquet` - Enables as [BATCH encoding](batch.md#encoding).
+- `testing` - Pytest dependencies required to use the [Tap & Target Testing Framework](testing.md).
+
 ## Resources
 
 ### Detailed Class Reference