Merge branch 'main' into edgarrmondragon/feat/python3.13

meltano · Sep 10, 2024 · 97d04e5 · 97d04e5
2 parents 3e34b18 + 6708cb9
commit 97d04e5
Show file tree

Hide file tree

Showing 40 changed files with 801 additions and 439 deletions.
diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml
@@ -13,7 +13,7 @@ body:
     attributes:
       label: Singer SDK Version
       description: Version of the library you are using
-      placeholder: "0.39.1"
+      placeholder: "0.40.0"
     validations:
       required: true
   - type: checkboxes

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -65,7 +65,7 @@ jobs:
         name: Packages
         path: dist
     - name: Publish
-      uses: pypa/gh-action-pypi-publish@v1.9.0
+      uses: pypa/gh-action-pypi-publish@v1.10.1
 
   upload-to-release:
     name: Upload files to release

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -106,6 +106,7 @@ jobs:
     - uses: actions/upload-artifact@v4
       if: always() && (matrix.session == 'tests')
       with:
+        include-hidden-files: true
         name: coverage-data-nox_${{ matrix.session }}-${{ matrix.os }}-py${{ matrix.python-version }}_sqlalchemy_${{ matrix.sqlalchemy }}
         path: ".coverage.*"
 

diff --git a/.github/workflows/version_bump.yml b/.github/workflows/version_bump.yml
@@ -84,7 +84,7 @@ jobs:
 
     - name: Create Pull Request
       if: ${{ github.event.inputs.dry_run == 'false' }}
-      uses: peter-evans/create-pull-request@v6
+      uses: peter-evans/create-pull-request@v7
       id: create-pull-request
       with:
         token: ${{ secrets.MELTYBOT_GITHUB_AUTH_TOKEN }}

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -50,7 +50,7 @@ repos:
   - id: check-readthedocs
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.6.2
+  rev: v0.6.4
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,40 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## v0.40.0 (2024-09-02)
+
+### ✨ New
+
+- [#2486](https://github.com/meltano/sdk/issues/2486) Emit target metrics
+- [#2567](https://github.com/meltano/sdk/issues/2567) A new `schema_is_valid` built-in tap test validates stream schemas against the JSON Schema specification
+- [#2598](https://github.com/meltano/sdk/issues/2598) Stream map expressions now have access to the `Faker` class, rather than just a faker instance
+- [#2549](https://github.com/meltano/sdk/issues/2549) Added a default user agent for REST and GraphQL taps
+
+### 🐛 Fixes
+
+- [#2613](https://github.com/meltano/sdk/issues/2613) Mismatch between timezone-aware and naive datetimes in start date and bookmarks is now correctly handled
+
+### ⚙️ Under the Hood
+
+- [#2628](https://github.com/meltano/sdk/issues/2628) Use context manager to read gzip batch files
+- [#2619](https://github.com/meltano/sdk/issues/2619) Default to UTC when parsing dates without a known timezone
+- [#2603](https://github.com/meltano/sdk/issues/2603) Backwards-compatible identifier quoting in fully qualified names
+- [#2601](https://github.com/meltano/sdk/issues/2601) Improved SQL identifier (de)normalization
+- [#2599](https://github.com/meltano/sdk/issues/2599) Remove `pytest-durations` dependency from `testing` and use native pytest option `--durations`
+- [#2597](https://github.com/meltano/sdk/issues/2597) Mark pagination classes with `@override` decorator
+- [#2596](https://github.com/meltano/sdk/issues/2596) Made `auth_headers` and `auth_params` of `APIAuthenticatorBase` simple instance attributes instead of decorated properties
+
+### 📚 Documentation Improvements
+
+- [#2639](https://github.com/meltano/sdk/issues/2639) Documented versions where `fake` and `Faker` objects were added to the stream maps context
+- [#2629](https://github.com/meltano/sdk/issues/2629) Reference `get_starting_timestamp` in incremental replication guide
+- [#2604](https://github.com/meltano/sdk/issues/2604) Update project sample links
+- [#2595](https://github.com/meltano/sdk/issues/2595) Documented examples of stream glob expressions and property aliasing
+
+### 📦 Packaging changes
+
+- [#2640](https://github.com/meltano/sdk/issues/2640) Remove upper constraint on `faker` extra
+
 ## v0.39.1 (2024-08-07)
 
 ### 🐛 Fixes

diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.pre-commit-config.yaml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.pre-commit-config.yaml
@@ -18,19 +18,19 @@ repos:
   - id: trailing-whitespace
 
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.29.1
+  rev: 0.29.2
   hooks:
   - id: check-dependabot
   - id: check-github-workflows
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.5.6
+  rev: v0.6.4
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]
   - id: ruff-format
 
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v1.11.1
+  rev: v1.11.2
   hooks:
   - id: mypy
diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml
@@ -31,12 +31,12 @@ packages = [
 
 [tool.poetry.dependencies]
 python = ">=3.8"
-singer-sdk = { version="~=0.39.1"{{ ', extras = ["faker"]' if cookiecutter.faker_extra }} }
+singer-sdk = { version="~=0.40.0"{{ ', extras = ["faker"]' if cookiecutter.faker_extra }} }
 fs-s3fs = { version = "~=1.1.1", optional = true }
 
 [tool.poetry.group.dev.dependencies]
 pytest = ">=8"
-singer-sdk = { version="~=0.39.1", extras = ["testing"] }
+singer-sdk = { version="~=0.40.0", extras = ["testing"] }
 
 [tool.poetry.extras]
 s3 = ["fs-s3fs"]

diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.pre-commit-config.yaml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.pre-commit-config.yaml
@@ -18,20 +18,20 @@ repos:
   - id: trailing-whitespace
 
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.29.1
+  rev: 0.29.2
   hooks:
   - id: check-dependabot
   - id: check-github-workflows
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.5.6
+  rev: v0.6.4
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]
   - id: ruff-format
 
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v1.11.1
+  rev: v1.11.2
   hooks:
   - id: mypy
     additional_dependencies:

diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml
@@ -31,7 +31,7 @@ packages = [
 [tool.poetry.dependencies]
 python = ">=3.8"
 importlib-resources = { version = "==6.4.*", python = "<3.9" }
-singer-sdk = { version="~=0.39.1", extras = [
+singer-sdk = { version="~=0.40.0", extras = [
     {%- if cookiecutter.auth_method == "JWT" -%}"jwt", {% endif -%}
     {%- if cookiecutter.faker_extra -%}"faker",{%- endif -%}
 ] }
@@ -43,9 +43,9 @@ requests = "~=2.32.3"
 [tool.poetry.group.dev.dependencies]
 pytest = ">=8"
 {%- if cookiecutter.auth_method == "JWT" %}
-singer-sdk = { version="~=0.39.1", extras = ["jwt", "testing"] }
+singer-sdk = { version="~=0.40.0", extras = ["jwt", "testing"] }
 {%- else %}
-singer-sdk = { version="~=0.39.1", extras = ["testing"] }
+singer-sdk = { version="~=0.40.0", extras = ["testing"] }
 {%- endif %}
 
 [tool.poetry.extras]

diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/.pre-commit-config.yaml b/cookiecutter/target-template/{{cookiecutter.target_id}}/.pre-commit-config.yaml
@@ -18,20 +18,20 @@ repos:
   - id: trailing-whitespace
 
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.29.1
+  rev: 0.29.2
   hooks:
   - id: check-dependabot
   - id: check-github-workflows
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.5.6
+  rev: v0.6.4
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]
   - id: ruff-format
 
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v1.11.1
+  rev: v1.11.2
   hooks:
   - id: mypy
     additional_dependencies:

diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml b/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml
@@ -30,15 +30,15 @@ packages = [
 
 [tool.poetry.dependencies]
 python = ">=3.8"
-singer-sdk = { version="~=0.39.1"{{ ', extras = ["faker"]' if cookiecutter.faker_extra }} }
+singer-sdk = { version="~=0.40.0"{{ ', extras = ["faker"]' if cookiecutter.faker_extra }} }
 fs-s3fs = { version = "~=1.1.1", optional = true }
 {%- if cookiecutter.serialization_method != "SQL" %}
 requests = "~=2.32.3"
 {%- endif %}
 
 [tool.poetry.dev-dependencies]
 pytest = ">=8"
-singer-sdk = { version="~=0.39.1", extras = ["testing"] }
+singer-sdk = { version="~=0.40.0", extras = ["testing"] }
 
 [tool.poetry.extras]
 s3 = ["fs-s3fs"]

diff --git a/docs/classes/singer_sdk.connectors.sql.SQLToJSONSchema.rst b/docs/classes/singer_sdk.connectors.sql.SQLToJSONSchema.rst
@@ -0,0 +1,8 @@
+singer_sdk.connectors.sql.SQLToJSONSchema
+=========================================
+
+.. currentmodule:: singer_sdk.connectors.sql
+
+.. autoclass:: SQLToJSONSchema
+    :members:
+    :special-members: __init__, __call__
diff --git a/docs/conf.py b/docs/conf.py
@@ -29,7 +29,7 @@
 author = "Meltano Core Team and Contributors"
 
 # The full version, including alpha/beta/rc tags
-release = "0.39.1"
+release = "0.40.0"
 
 
 # -- General configuration -------------------------------------------------------------
@@ -158,6 +158,10 @@
         "https://json-schema.org/understanding-json-schema/reference/%s",
         "%s",
     ),
+    "column_type": (
+        "https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.%s",
+        "%s",
+    ),
 }
 
 # -- Options for intersphinx -----------------------------------------------------------

diff --git a/docs/guides/index.md b/docs/guides/index.md
@@ -9,4 +9,5 @@ porting
 pagination-classes
 custom-clis
 config-schema
+sql-tap
 ```
diff --git a/docs/guides/sql-tap.md b/docs/guides/sql-tap.md
@@ -0,0 +1,58 @@
+# Building SQL taps
+
+## Mapping SQL types to JSON Schema
+
+Starting with version `0.41.0`, the Meltano Singer SDK provides a clean way to map SQL types to JSON Schema. This is useful when the SQL dialect you are using has custom types that need to be mapped accordingly to JSON Schema.
+
+### Default type mapping
+
+The Singer SDK automatically handles the most common SQLAlchemy column types, using [`functools.singledispatchmethod`](inv:python:py:class:#functools.singledispatchmethod) to process each type. See the [`SQLToJSONSchema`](connectors.sql.SQLToJSONSchema) reference documentation for details.
+
+### Custom type mapping
+
+If the class above doesn't cover all the types supported by the SQLAlchemy dialect in your tap, you can subclass it and override or extend with a new method for the type you need to support:
+
+```python
+import functools
+
+from sqlalchemy import Numeric
+from singer_sdk import typing as th
+from singer_sdk.connectors import SQLConnector
+from singer_sdk.connectors.sql import SQLToJSONSchema
+
+from my_sqlalchemy_dialect import VectorType
+
+
+class CustomSQLToJSONSchema(SQLToJSONSchema):
+    @SQLToJSONSchema.to_jsonschema.register
+    def custom_number_to_jsonschema(self, column_type: Numeric):
+        """Override the default mapping for NUMERIC columns.
+
+        For example, a scale of 4 translates to a multipleOf 0.0001.
+        """
+        return {"type": ["number"], "multipleOf": 10**-column_type.scale}
+
+    @SQLToJSONSchema.to_jsonschema.register(VectorType)
+    def vector_to_json_schema(self, column_type):
+        """Custom vector to JSON schema."""
+        return th.ArrayType(th.NumberType()).to_dict()
+```
+
+````{tip}
+You can also use a type annotation to specify the type of the column when registering a new method:
+
+```python
+@SQLToJSONSchema.to_jsonschema.register
+def vector_to_json_schema(self, column_type: VectorType):
+    return th.ArrayType(th.NumberType()).to_dict()
+```
+````
+
+Then, you need to use your custom type mapping in your connector:
+
+```python
+class MyConnector(SQLConnector):
+    @functools.cached_property
+    def sql_to_jsonschema(self):
+        return CustomSQLToJSONSchema()
+```
diff --git a/docs/reference.rst b/docs/reference.rst
@@ -143,3 +143,12 @@ Batch
 
     batch.BaseBatcher
     batch.JSONLinesBatcher
+
+Other
+-----
+
+.. autosummary::
+    :toctree: classes
+    :template: class.rst
+
+    connectors.sql.SQLToJSONSchema
diff --git a/docs/stream_maps.md b/docs/stream_maps.md
@@ -253,9 +253,17 @@ can be referenced directly by mapping expressions.
   masking by allowing users to call `Faker.seed()`.
 
   ```{tip}
-  The `fake` object is only available if the plugin specifies `faker` as an additional dependency (through the `singer-sdk` `faker` extra, or directly).
+  The `fake` object and `Faker` are only available if the plugin specifies `faker` as an additional dependency (through the `singer-sdk` `faker` extra, or directly).
   ```
 
+:::{versionadded} 0.35.0
+The `faker` object.
+:::
+
+:::{versionadded} 0.40.0
+The `Faker` class.
+:::
+
 #### Automatic Schema Detection
 
 For performance reasons, type detection is performed at runtime using text analysis