diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index d33518f65..0000da931 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -13,7 +13,7 @@ body: attributes: label: Singer SDK Version description: Version of the library you are using - placeholder: "0.34.1" + placeholder: "0.35.0" validations: required: true - type: checkboxes diff --git a/.github/workflows/constraints.txt b/.github/workflows/constraints.txt index cdfd9eb88..22e02b624 100644 --- a/.github/workflows/constraints.txt +++ b/.github/workflows/constraints.txt @@ -1,4 +1,4 @@ -pip==23.3.2 +pip==24.0 poetry==1.7.1 poetry-plugin-export==1.6.0 poetry-dynamic-versioning==1.2.0 diff --git a/.github/workflows/cookiecutter-e2e.yml b/.github/workflows/cookiecutter-e2e.yml index bb75980c7..025406eb5 100644 --- a/.github/workflows/cookiecutter-e2e.yml +++ b/.github/workflows/cookiecutter-e2e.yml @@ -30,7 +30,7 @@ jobs: fail-fast: true matrix: include: - - { python-version: "3.11", os: "ubuntu-latest" } + - { python-version: "3.12", os: "ubuntu-latest" } steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7ad37ef34..710a72171 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -51,9 +51,9 @@ jobs: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] sqlalchemy: ["2"] include: - - { session: tests, python-version: "3.11", os: "ubuntu-latest", sqlalchemy: "1" } - - { session: doctest, python-version: "3.11", os: "ubuntu-latest", sqlalchemy: "2" } - - { session: mypy, python-version: "3.11", os: "ubuntu-latest", sqlalchemy: "2" } + - { session: tests, python-version: "3.12", os: "ubuntu-latest", sqlalchemy: "1" } + - { session: doctest, python-version: "3.12", os: "ubuntu-latest", sqlalchemy: "2" } + - { session: mypy, python-version: "3.12", os: "ubuntu-latest", sqlalchemy: "2" } steps: - uses: actions/checkout@v4 @@ -109,7 +109,7 @@ jobs: runs-on: ubuntu-latest if: ${{ !github.event.pull_request.head.repo.fork }} env: - NOXPYTHON: "3.11" + NOXPYTHON: "3.12" NOXSESSION: tests SAMPLE_TAP_GITLAB_AUTH_TOKEN: ${{ secrets.SAMPLE_TAP_GITLAB_AUTH_TOKEN }} SAMPLE_TAP_GITLAB_GROUP_IDS: ${{ secrets.SAMPLE_TAP_GITLAB_GROUP_IDS }} @@ -180,7 +180,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: '3.12' cache: 'pip' cache-dependency-path: 'poetry.lock' @@ -213,7 +213,7 @@ jobs: run: | nox -r --no-install -- xml - - uses: codecov/codecov-action@v3 + - uses: codecov/codecov-action@v4 with: fail_ci_if_error: true token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/version_bump.yml b/.github/workflows/version_bump.yml index ed3edcf94..04d38c8ab 100644 --- a/.github/workflows/version_bump.yml +++ b/.github/workflows/version_bump.yml @@ -42,7 +42,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: "3.12" architecture: x64 - name: Bump version @@ -74,7 +74,7 @@ jobs: sudo chown -R $USER:$USER .git/objects - name: Create Pull Request - uses: peter-evans/create-pull-request@v5 + uses: peter-evans/create-pull-request@v6 id: create-pull-request with: token: ${{ secrets.MELTYBOT_GITHUB_AUTH_TOKEN }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0501629fb..345a262e3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,14 +42,14 @@ repos: )$ - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.27.3 + rev: 0.27.4 hooks: - id: check-dependabot - id: check-github-workflows - id: check-readthedocs - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.14 + rev: v0.2.0 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix, --show-fixes] diff --git a/CHANGELOG.md b/CHANGELOG.md index f1b205fc8..9c23c195d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,45 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## v0.35.0 (2024-02-02) + +### ✨ New + +- [#2208](https://github.com/meltano/sdk/issues/2208) Allow users to disable schema validation in targets +- [#2170](https://github.com/meltano/sdk/issues/2170) Generate fake data with stream maps -- _**Thanks @ReubenFrankel!**_ +- [#937](https://github.com/meltano/sdk/issues/937) Support validating configuration for any tap with a dynamic catalog +- [#2144](https://github.com/meltano/sdk/issues/2144) Support fanning out parent record into multiple child contexts/syncs +- [#1918](https://github.com/meltano/sdk/issues/1918) End RESTStream pagination if an empty page is received + +### 🐛 Fixes + +- [#2203](https://github.com/meltano/sdk/issues/2203) Fix serialization of arbitrary objects (e.g. `ObjectId` from mongoDB) during flattening -- _**Thanks @dgawlowsky!**_ +- [#2200](https://github.com/meltano/sdk/issues/2200) Quote column names in INSERT statement +- [#2195](https://github.com/meltano/sdk/issues/2195) Include empty `schemas` directory in REST tap cookiecutter +- [#2187](https://github.com/meltano/sdk/issues/2187) Replace use of deprecated `jsonschema._RefResolver` with recommended `referencing` library +- [#2184](https://github.com/meltano/sdk/issues/2184) Reduce amount of unnecessary whitespace in Singer output +- [#2183](https://github.com/meltano/sdk/issues/2183) Ensure `.vscode` directory is included when requested in cookiecutters and avoid failing if it does not exist +- [#2180](https://github.com/meltano/sdk/issues/2180) Limit supported Python versions in `--about` output to existing ones +- [#2108](https://github.com/meltano/sdk/issues/2108) Log sink name when an unhandled error occurs during setup +- [#2158](https://github.com/meltano/sdk/issues/2158) Fix pytest plugin declaration so it can be used without requiring defining `pytest_plugins` in `conftest.py` +- [#2105](https://github.com/meltano/sdk/issues/2105) Default handling of `ACTIVATE_VERSION` messages to soft deletes and add new `SQLConnector.delete_old_versions` method + +### ⚙️ Under the Hood + +- [#2189](https://github.com/meltano/sdk/issues/2189) Use `functools.lru_cache` instead of the stale `memoization` library (#1981) +- [#2188](https://github.com/meltano/sdk/issues/2188) Remove unused `logger` parameter from private catalog helper functions +- [#2143](https://github.com/meltano/sdk/issues/2143) Drop support for Python 3.7 +- [#2157](https://github.com/meltano/sdk/issues/2157) Remove `pytz` dependency and use `datetime.timezone.utc` instead of `pytz.UTC` where possible +- [#2136](https://github.com/meltano/sdk/issues/2136) Create interface for schema validation in sinks, and implement it for `python-jsonschema` -- _**Thanks @BuzzCutNorman!**_ +- [#2130](https://github.com/meltano/sdk/issues/2130) Allow loading stream schemas from `importlib.resources.abc.Traversable` types + +### 📚 Documentation Improvements + +- [#2204](https://github.com/meltano/sdk/issues/2204) Document supported package extras +- [#2186](https://github.com/meltano/sdk/issues/2186) Call out minimum recommended `cookiecutter` version +- [#2168](https://github.com/meltano/sdk/issues/2168) Explain `Progress is not resumable if interrupted` in docs FAQ +- [#2140](https://github.com/meltano/sdk/issues/2140) Update auth caching example to use `functools.cached_property` + ## v0.34.1 (2023-12-19) ### 🐛 Fixes diff --git a/cookiecutter/mapper-template/cookiecutter.json b/cookiecutter/mapper-template/cookiecutter.json index 9d7b6385f..073319d08 100644 --- a/cookiecutter/mapper-template/cookiecutter.json +++ b/cookiecutter/mapper-template/cookiecutter.json @@ -5,6 +5,7 @@ "mapper_id": "mapper-{{ cookiecutter.name.lower() }}", "library_name": "{{ cookiecutter.mapper_id.replace('-', '_') }}", "variant": "None (Skip)", + "faker_extra": false, "include_ci_files": ["GitHub", "None (Skip)"], "license": ["Apache-2.0"], "ide": ["VSCode", "None"], @@ -14,6 +15,7 @@ "admin_email": "Provide your [bold yellow]email[/]", "mapper_id": "The ID of the tap, in kebab-case", "library_name": "The name of the library, in snake_case. This is how the library will be imported in Python.", + "faker_extra": "Add [bold orange1][link=https://faker.readthedocs.io/en/master/]Faker[/link][/] as an extra dependency to support generating fake data in stream maps?", "include_ci_files": "Whether to include CI files for a common CI services", "license": "The license for the project", "ide": "Add configuration files for your preferred IDE" diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml index d0ae4b1a9..eaf71ebac 100644 --- a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml @@ -31,12 +31,12 @@ packages = [ [tool.poetry.dependencies] python = ">=3.8" -singer-sdk = { version="~=0.34.1" } +singer-sdk = { version="~=0.35.0"{{ ', extras = ["faker"]' if cookiecutter.faker_extra }} } fs-s3fs = { version = "~=1.1.1", optional = true } [tool.poetry.group.dev.dependencies] pytest = ">=7.4.0" -singer-sdk = { version="~=0.34.1", extras = ["testing"] } +singer-sdk = { version="~=0.35.0", extras = ["testing"] } [tool.poetry.extras] s3 = ["fs-s3fs"] diff --git a/cookiecutter/tap-template/cookiecutter.json b/cookiecutter/tap-template/cookiecutter.json index 4eae4d8ad..cd5c01bf9 100644 --- a/cookiecutter/tap-template/cookiecutter.json +++ b/cookiecutter/tap-template/cookiecutter.json @@ -14,6 +14,7 @@ "JWT", "Custom or N/A" ], + "faker_extra": false, "include_ci_files": ["GitHub", "None"], "license": ["Apache-2.0", "None"], "ide": ["VSCode", "None"], @@ -25,6 +26,7 @@ "library_name": "The name of the library, in snake_case. This is how the library will be imported in Python.", "stream_type": "The type of stream the source provides", "auth_method": "The [bold red]authentication[/] method used by the source, for REST and GraphQL sources", + "faker_extra": "Add [bold orange1][link=https://faker.readthedocs.io/en/master/]Faker[/link][/] as an extra dependency to support generating fake data in stream maps?", "include_ci_files": "Whether to include CI files for a common CI services", "license": "The license for the project", "ide": "Add configuration files for your preferred IDE" diff --git a/cookiecutter/tap-template/hooks/post_gen_project.py b/cookiecutter/tap-template/hooks/post_gen_project.py index dc7134238..05044d490 100644 --- a/cookiecutter/tap-template/hooks/post_gen_project.py +++ b/cookiecutter/tap-template/hooks/post_gen_project.py @@ -15,6 +15,9 @@ for client_py in PACKAGE_PATH.rglob("*-client.py"): client_py.unlink() + if "{{ cookiecutter.stream_type }}" != "REST": + shutil.rmtree(PACKAGE_PATH.joinpath("schemas"), ignore_errors=True) + if "{{ cookiecutter.auth_method }}" not in ("OAuth2", "JWT"): PACKAGE_PATH.joinpath("auth.py").unlink() diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml index b1902b724..c918ac15d 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml @@ -31,7 +31,7 @@ packages = [ [tool.poetry.dependencies] python = ">=3.8" importlib-resources = { version = "==6.1.*", python = "<3.9" } -singer-sdk = { version="~=0.34.1" } +singer-sdk = { version="~=0.35.0"{{ ', extras = ["faker"]' if cookiecutter.faker_extra }} } fs-s3fs = { version = "~=1.1.1", optional = true } {%- if cookiecutter.stream_type in ["REST", "GraphQL"] %} requests = "~=2.31.0" @@ -39,7 +39,7 @@ requests = "~=2.31.0" [tool.poetry.group.dev.dependencies] pytest = ">=7.4.0" -singer-sdk = { version="~=0.34.1", extras = ["testing"] } +singer-sdk = { version="~=0.35.0", extras = ["testing"] } [tool.poetry.extras] s3 = ["fs-s3fs"] diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/schemas/__init__.py b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/schemas/__init__.py new file mode 100644 index 000000000..06c0a1988 --- /dev/null +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/schemas/__init__.py @@ -0,0 +1 @@ +"""JSON schema files for the REST API.""" diff --git a/cookiecutter/target-template/cookiecutter.json b/cookiecutter/target-template/cookiecutter.json index 2490a31db..5c1ff2544 100644 --- a/cookiecutter/target-template/cookiecutter.json +++ b/cookiecutter/target-template/cookiecutter.json @@ -6,6 +6,7 @@ "library_name": "{{ cookiecutter.target_id.replace('-', '_') }}", "variant": "None (Skip)", "serialization_method": ["Per record", "Per batch", "SQL"], + "faker_extra": false, "include_ci_files": ["GitHub", "None (Skip)"], "license": ["Apache-2.0"], "ide": ["VSCode", "None"], @@ -16,6 +17,7 @@ "mapper_id": "The ID of the tap, in kebab-case", "library_name": "The name of the library, in snake_case. This is how the library will be imported in Python.", "serialization_method": "The serialization method to use for loading data", + "faker_extra": "Add [bold orange1][link=https://faker.readthedocs.io/en/master/]Faker[/link][/] as an extra dependency to support generating fake data in stream maps?", "include_ci_files": "Whether to include CI files for a common CI services", "license": "The license for the project", "ide": "Add configuration files for your preferred IDE" diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml b/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml index 7bcceaf69..417a8e8ab 100644 --- a/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml +++ b/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml @@ -30,7 +30,7 @@ packages = [ [tool.poetry.dependencies] python = ">=3.8" -singer-sdk = { version="~=0.34.1" } +singer-sdk = { version="~=0.35.0"{{ ', extras = ["faker"]' if cookiecutter.faker_extra }} } fs-s3fs = { version = "~=1.1.1", optional = true } {%- if cookiecutter.serialization_method != "SQL" %} requests = "~=2.31.0" @@ -38,7 +38,7 @@ requests = "~=2.31.0" [tool.poetry.dev-dependencies] pytest = ">=7.4.0" -singer-sdk = { version="~=0.34.1", extras = ["testing"] } +singer-sdk = { version="~=0.35.0", extras = ["testing"] } [tool.poetry.extras] s3 = ["fs-s3fs"] diff --git a/docs/batch.md b/docs/batch.md index df6ef1178..45b1fc1f1 100644 --- a/docs/batch.md +++ b/docs/batch.md @@ -52,7 +52,7 @@ AWS S3 ### `encoding` -The `encoding` field is used to specify the format and compression of the batch files. Currently only `jsonl` and `gzip` are supported, respectively. +The `encoding` field is used to specify the format and compression of the batch files. Currently `jsonl`, `gzip` and `parquet` are supported. ### `manifest` diff --git a/docs/classes/singer_sdk.authenticators.APIAuthenticatorBase.rst b/docs/classes/singer_sdk.authenticators.APIAuthenticatorBase.rst index 70e9a127f..1b3b608c5 100644 --- a/docs/classes/singer_sdk.authenticators.APIAuthenticatorBase.rst +++ b/docs/classes/singer_sdk.authenticators.APIAuthenticatorBase.rst @@ -5,4 +5,4 @@ .. autoclass:: APIAuthenticatorBase :members: - :special-members: __init__, __call__ + :special-members: __init__, __call__ \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index 519b2af0c..1b91724f0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -26,7 +26,7 @@ author = "Meltano Core Team and Contributors" # The full version, including alpha/beta/rc tags -release = "0.34.1" +release = "0.35.0" # -- General configuration --------------------------------------------------- @@ -39,6 +39,7 @@ "sphinx.ext.napoleon", "sphinx.ext.autosectionlabel", "sphinx.ext.autosummary", + "sphinx.ext.intersphinx", "sphinx_copybutton", "myst_parser", "sphinx_reredirects", @@ -55,7 +56,7 @@ exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # Show typehints in the description, along with parameter descriptions -autodoc_typehints = "signature" +autodoc_typehints = "description" autodoc_class_signature = "separated" autodoc_member_order = "groupwise" @@ -127,3 +128,10 @@ redirects = { "porting.html": "guides/porting.html", } + +# -- Options for intersphinx ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#configuration +intersphinx_mapping = { + "requests": ("https://requests.readthedocs.io/en/latest/", None), + "python": ("https://docs.python.org/3/", None), +} diff --git a/docs/dev_guide.md b/docs/dev_guide.md index 75dcf60dc..af198e9b8 100644 --- a/docs/dev_guide.md +++ b/docs/dev_guide.md @@ -177,6 +177,15 @@ Some APIs instead return the records as values inside an object where each key i ] ``` +## Extra features + +The following [extra features](https://packaging.python.org/en/latest/specifications/dependency-specifiers/#extras) are available for the Singer SDK: + +- `faker` - Enables the use of [Faker](https://faker.readthedocs.io/en/master/) in [stream maps](stream_maps.md). +- `s3` - Enables AWS S3 as a [BATCH storage](batch.md#the-batch-message). +- `parquet` - Enables as [BATCH encoding](batch.md#encoding). +- `testing` - Pytest dependencies required to use the [Tap & Target Testing Framework](testing.md). + ## Resources ### Detailed Class Reference diff --git a/docs/stream_maps.md b/docs/stream_maps.md index 8d84d9ea0..b7bffd36b 100644 --- a/docs/stream_maps.md +++ b/docs/stream_maps.md @@ -80,23 +80,13 @@ These capabilities are all out of scope _by design_: a transformation tool like [dbt](https://www.getdbt.com), or (b) create a custom mapper plugin with inline lookup logic. -### A feature for all Singer users, enabled by the SDK +## A feature for all Singer users, enabled by the SDK -The mapping features described here are create for the **_users_** of SDK-based taps and targets. +The mapping features described here are created for the **_users_** of SDK-based taps and targets, which support inline transformations with `stream_maps` and `stream_map_config` out-of-box. -Developers simply enable the feature using the instructions below, and then users can benefit from having inline transformation capabilities out-of-box on their favorite taps and targets. +**Note:** to support non-SDK taps and targets, the standalone inline mapper plugin [`meltano-map-transformer`](https://hub.meltano.com/mappers/meltano-map-transformer/) follows all specifications defined here and can apply mapping transformations between _any_ Singer tap and target, even if they are not built using the SDK. -**Note:** to support non-SDK taps and targets, we are also creating a standalone inline mapper plugin (`meltano-map-transform`), which follows all specifications defined here and can apply mapping transformations between _any_ Singer tap and target, even if they are not built using the SDK. - -## Enabling Stream Maps in SDK-Based Plugins - -To support inline mapping functions, the developer only needs to declare two plugin settings, -called `stream_maps` and `stream_map_config`, and declare both settings as `object` type. (For example: -`Property("stream_maps, ObjectType())` if using the python helper classes or -`"stream_maps": {"type": "object"}` if using native JSON Schema declarations.) - -If the `stream_maps` setting is detected, the following behaviors will be implemented -by the SDK automatically: +The following behaviors are implemented by the SDK automatically: 1. For taps, the SCHEMA and RECORD messages will automatically be transformed, duplicated, filtered, or aliased, as per the `stream_maps` config settings _after_ @@ -108,7 +98,7 @@ by the SDK automatically: setting _prior_ to any Sink processing functions. - This means that the target developer can assume that all streams and records are transformed, aliased, filtered, etc. _before_ any custom target code is executed. -3. The upcoming standalone mapper plugin (`meltano-map-transform`) is a hybrid tap/target which +3. The standalone mapper plugin [`meltano-map-transformer`](https://hub.meltano.com/mappers/meltano-map-transformer/) is a hybrid tap/target which simply receives input from a tap, transforms all stream and schema messages via the `stream_maps` config option, and then emits the resulting stream(s) to a downstream target. @@ -122,8 +112,7 @@ by the SDK automatically: The `stream_maps` config expects a mapping of stream names to a structured transform object. -Here is a sample `stream_maps` transformation which removes all references to `email` and -adds `email_domain` and `email_hash` as new properties: +Here is a sample `stream_maps` transformation which obfuscates `phone_number` with a fake value, removes all references to `email` and adds `email_domain` and `email_hash` as new properties: `meltano.yml` or `config.json`: @@ -138,9 +127,18 @@ stream_maps: email_domain: owner_email.split('@')[-1] # for uniqueness checks email_hash: md5(config['hash_seed'] + owner_email) + # generate a fake phone number + phone_number: fake.phone_number() stream_map_config: # hash outputs are not able to be replicated without the original seed: hash_seed: 01AWZh7A6DzGm6iJZZ2T +faker_config: + # set specific seed + seed: 0 + # set specific locales + locale: + - en_US + - en_GB ``` ```` @@ -151,11 +149,19 @@ stream_map_config: "customers": { "email": null, "email_domain": "owner_email.split('@')[-1]", - "email_hash": "md5(config['hash_seed'] + owner_email)" + "email_hash": "md5(config['hash_seed'] + owner_email)", + "phone_number": "fake.phone_number()" } }, "stream_map_config": { "hash_seed": "01AWZh7A6DzGm6iJZZ2T" + }, + "faker_config": { + "seed": 0, + "locale": [ + "en_US", + "en_GB" + ] } } ``` @@ -236,6 +242,11 @@ can be referenced directly by mapping expressions. - `record` - an alias for the record values dictionary in the current stream. - `_` - same as `record` but shorter to type - `self` - the existing property value if the property already exists +- `fake` - a [`Faker`](https://faker.readthedocs.io/en/master/) instance, configurable via `faker_config` (see previous example) - see the built-in [standard providers](https://faker.readthedocs.io/en/master/providers.html) for available methods + + ```{tip} + The `fake` object is only available if the plugin specifies `faker` as an addtional dependency (through the `singer-sdk` `faker` extra, or directly). + ``` #### Automatic Schema Detection diff --git a/docs/testing.md b/docs/testing.md index 19ad02ef0..1f8c97e33 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -10,7 +10,7 @@ The Meltano SDK test framework consists of 4 main components: 1. A runner class (`TapTestRunner` and `TargetTestRunner`), responsible for executing Taps/Targets and capturing their output. 1. A suite dataclass, containing a list of tests. 1. A test template classes (`TapTestTemplate`, `StreamTestTemplate`, `AttributeTestTemplate` and `TargetTestTemplate`), with methods to `.setup()`, `.test()`, `.validate()` and `.teardown()` (called in that order using `.run()`). -1. `get_tap_test_class` and `get_target_test_class` factory methods. These wrap a `get_test_class` factory method, which takes a runner and a list of suites and return a `pytest` test class. +1. {func}`get_tap_test_class ` and {func}`get_target_test_class ` factory methods. These wrap a `get_test_class` factory method, which takes a runner and a list of suites and return a `pytest` test class. ## Example Usage @@ -76,7 +76,7 @@ class TestTargetExample(StandardTargetTests): ## Configuring Tests -Test suite behaviors can be configured by passing a `SuiteConfig` instance to the `get_test_class` functions: +Test suite behaviors can be configured by passing a {func}`SuiteConfig ` instance to the `get_test_class` functions: ```python from singer_sdk.testing import SuiteConfig, get_tap_test_class @@ -101,7 +101,7 @@ TestTapStackExchange = get_tap_test_class( ) ``` -Check out [`singer_sdk/testing/config.py`](https://github.com/meltano/sdk/tree/main/singer_sdk/testing/config.py) for available config options. +Check out [the reference](#reference) for more information on the available configuration options. ## Writing New Tests @@ -127,6 +127,21 @@ my_custom_tap_tests = TestSuite( ) ``` -This suite can now be passed to `get_tap_test_class` or `get_target_test_class` in a list of `custom_suites` along with any other suites, to generate your custom test class. +This suite can now be passed to {func}`get_tap_test_class ` or {func}`get_target_test_class ` in a list of `custom_suites` along with any other suites, to generate your custom test class. If your new test covers a common or general case, consider contributing to the standard test library via a pull request to [meltano/sdk](https://github.com/meltano/sdk). + +## Reference + +```{eval-rst} +.. autofunction:: singer_sdk.testing.get_tap_test_class +``` + +```{eval-rst} +.. autofunction:: singer_sdk.testing.get_target_test_class +``` + +```{eval-rst} +.. autoclass:: singer_sdk.testing.SuiteConfig + :members: +``` diff --git a/e2e-tests/cookiecutters/mapper-base.json b/e2e-tests/cookiecutters/mapper-base.json index 01834631d..24d67ad61 100644 --- a/e2e-tests/cookiecutters/mapper-base.json +++ b/e2e-tests/cookiecutters/mapper-base.json @@ -6,6 +6,7 @@ "mapper_id": "mapper-base", "library_name": "mapper_base", "variant": "None (Skip)", + "faker_extra": false, "include_ci_files": "None (Skip)", "license": "Apache-2.0", "ide": "VSCode", diff --git a/e2e-tests/cookiecutters/tap-faker.json b/e2e-tests/cookiecutters/tap-faker.json new file mode 100644 index 000000000..c01d30642 --- /dev/null +++ b/e2e-tests/cookiecutters/tap-faker.json @@ -0,0 +1,18 @@ +{ + "cookiecutter": { + "source_name": "AutomaticTestTap", + "admin_name": "Automatic Tester", + "admin_email": "auto.tester@example.com", + "tap_id": "tap-faker", + "library_name": "tap_faker", + "variant": "None (Skip)", + "stream_type": "REST", + "auth_method": "Bearer Token", + "include_ci_files": "None (Skip)", + "faker_extra": true, + "license": "Apache-2.0", + "ide": "VSCode", + "_template": "../tap-template/", + "_output_dir": "." + } +} diff --git a/e2e-tests/cookiecutters/tap-graphql-jwt.json b/e2e-tests/cookiecutters/tap-graphql-jwt.json index 28b3dfee4..4fe00ef7b 100644 --- a/e2e-tests/cookiecutters/tap-graphql-jwt.json +++ b/e2e-tests/cookiecutters/tap-graphql-jwt.json @@ -9,6 +9,7 @@ "stream_type": "GraphQL", "auth_method": "JWT", "include_ci_files": "None (Skip)", + "faker_extra": false, "license": "Apache-2.0", "ide": "VSCode", "_template": "../tap-template/", diff --git a/e2e-tests/cookiecutters/tap-other-custom.json b/e2e-tests/cookiecutters/tap-other-custom.json index d0aabab09..b78fcd51b 100644 --- a/e2e-tests/cookiecutters/tap-other-custom.json +++ b/e2e-tests/cookiecutters/tap-other-custom.json @@ -9,6 +9,7 @@ "stream_type": "Other", "auth_method": "Custom or N/A", "include_ci_files": "None (Skip)", + "faker_extra": false, "license": "Apache-2.0", "ide": "VSCode", "_template": "../tap-template/", diff --git a/e2e-tests/cookiecutters/tap-rest-api_key-github.json b/e2e-tests/cookiecutters/tap-rest-api_key-github.json index cb162402a..1d474c2fd 100644 --- a/e2e-tests/cookiecutters/tap-rest-api_key-github.json +++ b/e2e-tests/cookiecutters/tap-rest-api_key-github.json @@ -9,6 +9,7 @@ "stream_type": "REST", "auth_method": "API Key", "include_ci_files": "GitHub", + "faker_extra": false, "license": "Apache-2.0", "ide": "VSCode", "_template": "../tap-template/", diff --git a/e2e-tests/cookiecutters/tap-rest-basic_auth.json b/e2e-tests/cookiecutters/tap-rest-basic_auth.json index aa91e3c0d..6c9a1a509 100644 --- a/e2e-tests/cookiecutters/tap-rest-basic_auth.json +++ b/e2e-tests/cookiecutters/tap-rest-basic_auth.json @@ -9,6 +9,7 @@ "stream_type": "REST", "auth_method": "Basic Auth", "include_ci_files": "None (Skip)", + "faker_extra": false, "license": "Apache-2.0", "ide": "VSCode", "_template": "../tap-template/", diff --git a/e2e-tests/cookiecutters/tap-rest-bearer_token.json b/e2e-tests/cookiecutters/tap-rest-bearer_token.json index 274039845..0e032babb 100644 --- a/e2e-tests/cookiecutters/tap-rest-bearer_token.json +++ b/e2e-tests/cookiecutters/tap-rest-bearer_token.json @@ -9,6 +9,7 @@ "stream_type": "REST", "auth_method": "Bearer Token", "include_ci_files": "None (Skip)", + "faker_extra": false, "license": "Apache-2.0", "ide": "VSCode", "_template": "../tap-template/", diff --git a/e2e-tests/cookiecutters/tap-rest-custom.json b/e2e-tests/cookiecutters/tap-rest-custom.json index 67d72dea9..9a75b7b69 100644 --- a/e2e-tests/cookiecutters/tap-rest-custom.json +++ b/e2e-tests/cookiecutters/tap-rest-custom.json @@ -9,6 +9,7 @@ "stream_type": "REST", "auth_method": "Custom or N/A", "include_ci_files": "None (Skip)", + "faker_extra": false, "license": "Apache-2.0", "ide": "VSCode", "_template": "../tap-template/", diff --git a/e2e-tests/cookiecutters/tap-rest-jwt.json b/e2e-tests/cookiecutters/tap-rest-jwt.json index aa3729388..572129c74 100644 --- a/e2e-tests/cookiecutters/tap-rest-jwt.json +++ b/e2e-tests/cookiecutters/tap-rest-jwt.json @@ -9,6 +9,7 @@ "stream_type": "REST", "auth_method": "JWT", "include_ci_files": "None (Skip)", + "faker_extra": false, "license": "Apache-2.0", "ide": "VSCode", "_template": "../tap-template/", diff --git a/e2e-tests/cookiecutters/tap-rest-oauth2.json b/e2e-tests/cookiecutters/tap-rest-oauth2.json index 905349aac..97b2ff41b 100644 --- a/e2e-tests/cookiecutters/tap-rest-oauth2.json +++ b/e2e-tests/cookiecutters/tap-rest-oauth2.json @@ -9,6 +9,7 @@ "stream_type": "REST", "auth_method": "OAuth2", "include_ci_files": "None (Skip)", + "faker_extra": false, "license": "Apache-2.0", "ide": "VSCode", "_template": "../tap-template/", diff --git a/e2e-tests/cookiecutters/tap-sql-custom.json b/e2e-tests/cookiecutters/tap-sql-custom.json index 81ac625d0..264d2d500 100644 --- a/e2e-tests/cookiecutters/tap-sql-custom.json +++ b/e2e-tests/cookiecutters/tap-sql-custom.json @@ -9,6 +9,7 @@ "stream_type": "SQL", "auth_method": "Custom or N/A", "include_ci_files": "None (Skip)", + "faker_extra": false, "license": "Apache-2.0", "ide": "VSCode", "_template": "../tap-template/", diff --git a/e2e-tests/cookiecutters/target-per_record.json b/e2e-tests/cookiecutters/target-per_record.json index 89f923911..9e55e598d 100644 --- a/e2e-tests/cookiecutters/target-per_record.json +++ b/e2e-tests/cookiecutters/target-per_record.json @@ -8,6 +8,7 @@ "variant": "None (Skip)", "serialization_method": "Per record", "include_ci_files": "None (Skip)", + "faker_extra": false, "license": "Apache-2.0", "ide": "VSCode", "_template": "./sdk/cookiecutter/target-template", diff --git a/e2e-tests/cookiecutters/target-sql.json b/e2e-tests/cookiecutters/target-sql.json index 881b3ebe4..edd037dae 100644 --- a/e2e-tests/cookiecutters/target-sql.json +++ b/e2e-tests/cookiecutters/target-sql.json @@ -8,6 +8,7 @@ "variant": "None (Skip)", "serialization_method": "SQL", "include_ci_files": "None (Skip)", + "faker_extra": false, "license": "Apache-2.0", "ide": "VSCode", "_template": "./sdk/cookiecutter/target-template", diff --git a/noxfile.py b/noxfile.py index 1fbb1627b..875e6670d 100644 --- a/noxfile.py +++ b/noxfile.py @@ -29,7 +29,7 @@ package = "singer_sdk" python_versions = ["3.12", "3.11", "3.10", "3.9", "3.8"] -main_python_version = "3.11" +main_python_version = "3.12" locations = "singer_sdk", "tests", "noxfile.py", "docs/conf.py" nox.options.sessions = ( "mypy", @@ -54,11 +54,18 @@ ] +def _clean_py312_deps(session: Session, dependencies: list[str]) -> None: + """Clean dependencies for Python 3.12.""" + if session.python == "3.12": + dependencies.remove("duckdb") + dependencies.remove("duckdb-engine") + + @session(python=main_python_version) def mypy(session: Session) -> None: """Check types with mypy.""" args = session.posargs or ["singer_sdk"] - session.install(".[s3,testing,parquet]") + session.install(".[faker,parquet,s3,testing]") session.install( "exceptiongroup", "mypy", @@ -79,7 +86,8 @@ def mypy(session: Session) -> None: @session(python=python_versions) def tests(session: Session) -> None: """Execute pytest tests and compute coverage.""" - session.install(".[s3,parquet]") + _clean_py312_deps(session, test_dependencies) + session.install(".[faker,parquet,s3]") session.install(*test_dependencies) sqlalchemy_version = os.environ.get("SQLALCHEMY_VERSION") @@ -112,6 +120,7 @@ def tests(session: Session) -> None: @session(python=main_python_version) def benches(session: Session) -> None: """Run benchmarks.""" + _clean_py312_deps(session, test_dependencies) session.install(".[s3]") session.install(*test_dependencies) sqlalchemy_version = os.environ.get("SQLALCHEMY_VERSION") @@ -134,7 +143,8 @@ def update_snapshots(session: Session) -> None: """Update pytest snapshots.""" args = session.posargs or ["-m", "snapshot"] - session.install(".") + _clean_py312_deps(session, test_dependencies) + session.install(".[faker]") session.install(*test_dependencies) session.run("pytest", "--snapshot-update", *args) diff --git a/poetry.lock b/poetry.lock index 87bc93474..1bc595cc3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -160,17 +160,17 @@ lxml = ["lxml"] [[package]] name = "boto3" -version = "1.34.23" +version = "1.34.31" description = "The AWS SDK for Python" optional = true python-versions = ">= 3.8" files = [ - {file = "boto3-1.34.23-py3-none-any.whl", hash = "sha256:364f942d38da283031cde08c46c9282129fd9ebf96fa244f2709886c31ccd49a"}, - {file = "boto3-1.34.23.tar.gz", hash = "sha256:2c96f6a4e9ce2f4d31fc7ab47a2b3a1808063fa3837d7d8548eb2031380f7498"}, + {file = "boto3-1.34.31-py3-none-any.whl", hash = "sha256:0d800130e43a5d4e71300cc6f91aabcef6fe6f26bc206bc61374bf695049587a"}, + {file = "boto3-1.34.31.tar.gz", hash = "sha256:c4dec7ea9bc9210ec783d39b56d332f5a266b0d1e31a96c5092f6bd5252361ba"}, ] [package.dependencies] -botocore = ">=1.34.23,<1.35.0" +botocore = ">=1.34.31,<1.35.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -179,13 +179,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.34.23" +version = "1.34.31" description = "Low-level, data-driven core of boto 3." optional = true python-versions = ">= 3.8" files = [ - {file = "botocore-1.34.23-py3-none-any.whl", hash = "sha256:1980411306593bbc2b0cd9b8d1dcacbd418b758077b82f68b932070ad902cfe9"}, - {file = "botocore-1.34.23.tar.gz", hash = "sha256:898fa169679782f396613f50a88b9b033845625c931275832063266110ea4297"}, + {file = "botocore-1.34.31-py3-none-any.whl", hash = "sha256:6ee1ba451ce3d640dccd485906f68a55d9e7f3534553876e4adc75d6060a05ac"}, + {file = "botocore-1.34.31.tar.gz", hash = "sha256:d5a2153dbe9687f510f179e03913bc9b4e266c865cabebe440c4d05ab923faa7"}, ] [package.dependencies] @@ -412,63 +412,63 @@ files = [ [[package]] name = "coverage" -version = "7.4.0" +version = "7.4.1" description = "Code coverage measurement for Python" optional = false python-versions = ">=3.8" files = [ - {file = "coverage-7.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:36b0ea8ab20d6a7564e89cb6135920bc9188fb5f1f7152e94e8300b7b189441a"}, - {file = "coverage-7.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0676cd0ba581e514b7f726495ea75aba3eb20899d824636c6f59b0ed2f88c471"}, - {file = "coverage-7.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0ca5c71a5a1765a0f8f88022c52b6b8be740e512980362f7fdbb03725a0d6b9"}, - {file = "coverage-7.4.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7c97726520f784239f6c62506bc70e48d01ae71e9da128259d61ca5e9788516"}, - {file = "coverage-7.4.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:815ac2d0f3398a14286dc2cea223a6f338109f9ecf39a71160cd1628786bc6f5"}, - {file = "coverage-7.4.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:80b5ee39b7f0131ebec7968baa9b2309eddb35b8403d1869e08f024efd883566"}, - {file = "coverage-7.4.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5b2ccb7548a0b65974860a78c9ffe1173cfb5877460e5a229238d985565574ae"}, - {file = "coverage-7.4.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:995ea5c48c4ebfd898eacb098164b3cc826ba273b3049e4a889658548e321b43"}, - {file = "coverage-7.4.0-cp310-cp310-win32.whl", hash = "sha256:79287fd95585ed36e83182794a57a46aeae0b64ca53929d1176db56aacc83451"}, - {file = "coverage-7.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:5b14b4f8760006bfdb6e08667af7bc2d8d9bfdb648351915315ea17645347137"}, - {file = "coverage-7.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:04387a4a6ecb330c1878907ce0dc04078ea72a869263e53c72a1ba5bbdf380ca"}, - {file = "coverage-7.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ea81d8f9691bb53f4fb4db603203029643caffc82bf998ab5b59ca05560f4c06"}, - {file = "coverage-7.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74775198b702868ec2d058cb92720a3c5a9177296f75bd97317c787daf711505"}, - {file = "coverage-7.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:76f03940f9973bfaee8cfba70ac991825611b9aac047e5c80d499a44079ec0bc"}, - {file = "coverage-7.4.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:485e9f897cf4856a65a57c7f6ea3dc0d4e6c076c87311d4bc003f82cfe199d25"}, - {file = "coverage-7.4.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6ae8c9d301207e6856865867d762a4b6fd379c714fcc0607a84b92ee63feff70"}, - {file = "coverage-7.4.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:bf477c355274a72435ceb140dc42de0dc1e1e0bf6e97195be30487d8eaaf1a09"}, - {file = "coverage-7.4.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:83c2dda2666fe32332f8e87481eed056c8b4d163fe18ecc690b02802d36a4d26"}, - {file = "coverage-7.4.0-cp311-cp311-win32.whl", hash = "sha256:697d1317e5290a313ef0d369650cfee1a114abb6021fa239ca12b4849ebbd614"}, - {file = "coverage-7.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:26776ff6c711d9d835557ee453082025d871e30b3fd6c27fcef14733f67f0590"}, - {file = "coverage-7.4.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:13eaf476ec3e883fe3e5fe3707caeb88268a06284484a3daf8250259ef1ba143"}, - {file = "coverage-7.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:846f52f46e212affb5bcf131c952fb4075b55aae6b61adc9856222df89cbe3e2"}, - {file = "coverage-7.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26f66da8695719ccf90e794ed567a1549bb2644a706b41e9f6eae6816b398c4a"}, - {file = "coverage-7.4.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:164fdcc3246c69a6526a59b744b62e303039a81e42cfbbdc171c91a8cc2f9446"}, - {file = "coverage-7.4.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:316543f71025a6565677d84bc4df2114e9b6a615aa39fb165d697dba06a54af9"}, - {file = "coverage-7.4.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:bb1de682da0b824411e00a0d4da5a784ec6496b6850fdf8c865c1d68c0e318dd"}, - {file = "coverage-7.4.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:0e8d06778e8fbffccfe96331a3946237f87b1e1d359d7fbe8b06b96c95a5407a"}, - {file = "coverage-7.4.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a56de34db7b7ff77056a37aedded01b2b98b508227d2d0979d373a9b5d353daa"}, - {file = "coverage-7.4.0-cp312-cp312-win32.whl", hash = "sha256:51456e6fa099a8d9d91497202d9563a320513fcf59f33991b0661a4a6f2ad450"}, - {file = "coverage-7.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:cd3c1e4cb2ff0083758f09be0f77402e1bdf704adb7f89108007300a6da587d0"}, - {file = "coverage-7.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e9d1bf53c4c8de58d22e0e956a79a5b37f754ed1ffdbf1a260d9dcfa2d8a325e"}, - {file = "coverage-7.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:109f5985182b6b81fe33323ab4707011875198c41964f014579cf82cebf2bb85"}, - {file = "coverage-7.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3cc9d4bc55de8003663ec94c2f215d12d42ceea128da8f0f4036235a119c88ac"}, - {file = "coverage-7.4.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc6d65b21c219ec2072c1293c505cf36e4e913a3f936d80028993dd73c7906b1"}, - {file = "coverage-7.4.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a10a4920def78bbfff4eff8a05c51be03e42f1c3735be42d851f199144897ba"}, - {file = "coverage-7.4.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b8e99f06160602bc64da35158bb76c73522a4010f0649be44a4e167ff8555952"}, - {file = "coverage-7.4.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:7d360587e64d006402b7116623cebf9d48893329ef035278969fa3bbf75b697e"}, - {file = "coverage-7.4.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:29f3abe810930311c0b5d1a7140f6395369c3db1be68345638c33eec07535105"}, - {file = "coverage-7.4.0-cp38-cp38-win32.whl", hash = "sha256:5040148f4ec43644702e7b16ca864c5314ccb8ee0751ef617d49aa0e2d6bf4f2"}, - {file = "coverage-7.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:9864463c1c2f9cb3b5db2cf1ff475eed2f0b4285c2aaf4d357b69959941aa555"}, - {file = "coverage-7.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:936d38794044b26c99d3dd004d8af0035ac535b92090f7f2bb5aa9c8e2f5cd42"}, - {file = "coverage-7.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:799c8f873794a08cdf216aa5d0531c6a3747793b70c53f70e98259720a6fe2d7"}, - {file = "coverage-7.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e7defbb9737274023e2d7af02cac77043c86ce88a907c58f42b580a97d5bcca9"}, - {file = "coverage-7.4.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a1526d265743fb49363974b7aa8d5899ff64ee07df47dd8d3e37dcc0818f09ed"}, - {file = "coverage-7.4.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf635a52fc1ea401baf88843ae8708591aa4adff875e5c23220de43b1ccf575c"}, - {file = "coverage-7.4.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:756ded44f47f330666843b5781be126ab57bb57c22adbb07d83f6b519783b870"}, - {file = "coverage-7.4.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:0eb3c2f32dabe3a4aaf6441dde94f35687224dfd7eb2a7f47f3fd9428e421058"}, - {file = "coverage-7.4.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bfd5db349d15c08311702611f3dccbef4b4e2ec148fcc636cf8739519b4a5c0f"}, - {file = "coverage-7.4.0-cp39-cp39-win32.whl", hash = "sha256:53d7d9158ee03956e0eadac38dfa1ec8068431ef8058fe6447043db1fb40d932"}, - {file = "coverage-7.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:cfd2a8b6b0d8e66e944d47cdec2f47c48fef2ba2f2dff5a9a75757f64172857e"}, - {file = "coverage-7.4.0-pp38.pp39.pp310-none-any.whl", hash = "sha256:c530833afc4707fe48524a44844493f36d8727f04dcce91fb978c414a8556cc6"}, - {file = "coverage-7.4.0.tar.gz", hash = "sha256:707c0f58cb1712b8809ece32b68996ee1e609f71bd14615bd8f87a1293cb610e"}, + {file = "coverage-7.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:077d366e724f24fc02dbfe9d946534357fda71af9764ff99d73c3c596001bbd7"}, + {file = "coverage-7.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0193657651f5399d433c92f8ae264aff31fc1d066deee4b831549526433f3f61"}, + {file = "coverage-7.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d17bbc946f52ca67adf72a5ee783cd7cd3477f8f8796f59b4974a9b59cacc9ee"}, + {file = "coverage-7.4.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a3277f5fa7483c927fe3a7b017b39351610265308f5267ac6d4c2b64cc1d8d25"}, + {file = "coverage-7.4.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6dceb61d40cbfcf45f51e59933c784a50846dc03211054bd76b421a713dcdf19"}, + {file = "coverage-7.4.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6008adeca04a445ea6ef31b2cbaf1d01d02986047606f7da266629afee982630"}, + {file = "coverage-7.4.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c61f66d93d712f6e03369b6a7769233bfda880b12f417eefdd4f16d1deb2fc4c"}, + {file = "coverage-7.4.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b9bb62fac84d5f2ff523304e59e5c439955fb3b7f44e3d7b2085184db74d733b"}, + {file = "coverage-7.4.1-cp310-cp310-win32.whl", hash = "sha256:f86f368e1c7ce897bf2457b9eb61169a44e2ef797099fb5728482b8d69f3f016"}, + {file = "coverage-7.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:869b5046d41abfea3e381dd143407b0d29b8282a904a19cb908fa24d090cc018"}, + {file = "coverage-7.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b8ffb498a83d7e0305968289441914154fb0ef5d8b3157df02a90c6695978295"}, + {file = "coverage-7.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3cacfaefe6089d477264001f90f55b7881ba615953414999c46cc9713ff93c8c"}, + {file = "coverage-7.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d6850e6e36e332d5511a48a251790ddc545e16e8beaf046c03985c69ccb2676"}, + {file = "coverage-7.4.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18e961aa13b6d47f758cc5879383d27b5b3f3dcd9ce8cdbfdc2571fe86feb4dd"}, + {file = "coverage-7.4.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dfd1e1b9f0898817babf840b77ce9fe655ecbe8b1b327983df485b30df8cc011"}, + {file = "coverage-7.4.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6b00e21f86598b6330f0019b40fb397e705135040dbedc2ca9a93c7441178e74"}, + {file = "coverage-7.4.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:536d609c6963c50055bab766d9951b6c394759190d03311f3e9fcf194ca909e1"}, + {file = "coverage-7.4.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7ac8f8eb153724f84885a1374999b7e45734bf93a87d8df1e7ce2146860edef6"}, + {file = "coverage-7.4.1-cp311-cp311-win32.whl", hash = "sha256:f3771b23bb3675a06f5d885c3630b1d01ea6cac9e84a01aaf5508706dba546c5"}, + {file = "coverage-7.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:9d2f9d4cc2a53b38cabc2d6d80f7f9b7e3da26b2f53d48f05876fef7956b6968"}, + {file = "coverage-7.4.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f68ef3660677e6624c8cace943e4765545f8191313a07288a53d3da188bd8581"}, + {file = "coverage-7.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:23b27b8a698e749b61809fb637eb98ebf0e505710ec46a8aa6f1be7dc0dc43a6"}, + {file = "coverage-7.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e3424c554391dc9ef4a92ad28665756566a28fecf47308f91841f6c49288e66"}, + {file = "coverage-7.4.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e0860a348bf7004c812c8368d1fc7f77fe8e4c095d661a579196a9533778e156"}, + {file = "coverage-7.4.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe558371c1bdf3b8fa03e097c523fb9645b8730399c14fe7721ee9c9e2a545d3"}, + {file = "coverage-7.4.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3468cc8720402af37b6c6e7e2a9cdb9f6c16c728638a2ebc768ba1ef6f26c3a1"}, + {file = "coverage-7.4.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:02f2edb575d62172aa28fe00efe821ae31f25dc3d589055b3fb64d51e52e4ab1"}, + {file = "coverage-7.4.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ca6e61dc52f601d1d224526360cdeab0d0712ec104a2ce6cc5ccef6ed9a233bc"}, + {file = "coverage-7.4.1-cp312-cp312-win32.whl", hash = "sha256:ca7b26a5e456a843b9b6683eada193fc1f65c761b3a473941efe5a291f604c74"}, + {file = "coverage-7.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:85ccc5fa54c2ed64bd91ed3b4a627b9cce04646a659512a051fa82a92c04a448"}, + {file = "coverage-7.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8bdb0285a0202888d19ec6b6d23d5990410decb932b709f2b0dfe216d031d218"}, + {file = "coverage-7.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:918440dea04521f499721c039863ef95433314b1db00ff826a02580c1f503e45"}, + {file = "coverage-7.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:379d4c7abad5afbe9d88cc31ea8ca262296480a86af945b08214eb1a556a3e4d"}, + {file = "coverage-7.4.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b094116f0b6155e36a304ff912f89bbb5067157aff5f94060ff20bbabdc8da06"}, + {file = "coverage-7.4.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2f5968608b1fe2a1d00d01ad1017ee27efd99b3437e08b83ded9b7af3f6f766"}, + {file = "coverage-7.4.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:10e88e7f41e6197ea0429ae18f21ff521d4f4490aa33048f6c6f94c6045a6a75"}, + {file = "coverage-7.4.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a4a3907011d39dbc3e37bdc5df0a8c93853c369039b59efa33a7b6669de04c60"}, + {file = "coverage-7.4.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6d224f0c4c9c98290a6990259073f496fcec1b5cc613eecbd22786d398ded3ad"}, + {file = "coverage-7.4.1-cp38-cp38-win32.whl", hash = "sha256:23f5881362dcb0e1a92b84b3c2809bdc90db892332daab81ad8f642d8ed55042"}, + {file = "coverage-7.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:a07f61fc452c43cd5328b392e52555f7d1952400a1ad09086c4a8addccbd138d"}, + {file = "coverage-7.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8e738a492b6221f8dcf281b67129510835461132b03024830ac0e554311a5c54"}, + {file = "coverage-7.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:46342fed0fff72efcda77040b14728049200cbba1279e0bf1188f1f2078c1d70"}, + {file = "coverage-7.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9641e21670c68c7e57d2053ddf6c443e4f0a6e18e547e86af3fad0795414a628"}, + {file = "coverage-7.4.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aeb2c2688ed93b027eb0d26aa188ada34acb22dceea256d76390eea135083950"}, + {file = "coverage-7.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d12c923757de24e4e2110cf8832d83a886a4cf215c6e61ed506006872b43a6d1"}, + {file = "coverage-7.4.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0491275c3b9971cdbd28a4595c2cb5838f08036bca31765bad5e17edf900b2c7"}, + {file = "coverage-7.4.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:8dfc5e195bbef80aabd81596ef52a1277ee7143fe419efc3c4d8ba2754671756"}, + {file = "coverage-7.4.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1a78b656a4d12b0490ca72651fe4d9f5e07e3c6461063a9b6265ee45eb2bdd35"}, + {file = "coverage-7.4.1-cp39-cp39-win32.whl", hash = "sha256:f90515974b39f4dea2f27c0959688621b46d96d5a626cf9c53dbc653a895c05c"}, + {file = "coverage-7.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:64e723ca82a84053dd7bfcc986bdb34af8d9da83c521c19d6b472bc6880e191a"}, + {file = "coverage-7.4.1-pp38.pp39.pp310-none-any.whl", hash = "sha256:32a8d985462e37cfdab611a6f95b09d7c091d07668fdc26e47a725ee575fe166"}, + {file = "coverage-7.4.1.tar.gz", hash = "sha256:1ed4b95480952b1a26d863e546fa5094564aa0065e1e5f0d4d0041f293251d04"}, ] [package.dependencies] @@ -479,43 +479,43 @@ toml = ["tomli"] [[package]] name = "cryptography" -version = "42.0.1" +version = "42.0.2" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" files = [ - {file = "cryptography-42.0.1-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:265bdc693570b895eb641410b8fc9e8ddbce723a669236162b9d9cfb70bd8d77"}, - {file = "cryptography-42.0.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:160fa08dfa6dca9cb8ad9bd84e080c0db6414ba5ad9a7470bc60fb154f60111e"}, - {file = "cryptography-42.0.1-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:727387886c9c8de927c360a396c5edcb9340d9e960cda145fca75bdafdabd24c"}, - {file = "cryptography-42.0.1-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4d84673c012aa698555d4710dcfe5f8a0ad76ea9dde8ef803128cc669640a2e0"}, - {file = "cryptography-42.0.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e6edc3a568667daf7d349d7e820783426ee4f1c0feab86c29bd1d6fe2755e009"}, - {file = "cryptography-42.0.1-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:d50718dd574a49d3ef3f7ef7ece66ef281b527951eb2267ce570425459f6a404"}, - {file = "cryptography-42.0.1-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:9544492e8024f29919eac2117edd8c950165e74eb551a22c53f6fdf6ba5f4cb8"}, - {file = "cryptography-42.0.1-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:ab6b302d51fbb1dd339abc6f139a480de14d49d50f65fdc7dff782aa8631d035"}, - {file = "cryptography-42.0.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2fe16624637d6e3e765530bc55caa786ff2cbca67371d306e5d0a72e7c3d0407"}, - {file = "cryptography-42.0.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ed1b2130f5456a09a134cc505a17fc2830a1a48ed53efd37dcc904a23d7b82fa"}, - {file = "cryptography-42.0.1-cp37-abi3-win32.whl", hash = "sha256:e5edf189431b4d51f5c6fb4a95084a75cef6b4646c934eb6e32304fc720e1453"}, - {file = "cryptography-42.0.1-cp37-abi3-win_amd64.whl", hash = "sha256:6bfd823b336fdcd8e06285ae8883d3d2624d3bdef312a0e2ef905f332f8e9302"}, - {file = "cryptography-42.0.1-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:351db02c1938c8e6b1fee8a78d6b15c5ccceca7a36b5ce48390479143da3b411"}, - {file = "cryptography-42.0.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:430100abed6d3652208ae1dd410c8396213baee2e01a003a4449357db7dc9e14"}, - {file = "cryptography-42.0.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2dff7a32880a51321f5de7869ac9dde6b1fca00fc1fef89d60e93f215468e824"}, - {file = "cryptography-42.0.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:b512f33c6ab195852595187af5440d01bb5f8dd57cb7a91e1e009a17f1b7ebca"}, - {file = "cryptography-42.0.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:95d900d19a370ae36087cc728e6e7be9c964ffd8cbcb517fd1efb9c9284a6abc"}, - {file = "cryptography-42.0.1-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:6ac8924085ed8287545cba89dc472fc224c10cc634cdf2c3e2866fe868108e77"}, - {file = "cryptography-42.0.1-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:cb2861a9364fa27d24832c718150fdbf9ce6781d7dc246a516435f57cfa31fe7"}, - {file = "cryptography-42.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:25ec6e9e81de5d39f111a4114193dbd39167cc4bbd31c30471cebedc2a92c323"}, - {file = "cryptography-42.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9d61fcdf37647765086030d81872488e4cb3fafe1d2dda1d487875c3709c0a49"}, - {file = "cryptography-42.0.1-cp39-abi3-win32.whl", hash = "sha256:16b9260d04a0bfc8952b00335ff54f471309d3eb9d7e8dbfe9b0bd9e26e67881"}, - {file = "cryptography-42.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:7911586fc69d06cd0ab3f874a169433db1bc2f0e40988661408ac06c4527a986"}, - {file = "cryptography-42.0.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:d3594947d2507d4ef7a180a7f49a6db41f75fb874c2fd0e94f36b89bfd678bf2"}, - {file = "cryptography-42.0.1-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:8d7efb6bf427d2add2f40b6e1e8e476c17508fa8907234775214b153e69c2e11"}, - {file = "cryptography-42.0.1-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:126e0ba3cc754b200a2fb88f67d66de0d9b9e94070c5bc548318c8dab6383cb6"}, - {file = "cryptography-42.0.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:802d6f83233cf9696b59b09eb067e6b4d5ae40942feeb8e13b213c8fad47f1aa"}, - {file = "cryptography-42.0.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0b7cacc142260ada944de070ce810c3e2a438963ee3deb45aa26fd2cee94c9a4"}, - {file = "cryptography-42.0.1-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:32ea63ceeae870f1a62e87f9727359174089f7b4b01e4999750827bf10e15d60"}, - {file = "cryptography-42.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d3902c779a92151f134f68e555dd0b17c658e13429f270d8a847399b99235a3f"}, - {file = "cryptography-42.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:50aecd93676bcca78379604ed664c45da82bc1241ffb6f97f6b7392ed5bc6f04"}, - {file = "cryptography-42.0.1.tar.gz", hash = "sha256:fd33f53809bb363cf126bebe7a99d97735988d9b0131a2be59fbf83e1259a5b7"}, + {file = "cryptography-42.0.2-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:701171f825dcab90969596ce2af253143b93b08f1a716d4b2a9d2db5084ef7be"}, + {file = "cryptography-42.0.2-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:61321672b3ac7aade25c40449ccedbc6db72c7f5f0fdf34def5e2f8b51ca530d"}, + {file = "cryptography-42.0.2-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea2c3ffb662fec8bbbfce5602e2c159ff097a4631d96235fcf0fb00e59e3ece4"}, + {file = "cryptography-42.0.2-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b15c678f27d66d247132cbf13df2f75255627bcc9b6a570f7d2fd08e8c081d2"}, + {file = "cryptography-42.0.2-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:8e88bb9eafbf6a4014d55fb222e7360eef53e613215085e65a13290577394529"}, + {file = "cryptography-42.0.2-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:a047682d324ba56e61b7ea7c7299d51e61fd3bca7dad2ccc39b72bd0118d60a1"}, + {file = "cryptography-42.0.2-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:36d4b7c4be6411f58f60d9ce555a73df8406d484ba12a63549c88bd64f7967f1"}, + {file = "cryptography-42.0.2-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:a00aee5d1b6c20620161984f8ab2ab69134466c51f58c052c11b076715e72929"}, + {file = "cryptography-42.0.2-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b97fe7d7991c25e6a31e5d5e795986b18fbbb3107b873d5f3ae6dc9a103278e9"}, + {file = "cryptography-42.0.2-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5fa82a26f92871eca593b53359c12ad7949772462f887c35edaf36f87953c0e2"}, + {file = "cryptography-42.0.2-cp37-abi3-win32.whl", hash = "sha256:4b063d3413f853e056161eb0c7724822a9740ad3caa24b8424d776cebf98e7ee"}, + {file = "cryptography-42.0.2-cp37-abi3-win_amd64.whl", hash = "sha256:841ec8af7a8491ac76ec5a9522226e287187a3107e12b7d686ad354bb78facee"}, + {file = "cryptography-42.0.2-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:55d1580e2d7e17f45d19d3b12098e352f3a37fe86d380bf45846ef257054b242"}, + {file = "cryptography-42.0.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28cb2c41f131a5758d6ba6a0504150d644054fd9f3203a1e8e8d7ac3aea7f73a"}, + {file = "cryptography-42.0.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b9097a208875fc7bbeb1286d0125d90bdfed961f61f214d3f5be62cd4ed8a446"}, + {file = "cryptography-42.0.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:44c95c0e96b3cb628e8452ec060413a49002a247b2b9938989e23a2c8291fc90"}, + {file = "cryptography-42.0.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2f9f14185962e6a04ab32d1abe34eae8a9001569ee4edb64d2304bf0d65c53f3"}, + {file = "cryptography-42.0.2-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:09a77e5b2e8ca732a19a90c5bca2d124621a1edb5438c5daa2d2738bfeb02589"}, + {file = "cryptography-42.0.2-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:ad28cff53f60d99a928dfcf1e861e0b2ceb2bc1f08a074fdd601b314e1cc9e0a"}, + {file = "cryptography-42.0.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:130c0f77022b2b9c99d8cebcdd834d81705f61c68e91ddd614ce74c657f8b3ea"}, + {file = "cryptography-42.0.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:fa3dec4ba8fb6e662770b74f62f1a0c7d4e37e25b58b2bf2c1be4c95372b4a33"}, + {file = "cryptography-42.0.2-cp39-abi3-win32.whl", hash = "sha256:3dbd37e14ce795b4af61b89b037d4bc157f2cb23e676fa16932185a04dfbf635"}, + {file = "cryptography-42.0.2-cp39-abi3-win_amd64.whl", hash = "sha256:8a06641fb07d4e8f6c7dda4fc3f8871d327803ab6542e33831c7ccfdcb4d0ad6"}, + {file = "cryptography-42.0.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:087887e55e0b9c8724cf05361357875adb5c20dec27e5816b653492980d20380"}, + {file = "cryptography-42.0.2-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:a7ef8dd0bf2e1d0a27042b231a3baac6883cdd5557036f5e8df7139255feaac6"}, + {file = "cryptography-42.0.2-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4383b47f45b14459cab66048d384614019965ba6c1a1a141f11b5a551cace1b2"}, + {file = "cryptography-42.0.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:fbeb725c9dc799a574518109336acccaf1303c30d45c075c665c0793c2f79a7f"}, + {file = "cryptography-42.0.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:320948ab49883557a256eab46149df79435a22d2fefd6a66fe6946f1b9d9d008"}, + {file = "cryptography-42.0.2-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:5ef9bc3d046ce83c4bbf4c25e1e0547b9c441c01d30922d812e887dc5f125c12"}, + {file = "cryptography-42.0.2-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:52ed9ebf8ac602385126c9a2fe951db36f2cb0c2538d22971487f89d0de4065a"}, + {file = "cryptography-42.0.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:141e2aa5ba100d3788c0ad7919b288f89d1fe015878b9659b307c9ef867d3a65"}, + {file = "cryptography-42.0.2.tar.gz", hash = "sha256:e0ec52ba3c7f1b7d813cd52649a5b3ef1fc0d433219dc8c93827c57eab6cf888"}, ] [package.dependencies] @@ -544,52 +544,61 @@ files = [ [[package]] name = "duckdb" -version = "0.9.3.dev2938" -description = "DuckDB in-process database" +version = "0.9.2" +description = "DuckDB embedded database" optional = false python-versions = ">=3.7.0" files = [ - {file = "duckdb-0.9.3.dev2938-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:51ad6a6fb2e0a580951f92c8d4b7ef3929779d6f4cb08916e824a340bb2e7eda"}, - {file = "duckdb-0.9.3.dev2938-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0385d40d9f2c079c3b1da4e051ffaaa060f13004bde966cf423f1c8f16df5a11"}, - {file = "duckdb-0.9.3.dev2938-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e141d1ab451d9fc1e1ebee2e3b68e1955f83667be2a75b4474e15e1c4f24e2c0"}, - {file = "duckdb-0.9.3.dev2938-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:73c7420e3f8591a4e0e2074e9256d27b9ed4cd4c734fb4dd8341a975c990622b"}, - {file = "duckdb-0.9.3.dev2938-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:abcf08e30d4c7ceace67fee5346ffd12184d98b8d67277cde9ee6887b7afd4ab"}, - {file = "duckdb-0.9.3.dev2938-cp310-cp310-win_amd64.whl", hash = "sha256:328534e1978d2df6eaaf31f94b1ff61573e0343b313165fe174ff30d67456dc8"}, - {file = "duckdb-0.9.3.dev2938-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:1ec6c1ba75d9f06db4cef4d91e47df485368a8b345ace8bdfb35ad66d1bafa54"}, - {file = "duckdb-0.9.3.dev2938-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b2737fb6ddddc90684e42af11615849f56ad789f54827edc298d9920da80303d"}, - {file = "duckdb-0.9.3.dev2938-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b6e712c73c6e5547310ca92ffb1849df68f323c2a74dca86f690986a49e505f9"}, - {file = "duckdb-0.9.3.dev2938-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:09e319d1b893fd61bad179bce86246171d9174e93f577f3f97a3fc634395ad62"}, - {file = "duckdb-0.9.3.dev2938-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2152156ffb3d7113b590b28fda327700e48431b07e0f75ecfeaa3b5c1ee465de"}, - {file = "duckdb-0.9.3.dev2938-cp311-cp311-win_amd64.whl", hash = "sha256:7cb671a43753c74ce83dffc3da6efcad678f64f153bd84b1f0da4fc6ef054d3c"}, - {file = "duckdb-0.9.3.dev2938-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:7a30d6b4ea17452e031d77ad9d8f149ee771052beba60b7a9a0f40edf8b5e2a8"}, - {file = "duckdb-0.9.3.dev2938-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:7094ca12f0e80c31e48046a07715c1dfaa854cf9c02a89fe9cc3ae8d398744a5"}, - {file = "duckdb-0.9.3.dev2938-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:199f222453c9da74b445e9faa3d23447fd3d1617f7f0e86c608581b9cd298f1d"}, - {file = "duckdb-0.9.3.dev2938-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64099c59be728bc2815c96fd7f16a6f1208f04898da1a464f472c35601b101a3"}, - {file = "duckdb-0.9.3.dev2938-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1b0b12e490517707ef9493e052dd4d4864c5d8571330e3416e7bcaf6603cc1a9"}, - {file = "duckdb-0.9.3.dev2938-cp312-cp312-win_amd64.whl", hash = "sha256:79ddd65e09173539b3c8bce4036f0ffb556a13f67ea71b46b051f41097bbc815"}, - {file = "duckdb-0.9.3.dev2938-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c590f5ab3a4554df9bafb1c5600500052957f430b0b2b185b2e61c1bcdb3107f"}, - {file = "duckdb-0.9.3.dev2938-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cdff96665f475a3e34798d3a4bab4be6961faf2b009cb070febe8c0e4d3db712"}, - {file = "duckdb-0.9.3.dev2938-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:138e04fb4038d2b87856edca962f62cdbae0d84a29cde3340b3c56083f9bffbd"}, - {file = "duckdb-0.9.3.dev2938-cp37-cp37m-win_amd64.whl", hash = "sha256:40d4705278b2d7d86909d26bacb2f9aa6a58f461355f17faf64cccfd219d9030"}, - {file = "duckdb-0.9.3.dev2938-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:989b7e3f53ebab8359249e3799a6348231d627da87028b54209243ca9a7ecb42"}, - {file = "duckdb-0.9.3.dev2938-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:65239eb07ce839866c2bfa0e65dcfe9fedcfc8903a6ac497367254be93454084"}, - {file = "duckdb-0.9.3.dev2938-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:8b74a836892d97992705f85f906e342b8215e0f1c61f51719a6737dc4efbb086"}, - {file = "duckdb-0.9.3.dev2938-cp38-cp38-win_amd64.whl", hash = "sha256:714d5d40a81c96cd1f71c165d666d3456d8f0d5525a00523a916a97e69051c32"}, - {file = "duckdb-0.9.3.dev2938-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7fda60cc28cd04fdab651911d5c913d8447311f03c74e8d7f2123fdc919bf852"}, - {file = "duckdb-0.9.3.dev2938-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d4051bdd32542c7270b28eb5ed85e3486713d0b43770e12286cb1fc007000a40"}, - {file = "duckdb-0.9.3.dev2938-cp39-cp39-win_amd64.whl", hash = "sha256:b0df736d08165fc3e9a22741166fbcca24d936f6946ae3c58b17202de7b47324"}, - {file = "duckdb-0.9.3.dev2938.tar.gz", hash = "sha256:7a2adfc9f125ab507d22854913a5f86a274bd7d985bd8b6923fd866c6ecffe82"}, + {file = "duckdb-0.9.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:aadcea5160c586704c03a8a796c06a8afffbefefb1986601104a60cb0bfdb5ab"}, + {file = "duckdb-0.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:08215f17147ed83cbec972175d9882387366de2ed36c21cbe4add04b39a5bcb4"}, + {file = "duckdb-0.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ee6c2a8aba6850abef5e1be9dbc04b8e72a5b2c2b67f77892317a21fae868fe7"}, + {file = "duckdb-0.9.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1ff49f3da9399900fd58b5acd0bb8bfad22c5147584ad2427a78d937e11ec9d0"}, + {file = "duckdb-0.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd5ac5baf8597efd2bfa75f984654afcabcd698342d59b0e265a0bc6f267b3f0"}, + {file = "duckdb-0.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:81c6df905589a1023a27e9712edb5b724566587ef280a0c66a7ec07c8083623b"}, + {file = "duckdb-0.9.2-cp310-cp310-win32.whl", hash = "sha256:a298cd1d821c81d0dec8a60878c4b38c1adea04a9675fb6306c8f9083bbf314d"}, + {file = "duckdb-0.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:492a69cd60b6cb4f671b51893884cdc5efc4c3b2eb76057a007d2a2295427173"}, + {file = "duckdb-0.9.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:061a9ea809811d6e3025c5de31bc40e0302cfb08c08feefa574a6491e882e7e8"}, + {file = "duckdb-0.9.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a43f93be768af39f604b7b9b48891f9177c9282a408051209101ff80f7450d8f"}, + {file = "duckdb-0.9.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ac29c8c8f56fff5a681f7bf61711ccb9325c5329e64f23cb7ff31781d7b50773"}, + {file = "duckdb-0.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b14d98d26bab139114f62ade81350a5342f60a168d94b27ed2c706838f949eda"}, + {file = "duckdb-0.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:796a995299878913e765b28cc2b14c8e44fae2f54ab41a9ee668c18449f5f833"}, + {file = "duckdb-0.9.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6cb64ccfb72c11ec9c41b3cb6181b6fd33deccceda530e94e1c362af5f810ba1"}, + {file = "duckdb-0.9.2-cp311-cp311-win32.whl", hash = "sha256:930740cb7b2cd9e79946e1d3a8f66e15dc5849d4eaeff75c8788d0983b9256a5"}, + {file = "duckdb-0.9.2-cp311-cp311-win_amd64.whl", hash = "sha256:c28f13c45006fd525001b2011cdf91fa216530e9751779651e66edc0e446be50"}, + {file = "duckdb-0.9.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:fbce7bbcb4ba7d99fcec84cec08db40bc0dd9342c6c11930ce708817741faeeb"}, + {file = "duckdb-0.9.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:15a82109a9e69b1891f0999749f9e3265f550032470f51432f944a37cfdc908b"}, + {file = "duckdb-0.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9490fb9a35eb74af40db5569d90df8a04a6f09ed9a8c9caa024998c40e2506aa"}, + {file = "duckdb-0.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:696d5c6dee86c1a491ea15b74aafe34ad2b62dcd46ad7e03b1d00111ca1a8c68"}, + {file = "duckdb-0.9.2-cp37-cp37m-win32.whl", hash = "sha256:4f0935300bdf8b7631ddfc838f36a858c1323696d8c8a2cecbd416bddf6b0631"}, + {file = "duckdb-0.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:0aab900f7510e4d2613263865570203ddfa2631858c7eb8cbed091af6ceb597f"}, + {file = "duckdb-0.9.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:7d8130ed6a0c9421b135d0743705ea95b9a745852977717504e45722c112bf7a"}, + {file = "duckdb-0.9.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:974e5de0294f88a1a837378f1f83330395801e9246f4e88ed3bfc8ada65dcbee"}, + {file = "duckdb-0.9.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4fbc297b602ef17e579bb3190c94d19c5002422b55814421a0fc11299c0c1100"}, + {file = "duckdb-0.9.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1dd58a0d84a424924a35b3772419f8cd78a01c626be3147e4934d7a035a8ad68"}, + {file = "duckdb-0.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11a1194a582c80dfb57565daa06141727e415ff5d17e022dc5f31888a5423d33"}, + {file = "duckdb-0.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:be45d08541002a9338e568dca67ab4f20c0277f8f58a73dfc1435c5b4297c996"}, + {file = "duckdb-0.9.2-cp38-cp38-win32.whl", hash = "sha256:dd6f88aeb7fc0bfecaca633629ff5c986ac966fe3b7dcec0b2c48632fd550ba2"}, + {file = "duckdb-0.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:28100c4a6a04e69aa0f4a6670a6d3d67a65f0337246a0c1a429f3f28f3c40b9a"}, + {file = "duckdb-0.9.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7ae5bf0b6ad4278e46e933e51473b86b4b932dbc54ff097610e5b482dd125552"}, + {file = "duckdb-0.9.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e5d0bb845a80aa48ed1fd1d2d285dd352e96dc97f8efced2a7429437ccd1fe1f"}, + {file = "duckdb-0.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4ce262d74a52500d10888110dfd6715989926ec936918c232dcbaddb78fc55b4"}, + {file = "duckdb-0.9.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6935240da090a7f7d2666f6d0a5e45ff85715244171ca4e6576060a7f4a1200e"}, + {file = "duckdb-0.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5cfb93e73911696a98b9479299d19cfbc21dd05bb7ab11a923a903f86b4d06e"}, + {file = "duckdb-0.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:64e3bc01751f31e7572d2716c3e8da8fe785f1cdc5be329100818d223002213f"}, + {file = "duckdb-0.9.2-cp39-cp39-win32.whl", hash = "sha256:6e5b80f46487636368e31b61461940e3999986359a78660a50dfdd17dd72017c"}, + {file = "duckdb-0.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:e6142a220180dbeea4f341708bd5f9501c5c962ce7ef47c1cadf5e8810b4cb13"}, + {file = "duckdb-0.9.2.tar.gz", hash = "sha256:3843afeab7c3fc4a4c0b53686a4cc1d9cdbdadcbb468d60fef910355ecafd447"}, ] [[package]] name = "duckdb-engine" -version = "0.10.0" +version = "0.11.1" description = "SQLAlchemy driver for duckdb" optional = false python-versions = ">=3.7" files = [ - {file = "duckdb_engine-0.10.0-py3-none-any.whl", hash = "sha256:c408d002e83630b6bbb05fc3b26a43406085b1c22dd43e8cab00bf0b9c011ea8"}, - {file = "duckdb_engine-0.10.0.tar.gz", hash = "sha256:5e3dad3b3513f055a4f5ec5430842249cfe03015743a7597ed1dcc0447dca565"}, + {file = "duckdb_engine-0.11.1-py3-none-any.whl", hash = "sha256:ab69a00472fb34b0c57329461f1765e6fc528a0b42edbd2d566dad9b760a0d61"}, + {file = "duckdb_engine-0.11.1.tar.gz", hash = "sha256:ea7004c689e1f17b2e2d1734c522aa8a7d56366bfbeeddbe8a02dc6a6c36dc15"}, ] [package.dependencies] @@ -610,6 +619,21 @@ files = [ [package.extras] test = ["pytest (>=6)"] +[[package]] +name = "faker" +version = "23.1.0" +description = "Faker is a Python package that generates fake data for you." +optional = true +python-versions = ">=3.8" +files = [ + {file = "Faker-23.1.0-py3-none-any.whl", hash = "sha256:60e89e5c0b584e285a7db05eceba35011a241954afdab2853cb246c8a56700a2"}, + {file = "Faker-23.1.0.tar.gz", hash = "sha256:b7f76bb1b2ac4cdc54442d955e36e477c387000f31ce46887fb9722a041be60b"}, +] + +[package.dependencies] +python-dateutil = ">=2.4" +typing-extensions = {version = ">=3.10.0.1", markers = "python_version <= \"3.8\""} + [[package]] name = "fastjsonschema" version = "2.19.0" @@ -677,13 +701,13 @@ six = ">=1.10,<2.0" [[package]] name = "furo" -version = "2023.9.10" +version = "2024.1.29" description = "A clean customisable Sphinx documentation theme." optional = true python-versions = ">=3.8" files = [ - {file = "furo-2023.9.10-py3-none-any.whl", hash = "sha256:513092538537dc5c596691da06e3c370714ec99bc438680edc1debffb73e5bfc"}, - {file = "furo-2023.9.10.tar.gz", hash = "sha256:5707530a476d2a63b8cad83b4f961f3739a69f4b058bcf38a03a39fa537195b2"}, + {file = "furo-2024.1.29-py3-none-any.whl", hash = "sha256:3548be2cef45a32f8cdc0272d415fcb3e5fa6a0eb4ddfe21df3ecf1fe45a13cf"}, + {file = "furo-2024.1.29.tar.gz", hash = "sha256:4d6b2fe3f10a6e36eb9cc24c1e7beb38d7a23fc7b3c382867503b7fcac8a1e02"}, ] [package.dependencies] @@ -1364,13 +1388,13 @@ files = [ [[package]] name = "pluggy" -version = "1.3.0" +version = "1.4.0" description = "plugin and hook calling mechanisms for python" optional = false python-versions = ">=3.8" files = [ - {file = "pluggy-1.3.0-py3-none-any.whl", hash = "sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7"}, - {file = "pluggy-1.3.0.tar.gz", hash = "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12"}, + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, ] [package.extras] @@ -1492,13 +1516,13 @@ tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] [[package]] name = "pytest" -version = "7.4.4" +version = "8.0.0" description = "pytest: simple powerful testing with Python" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8"}, - {file = "pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280"}, + {file = "pytest-8.0.0-py3-none-any.whl", hash = "sha256:50fb9cbe836c3f20f0dfa99c565201fb75dc54c8d76373cd1bde06b06657bdb6"}, + {file = "pytest-8.0.0.tar.gz", hash = "sha256:249b1b0864530ba251b7438274c4d251c58d868edaaec8762893ad4a0d71c36c"}, ] [package.dependencies] @@ -1506,7 +1530,7 @@ colorama = {version = "*", markers = "sys_platform == \"win32\""} exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} iniconfig = "*" packaging = "*" -pluggy = ">=0.12,<2.0" +pluggy = ">=1.3.0,<2.0" tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] @@ -1612,13 +1636,13 @@ cli = ["click (>=5.0)"] [[package]] name = "pytz" -version = "2023.4" +version = "2024.1" description = "World timezone definitions, modern and historical" optional = false python-versions = "*" files = [ - {file = "pytz-2023.4-py2.py3-none-any.whl", hash = "sha256:f90ef520d95e7c46951105338d918664ebfd6f1d995bd7d153127ce90efafa6a"}, - {file = "pytz-2023.4.tar.gz", hash = "sha256:31d4583c4ed539cd037956140d695e42c033a19e984bfce9964a3f7d59bc2b40"}, + {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, + {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, ] [[package]] @@ -1633,6 +1657,7 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1640,8 +1665,16 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -1658,6 +1691,7 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -1665,6 +1699,7 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -2472,13 +2507,13 @@ files = [ [[package]] name = "types-pytz" -version = "2023.3.1.1" +version = "2024.1.0.20240203" description = "Typing stubs for pytz" optional = false -python-versions = "*" +python-versions = ">=3.8" files = [ - {file = "types-pytz-2023.3.1.1.tar.gz", hash = "sha256:cc23d0192cd49c8f6bba44ee0c81e4586a8f30204970fc0894d209a6b08dab9a"}, - {file = "types_pytz-2023.3.1.1-py3-none-any.whl", hash = "sha256:1999a123a3dc0e39a2ef6d19f3f8584211de9e6a77fe7a0259f04a524e90a5cf"}, + {file = "types-pytz-2024.1.0.20240203.tar.gz", hash = "sha256:c93751ee20dfc6e054a0148f8f5227b9a00b79c90a4d3c9f464711a73179c89e"}, + {file = "types_pytz-2024.1.0.20240203-py3-none-any.whl", hash = "sha256:9679eef0365db3af91ef7722c199dbb75ee5c1b67e3c4dd7bfbeb1b8a71c21a3"}, ] [[package]] @@ -2568,13 +2603,13 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "xdoctest" -version = "1.1.2" +version = "1.1.3" description = "A rewrite of the builtin doctest module" optional = false python-versions = ">=3.6" files = [ - {file = "xdoctest-1.1.2-py3-none-any.whl", hash = "sha256:ebe133222534f09597cbe461f97cc5f95ad7b36e5d31f3437caffb9baaddbddb"}, - {file = "xdoctest-1.1.2.tar.gz", hash = "sha256:267d3d4e362547fa917d3deabaf6888232bbf43c8d30298faeb957dbfa7e0ba3"}, + {file = "xdoctest-1.1.3-py3-none-any.whl", hash = "sha256:9360535bd1a971ffc216d9613898cedceb81d0fd024587cc3c03c74d14c00a31"}, + {file = "xdoctest-1.1.3.tar.gz", hash = "sha256:84e76a42a11a5926ff66d9d84c616bc101821099672550481ad96549cbdd02ae"}, ] [package.extras] @@ -2605,7 +2640,8 @@ docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.link testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy (>=0.9.1)", "pytest-ruff"] [extras] -docs = ["furo", "myst-parser", "sphinx", "sphinx-autobuild", "sphinx-copybutton", "sphinx-inline-tabs", "sphinx-notfound-page", "sphinx-reredirects"] +docs = ["furo", "myst-parser", "pytest", "sphinx", "sphinx-autobuild", "sphinx-copybutton", "sphinx-inline-tabs", "sphinx-notfound-page", "sphinx-reredirects"] +faker = ["faker"] parquet = ["numpy", "numpy", "pyarrow"] s3 = ["fs-s3fs"] testing = ["pytest", "pytest-durations"] @@ -2613,4 +2649,4 @@ testing = ["pytest", "pytest-durations"] [metadata] lock-version = "2.0" python-versions = ">=3.8" -content-hash = "80c16c4028b74311deea375a9a441a5f63dff73fe7b6843005a66045b92830bc" +content-hash = "9833a1255c2965ea7f76b1d5fae01182ee8a0b52520078ca12861a4e19fc56ca" diff --git a/pyproject.toml b/pyproject.toml index 1899bdd0c..89c61d512 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -91,12 +91,16 @@ pyarrow = { version = ">=13", optional = true } pytest = {version=">=7.2.1", optional = true} pytest-durations = {version = ">=1.2.0", optional = true} +# installed as optional 'faker' extra +faker = {version = ">=22.5,<24.0", optional = true} + [tool.poetry.extras] docs = [ - "sphinx", "furo", - "sphinx-copybutton", "myst-parser", + "pytest", + "sphinx", + "sphinx-copybutton", "sphinx-autobuild", "sphinx-inline-tabs", "sphinx-notfound-page", @@ -108,12 +112,14 @@ testing = [ "pytest-durations" ] parquet = ["numpy", "pyarrow"] +faker = ["faker"] [tool.poetry.group.dev.dependencies] coverage = {extras = ["toml"], version = ">=7.4"} -duckdb = { version = ">=0.9.3.dev2490", allow-prereleases = true } -duckdb-engine = { version = ">=0.9.4" } +# TODO: Remove the Python 3.12 marker when DuckDB supports it +duckdb = { version = ">=0.8.0", python = "<3.12" } +duckdb-engine = { version = ">=0.9.4", python = "<3.12" } mypy = ">=1.0" pytest-benchmark = ">=4.0.0" @@ -135,6 +141,26 @@ pytest-codspeed = ">=2.2.0" [tool.pytest.ini_options] addopts = '--ignore=singer_sdk/helpers/_simpleeval.py -m "not external"' +filterwarnings = [ + "error", + "ignore:Could not configure external gitlab tests:UserWarning", + "ignore:No records were available to test:UserWarning", + # https://github.com/meltano/sdk/issues/1354 + "ignore:The function singer_sdk.testing.get_standard_tap_tests is deprecated:DeprecationWarning", + # https://docs.python.org/3.12/whatsnew/3.12.html#deprecated + # https://github.com/danthedeckie/simpleeval/blob/master/simpleeval.py + "ignore:ast.NameConstant is deprecated:DeprecationWarning:simpleeval", + # https://docs.python.org/3.12/whatsnew/3.12.html#deprecated + # https://github.com/danthedeckie/simpleeval/blob/master/simpleeval.py + "ignore:ast.Num is deprecated:DeprecationWarning:simpleeval", + # https://github.com/joblib/joblib/pull/1518 + "ignore:ast.Num is deprecated:DeprecationWarning:joblib._utils", + # https://docs.python.org/3.12/whatsnew/3.12.html#deprecated + # https://github.com/danthedeckie/simpleeval/blob/master/simpleeval.py + "ignore:ast.Str is deprecated:DeprecationWarning:simpleeval", + # https://github.com/joblib/joblib/pull/1518 + "ignore:Attribute n is deprecated:DeprecationWarning:joblib._utils", +] markers = [ "external: Tests relying on external resources", "windows: Tests that only run on Windows", @@ -145,7 +171,7 @@ norecursedirs = "cookiecutter" [tool.commitizen] name = "cz_version_bump" -version = "0.34.1" +version = "0.35.0" changelog_merge_prerelease = true prerelease_offset = 1 tag_format = "v$major.$minor.$patch$prerelease" diff --git a/samples/sample_mapper/mapper.py b/samples/sample_mapper/mapper.py index e079bac08..1fc3c0f94 100644 --- a/samples/sample_mapper/mapper.py +++ b/samples/sample_mapper/mapper.py @@ -7,7 +7,7 @@ import singer_sdk._singerlib as singer import singer_sdk.typing as th from singer_sdk.helpers._util import utc_now -from singer_sdk.mapper import PluginMapper +from singer_sdk.mapper import PluginMapper, RemoveRecordTransform from singer_sdk.mapper_base import InlineMapper if t.TYPE_CHECKING: @@ -90,13 +90,15 @@ def map_schema_message( message_dict.get("key_properties", []), ) for stream_map in self.mapper.stream_maps[stream_id]: - schema_message = singer.SchemaMessage( + if isinstance(stream_map, RemoveRecordTransform): + # Don't emit schema if the stream's records are all ignored. + continue + yield singer.SchemaMessage( stream_map.stream_alias, stream_map.transformed_schema, stream_map.transformed_key_properties, message_dict.get("bookmark_keys", []), ) - yield schema_message def map_record_message( self, diff --git a/samples/sample_tap_gitlab/gitlab_rest_streams.py b/samples/sample_tap_gitlab/gitlab_rest_streams.py index 0d5968132..53e0c8fa1 100644 --- a/samples/sample_tap_gitlab/gitlab_rest_streams.py +++ b/samples/sample_tap_gitlab/gitlab_rest_streams.py @@ -13,6 +13,7 @@ DateTimeType, DateType, IntegerType, + ObjectType, PropertiesList, Property, StringType, @@ -158,7 +159,6 @@ class EpicsStream(ProjectBasedStream): Property("title", StringType), Property("description", StringType), Property("state", StringType), - Property("author_id", IntegerType), Property("start_date", DateType), Property("end_date", DateType), Property("due_date", DateType), @@ -167,6 +167,17 @@ class EpicsStream(ProjectBasedStream): Property("labels", ArrayType(StringType)), Property("upvotes", IntegerType), Property("downvotes", IntegerType), + Property( + "author", + ObjectType( + Property("id", IntegerType), + Property("name", StringType), + Property("username", StringType), + Property("state", StringType), + Property("avatar_url", StringType), + Property("web_url", StringType), + ), + ), ).to_dict() def get_child_context( diff --git a/samples/sample_tap_gitlab/schemas/issues.json b/samples/sample_tap_gitlab/schemas/issues.json index 98119c7f8..3bdd3cf7b 100644 --- a/samples/sample_tap_gitlab/schemas/issues.json +++ b/samples/sample_tap_gitlab/schemas/issues.json @@ -1,223 +1,365 @@ { - "type": "object", - "properties": { + "type": "object", + "properties": { + "project_id": { + "type": "integer" + }, + "id": { + "type": "integer" + }, + "iid": { + "type": "integer" + }, + "milestone": { + "type": [ + "null", + "object" + ], + "properties": { "project_id": { - "type": "integer" + "type": "integer" }, - "id": { - "type": "integer" + "description": { + "type": [ + "null", + "string" + ] + }, + "state": { + "type": [ + "null", + "string" + ] + }, + "due_date": { + "type": [ + "null", + "string" + ], + "format": "date" }, "iid": { - "type": "integer" + "type": "integer" }, - "milestone_id": { - "type": [ - "null", - "integer" - ] + "created_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" }, - "author_id": { - "type": [ - "null", - "integer" - ] + "title": { + "type": [ + "null", + "string" + ] }, - "assignee_id": { - "type": [ - "null", - "integer" - ] + "id": { + "type": "integer" }, - "assignees": { - "type": "array", - "items": { - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "name": { - "type": "string" - }, - "username": { - "type": "string" - }, - "state": { - "type": "string" - }, - "avatar_url": { - "type": [ - "null", - "string" - ] - }, - "web_url": { - "type": [ - "null", - "string" - ] - } - } - } - }, - "closed_by_id": { - "type": [ - "null", - "integer" - ] + "updated_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + } + }, + "example": { + "project_id" : 1, + "description" : "Ducimus nam enim ex consequatur cumque ratione.", + "state" : "closed", + "due_date" : null, + "iid" : 2, + "created_at" : "2016-01-04T15:31:39.996Z", + "title" : "v4.0", + "id" : 17, + "updated_at" : "2016-01-04T15:31:39.996Z" + } + }, + "author": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": "integer" }, - "title": { - "type": [ - "null", - "string" - ] + "name": { + "type": "string" }, - "description": { - "type": [ - "null", - "string" - ] + "username": { + "type": "string" }, "state": { - "type": [ - "null", - "string" - ] + "type": "string" }, - "labels": { - "type": "array", - "items": { - "type": "string" - } + "avatar_url": { + "type": [ + "null", + "string" + ] }, - "created_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] + "web_url": { + "type": [ + "null", + "string" + ] + } + } + }, + "assignee": { + "type": [ + "null", + "object" + ], + "properties": { + "state": { + "type": "string" }, - "updated_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] + "id": { + "type": "integer" }, - "closed_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] + "name": { + "type": "string" }, - "subscribed": { - "type": [ - "null", - "boolean" - ] - }, - "upvotes": { - "type": [ - "null", - "integer" - ] + "web_url": { + "type": [ + "null", + "string" + ] }, - "downvotes": { - "type": [ - "null", - "integer" - ] + "avatar_url": { + "type": [ + "null", + "string" + ] }, - "merge_requests_count": { + "username": { + "type": "string" + } + } + }, + "assignees": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "name": { + "type": "string" + }, + "username": { + "type": "string" + }, + "state": { + "type": "string" + }, + "avatar_url": { "type": [ - "null", - "integer" + "null", + "string" ] - }, - "user_notes_count": { + }, + "web_url": { "type": [ - "null", - "integer" + "null", + "string" ] + } + } + } + }, + "closed_by": { + "type": [ + "null", + "object" + ], + "properties": { + "state": { + "type": "string" }, - "due_date": { - "type": [ - "null", - "string" - ] + "id": { + "type": "integer" }, - "weight": { - "type": [ - "null", - "integer" - ] + "name": { + "type": "string" }, "web_url": { - "type": [ - "null", - "string" - ] + "type": [ + "null", + "string" + ] }, - "confidential": { - "type": [ - "null", - "boolean" - ] + "avatar_url": { + "type": [ + "null", + "string" + ] }, - "discussion_locked": { - "type": [ - "null", - "boolean" - ] + "username": { + "type": "string" + } + } + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "description": { + "type": [ + "null", + "string" + ] + }, + "state": { + "type": [ + "null", + "string" + ] + }, + "labels": { + "type": "array", + "items": { + "type": "string" + } + }, + "created_at": { + "anyOf": [ + { + "type": "string", + "format": "date-time" }, - "has_tasks": { - "type": [ - "null", - "boolean" - ] + { + "type": "null" + } + ] + }, + "updated_at": { + "anyOf": [ + { + "type": "string", + "format": "date-time" }, - "task_status": { - "type": [ - "null", - "string" - ] + { + "type": "null" + } + ] + }, + "closed_at": { + "anyOf": [ + { + "type": "string", + "format": "date-time" }, + { + "type": "null" + } + ] + }, + "upvotes": { + "type": [ + "null", + "integer" + ] + }, + "downvotes": { + "type": [ + "null", + "integer" + ] + }, + "merge_requests_count": { + "type": [ + "null", + "integer" + ] + }, + "user_notes_count": { + "type": [ + "null", + "integer" + ] + }, + "due_date": { + "type": [ + "null", + "string" + ], + "format": "date" + }, + "weight": { + "type": [ + "null", + "integer" + ] + }, + "web_url": { + "type": [ + "null", + "string" + ] + }, + "confidential": { + "type": [ + "null", + "boolean" + ] + }, + "discussion_locked": { + "type": [ + "null", + "boolean" + ] + }, + "has_tasks": { + "type": [ + "null", + "boolean" + ] + }, + "task_status": { + "type": [ + "null", + "string" + ] + }, + "time_stats": { + "type": [ + "null", + "object" + ], + "properties": { "time_estimate": { - "type": [ - "null", - "integer" - ] + "type": [ + "null", + "integer" + ] }, "total_time_spent": { - "type": [ - "null", - "integer" - ] + "type": [ + "null", + "integer" + ] }, "human_time_estimate": { - "type": [ - "null", - "string" - ] + "type": [ + "null", + "string" + ] }, "human_total_time_spent": { - "type": [ - "null", - "string" - ] + "type": [ + "null", + "string" + ] } + } } -} \ No newline at end of file + } +} diff --git a/samples/sample_tap_gitlab/schemas/projects.json b/samples/sample_tap_gitlab/schemas/projects.json index d02858e5f..2f9ececc4 100644 --- a/samples/sample_tap_gitlab/schemas/projects.json +++ b/samples/sample_tap_gitlab/schemas/projects.json @@ -10,8 +10,9 @@ "avatar_url": { "type": ["null", "string"] }, - "builds_enabled": { - "type": ["null", "boolean"] + "builds_access_level": { + "type": ["null", "string"], + "enum": ["disabled", "private", "enabled"] }, "container_registry_enabled": { "type": ["null", "boolean"] @@ -86,15 +87,12 @@ "only_allow_merge_if_all_discussions_are_resolved": { "type": ["null", "boolean"] }, - "only_allow_merge_if_build_succeeds": { + "only_allow_merge_if_pipeline_succeeds": { "type": ["null", "boolean"] }, "open_issues_count": { "type": ["null", "integer"] }, - "owner_id": { - "type": ["null", "integer"] - }, "path": { "type": ["null", "string"] }, @@ -128,10 +126,7 @@ } } }, - "public": { - "type": ["null", "boolean"] - }, - "public_builds": { + "public_jobs": { "type": ["null", "boolean"] }, "request_access_enabled": { @@ -166,37 +161,15 @@ "star_count": { "type": ["null", "integer"] }, - "statistics": { - "type": ["object", "null"], - "properties": { - "commit_count": { - "type": ["null", "integer"] - }, - "storage_size": { - "type": ["null", "integer"] - }, - "repository_size": { - "type": ["null", "integer"] - }, - "lfs_objects_size": { - "type": ["null", "integer"] - }, - "job_artifacts_size": { - "type": ["null", "integer"] - } - } - }, "tag_list": { "type": ["array", "null"], "items": { "type": "string" } }, - "visibility_level": { - "type": ["null", "integer"] - }, "visibility": { - "type": ["null", "string"] + "type": ["null", "string"], + "enum": ["private", "internal", "public"] }, "web_url": { "type": ["null", "string"] diff --git a/samples/sample_tap_gitlab/schemas/releases.json b/samples/sample_tap_gitlab/schemas/releases.json index 9165c802d..972a1f182 100644 --- a/samples/sample_tap_gitlab/schemas/releases.json +++ b/samples/sample_tap_gitlab/schemas/releases.json @@ -4,20 +4,102 @@ "tag_name": { "type": "string" }, - "author_id": { - "type": ["null", "integer"] + "author": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": "integer" + }, + "name": { + "type": "string" + }, + "username": { + "type": "string" + }, + "state": { + "type": "string" + }, + "avatar_url": { + "type": [ + "null", + "string" + ] + }, + "web_url": { + "type": [ + "null", + "string" + ] + } + } }, - "commit_id": { - "type": ["string", "null"] + "commit": { + "type": [ + "object", + "null" + ], + "properties": { + "id": { + "type": "string" + }, + "short_id": { + "type": "string" + }, + "title": { + "type": "string" + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "parent_ids": { + "type": "array", + "items": { + "type": "string" + } + }, + "message": { + "type": "string" + }, + "author_name": { + "type": "string" + }, + "author_email": { + "type": "string" + }, + "authored_date": { + "type": "string", + "format": "date-time" + }, + "committer_name": { + "type": "string" + }, + "committer_email": { + "type": "string" + }, + "committed_date": { + "type": "string", + "format": "date-time" + } + } }, "project_id": { "type": "string" }, "description": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "name": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "created_at": { "anyOf": [ diff --git a/samples/sample_tap_google_analytics/schemas/simple-sample.json b/samples/sample_tap_google_analytics/schemas/simple-sample.json index 794869260..0a5f97f4d 100644 --- a/samples/sample_tap_google_analytics/schemas/simple-sample.json +++ b/samples/sample_tap_google_analytics/schemas/simple-sample.json @@ -1,102 +1,11 @@ { "type": "object", "properties": { - "id": { - "type": ["string", "null"] - }, - "project_id": { - "type": ["integer", "null"] - }, - "short_id": { - "type": ["string", "null"] - }, - "title": { - "type": ["null", "string"] - }, - "author_name": { - "type": ["null", "string"] - }, - "author_email": { - "type": ["null", "string"] - }, - "authored_date": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - }, - "committer_name": { - "type": ["null", "string"] - }, - "committer_email": { - "type": ["null", "string"] - }, - "committed_date": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - }, - "created_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - }, - "message": { - "type": ["null", "string"] - }, - "allow_failure": { - "type": ["null", "boolean"] - }, - "parent_ids": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] - }, - "stats": { - "anyOf": [ - { - "type": "object", - "properties": { - "additions": { - "type": "integer" - }, - "deletions": { - "type": "integer" - }, - "total": { - "type": "integer" - } - } - }, - { - "type": "null" - } - ] + "totals": { + "type": "array", + "items": { + "type": "string" + } } } } diff --git a/singer_sdk/authenticators.py b/singer_sdk/authenticators.py index d31ab1dee..82e0556bc 100644 --- a/singer_sdk/authenticators.py +++ b/singer_sdk/authenticators.py @@ -131,13 +131,10 @@ def authenticate_request( """Authenticate a request. Args: - request: A `request object`_. + request: A :class:`requests.PreparedRequest` object. Returns: The authenticated request object. - - .. _request object: - https://requests.readthedocs.io/en/latest/api/#requests.PreparedRequest """ request.headers.update(self.auth_headers) @@ -154,13 +151,10 @@ def __call__(self, r: requests.PreparedRequest) -> requests.PreparedRequest: and returns the result. Args: - r: A `request object`_. + r: A :class:`requests.PreparedRequest` object. Returns: The authenticated request object. - - .. _request object: - https://requests.readthedocs.io/en/latest/api/#requests.PreparedRequest """ return self.authenticate_request(r) diff --git a/singer_sdk/connectors/sql.py b/singer_sdk/connectors/sql.py index 02c8e8d16..e097a4ce9 100644 --- a/singer_sdk/connectors/sql.py +++ b/singer_sdk/connectors/sql.py @@ -464,7 +464,8 @@ def discover_catalog_entry( th.Property( name=column_name, wrapped=th.CustomType(jsonschema_type), - required=not is_nullable, + nullable=is_nullable, + required=column_name in key_properties if key_properties else False, ), ) schema = table_schema.to_dict() diff --git a/singer_sdk/helpers/_flattening.py b/singer_sdk/helpers/_flattening.py index 866eb8a48..77e3935b9 100644 --- a/singer_sdk/helpers/_flattening.py +++ b/singer_sdk/helpers/_flattening.py @@ -33,7 +33,7 @@ def get_flattening_options( Returns: A new FlatteningOptions object or None if flattening is disabled. """ - if "flattening_enabled" in plugin_config and plugin_config["flattening_enabled"]: + if plugin_config.get("flattening_enabled"): return FlatteningOptions(max_level=int(plugin_config["flattening_max_depth"])) return None @@ -430,7 +430,7 @@ def _flatten_record( items.append( ( new_key, - json.dumps(v, use_decimal=True) + json.dumps(v, use_decimal=True, default=str) if _should_jsondump_value(k, v, flattened_schema) else v, ), diff --git a/singer_sdk/helpers/capabilities.py b/singer_sdk/helpers/capabilities.py index b18d53748..b48d290a4 100644 --- a/singer_sdk/helpers/capabilities.py +++ b/singer_sdk/helpers/capabilities.py @@ -7,9 +7,12 @@ from warnings import warn from singer_sdk.typing import ( + ArrayType, BooleanType, IntegerType, + NumberType, ObjectType, + OneOf, PropertiesList, Property, StringType, @@ -34,6 +37,33 @@ ObjectType(), description="User-defined config values to be used within map expressions.", ), + Property( + "faker_config", + ObjectType( + Property( + "seed", + OneOf(NumberType, StringType, BooleanType), + description=( + "Value to seed the Faker generator for deterministic output: " + "https://faker.readthedocs.io/en/master/#seeding-the-generator" + ), + ), + Property( + "locale", + OneOf(StringType, ArrayType(StringType)), + description=( + "One or more LCID locale strings to produce localized output for: " + "https://faker.readthedocs.io/en/master/#localization" + ), + ), + ), + description=( + "Config for the [`Faker`](https://faker.readthedocs.io/en/master/) " + "instance variable `fake` used within map expressions. Only applicable if " + "the plugin specifies `faker` as an addtional dependency (through the " + "`singer-sdk` `faker` extra or directly)." + ), + ), ).to_dict() FLATTENING_CONFIG = PropertiesList( Property( @@ -114,6 +144,14 @@ default=False, ), ).to_dict() +TARGET_VALIDATE_RECORDS_CONFIG = PropertiesList( + Property( + "validate_records", + BooleanType(), + description="Whether to validate the schema of the incoming streams.", + default=True, + ), +).to_dict() class TargetLoadMethods(str, Enum): @@ -325,3 +363,6 @@ class TargetCapabilities(CapabilitiesEnum): #: Allow setting the target schema. TARGET_SCHEMA = "target-schema" + + #: Validate the schema of the incoming records. + VALIDATE_RECORDS = "validate-records" diff --git a/singer_sdk/mapper.py b/singer_sdk/mapper.py index d3cd9400d..cd5faf93c 100644 --- a/singer_sdk/mapper.py +++ b/singer_sdk/mapper.py @@ -10,6 +10,7 @@ import copy import datetime import hashlib +import importlib.util import logging import typing as t @@ -28,6 +29,8 @@ if t.TYPE_CHECKING: import sys + from faker import Faker + if sys.version_info >= (3, 10): from typing import TypeAlias # noqa: ICN003 else: @@ -231,6 +234,7 @@ def __init__( self, stream_alias: str, map_config: dict, + faker_config: dict, raw_schema: dict, key_properties: t.Sequence[str] | None, map_transform: dict, @@ -241,6 +245,7 @@ def __init__( Args: stream_alias: Stream name. map_config: Stream map configuration. + faker_config: Faker configuration. raw_schema: Original stream's JSON schema. key_properties: Primary key of the source stream. map_transform: Dictionary of transformations to apply to the stream. @@ -254,6 +259,8 @@ def __init__( ) self.map_config = map_config + self.faker_config = faker_config + self._transform_fn: t.Callable[[dict], dict | None] self._filter_fn: t.Callable[[dict], bool] ( @@ -262,6 +269,7 @@ def __init__( self.transformed_schema, ) = self._init_functions_and_schema(stream_map=map_transform) self.expr_evaluator = simpleeval.EvalWithCompoundTypes(functions=self.functions) + self.fake = self._init_faker_instance() def transform(self, record: dict) -> dict | None: """Return a transformed record. @@ -324,6 +332,10 @@ def _eval( names["_"] = record # Add a shorthand alias in case of reserved words in names names["record"] = record # ...and a longhand alias names["config"] = self.map_config # Allow map config access within transform + + if self.fake: + names["fake"] = self.fake + if property_name and property_name in record: # Allow access to original property value if applicable names["self"] = record[property_name] @@ -593,6 +605,24 @@ def transform_fn(record: dict) -> dict | None: return filter_fn, transform_fn, transformed_schema + def _init_faker_instance(self) -> Faker | None: + if not importlib.util.find_spec("faker"): + return None + + from faker import Faker + + if self.faker_config: + faker_seed = self.faker_config.get("seed") + faker_locale = self.faker_config.get("locale") + + if faker_seed is not None: + Faker.seed(faker_seed) + + if faker_locale is not None: + return Faker(faker_locale) + + return Faker() + class PluginMapper: """Inline map tranformer.""" @@ -613,6 +643,7 @@ def __init__( """ self.stream_maps: dict[str, list[StreamMap]] = {} self.map_config = plugin_config.get("stream_map_config", {}) + self.faker_config = plugin_config.get("faker_config", {}) self.flattening_options = get_flattening_options(plugin_config) self.default_mapper_type: type[DefaultStreamMap] = SameRecordTransform self.logger = logger @@ -751,6 +782,7 @@ def register_raw_stream_schema( # noqa: PLR0912, C901 stream_alias=stream_alias, map_transform=stream_def, map_config=self.map_config, + faker_config=self.faker_config, raw_schema=schema, key_properties=key_properties, flattening_options=self.flattening_options, diff --git a/singer_sdk/pagination.py b/singer_sdk/pagination.py index 51ce52489..5dc36575d 100644 --- a/singer_sdk/pagination.py +++ b/singer_sdk/pagination.py @@ -181,7 +181,7 @@ class BaseHATEOASPaginator( like "https://api.com/link/to/next-item". The :attr:`~singer_sdk.pagination.BaseAPIPaginator.current_value` attribute of - this paginator is a `urllib.parse.ParseResult`_ object. This object + this paginator is a :class:`urllib.parse.ParseResult` object. This object contains the following attributes: - scheme @@ -208,9 +208,6 @@ def get_url_params(self, next_page_token) -> dict: if next_page_token: return dict(parse_qsl(next_page_token.query)) return {} - - .. _`urllib.parse.ParseResult`: - https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urlparse """ def __init__(self, *args: t.Any, **kwargs: t.Any) -> None: @@ -390,7 +387,6 @@ def get_next_page_token( response: API response object. previous_token: Previous page token. """ - ... # pragma: no cover class LegacyStreamPaginator( diff --git a/singer_sdk/plugin_base.py b/singer_sdk/plugin_base.py index fc1ce9415..1c2f46c97 100644 --- a/singer_sdk/plugin_base.py +++ b/singer_sdk/plugin_base.py @@ -403,9 +403,7 @@ def print_version( Args: print_fn: A function to use to display the plugin version. - Defaults to `print`_. - - .. _print: https://docs.python.org/3/library/functions.html#print + Defaults to :py:func:`print`. """ print_fn(f"{cls.name} v{cls.plugin_version}, Meltano SDK v{cls.sdk_version}") diff --git a/singer_sdk/sinks/core.py b/singer_sdk/sinks/core.py index a1ab9c8bb..ef623e01b 100644 --- a/singer_sdk/sinks/core.py +++ b/singer_sdk/sinks/core.py @@ -9,6 +9,7 @@ import json import time import typing as t +from functools import cached_property from gzip import GzipFile from gzip import open as gzip_open from types import MappingProxyType @@ -122,9 +123,6 @@ class Sink(metaclass=abc.ABCMeta): MAX_SIZE_DEFAULT = 10000 - validate_schema = True - """Enable JSON schema record validation.""" - validate_field_string_format = False """Enable JSON schema format validation, for example `date-time` string fields.""" @@ -177,6 +175,15 @@ def __init__( self._validator: BaseJSONSchemaValidator | None = self.get_validator() + @cached_property + def validate_schema(self) -> bool: + """Enable JSON schema record validation. + + Returns: + True if JSON schema validation is enabled. + """ + return self.config.get("validate_records", True) + def get_validator(self) -> BaseJSONSchemaValidator | None: """Get a record validator for this sink. diff --git a/singer_sdk/sinks/sql.py b/singer_sdk/sinks/sql.py index 3a37570d6..991ba431a 100644 --- a/singer_sdk/sinks/sql.py +++ b/singer_sdk/sinks/sql.py @@ -10,6 +10,7 @@ import sqlalchemy as sa from pendulum import now +from sqlalchemy.sql import quoted_name from sqlalchemy.sql.expression import bindparam from singer_sdk.connectors import SQLConnector @@ -279,10 +280,14 @@ def generate_insert_statement( An insert statement. """ property_names = list(self.conform_schema(schema)["properties"].keys()) + column_identifiers = [ + self.connector.quote(quoted_name(name, quote=True)) + for name in property_names + ] statement = dedent( f"""\ INSERT INTO {full_table_name} - ({", ".join(property_names)}) + ({", ".join(column_identifiers)}) VALUES ({", ".join([f":{name}" for name in property_names])}) """, # noqa: S608 ) diff --git a/singer_sdk/streams/core.py b/singer_sdk/streams/core.py index 064b78361..e57d93a62 100644 --- a/singer_sdk/streams/core.py +++ b/singer_sdk/streams/core.py @@ -1072,7 +1072,6 @@ def _sync_records( # noqa: C901 record_counter.context = context_element timer.context = context_element - partition_record_index = 0 current_context = context_element or None state = self.get_context_state(current_context) state_partition_context = self._get_state_partition_context( @@ -1083,7 +1082,7 @@ def _sync_records( # noqa: C901 None if current_context is None else copy.copy(current_context) ) - for record_result in self.get_records(current_context): + for idx, record_result in enumerate(self.get_records(current_context)): self._check_max_record_limit(current_record_index=record_index) if isinstance(record_result, tuple): @@ -1102,7 +1101,7 @@ def _sync_records( # noqa: C901 log_fn=self.logger.error, ex=ex, record_count=record_index + 1, - partition_record_count=partition_record_index + 1, + partition_record_count=idx + 1, current_context=current_context, state_partition_context=state_partition_context, stream_name=self.name, @@ -1123,7 +1122,6 @@ def _sync_records( # noqa: C901 yield record record_index += 1 - partition_record_index += 1 if current_context == state_partition_context: # Finalize per-partition state only if 1:1 with context diff --git a/singer_sdk/streams/rest.py b/singer_sdk/streams/rest.py index a9cfa568a..87e1001d9 100644 --- a/singer_sdk/streams/rest.py +++ b/singer_sdk/streams/rest.py @@ -137,10 +137,7 @@ def requests_session(self) -> requests.Session: """Get requests session. Returns: - The `requests.Session`_ object for HTTP requests. - - .. _requests.Session: - https://requests.readthedocs.io/en/latest/api.html#requests.Session + The :class:`requests.Session` object for HTTP requests. """ if not self._requests_session: self._requests_session = requests.Session() @@ -168,14 +165,11 @@ def validate_response(self, response: requests.Response) -> None: .. image:: ../images/200.png Args: - response: A `requests.Response`_ object. + response: A :class:`requests.Response` object. Raises: FatalAPIError: If the request is not retriable. RetriableAPIError: If the request is retriable. - - .. _requests.Response: - https://requests.readthedocs.io/en/latest/api.html#requests.Response """ if ( response.status_code in self.extra_retry_statuses @@ -198,7 +192,7 @@ def response_error_message(self, response: requests.Response) -> str: WARNING - Override this method when the URL path may contain secrets or PII Args: - response: A `requests.Response`_ object. + response: A :class:`requests.Response` object. Returns: str: The error message @@ -286,7 +280,7 @@ def get_url_params( If your source needs special handling and, for example, parentheses should not be encoded, you can return a string constructed with - `urllib.parse.urlencode`_: + :py:func:`urllib.parse.urlencode`: .. code-block:: python @@ -306,9 +300,6 @@ def get_url_params(self, context, next_page_token): Returns: Dictionary or encoded string with URL query parameters to use in the request. - - .. _urllib.parse.urlencode: - https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urlencode """ return {} @@ -322,16 +313,11 @@ def build_prepared_request( Uses the authenticator instance to mutate the request with authentication. Args: - *args: Arguments to pass to `requests.Request`_. - **kwargs: Keyword arguments to pass to `requests.Request`_. + *args: Arguments to pass to :class:`requests.Request`. + **kwargs: Keyword arguments to pass to :class:`requests.Request`. Returns: - A `requests.PreparedRequest`_ object. - - .. _requests.PreparedRequest: - https://requests.readthedocs.io/en/latest/api.html#requests.PreparedRequest - .. _requests.Request: - https://requests.readthedocs.io/en/latest/api.html#requests.Request + A :class:`requests.PreparedRequest` object. """ request = requests.Request(*args, **kwargs) self.requests_session.auth = self.authenticator @@ -460,7 +446,7 @@ def update_sync_costs( Args: request: the Request object that was just called. - response: the `requests.Response` object + response: the :class:`requests.Response` object context: the context passed to the call Returns: @@ -498,7 +484,7 @@ def calculate_sync_cost( Args: request: the API Request object that was just called. - response: the `requests.Response` object + response: the :class:`requests.Response` object context: the context passed to the call Returns: @@ -596,13 +582,10 @@ def parse_response(self, response: requests.Response) -> t.Iterable[dict]: """Parse the response and return an iterator of result records. Args: - response: A raw `requests.Response`_ object. + response: A raw :class:`requests.Response` Yields: One item for every item found in the response. - - .. _requests.Response: - https://requests.readthedocs.io/en/latest/api.html#requests.Response """ yield from extract_jsonpath(self.records_jsonpath, input=response.json()) diff --git a/singer_sdk/target_base.py b/singer_sdk/target_base.py index 4ab959ff7..5a78bf362 100644 --- a/singer_sdk/target_base.py +++ b/singer_sdk/target_base.py @@ -21,6 +21,7 @@ TARGET_HARD_DELETE_CONFIG, TARGET_LOAD_METHOD_CONFIG, TARGET_SCHEMA_CONFIG, + TARGET_VALIDATE_RECORDS_CONFIG, CapabilitiesEnum, PluginCapabilities, TargetCapabilities, @@ -104,6 +105,7 @@ def capabilities(self) -> list[CapabilitiesEnum]: PluginCapabilities.ABOUT, PluginCapabilities.STREAM_MAPS, PluginCapabilities.FLATTENING, + TargetCapabilities.VALIDATE_RECORDS, ] @property @@ -614,6 +616,9 @@ def _merge_missing(source_jsonschema: dict, target_jsonschema: dict) -> None: if PluginCapabilities.BATCH in capabilities: _merge_missing(BATCH_CONFIG, config_jsonschema) + if TargetCapabilities.VALIDATE_RECORDS in capabilities: + _merge_missing(TARGET_VALIDATE_RECORDS_CONFIG, config_jsonschema) + super().append_builtin_config(config_jsonschema) diff --git a/singer_sdk/typing.py b/singer_sdk/typing.py index 80e553574..7a412fe81 100644 --- a/singer_sdk/typing.py +++ b/singer_sdk/typing.py @@ -517,7 +517,7 @@ class Property(JSONTypeHelper[T], t.Generic[T]): """Generic Property. Should be nested within a `PropertiesList`.""" # TODO: Make some of these arguments keyword-only. This is a breaking change. - def __init__( + def __init__( # noqa: PLR0913 self, name: str, wrapped: JSONTypeHelper[T] | type[JSONTypeHelper[T]], @@ -527,6 +527,8 @@ def __init__( secret: bool | None = False, # noqa: FBT002 allowed_values: list[T] | None = None, examples: list[T] | None = None, + *, + nullable: bool | None = None, ) -> None: """Initialize Property object. @@ -547,6 +549,7 @@ def __init__( are permitted. This will define the type as an 'enum'. examples: Optional. A list of one or more sample values. These may be displayed to the user as hints of the expected format of inputs. + nullable: If True, the property may be null. """ self.name = name self.wrapped = wrapped @@ -556,6 +559,7 @@ def __init__( self.secret = secret self.allowed_values = allowed_values or None self.examples = examples or None + self.nullable = nullable @property def type_dict(self) -> dict: # type: ignore[override] @@ -585,7 +589,7 @@ def to_dict(self) -> dict: A JSON Schema dictionary describing the object. """ type_dict = self.type_dict - if self.optional: + if self.nullable or self.optional: type_dict = append_type(type_dict, "null") if self.default is not None: type_dict.update({"default": self.default}) diff --git a/tests/core/configuration/test_dict_config.py b/tests/core/configuration/test_dict_config.py index 7d94b06f4..62b525303 100644 --- a/tests/core/configuration/test_dict_config.py +++ b/tests/core/configuration/test_dict_config.py @@ -1,9 +1,7 @@ from __future__ import annotations import json -import os from pathlib import Path -from unittest import mock import pytest @@ -38,16 +36,12 @@ def config_file2(tmpdir) -> str: return filepath -def test_get_env_var_config(): +def test_get_env_var_config(monkeypatch: pytest.MonkeyPatch): """Test settings parsing from environment variables.""" - with mock.patch.dict( - os.environ, - { - "PLUGIN_TEST_PROP1": "hello", - "PLUGIN_TEST_PROP3": "val1,val2", - "PLUGIN_TEST_PROP4": "not-a-tap-setting", - }, - ): + with monkeypatch.context() as m: + m.setenv("PLUGIN_TEST_PROP1", "hello") + m.setenv("PLUGIN_TEST_PROP3", "val1,val2") + m.setenv("PLUGIN_TEST_PROP4", "not-a-tap-setting") env_config = parse_environment_config(CONFIG_JSONSCHEMA, "PLUGIN_TEST_") assert env_config["prop1"] == "hello" assert env_config["prop3"] == ["val1", "val2"] @@ -68,9 +62,8 @@ def test_get_env_var_config(): assert "PROP4" not in env_config -def test_get_dotenv_config(tmpdir, monkeypatch: pytest.MonkeyPatch): - monkeypatch.chdir(tmpdir) - dotenv = Path(tmpdir) / ".env" +def test_get_dotenv_config(tmp_path: Path): + dotenv = tmp_path / ".env" dotenv.write_text("PLUGIN_TEST_PROP1=hello\n") dotenv_config = parse_environment_config( CONFIG_JSONSCHEMA, @@ -81,27 +74,24 @@ def test_get_dotenv_config(tmpdir, monkeypatch: pytest.MonkeyPatch): assert dotenv_config["prop1"] == "hello" -def test_get_env_var_config_not_parsable(): +def test_get_env_var_config_not_parsable(monkeypatch: pytest.MonkeyPatch): """Test settings parsing from environment variables with a non-parsable value.""" - with mock.patch.dict( - os.environ, - { - "PLUGIN_TEST_PROP1": "hello", - "PLUGIN_TEST_PROP3": '["repeated"]', - }, - ), pytest.raises(ValueError, match="A bracketed list was detected"): - parse_environment_config(CONFIG_JSONSCHEMA, "PLUGIN_TEST_") - - -def test_merge_config_sources(config_file1, config_file2): + with monkeypatch.context() as m: + m.setenv("PLUGIN_TEST_PROP1", "hello") + m.setenv("PLUGIN_TEST_PROP3", '["repeated"]') + with pytest.raises(ValueError, match="A bracketed list was detected"): + parse_environment_config(CONFIG_JSONSCHEMA, "PLUGIN_TEST_") + + +def test_merge_config_sources( + config_file1, + config_file2, + monkeypatch: pytest.MonkeyPatch, +): """Test merging multiple configuration sources.""" - with mock.patch.dict( - os.environ, - { - "PLUGIN_TEST_PROP1": "from-env", - "PLUGIN_TEST_PROP4": "not-a-tap-setting", - }, - ): + with monkeypatch.context() as m: + m.setenv("PLUGIN_TEST_PROP1", "from-env") + m.setenv("PLUGIN_TEST_PROP4", "not-a-tap-setting") config = merge_config_sources( [config_file1, config_file2, "ENV"], CONFIG_JSONSCHEMA, diff --git a/tests/core/test_connector_sql.py b/tests/core/test_connector_sql.py index 816086582..8d805ebd8 100644 --- a/tests/core/test_connector_sql.py +++ b/tests/core/test_connector_sql.py @@ -1,5 +1,6 @@ from __future__ import annotations +import sys import typing as t from decimal import Decimal from unittest import mock @@ -7,6 +8,7 @@ import pytest import sqlalchemy as sa from sqlalchemy.dialects import registry, sqlite +from sqlalchemy.exc import NoSuchModuleError from singer_sdk.connectors import SQLConnector from singer_sdk.exceptions import ConfigValidationError @@ -274,7 +276,7 @@ def test_engine_json_serialization(self, connector: SQLConnector): sa.Column("attrs", sa.JSON), ) meta.create_all(engine) - with engine.connect() as conn: + with engine.connect() as conn, conn.begin(): conn.execute( table.insert(), [ @@ -308,6 +310,11 @@ def get_column_alter_ddl( ) +@pytest.mark.xfail( + reason="DuckDB does not build on Python 3.12 yet", + condition=sys.version_info >= (3, 12), + raises=NoSuchModuleError, +) class TestDuckDBConnector: @pytest.fixture def connector(self): diff --git a/tests/core/test_mapper.py b/tests/core/test_mapper.py index ab8c5f1fc..2f17684d6 100644 --- a/tests/core/test_mapper.py +++ b/tests/core/test_mapper.py @@ -505,6 +505,14 @@ def _test_transform( ) +class CustomObj: + def __init__(self, value: str): + self.value = value + + def __str__(self) -> str: + return f"obj-{self.value}" + + class MappedStream(Stream): """A stream to be mapped.""" @@ -516,7 +524,13 @@ class MappedStream(Stream): "user", ObjectType( Property("id", IntegerType()), - Property("sub", ObjectType(Property("num", IntegerType()))), + Property( + "sub", + ObjectType( + Property("num", IntegerType()), + Property("custom_obj", StringType), + ), + ), Property("some_numbers", ArrayType(NumberType())), ), ), @@ -528,7 +542,7 @@ def get_records(self, context): # noqa: ARG002 "count": 21, "user": { "id": 1, - "sub": {"num": 1}, + "sub": {"num": 1, "custom_obj": CustomObj("hello")}, "some_numbers": [Decimal("3.14"), Decimal("2.718")], }, } @@ -537,7 +551,7 @@ def get_records(self, context): # noqa: ARG002 "count": 13, "user": { "id": 2, - "sub": {"num": 2}, + "sub": {"num": 2, "custom_obj": CustomObj("world")}, "some_numbers": [Decimal("10.32"), Decimal("1.618")], }, } @@ -546,7 +560,7 @@ def get_records(self, context): # noqa: ARG002 "count": 19, "user": { "id": 3, - "sub": {"num": 3}, + "sub": {"num": 3, "custom_obj": CustomObj("hello")}, "some_numbers": [Decimal("1.414"), Decimal("1.732")], }, } @@ -568,12 +582,11 @@ def discover_streams(self): ) @pytest.mark.snapshot() @pytest.mark.parametrize( - "stream_maps,flatten,flatten_max_depth,snapshot_name", + "stream_maps,config,snapshot_name", [ pytest.param( {}, - False, - 0, + {"flattening_enabled": False, "flattening_max_depth": 0}, "no_map.jsonl", id="no_map", ), @@ -583,8 +596,7 @@ def discover_streams(self): "email_hash": "md5(email)", }, }, - False, - 0, + {"flattening_enabled": False, "flattening_max_depth": 0}, "keep_all_fields.jsonl", id="keep_all_fields", ), @@ -596,8 +608,7 @@ def discover_streams(self): "__else__": None, }, }, - False, - 0, + {"flattening_enabled": False, "flattening_max_depth": 0}, "only_mapped_fields.jsonl", id="only_mapped_fields", ), @@ -609,8 +620,7 @@ def discover_streams(self): "__else__": "__NULL__", }, }, - False, - 0, + {"flattening_enabled": False, "flattening_max_depth": 0}, "only_mapped_fields_null_string.jsonl", id="only_mapped_fields_null_string", ), @@ -622,57 +632,49 @@ def discover_streams(self): "__else__": None, }, }, - False, - 0, + {"flattening_enabled": False, "flattening_max_depth": 0}, "changed_key_properties.jsonl", id="changed_key_properties", ), pytest.param( {"mystream": None, "sourced_stream_1": {"__source__": "mystream"}}, - False, - 0, + {"flattening_enabled": False, "flattening_max_depth": 0}, "sourced_stream_1.jsonl", id="sourced_stream_1", ), pytest.param( {"mystream": "__NULL__", "sourced_stream_1": {"__source__": "mystream"}}, - False, - 0, + {"flattening_enabled": False, "flattening_max_depth": 0}, "sourced_stream_1_null_string.jsonl", id="sourced_stream_1_null_string", ), pytest.param( {"sourced_stream_2": {"__source__": "mystream"}, "__else__": None}, - False, - 0, + {"flattening_enabled": False, "flattening_max_depth": 0}, "sourced_stream_2.jsonl", id="sourced_stream_2", ), pytest.param( {"mystream": {"__alias__": "aliased_stream"}}, - False, - 0, + {"flattening_enabled": False, "flattening_max_depth": 0}, "aliased_stream.jsonl", id="aliased_stream", ), pytest.param( {}, - True, - 0, + {"flattening_enabled": True, "flattening_max_depth": 0}, "flatten_depth_0.jsonl", id="flatten_depth_0", ), pytest.param( {}, - True, - 1, + {"flattening_enabled": True, "flattening_max_depth": 1}, "flatten_depth_1.jsonl", id="flatten_depth_1", ), pytest.param( {}, - True, - 10, + {"flattening_enabled": True, "flattening_max_depth": 2}, "flatten_all.jsonl", id="flatten_all", ), @@ -683,8 +685,7 @@ def discover_streams(self): "__key_properties__": ["email_hash"], }, }, - True, - 10, + {"flattening_enabled": True, "flattening_max_depth": 10}, "map_and_flatten.jsonl", id="map_and_flatten", ), @@ -694,15 +695,13 @@ def discover_streams(self): "email": None, }, }, - False, - 0, + {"flattening_enabled": False, "flattening_max_depth": 0}, "drop_property.jsonl", id="drop_property", ), pytest.param( {"mystream": {"email": "__NULL__"}}, - False, - 0, + {"flattening_enabled": False, "flattening_max_depth": 0}, "drop_property_null_string.jsonl", id="drop_property_null_string", ), @@ -713,8 +712,7 @@ def discover_streams(self): "__else__": None, }, }, - False, - 0, + {"flattening_enabled": False, "flattening_max_depth": 0}, "non_pk_passthrough.jsonl", id="non_pk_passthrough", ), @@ -725,29 +723,41 @@ def discover_streams(self): "__else__": None, }, }, - False, - 0, + {"flattening_enabled": False, "flattening_max_depth": 0}, "record_to_column.jsonl", id="record_to_column", ), + pytest.param( + { + "mystream": { + "cc": "fake.credit_card_number()", + "__else__": None, + }, + }, + { + "flattening_enabled": False, + "flattening_max_depth": 0, + "faker_config": { + "locale": "en_US", + "seed": 123456, + }, + }, + "fake_credit_card_number.jsonl", + id="fake_credit_card_number", + ), ], ) def test_mapped_stream( snapshot: Snapshot, snapshot_dir: Path, stream_maps: dict, - flatten: bool, - flatten_max_depth: int | None, + config: dict, snapshot_name: str, ): snapshot.snapshot_dir = snapshot_dir.joinpath("mapped_stream") tap = MappedTap( - config={ - "stream_maps": stream_maps, - "flattening_enabled": flatten, - "flattening_max_depth": flatten_max_depth, - }, + config={"stream_maps": stream_maps, **config}, ) buf = io.StringIO() with redirect_stdout(buf): diff --git a/tests/core/test_plugin_base.py b/tests/core/test_plugin_base.py index 0ea87f036..04b6a1665 100644 --- a/tests/core/test_plugin_base.py +++ b/tests/core/test_plugin_base.py @@ -1,8 +1,5 @@ from __future__ import annotations -import os -from unittest import mock - import pytest from singer_sdk.plugin_base import SDK_PACKAGE_NAME, MapperNotInitialized, PluginBase @@ -19,15 +16,17 @@ class PluginTest(PluginBase): ).to_dict() -def test_get_env_var_config(): +def test_get_env_var_config(monkeypatch: pytest.MonkeyPatch): """Test settings parsing from environment variables.""" - with mock.patch.dict( - os.environ, - { - "PLUGIN_TEST_PROP1": "hello", - "PLUGIN_TEST_PROP3": "not-a-tap-setting", - }, - ): + monkeypatch.delenv("PLUGIN_TEST_PROP1", raising=False) + monkeypatch.delenv("PLUGIN_TEST_PROP2", raising=False) + monkeypatch.delenv("PLUGIN_TEST_PROP3", raising=False) + monkeypatch.delenv("PLUGIN_TEST_PROP4", raising=False) + + with monkeypatch.context() as m: + m.setenv("PLUGIN_TEST_PROP1", "hello") + m.setenv("PLUGIN_TEST_PROP3", "not-a-tap-setting") + m.setenv("PLUGIN_TEST_PROP4", "not-a-tap-setting") env_config = PluginTest._env_var_config assert env_config["prop1"] == "hello" assert "PROP1" not in env_config diff --git a/tests/core/test_target_base.py b/tests/core/test_target_base.py index de344c7e3..eaff6d6a1 100644 --- a/tests/core/test_target_base.py +++ b/tests/core/test_target_base.py @@ -8,7 +8,7 @@ MissingKeyPropertiesError, RecordsWithoutSchemaException, ) -from singer_sdk.helpers.capabilities import PluginCapabilities +from singer_sdk.helpers.capabilities import PluginCapabilities, TargetCapabilities from tests.conftest import BatchSinkMock, SQLSinkMock, SQLTargetMock, TargetMock @@ -67,6 +67,7 @@ def test_target_about_info(): PluginCapabilities.ABOUT, PluginCapabilities.STREAM_MAPS, PluginCapabilities.FLATTENING, + TargetCapabilities.VALIDATE_RECORDS, PluginCapabilities.BATCH, ] diff --git a/tests/samples/conftest.py b/tests/samples/conftest.py index b9ce33319..52c0857be 100644 --- a/tests/samples/conftest.py +++ b/tests/samples/conftest.py @@ -25,7 +25,14 @@ def _sqlite_sample_db(sqlite_connector): for t in range(3): conn.execute(sa.text(f"DROP TABLE IF EXISTS t{t}")) conn.execute( - sa.text(f"CREATE TABLE t{t} (c1 int PRIMARY KEY, c2 varchar(10))"), + sa.text( + f""" + CREATE TABLE t{t} ( + c1 int PRIMARY KEY NOT NULL, + c2 varchar(10) NOT NULL + ) + """ + ), ) for x in range(100): conn.execute( diff --git a/tests/samples/test_tap_sqlite.py b/tests/samples/test_tap_sqlite.py index b5ed7b549..2c1094b75 100644 --- a/tests/samples/test_tap_sqlite.py +++ b/tests/samples/test_tap_sqlite.py @@ -80,6 +80,8 @@ def test_sqlite_discovery(sqlite_sample_tap: SQLTap): assert stream.metadata.root.table_key_properties == ["c1"] assert stream.primary_keys == ["c1"] + assert stream.schema["properties"]["c1"] == {"type": ["integer"]} + assert stream.schema["required"] == ["c1"] def test_sqlite_input_catalog(sqlite_sample_tap: SQLTap): @@ -90,7 +92,7 @@ def test_sqlite_input_catalog(sqlite_sample_tap: SQLTap): for schema in [stream.schema, stream.stream_maps[0].transformed_schema]: assert len(schema["properties"]) == 2 - assert schema["properties"]["c1"] == {"type": ["integer", "null"]} + assert schema["properties"]["c1"] == {"type": ["integer"]} assert schema["properties"]["c2"] == {"type": ["string", "null"]} assert stream.name == stream.tap_stream_id == "main-t1" diff --git a/tests/samples/test_target_sqlite.py b/tests/samples/test_target_sqlite.py index 0d34c60a1..edf88ee92 100644 --- a/tests/samples/test_target_sqlite.py +++ b/tests/samples/test_target_sqlite.py @@ -484,14 +484,15 @@ def test_record_with_missing_properties( "properties": { "id": {"type": "integer"}, "name": {"type": "string"}, + "table": {"type": "string"}, }, }, [], dedent( """\ INSERT INTO test_stream - (id, name) - VALUES (:id, :name)""", + (id, name, "table") + VALUES (:id, :name, :table)""", ), ), ], diff --git a/tests/snapshots/mapped_stream/aliased_stream.jsonl b/tests/snapshots/mapped_stream/aliased_stream.jsonl index e353c0717..8956e8020 100644 --- a/tests/snapshots/mapped_stream/aliased_stream.jsonl +++ b/tests/snapshots/mapped_stream/aliased_stream.jsonl @@ -1,6 +1,6 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"aliased_stream","schema":{"properties":{"email":{"type":["string","null"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]}},"type":"object"},"key_properties":[]} -{"type":"RECORD","stream":"aliased_stream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1},"some_numbers":[3.14,2.718]}},"time_extracted":"2022-01-01T00:00:00+00:00"} -{"type":"RECORD","stream":"aliased_stream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2},"some_numbers":[10.32,1.618]}},"time_extracted":"2022-01-01T00:00:00+00:00"} -{"type":"RECORD","stream":"aliased_stream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3},"some_numbers":[1.414,1.732]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"SCHEMA","stream":"aliased_stream","schema":{"properties":{"email":{"type":["string","null"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]}},"type":"object"},"key_properties":[]} +{"type":"RECORD","stream":"aliased_stream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"aliased_stream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"aliased_stream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]}},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"STATE","value":{"bookmarks":{"mystream":{}}}} diff --git a/tests/snapshots/mapped_stream/drop_property.jsonl b/tests/snapshots/mapped_stream/drop_property.jsonl index 03c663b05..9b6c9b61a 100644 --- a/tests/snapshots/mapped_stream/drop_property.jsonl +++ b/tests/snapshots/mapped_stream/drop_property.jsonl @@ -1,6 +1,6 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]}},"type":"object"},"key_properties":[]} -{"type":"RECORD","stream":"mystream","record":{"count":21,"user":{"id":1,"sub":{"num":1},"some_numbers":[3.14,2.718]}},"time_extracted":"2022-01-01T00:00:00+00:00"} -{"type":"RECORD","stream":"mystream","record":{"count":13,"user":{"id":2,"sub":{"num":2},"some_numbers":[10.32,1.618]}},"time_extracted":"2022-01-01T00:00:00+00:00"} -{"type":"RECORD","stream":"mystream","record":{"count":19,"user":{"id":3,"sub":{"num":3},"some_numbers":[1.414,1.732]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]}},"type":"object"},"key_properties":[]} +{"type":"RECORD","stream":"mystream","record":{"count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]}},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"STATE","value":{"bookmarks":{"mystream":{}}}} diff --git a/tests/snapshots/mapped_stream/drop_property_null_string.jsonl b/tests/snapshots/mapped_stream/drop_property_null_string.jsonl index 03c663b05..9b6c9b61a 100644 --- a/tests/snapshots/mapped_stream/drop_property_null_string.jsonl +++ b/tests/snapshots/mapped_stream/drop_property_null_string.jsonl @@ -1,6 +1,6 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]}},"type":"object"},"key_properties":[]} -{"type":"RECORD","stream":"mystream","record":{"count":21,"user":{"id":1,"sub":{"num":1},"some_numbers":[3.14,2.718]}},"time_extracted":"2022-01-01T00:00:00+00:00"} -{"type":"RECORD","stream":"mystream","record":{"count":13,"user":{"id":2,"sub":{"num":2},"some_numbers":[10.32,1.618]}},"time_extracted":"2022-01-01T00:00:00+00:00"} -{"type":"RECORD","stream":"mystream","record":{"count":19,"user":{"id":3,"sub":{"num":3},"some_numbers":[1.414,1.732]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]}},"type":"object"},"key_properties":[]} +{"type":"RECORD","stream":"mystream","record":{"count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]}},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"STATE","value":{"bookmarks":{"mystream":{}}}} diff --git a/tests/snapshots/mapped_stream/fake_credit_card_number.jsonl b/tests/snapshots/mapped_stream/fake_credit_card_number.jsonl new file mode 100644 index 000000000..6db048524 --- /dev/null +++ b/tests/snapshots/mapped_stream/fake_credit_card_number.jsonl @@ -0,0 +1,6 @@ +{"type":"STATE","value":{}} +{"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"cc":{"type":["string","null"]}}},"key_properties":[]} +{"type":"RECORD","stream":"mystream","record":{"cc":"4201040137208265027"},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"cc":"675987782884"},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"cc":"502011811259"},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"STATE","value":{"bookmarks":{"mystream":{}}}} diff --git a/tests/snapshots/mapped_stream/flatten_all.jsonl b/tests/snapshots/mapped_stream/flatten_all.jsonl index ddb313b01..21504a38f 100644 --- a/tests/snapshots/mapped_stream/flatten_all.jsonl +++ b/tests/snapshots/mapped_stream/flatten_all.jsonl @@ -1,6 +1,6 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string","null"]},"count":{"type":["integer","null"]},"user__id":{"type":["integer","null"]},"user__sub__num":{"type":["integer","null"]},"user__some_numbers":{"type":["string","null"]}},"type":"object"},"key_properties":[]} -{"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user__id":1,"user__sub__num":1,"user__some_numbers":"[3.14, 2.718]"},"time_extracted":"2022-01-01T00:00:00+00:00"} -{"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user__id":2,"user__sub__num":2,"user__some_numbers":"[10.32, 1.618]"},"time_extracted":"2022-01-01T00:00:00+00:00"} -{"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user__id":3,"user__sub__num":3,"user__some_numbers":"[1.414, 1.732]"},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string","null"]},"count":{"type":["integer","null"]},"user__id":{"type":["integer","null"]},"user__sub__num":{"type":["integer","null"]},"user__sub__custom_obj":{"type":["string","null"]},"user__some_numbers":{"type":["string","null"]}},"type":"object"},"key_properties":[]} +{"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user__id":1,"user__sub__num":1,"user__sub__custom_obj":"obj-hello","user__some_numbers":"[3.14, 2.718]"},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user__id":2,"user__sub__num":2,"user__sub__custom_obj":"obj-world","user__some_numbers":"[10.32, 1.618]"},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user__id":3,"user__sub__num":3,"user__sub__custom_obj":"obj-hello","user__some_numbers":"[1.414, 1.732]"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"STATE","value":{"bookmarks":{"mystream":{}}}} diff --git a/tests/snapshots/mapped_stream/flatten_depth_0.jsonl b/tests/snapshots/mapped_stream/flatten_depth_0.jsonl index 4107f9d19..aaec30f1b 100644 --- a/tests/snapshots/mapped_stream/flatten_depth_0.jsonl +++ b/tests/snapshots/mapped_stream/flatten_depth_0.jsonl @@ -1,6 +1,6 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string","null"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]}},"type":"object"},"key_properties":[]} -{"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1},"some_numbers":[3.14,2.718]}},"time_extracted":"2022-01-01T00:00:00+00:00"} -{"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2},"some_numbers":[10.32,1.618]}},"time_extracted":"2022-01-01T00:00:00+00:00"} -{"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3},"some_numbers":[1.414,1.732]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string","null"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]}},"type":"object"},"key_properties":[]} +{"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]}},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"STATE","value":{"bookmarks":{"mystream":{}}}} diff --git a/tests/snapshots/mapped_stream/flatten_depth_1.jsonl b/tests/snapshots/mapped_stream/flatten_depth_1.jsonl index 68022e1f0..317008dd8 100644 --- a/tests/snapshots/mapped_stream/flatten_depth_1.jsonl +++ b/tests/snapshots/mapped_stream/flatten_depth_1.jsonl @@ -1,6 +1,6 @@ {"type":"STATE","value":{}} {"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string","null"]},"count":{"type":["integer","null"]},"user__id":{"type":["integer","null"]},"user__sub":{"type":["string","null"]},"user__some_numbers":{"type":["string","null"]}},"type":"object"},"key_properties":[]} -{"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user__id":1,"user__sub":"{\"num\": 1}","user__some_numbers":"[3.14, 2.718]"},"time_extracted":"2022-01-01T00:00:00+00:00"} -{"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user__id":2,"user__sub":"{\"num\": 2}","user__some_numbers":"[10.32, 1.618]"},"time_extracted":"2022-01-01T00:00:00+00:00"} -{"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user__id":3,"user__sub":"{\"num\": 3}","user__some_numbers":"[1.414, 1.732]"},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user__id":1,"user__sub":"{\"num\": 1, \"custom_obj\": \"obj-hello\"}","user__some_numbers":"[3.14, 2.718]"},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user__id":2,"user__sub":"{\"num\": 2, \"custom_obj\": \"obj-world\"}","user__some_numbers":"[10.32, 1.618]"},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user__id":3,"user__sub":"{\"num\": 3, \"custom_obj\": \"obj-hello\"}","user__some_numbers":"[1.414, 1.732]"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"STATE","value":{"bookmarks":{"mystream":{}}}} diff --git a/tests/snapshots/mapped_stream/keep_all_fields.jsonl b/tests/snapshots/mapped_stream/keep_all_fields.jsonl index 4c337d0fa..7c9d13622 100644 --- a/tests/snapshots/mapped_stream/keep_all_fields.jsonl +++ b/tests/snapshots/mapped_stream/keep_all_fields.jsonl @@ -1,6 +1,6 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string","null"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"email_hash":{"type":["string","null"]}},"type":"object"},"key_properties":[]} -{"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1},"some_numbers":[3.14,2.718]},"email_hash":"c160f8cc69a4f0bf2b0362752353d060"},"time_extracted":"2022-01-01T00:00:00+00:00"} -{"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2},"some_numbers":[10.32,1.618]},"email_hash":"4b9bb80620f03eb3719e0a061c14283d"},"time_extracted":"2022-01-01T00:00:00+00:00"} -{"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3},"some_numbers":[1.414,1.732]},"email_hash":"426b189df1e2f359efe6ee90f2d2030f"},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string","null"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"email_hash":{"type":["string","null"]}},"type":"object"},"key_properties":[]} +{"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"email_hash":"c160f8cc69a4f0bf2b0362752353d060"},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"email_hash":"4b9bb80620f03eb3719e0a061c14283d"},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"email_hash":"426b189df1e2f359efe6ee90f2d2030f"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"STATE","value":{"bookmarks":{"mystream":{}}}} diff --git a/tests/snapshots/mapped_stream/map_and_flatten.jsonl b/tests/snapshots/mapped_stream/map_and_flatten.jsonl index 4d1ecd5d4..89397a046 100644 --- a/tests/snapshots/mapped_stream/map_and_flatten.jsonl +++ b/tests/snapshots/mapped_stream/map_and_flatten.jsonl @@ -1,6 +1,6 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string","null"]},"count":{"type":["integer","null"]},"user__id":{"type":["integer","null"]},"user__sub__num":{"type":["integer","null"]},"user__some_numbers":{"type":["string","null"]},"email_hash":{"type":["string","null"]}},"type":"object"},"key_properties":["email_hash"]} -{"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user__id":1,"user__sub__num":1,"user__some_numbers":"[3.14, 2.718]","email_hash":"c160f8cc69a4f0bf2b0362752353d060"},"time_extracted":"2022-01-01T00:00:00+00:00"} -{"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user__id":2,"user__sub__num":2,"user__some_numbers":"[10.32, 1.618]","email_hash":"4b9bb80620f03eb3719e0a061c14283d"},"time_extracted":"2022-01-01T00:00:00+00:00"} -{"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user__id":3,"user__sub__num":3,"user__some_numbers":"[1.414, 1.732]","email_hash":"426b189df1e2f359efe6ee90f2d2030f"},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string","null"]},"count":{"type":["integer","null"]},"user__id":{"type":["integer","null"]},"user__sub__num":{"type":["integer","null"]},"user__sub__custom_obj":{"type":["string","null"]},"user__some_numbers":{"type":["string","null"]},"email_hash":{"type":["string","null"]}},"type":"object"},"key_properties":["email_hash"]} +{"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user__id":1,"user__sub__num":1,"user__sub__custom_obj":"obj-hello","user__some_numbers":"[3.14, 2.718]","email_hash":"c160f8cc69a4f0bf2b0362752353d060"},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user__id":2,"user__sub__num":2,"user__sub__custom_obj":"obj-world","user__some_numbers":"[10.32, 1.618]","email_hash":"4b9bb80620f03eb3719e0a061c14283d"},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user__id":3,"user__sub__num":3,"user__sub__custom_obj":"obj-hello","user__some_numbers":"[1.414, 1.732]","email_hash":"426b189df1e2f359efe6ee90f2d2030f"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"STATE","value":{"bookmarks":{"mystream":{}}}} diff --git a/tests/snapshots/mapped_stream/no_map.jsonl b/tests/snapshots/mapped_stream/no_map.jsonl index 4107f9d19..aaec30f1b 100644 --- a/tests/snapshots/mapped_stream/no_map.jsonl +++ b/tests/snapshots/mapped_stream/no_map.jsonl @@ -1,6 +1,6 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string","null"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]}},"type":"object"},"key_properties":[]} -{"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1},"some_numbers":[3.14,2.718]}},"time_extracted":"2022-01-01T00:00:00+00:00"} -{"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2},"some_numbers":[10.32,1.618]}},"time_extracted":"2022-01-01T00:00:00+00:00"} -{"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3},"some_numbers":[1.414,1.732]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string","null"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]}},"type":"object"},"key_properties":[]} +{"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]}},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"STATE","value":{"bookmarks":{"mystream":{}}}} diff --git a/tests/snapshots/mapped_stream/record_to_column.jsonl b/tests/snapshots/mapped_stream/record_to_column.jsonl index 74e0326b7..622f5f23b 100644 --- a/tests/snapshots/mapped_stream/record_to_column.jsonl +++ b/tests/snapshots/mapped_stream/record_to_column.jsonl @@ -1,6 +1,6 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"_data":{"properties":{"email":{"type":["string","null"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]}},"type":["object","null"]}}},"key_properties":[]} -{"type":"RECORD","stream":"mystream","record":{"_data":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1},"some_numbers":[3.14,2.718]}}},"time_extracted":"2022-01-01T00:00:00+00:00"} -{"type":"RECORD","stream":"mystream","record":{"_data":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2},"some_numbers":[10.32,1.618]}}},"time_extracted":"2022-01-01T00:00:00+00:00"} -{"type":"RECORD","stream":"mystream","record":{"_data":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3},"some_numbers":[1.414,1.732]}}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"_data":{"properties":{"email":{"type":["string","null"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]}},"type":["object","null"]}}},"key_properties":[]} +{"type":"RECORD","stream":"mystream","record":{"_data":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]}}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"_data":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]}}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"mystream","record":{"_data":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]}}},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"STATE","value":{"bookmarks":{"mystream":{}}}} diff --git a/tests/snapshots/mapped_stream/sourced_stream_1.jsonl b/tests/snapshots/mapped_stream/sourced_stream_1.jsonl index 8868c39cd..d5dac940b 100644 --- a/tests/snapshots/mapped_stream/sourced_stream_1.jsonl +++ b/tests/snapshots/mapped_stream/sourced_stream_1.jsonl @@ -1,6 +1,6 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"sourced_stream_1","schema":{"properties":{"email":{"type":["string","null"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]}},"type":"object"},"key_properties":[]} -{"type":"RECORD","stream":"sourced_stream_1","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1},"some_numbers":[3.14,2.718]}},"time_extracted":"2022-01-01T00:00:00+00:00"} -{"type":"RECORD","stream":"sourced_stream_1","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2},"some_numbers":[10.32,1.618]}},"time_extracted":"2022-01-01T00:00:00+00:00"} -{"type":"RECORD","stream":"sourced_stream_1","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3},"some_numbers":[1.414,1.732]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"SCHEMA","stream":"sourced_stream_1","schema":{"properties":{"email":{"type":["string","null"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]}},"type":"object"},"key_properties":[]} +{"type":"RECORD","stream":"sourced_stream_1","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"sourced_stream_1","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"sourced_stream_1","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]}},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"STATE","value":{"bookmarks":{"mystream":{}}}} diff --git a/tests/snapshots/mapped_stream/sourced_stream_1_null_string.jsonl b/tests/snapshots/mapped_stream/sourced_stream_1_null_string.jsonl index 8868c39cd..d5dac940b 100644 --- a/tests/snapshots/mapped_stream/sourced_stream_1_null_string.jsonl +++ b/tests/snapshots/mapped_stream/sourced_stream_1_null_string.jsonl @@ -1,6 +1,6 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"sourced_stream_1","schema":{"properties":{"email":{"type":["string","null"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]}},"type":"object"},"key_properties":[]} -{"type":"RECORD","stream":"sourced_stream_1","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1},"some_numbers":[3.14,2.718]}},"time_extracted":"2022-01-01T00:00:00+00:00"} -{"type":"RECORD","stream":"sourced_stream_1","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2},"some_numbers":[10.32,1.618]}},"time_extracted":"2022-01-01T00:00:00+00:00"} -{"type":"RECORD","stream":"sourced_stream_1","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3},"some_numbers":[1.414,1.732]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"SCHEMA","stream":"sourced_stream_1","schema":{"properties":{"email":{"type":["string","null"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]}},"type":"object"},"key_properties":[]} +{"type":"RECORD","stream":"sourced_stream_1","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"sourced_stream_1","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"sourced_stream_1","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]}},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"STATE","value":{"bookmarks":{"mystream":{}}}} diff --git a/tests/snapshots/mapped_stream/sourced_stream_2.jsonl b/tests/snapshots/mapped_stream/sourced_stream_2.jsonl index 3bdc0db77..530c2c31f 100644 --- a/tests/snapshots/mapped_stream/sourced_stream_2.jsonl +++ b/tests/snapshots/mapped_stream/sourced_stream_2.jsonl @@ -1,6 +1,6 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"sourced_stream_2","schema":{"properties":{"email":{"type":["string","null"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]}},"type":"object"},"key_properties":[]} -{"type":"RECORD","stream":"sourced_stream_2","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1},"some_numbers":[3.14,2.718]}},"time_extracted":"2022-01-01T00:00:00+00:00"} -{"type":"RECORD","stream":"sourced_stream_2","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2},"some_numbers":[10.32,1.618]}},"time_extracted":"2022-01-01T00:00:00+00:00"} -{"type":"RECORD","stream":"sourced_stream_2","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3},"some_numbers":[1.414,1.732]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"SCHEMA","stream":"sourced_stream_2","schema":{"properties":{"email":{"type":["string","null"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]}},"type":"object"},"key_properties":[]} +{"type":"RECORD","stream":"sourced_stream_2","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"sourced_stream_2","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]}},"time_extracted":"2022-01-01T00:00:00+00:00"} +{"type":"RECORD","stream":"sourced_stream_2","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]}},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"STATE","value":{"bookmarks":{"mystream":{}}}}