From e3565eea17bcdaa22a756fe6c1c040bc361e4d58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= <16805946+edgarrmondragon@users.noreply.github.com> Date: Thu, 15 Feb 2024 15:11:37 -0600 Subject: [PATCH 1/4] fix: Instances of `oneOf` are now handled by null-appending logic (#2245) fix: Instance of `oneOf` are now handled by null-appending logic --- singer_sdk/helpers/_typing.py | 4 ++++ tests/core/test_typing.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/singer_sdk/helpers/_typing.py b/singer_sdk/helpers/_typing.py index 6df38e83b..f3ea68e1b 100644 --- a/singer_sdk/helpers/_typing.py +++ b/singer_sdk/helpers/_typing.py @@ -59,6 +59,10 @@ def append_type(type_dict: dict, new_type: str) -> dict: result["anyOf"] = [result["anyOf"], new_type] return result + if "oneOf" in result: + result["oneOf"].append(new_type) + return result + if "type" in result: type_array = ( result["type"] if isinstance(result["type"], list) else [result["type"]] diff --git a/tests/core/test_typing.py b/tests/core/test_typing.py index 7bb0ab362..b985aeecc 100644 --- a/tests/core/test_typing.py +++ b/tests/core/test_typing.py @@ -19,6 +19,7 @@ PropertiesList, Property, StringType, + append_type, to_sql_type, ) @@ -318,3 +319,33 @@ def test_conform_primitives(): ) def test_to_sql_type(jsonschema_type, expected): assert isinstance(to_sql_type(jsonschema_type), expected) + + +@pytest.mark.parametrize( + "type_dict,expected", + [ + pytest.param({"type": "string"}, {"type": ["string", "null"]}, id="string"), + pytest.param({"type": "integer"}, {"type": ["integer", "null"]}, id="integer"), + pytest.param({"type": "number"}, {"type": ["number", "null"]}, id="number"), + pytest.param({"type": "boolean"}, {"type": ["boolean", "null"]}, id="boolean"), + pytest.param( + {"type": "object", "properties": {}}, + {"type": ["object", "null"], "properties": {}}, + id="object", + ), + pytest.param({"type": "array"}, {"type": ["array", "null"]}, id="array"), + pytest.param( + {"anyOf": [{"type": "integer"}, {"type": "number"}]}, + {"anyOf": [{"type": "integer"}, {"type": "number"}, "null"]}, + id="anyOf", + ), + pytest.param( + {"oneOf": [{"type": "integer"}, {"type": "number"}]}, + {"oneOf": [{"type": "integer"}, {"type": "number"}, "null"]}, + id="oneOf", + ), + ], +) +def test_append_null(type_dict: dict, expected: dict): + result = append_type(type_dict, "null") + assert result == expected From 15a22efafab207371c5040ae63b7fec2a79f0b74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= <16805946+edgarrmondragon@users.noreply.github.com> Date: Thu, 15 Feb 2024 15:21:36 -0600 Subject: [PATCH 2/4] chore: Cleaned up some unused files (#2240) chore: Clean up unused files --- .bumpversion.cfg | 27 -------------- .gitlab-ci.yml | 95 ------------------------------------------------ 2 files changed, 122 deletions(-) delete mode 100644 .bumpversion.cfg delete mode 100644 .gitlab-ci.yml diff --git a/.bumpversion.cfg b/.bumpversion.cfg deleted file mode 100644 index 580a19e62..000000000 --- a/.bumpversion.cfg +++ /dev/null @@ -1,27 +0,0 @@ -[bumpversion] -current_version = 0.5.0 -commit = True -tag = False -parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+)(?P\d+))? -serialize = - {major}.{minor}.{patch}-{release}{build} - {major}.{minor}.{patch} - -[bumpversion:part:release] -values = dev - -[bumpversion:file:pyproject.toml] -search = version = "{current_version}" -replace = version = "{new_version}" - -[bumpversion:file:docs/conf.py] -search = release = "{current_version}" -replace = release = "{new_version}" - -[bumpversion:file:cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml] -search = singer-sdk = "^{current_version}" -replace = singer-sdk = "^{new_version}" - -[bumpversion:file:cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml] -search = singer-sdk = "^{current_version}" -replace = singer-sdk = "^{new_version}" diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index f944e5329..000000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,95 +0,0 @@ -stages: - - setup - - test - - publish - -include: - - template: Security/License-Scanning.gitlab-ci.yml - -default: - before_script: - - python -V - - python -m pip install pipx - - python -m pipx ensurepath - - python -m pipx install poetry - # Force update PATH to include pipx executables - - export PATH=$PATH:/root/.local/bin - # Create virtual environment and make sure pip and setuptools are up-to-date - - poetry env use python - - poetry run pip install --upgrade pip setuptools - - poetry install --no-root - -variables: - PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" - LM_PYTHON_VERSION: 3 - -cache: - paths: - - .cache/pip - - venv/ - -generate_requirements: - stage: setup - image: - name: python:3.8 - before_script: - # Install poetry - - python -V - - python -m pip install pipx - - python -m pipx ensurepath - - python -m pipx install poetry - # Force update PATH to include pipx executables - - export PATH=$PATH:/root/.local/bin - script: - - poetry export --format requirements.txt --output requirements.txt - artifacts: - paths: - - requirements.txt - -license_scanning: - stage: test - before_script: [] # negate defaults above - dependencies: - - generate_requirements - -pypi_publish: - # release: - # name: '$CI_COMMIT_TAG' - # tag_name: '$CI_COMMIT_TAG' - # description: '$CI_COMMIT_TAG' - only: - - tags - stage: publish - parallel: - matrix: - - PYTHON_VERSION: ["3.8"] - image: python:${PYTHON_VERSION} - script: - - | - echo "Publishing tag '$CI_COMMIT_TAG' to PyPi, Ref='$CI_COMMIT_REF_NAME' and Namespace='$CI_PROJECT_NAMESPACE'..." - echo "Detected poetry version: v$(poetry version --short)" - if [[ "$CI_COMMIT_TAG" == "v$(poetry version --short)" ]]; - then - echo -e "\nPublishing to version ref 'v$(poetry version --short)'...\n\n" - poetry publish --build - else - echo -e "\nERROR - Tag '$CI_COMMIT_TAG' did not match detected version 'v$(poetry version --short)'." - exit 1 - fi - -pypi_prerelease: - when: manual - except: - - tags - stage: publish - parallel: - matrix: - - PYTHON_VERSION: ["3.8"] - image: python:${PYTHON_VERSION} - script: - - | - echo "Publishing to PyPi, Ref='$CI_COMMIT_REF_NAME' and Namespace='$CI_PROJECT_NAMESPACE'..." - poetry version $(poetry version --short)-dev.$CI_JOB_ID - poetry version --short - echo -e "\nPublishing to version ref '$(poetry version --short)'...\n\n" - poetry publish --build From d9484a714f299d5c38848efafc07e0af5216d637 Mon Sep 17 00:00:00 2001 From: Joao Amaral <7281460+joaopamaral@users.noreply.github.com> Date: Thu, 15 Feb 2024 19:41:47 -0300 Subject: [PATCH 3/4] fix: Force flattened record according to provided flattened schema (#2243) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Force flattened record according to provided flattened schema * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix import * fix * revert collections change * ruff linter fixes * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * ruff linter fixes * fix * fix * add test * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * removing None option for flatten schema and max level * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * revert typing * fix check * Add a short comment --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Edgar Ramírez Mondragón <16805946+edgarrmondragon@users.noreply.github.com> Co-authored-by: Edgar Ramírez-Mondragón --- singer_sdk/helpers/_flattening.py | 9 +++- tests/core/test_flattening.py | 74 +++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 tests/core/test_flattening.py diff --git a/singer_sdk/helpers/_flattening.py b/singer_sdk/helpers/_flattening.py index 77e3935b9..2a3e194d0 100644 --- a/singer_sdk/helpers/_flattening.py +++ b/singer_sdk/helpers/_flattening.py @@ -415,7 +415,14 @@ def _flatten_record( items: list[tuple[str, t.Any]] = [] for k, v in record_node.items(): new_key = flatten_key(k, parent_key, separator) - if isinstance(v, collections.abc.MutableMapping) and level < max_level: + # If the value is a dictionary, and the key is not in the schema, and the + # level is less than the max level, then we should continue to flatten. + if ( + isinstance(v, collections.abc.MutableMapping) + and flattened_schema + and new_key not in flattened_schema.get("properties", {}) + and (level < max_level) + ): items.extend( _flatten_record( v, diff --git a/tests/core/test_flattening.py b/tests/core/test_flattening.py new file mode 100644 index 000000000..73169eab3 --- /dev/null +++ b/tests/core/test_flattening.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import pytest + +from singer_sdk.helpers._flattening import flatten_record + + +@pytest.mark.parametrize( + "flattened_schema, max_level, expected", + [ + pytest.param( + { + "properties": { + "key_1": {"type": ["null", "integer"]}, + "key_2__key_3": {"type": ["null", "string"]}, + "key_2__key_4": {"type": ["null", "object"]}, + } + }, + 99, + { + "key_1": 1, + "key_2__key_3": "value", + "key_2__key_4": '{"key_5": 1, "key_6": ["a", "b"]}', + }, + id="flattened schema limiting the max level", + ), + pytest.param( + { + "properties": { + "key_1": {"type": ["null", "integer"]}, + "key_2__key_3": {"type": ["null", "string"]}, + "key_2__key_4__key_5": {"type": ["null", "integer"]}, + "key_2__key_4__key_6": {"type": ["null", "array"]}, + } + }, + 99, + { + "key_1": 1, + "key_2__key_3": "value", + "key_2__key_4__key_5": 1, + "key_2__key_4__key_6": '["a", "b"]', + }, + id="flattened schema not limiting the max level", + ), + pytest.param( + { + "properties": { + "key_1": {"type": ["null", "integer"]}, + "key_2__key_3": {"type": ["null", "string"]}, + "key_2__key_4__key_5": {"type": ["null", "integer"]}, + "key_2__key_4__key_6": {"type": ["null", "array"]}, + } + }, + 1, + { + "key_1": 1, + "key_2__key_3": "value", + "key_2__key_4": '{"key_5": 1, "key_6": ["a", "b"]}', + }, + id="max level limiting flattened schema", + ), + ], +) +def test_flatten_record(flattened_schema, max_level, expected): + """Test flatten_record to obey the max_level and flattened_schema parameters.""" + record = { + "key_1": 1, + "key_2": {"key_3": "value", "key_4": {"key_5": 1, "key_6": ["a", "b"]}}, + } + + result = flatten_record( + record, max_level=max_level, flattened_schema=flattened_schema + ) + assert expected == result From 364f490a566f24d20f4892f8fccafe32abee36f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= <16805946+edgarrmondragon@users.noreply.github.com> Date: Thu, 15 Feb 2024 17:05:26 -0600 Subject: [PATCH 4/4] docs: Linked reference docs to source code (#2239) docs: Link to source code Closes https://github.com/meltano/sdk/issues/149 --- docs/conf.py | 52 +++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 9 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 1b91724f0..ccd6fa98d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,6 +12,7 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # +from __future__ import annotations import sys from pathlib import Path @@ -29,7 +30,7 @@ release = "0.35.0" -# -- General configuration --------------------------------------------------- +# -- General configuration ------------------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom @@ -40,6 +41,7 @@ "sphinx.ext.autosectionlabel", "sphinx.ext.autosummary", "sphinx.ext.intersphinx", + "sphinx.ext.linkcode", "sphinx_copybutton", "myst_parser", "sphinx_reredirects", @@ -55,12 +57,7 @@ # This pattern also affects html_static_path and html_extra_path. exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] -# Show typehints in the description, along with parameter descriptions -autodoc_typehints = "description" -autodoc_class_signature = "separated" -autodoc_member_order = "groupwise" - -# -- Options for HTML output ------------------------------------------------- +# -- Options for HTML output ----------------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. @@ -118,7 +115,19 @@ "css/custom.css", ] -# -- Options for MyST -------------------------------------------------------- +# -- Options for AutoDoc --------------------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html#configuration + +# Show typehints in the description +autodoc_typehints = "description" + +# Display the signature as a method. +autodoc_class_signature = "separated" + +# Sort members by type. +autodoc_member_order = "groupwise" + +# -- Options for MyST ------------------------------------------------------------------ # https://myst-parser.readthedocs.io/en/latest/configuration.html myst_heading_anchors = 3 myst_enable_extensions = { @@ -129,9 +138,34 @@ "porting.html": "guides/porting.html", } -# -- Options for intersphinx ------------------------------------------------- +# -- Options for intersphinx ----------------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#configuration intersphinx_mapping = { "requests": ("https://requests.readthedocs.io/en/latest/", None), "python": ("https://docs.python.org/3/", None), } + +# -- Options for linkcode -------------------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/extensions/linkcode.html#configuration + + +def linkcode_resolve(domain: str, info: dict) -> str | None: + """Get URL to source code. + + Args: + domain: Language domain the object is in. + info: A dictionary with domain-specific keys. + + Returns: + A URL. + """ + if domain != "py": + return None + if not info["module"]: + return None + filename = info["module"].replace(".", "/") + + if filename == "singer_sdk": + filename = "singer_sdk/__init__" + + return f"https://github.com/meltano/sdk/tree/main/{filename}.py"