diff --git a/.bumpversion.cfg b/.bumpversion.cfg deleted file mode 100644 index 580a19e62..000000000 --- a/.bumpversion.cfg +++ /dev/null @@ -1,27 +0,0 @@ -[bumpversion] -current_version = 0.5.0 -commit = True -tag = False -parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+)(?P\d+))? -serialize = - {major}.{minor}.{patch}-{release}{build} - {major}.{minor}.{patch} - -[bumpversion:part:release] -values = dev - -[bumpversion:file:pyproject.toml] -search = version = "{current_version}" -replace = version = "{new_version}" - -[bumpversion:file:docs/conf.py] -search = release = "{current_version}" -replace = release = "{new_version}" - -[bumpversion:file:cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml] -search = singer-sdk = "^{current_version}" -replace = singer-sdk = "^{new_version}" - -[bumpversion:file:cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml] -search = singer-sdk = "^{current_version}" -replace = singer-sdk = "^{new_version}" diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index f944e5329..000000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,95 +0,0 @@ -stages: - - setup - - test - - publish - -include: - - template: Security/License-Scanning.gitlab-ci.yml - -default: - before_script: - - python -V - - python -m pip install pipx - - python -m pipx ensurepath - - python -m pipx install poetry - # Force update PATH to include pipx executables - - export PATH=$PATH:/root/.local/bin - # Create virtual environment and make sure pip and setuptools are up-to-date - - poetry env use python - - poetry run pip install --upgrade pip setuptools - - poetry install --no-root - -variables: - PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" - LM_PYTHON_VERSION: 3 - -cache: - paths: - - .cache/pip - - venv/ - -generate_requirements: - stage: setup - image: - name: python:3.8 - before_script: - # Install poetry - - python -V - - python -m pip install pipx - - python -m pipx ensurepath - - python -m pipx install poetry - # Force update PATH to include pipx executables - - export PATH=$PATH:/root/.local/bin - script: - - poetry export --format requirements.txt --output requirements.txt - artifacts: - paths: - - requirements.txt - -license_scanning: - stage: test - before_script: [] # negate defaults above - dependencies: - - generate_requirements - -pypi_publish: - # release: - # name: '$CI_COMMIT_TAG' - # tag_name: '$CI_COMMIT_TAG' - # description: '$CI_COMMIT_TAG' - only: - - tags - stage: publish - parallel: - matrix: - - PYTHON_VERSION: ["3.8"] - image: python:${PYTHON_VERSION} - script: - - | - echo "Publishing tag '$CI_COMMIT_TAG' to PyPi, Ref='$CI_COMMIT_REF_NAME' and Namespace='$CI_PROJECT_NAMESPACE'..." - echo "Detected poetry version: v$(poetry version --short)" - if [[ "$CI_COMMIT_TAG" == "v$(poetry version --short)" ]]; - then - echo -e "\nPublishing to version ref 'v$(poetry version --short)'...\n\n" - poetry publish --build - else - echo -e "\nERROR - Tag '$CI_COMMIT_TAG' did not match detected version 'v$(poetry version --short)'." - exit 1 - fi - -pypi_prerelease: - when: manual - except: - - tags - stage: publish - parallel: - matrix: - - PYTHON_VERSION: ["3.8"] - image: python:${PYTHON_VERSION} - script: - - | - echo "Publishing to PyPi, Ref='$CI_COMMIT_REF_NAME' and Namespace='$CI_PROJECT_NAMESPACE'..." - poetry version $(poetry version --short)-dev.$CI_JOB_ID - poetry version --short - echo -e "\nPublishing to version ref '$(poetry version --short)'...\n\n" - poetry publish --build diff --git a/docs/conf.py b/docs/conf.py index 1b91724f0..ccd6fa98d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,6 +12,7 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # +from __future__ import annotations import sys from pathlib import Path @@ -29,7 +30,7 @@ release = "0.35.0" -# -- General configuration --------------------------------------------------- +# -- General configuration ------------------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom @@ -40,6 +41,7 @@ "sphinx.ext.autosectionlabel", "sphinx.ext.autosummary", "sphinx.ext.intersphinx", + "sphinx.ext.linkcode", "sphinx_copybutton", "myst_parser", "sphinx_reredirects", @@ -55,12 +57,7 @@ # This pattern also affects html_static_path and html_extra_path. exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] -# Show typehints in the description, along with parameter descriptions -autodoc_typehints = "description" -autodoc_class_signature = "separated" -autodoc_member_order = "groupwise" - -# -- Options for HTML output ------------------------------------------------- +# -- Options for HTML output ----------------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. @@ -118,7 +115,19 @@ "css/custom.css", ] -# -- Options for MyST -------------------------------------------------------- +# -- Options for AutoDoc --------------------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html#configuration + +# Show typehints in the description +autodoc_typehints = "description" + +# Display the signature as a method. +autodoc_class_signature = "separated" + +# Sort members by type. +autodoc_member_order = "groupwise" + +# -- Options for MyST ------------------------------------------------------------------ # https://myst-parser.readthedocs.io/en/latest/configuration.html myst_heading_anchors = 3 myst_enable_extensions = { @@ -129,9 +138,34 @@ "porting.html": "guides/porting.html", } -# -- Options for intersphinx ------------------------------------------------- +# -- Options for intersphinx ----------------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#configuration intersphinx_mapping = { "requests": ("https://requests.readthedocs.io/en/latest/", None), "python": ("https://docs.python.org/3/", None), } + +# -- Options for linkcode -------------------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/extensions/linkcode.html#configuration + + +def linkcode_resolve(domain: str, info: dict) -> str | None: + """Get URL to source code. + + Args: + domain: Language domain the object is in. + info: A dictionary with domain-specific keys. + + Returns: + A URL. + """ + if domain != "py": + return None + if not info["module"]: + return None + filename = info["module"].replace(".", "/") + + if filename == "singer_sdk": + filename = "singer_sdk/__init__" + + return f"https://github.com/meltano/sdk/tree/main/{filename}.py" diff --git a/singer_sdk/helpers/_flattening.py b/singer_sdk/helpers/_flattening.py index 77e3935b9..2a3e194d0 100644 --- a/singer_sdk/helpers/_flattening.py +++ b/singer_sdk/helpers/_flattening.py @@ -415,7 +415,14 @@ def _flatten_record( items: list[tuple[str, t.Any]] = [] for k, v in record_node.items(): new_key = flatten_key(k, parent_key, separator) - if isinstance(v, collections.abc.MutableMapping) and level < max_level: + # If the value is a dictionary, and the key is not in the schema, and the + # level is less than the max level, then we should continue to flatten. + if ( + isinstance(v, collections.abc.MutableMapping) + and flattened_schema + and new_key not in flattened_schema.get("properties", {}) + and (level < max_level) + ): items.extend( _flatten_record( v, diff --git a/singer_sdk/helpers/_typing.py b/singer_sdk/helpers/_typing.py index 6df38e83b..f3ea68e1b 100644 --- a/singer_sdk/helpers/_typing.py +++ b/singer_sdk/helpers/_typing.py @@ -59,6 +59,10 @@ def append_type(type_dict: dict, new_type: str) -> dict: result["anyOf"] = [result["anyOf"], new_type] return result + if "oneOf" in result: + result["oneOf"].append(new_type) + return result + if "type" in result: type_array = ( result["type"] if isinstance(result["type"], list) else [result["type"]] diff --git a/tests/core/test_flattening.py b/tests/core/test_flattening.py new file mode 100644 index 000000000..73169eab3 --- /dev/null +++ b/tests/core/test_flattening.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import pytest + +from singer_sdk.helpers._flattening import flatten_record + + +@pytest.mark.parametrize( + "flattened_schema, max_level, expected", + [ + pytest.param( + { + "properties": { + "key_1": {"type": ["null", "integer"]}, + "key_2__key_3": {"type": ["null", "string"]}, + "key_2__key_4": {"type": ["null", "object"]}, + } + }, + 99, + { + "key_1": 1, + "key_2__key_3": "value", + "key_2__key_4": '{"key_5": 1, "key_6": ["a", "b"]}', + }, + id="flattened schema limiting the max level", + ), + pytest.param( + { + "properties": { + "key_1": {"type": ["null", "integer"]}, + "key_2__key_3": {"type": ["null", "string"]}, + "key_2__key_4__key_5": {"type": ["null", "integer"]}, + "key_2__key_4__key_6": {"type": ["null", "array"]}, + } + }, + 99, + { + "key_1": 1, + "key_2__key_3": "value", + "key_2__key_4__key_5": 1, + "key_2__key_4__key_6": '["a", "b"]', + }, + id="flattened schema not limiting the max level", + ), + pytest.param( + { + "properties": { + "key_1": {"type": ["null", "integer"]}, + "key_2__key_3": {"type": ["null", "string"]}, + "key_2__key_4__key_5": {"type": ["null", "integer"]}, + "key_2__key_4__key_6": {"type": ["null", "array"]}, + } + }, + 1, + { + "key_1": 1, + "key_2__key_3": "value", + "key_2__key_4": '{"key_5": 1, "key_6": ["a", "b"]}', + }, + id="max level limiting flattened schema", + ), + ], +) +def test_flatten_record(flattened_schema, max_level, expected): + """Test flatten_record to obey the max_level and flattened_schema parameters.""" + record = { + "key_1": 1, + "key_2": {"key_3": "value", "key_4": {"key_5": 1, "key_6": ["a", "b"]}}, + } + + result = flatten_record( + record, max_level=max_level, flattened_schema=flattened_schema + ) + assert expected == result diff --git a/tests/core/test_typing.py b/tests/core/test_typing.py index 7bb0ab362..b985aeecc 100644 --- a/tests/core/test_typing.py +++ b/tests/core/test_typing.py @@ -19,6 +19,7 @@ PropertiesList, Property, StringType, + append_type, to_sql_type, ) @@ -318,3 +319,33 @@ def test_conform_primitives(): ) def test_to_sql_type(jsonschema_type, expected): assert isinstance(to_sql_type(jsonschema_type), expected) + + +@pytest.mark.parametrize( + "type_dict,expected", + [ + pytest.param({"type": "string"}, {"type": ["string", "null"]}, id="string"), + pytest.param({"type": "integer"}, {"type": ["integer", "null"]}, id="integer"), + pytest.param({"type": "number"}, {"type": ["number", "null"]}, id="number"), + pytest.param({"type": "boolean"}, {"type": ["boolean", "null"]}, id="boolean"), + pytest.param( + {"type": "object", "properties": {}}, + {"type": ["object", "null"], "properties": {}}, + id="object", + ), + pytest.param({"type": "array"}, {"type": ["array", "null"]}, id="array"), + pytest.param( + {"anyOf": [{"type": "integer"}, {"type": "number"}]}, + {"anyOf": [{"type": "integer"}, {"type": "number"}, "null"]}, + id="anyOf", + ), + pytest.param( + {"oneOf": [{"type": "integer"}, {"type": "number"}]}, + {"oneOf": [{"type": "integer"}, {"type": "number"}, "null"]}, + id="oneOf", + ), + ], +) +def test_append_null(type_dict: dict, expected: dict): + result = append_type(type_dict, "null") + assert result == expected