Merge branch 'main' into 2155-target-set-batch-size-rows

docs/conf.py

@@ -12,6 +12,7 @@
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
+from __future__ import annotations
 
 import sys
 from pathlib import Path
@@ -29,7 +30,7 @@
 release = "0.35.0"
 
 
-# -- General configuration ---------------------------------------------------
+# -- General configuration -------------------------------------------------------------
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
@@ -40,6 +41,7 @@
     "sphinx.ext.autosectionlabel",
     "sphinx.ext.autosummary",
     "sphinx.ext.intersphinx",
+    "sphinx.ext.linkcode",
     "sphinx_copybutton",
     "myst_parser",
     "sphinx_reredirects",
@@ -55,12 +57,7 @@
 # This pattern also affects html_static_path and html_extra_path.
 exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 
-# Show typehints in the description, along with parameter descriptions
-autodoc_typehints = "description"
-autodoc_class_signature = "separated"
-autodoc_member_order = "groupwise"
-
-# -- Options for HTML output -------------------------------------------------
+# -- Options for HTML output -----------------------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
@@ -118,7 +115,19 @@
     "css/custom.css",
 ]
 
-# -- Options for MyST --------------------------------------------------------
+# -- Options for AutoDoc ---------------------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html#configuration
+
+# Show typehints in the description
+autodoc_typehints = "description"
+
+# Display the signature as a method.
+autodoc_class_signature = "separated"
+
+# Sort members by type.
+autodoc_member_order = "groupwise"
+
+# -- Options for MyST ------------------------------------------------------------------
 # https://myst-parser.readthedocs.io/en/latest/configuration.html
 myst_heading_anchors = 3
 myst_enable_extensions = {
@@ -129,9 +138,34 @@
     "porting.html": "guides/porting.html",
 }
 
-# -- Options for intersphinx -------------------------------------------------
+# -- Options for intersphinx -----------------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#configuration
 intersphinx_mapping = {
     "requests": ("https://requests.readthedocs.io/en/latest/", None),
     "python": ("https://docs.python.org/3/", None),
 }
+
+# -- Options for linkcode --------------------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/extensions/linkcode.html#configuration
+
+
+def linkcode_resolve(domain: str, info: dict) -> str | None:
+    """Get URL to source code.
+
+    Args:
+        domain: Language domain the object is in.
+        info: A dictionary with domain-specific keys.
+
+    Returns:
+        A URL.
+    """
+    if domain != "py":
+        return None
+    if not info["module"]:
+        return None
+    filename = info["module"].replace(".", "/")
+
+    if filename == "singer_sdk":
+        filename = "singer_sdk/__init__"
+
+    return f"https://github.com/meltano/sdk/tree/main/{filename}.py"

singer_sdk/helpers/_flattening.py

@@ -415,7 +415,14 @@ def _flatten_record(
     items: list[tuple[str, t.Any]] = []
     for k, v in record_node.items():
         new_key = flatten_key(k, parent_key, separator)
-        if isinstance(v, collections.abc.MutableMapping) and level < max_level:
+        # If the value is a dictionary, and the key is not in the schema, and the
+        # level is less than the max level, then we should continue to flatten.
+        if (
+            isinstance(v, collections.abc.MutableMapping)
+            and flattened_schema
+            and new_key not in flattened_schema.get("properties", {})
+            and (level < max_level)
+        ):
             items.extend(
                 _flatten_record(
                     v,

singer_sdk/helpers/_typing.py

@@ -59,6 +59,10 @@ def append_type(type_dict: dict, new_type: str) -> dict:
             result["anyOf"] = [result["anyOf"], new_type]
         return result
 
+    if "oneOf" in result:
+        result["oneOf"].append(new_type)
+        return result
+
     if "type" in result:
         type_array = (
             result["type"] if isinstance(result["type"], list) else [result["type"]]

tests/core/test_flattening.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+import pytest
+
+from singer_sdk.helpers._flattening import flatten_record
+
+
+@pytest.mark.parametrize(
+    "flattened_schema, max_level, expected",
+    [
+        pytest.param(
+            {
+                "properties": {
+                    "key_1": {"type": ["null", "integer"]},
+                    "key_2__key_3": {"type": ["null", "string"]},
+                    "key_2__key_4": {"type": ["null", "object"]},
+                }
+            },
+            99,
+            {
+                "key_1": 1,
+                "key_2__key_3": "value",
+                "key_2__key_4": '{"key_5": 1, "key_6": ["a", "b"]}',
+            },
+            id="flattened schema limiting the max level",
+        ),
+        pytest.param(
+            {
+                "properties": {
+                    "key_1": {"type": ["null", "integer"]},
+                    "key_2__key_3": {"type": ["null", "string"]},
+                    "key_2__key_4__key_5": {"type": ["null", "integer"]},
+                    "key_2__key_4__key_6": {"type": ["null", "array"]},
+                }
+            },
+            99,
+            {
+                "key_1": 1,
+                "key_2__key_3": "value",
+                "key_2__key_4__key_5": 1,
+                "key_2__key_4__key_6": '["a", "b"]',
+            },
+            id="flattened schema not limiting the max level",
+        ),
+        pytest.param(
+            {
+                "properties": {
+                    "key_1": {"type": ["null", "integer"]},
+                    "key_2__key_3": {"type": ["null", "string"]},
+                    "key_2__key_4__key_5": {"type": ["null", "integer"]},
+                    "key_2__key_4__key_6": {"type": ["null", "array"]},
+                }
+            },
+            1,
+            {
+                "key_1": 1,
+                "key_2__key_3": "value",
+                "key_2__key_4": '{"key_5": 1, "key_6": ["a", "b"]}',
+            },
+            id="max level limiting flattened schema",
+        ),
+    ],
+)
+def test_flatten_record(flattened_schema, max_level, expected):
+    """Test flatten_record to obey the max_level and flattened_schema parameters."""
+    record = {
+        "key_1": 1,
+        "key_2": {"key_3": "value", "key_4": {"key_5": 1, "key_6": ["a", "b"]}},
+    }
+
+    result = flatten_record(
+        record, max_level=max_level, flattened_schema=flattened_schema
+    )
+    assert expected == result

tests/core/test_typing.py

@@ -19,6 +19,7 @@
     PropertiesList,
     Property,
     StringType,
+    append_type,
     to_sql_type,
 )
 
@@ -318,3 +319,33 @@ def test_conform_primitives():
 )
 def test_to_sql_type(jsonschema_type, expected):
     assert isinstance(to_sql_type(jsonschema_type), expected)
+
+
+@pytest.mark.parametrize(
+    "type_dict,expected",
+    [
+        pytest.param({"type": "string"}, {"type": ["string", "null"]}, id="string"),
+        pytest.param({"type": "integer"}, {"type": ["integer", "null"]}, id="integer"),
+        pytest.param({"type": "number"}, {"type": ["number", "null"]}, id="number"),
+        pytest.param({"type": "boolean"}, {"type": ["boolean", "null"]}, id="boolean"),
+        pytest.param(
+            {"type": "object", "properties": {}},
+            {"type": ["object", "null"], "properties": {}},
+            id="object",
+        ),
+        pytest.param({"type": "array"}, {"type": ["array", "null"]}, id="array"),
+        pytest.param(
+            {"anyOf": [{"type": "integer"}, {"type": "number"}]},
+            {"anyOf": [{"type": "integer"}, {"type": "number"}, "null"]},
+            id="anyOf",
+        ),
+        pytest.param(
+            {"oneOf": [{"type": "integer"}, {"type": "number"}]},
+            {"oneOf": [{"type": "integer"}, {"type": "number"}, "null"]},
+            id="oneOf",
+        ),
+    ],
+)
+def test_append_null(type_dict: dict, expected: dict):
+    result = append_type(type_dict, "null")
+    assert result == expected