diff --git a/target_snowflake/flattening.py b/target_snowflake/flattening.py index 40ab48ec..d86964e1 100644 --- a/target_snowflake/flattening.py +++ b/target_snowflake/flattening.py @@ -53,6 +53,7 @@ def flatten_schema(d, parent_key=None, sep='__', level=0, max_level=0): else: items.append((new_key, v)) else: + # handle situations where, e.g., an `anyOf` value is passed if len(v.values()) > 0: if list(v.values())[0][0]['type'] == 'string': list(v.values())[0][0]['type'] = ['null', 'string'] @@ -63,6 +64,9 @@ def flatten_schema(d, parent_key=None, sep='__', level=0, max_level=0): elif list(v.values())[0][0]['type'] == 'object': list(v.values())[0][0]['type'] = ['null', 'object'] items.append((new_key, list(v.values())[0][0])) + else: + list(v.values())[0][0]['type'] = ['null', 'string'] + items.append((new_key, list(v.values())[0][0])) key_func = lambda item: item[0] sorted_items = sorted(items, key=key_func) diff --git a/tests/unit/test_flattening.py b/tests/unit/test_flattening.py index 99f47d4c..5b04204b 100644 --- a/tests/unit/test_flattening.py +++ b/tests/unit/test_flattening.py @@ -61,6 +61,24 @@ def test_flatten_schema(self): } } + not_nested_schema_with_anyof_property_type = { + "type": "object", + "properties": { + "object_col": {"anyOf": [{"type": "object"}, {"type": ["null", "string"]}]}, + "array_col": {"anyOf": [{"type": "array"}, {"type": ["null", "string"]}]}, + "bool_col": {"anyOf": [{"type": ["boolean", "null"]}, {"type": ["null", "string"]}]} + } + } + flattened_schema_with_anyof_property_type = { + "object_col": {"type": ["null", "object"]}, + "array_col": {"type": ["null", "array"]}, + "bool_col": {"type": ["null", "string"]} + } + + # NO FLATTENING - Schema with anyOf properties should be cast to a single data type + self.assertEqual(flatten_schema(not_nested_schema_with_anyof_property_type), + flattened_schema_with_anyof_property_type) + # NO FLATTENING - Schema with object type property but without further properties should be a plain dictionary # No flattening (default) self.assertEqual(flatten_schema(nested_schema_with_properties), nested_schema_with_properties['properties'])