From 578c3a8f4fc4aa1c0d5221f4a54ee459dd0bc819 Mon Sep 17 00:00:00 2001 From: Alberto Costa Date: Mon, 29 Jun 2020 14:58:09 +0200 Subject: [PATCH] Fix: accept empty schema --- target_snowflake/__init__.py | 2 +- target_snowflake/db_sync.py | 13 +++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/target_snowflake/__init__.py b/target_snowflake/__init__.py index 700e1c15..d2eb582c 100644 --- a/target_snowflake/__init__.py +++ b/target_snowflake/__init__.py @@ -145,7 +145,7 @@ def reset_new_value(record: Dict, key: str, format: str): reset_new_value(record, key, type_dict['format']) break else: - if 'string' in schema['properties'][key]['type'] and \ + if 'string' in schema['properties'][key].get('type', []) and \ schema['properties'][key].get('format', None) in {'date-time', 'time', 'date'}: reset_new_value(record, key, schema['properties'][key]['format']) diff --git a/target_snowflake/db_sync.py b/target_snowflake/db_sync.py index a0ae6bdb..05c5ea8f 100644 --- a/target_snowflake/db_sync.py +++ b/target_snowflake/db_sync.py @@ -51,10 +51,10 @@ def validate_config(config): def column_type(schema_property): - property_type = schema_property['type'] + property_type = schema_property.get('type', []) property_format = schema_property['format'] if 'format' in schema_property else None column_type = 'text' - if 'object' in property_type or 'array' in property_type: + if 'object' in property_type or 'array' in property_type or property_type == []: column_type = 'variant' # Every date-time JSON value is currently mapped to TIMESTAMP_NTZ @@ -80,9 +80,11 @@ def column_type(schema_property): def column_trans(schema_property): - property_type = schema_property['type'] + property_type = schema_property.get('type', []) column_trans = '' - if 'object' in property_type or 'array' in property_type: + if property_type == []: + column_trans = 'to_variant' + elif 'object' in property_type or 'array' in property_type: column_trans = 'parse_json' elif schema_property.get('format') == 'binary': column_trans = 'to_binary' @@ -135,6 +137,9 @@ def flatten_schema(d, parent_key=[], sep='__', level=0, max_level=0): elif list(v.values())[0][0]['type'] == 'object': list(v.values())[0][0]['type'] = ['null', 'object'] items.append((new_key, list(v.values())[0][0])) + else: + # In case of empty schema {} + items.append((new_key, {})) key_func = lambda item: item[0] sorted_items = sorted(items, key=key_func)