Skip to content
This repository has been archived by the owner on Sep 23, 2024. It is now read-only.

Schema fix #236

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Schema fix #236

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
92fcb94
Merge pull request #3 from transferwise/master
elephantum Jun 7, 2024
9561277
object, array, json, jsonb -> string
SergeyZ06 Jun 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 20 additions & 10 deletions tap_postgres/discovery_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,11 +154,16 @@ def discover_columns(connection, table_info):

column_schemas = {col_name: schema_for_column(col_info) for col_name, col_info in columns.items()}

schema = {'type': 'object',
'properties': column_schemas,
'definitions': {}}
# schema = {'type': 'object',
# 'properties': column_schemas,
# 'definitions': {}}

schema = include_array_schemas(columns, schema)
schema = {
'type': 'object',
'properties': column_schemas
}

# schema = include_array_schemas(columns, schema)

for c_name in column_schemas.keys():
mdata = write_sql_data_type_md(mdata, columns[c_name])
Expand Down Expand Up @@ -219,16 +224,18 @@ def schema_for_column_datatype(col):
return schema

if data_type == 'hstore':
schema['type'] = nullable_column('object', col.is_primary_key)
schema['properties'] = {}
# schema['type'] = nullable_column('object', col.is_primary_key)
# schema['properties'] = {}
schema['type'] = nullable_columns('string', col.is_primary_key)
return schema

if data_type == 'citext':
schema['type'] = nullable_column('string', col.is_primary_key)
return schema

if data_type in JSON_TYPES:
schema['type'] = nullable_columns(['object', 'array'], col.is_primary_key)
# schema['type'] = nullable_columns(['object', 'array'], col.is_primary_key)
schema['type'] = nullable_columns(['string'], col.is_primary_key)
return schema

if data_type == 'numeric':
Expand Down Expand Up @@ -291,9 +298,12 @@ def schema_for_column(col_info):
# either. These means we can say nothing about an array column. its items may be more arrays or primitive types
# like integers and this can vary on a row by row basis

column_schema = {'type': ["null", "array"]}
if not col_info.is_array:
return schema_for_column_datatype(col_info)
# column_schema = {'type': ["null", "array"]}
column_schema = {'type': ["null", "string"]}

# if not col_info.is_array:
# return schema_for_column_datatype(col_info)
return schema_for_column_datatype(col_info)

if col_info.sql_data_type == 'integer[]':
column_schema['items'] = {'$ref': '#/definitions/sdc_recursive_integer_array'}
Expand Down