Skip to content

Commit

Permalink
Allow null datetimes (NoneType) (#18)
Browse files Browse the repository at this point in the history
* Allow null datetimes (NoneType)

* Add null check for all other property types

* One more

* Respond to comments
  • Loading branch information
NiallRees authored Nov 19, 2020
1 parent 9e41d3c commit c23108e
Showing 1 changed file with 12 additions and 9 deletions.
21 changes: 12 additions & 9 deletions tap_bigquery/sync_bigquery.py
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,14 @@ def do_sync(config, state, stream):
BATCH_TIMESTAMP]:
continue

if prop.format == "date-time":
if row[key] is None:
if prop.type[0] != "null":
raise ValueError(
"NULL value not allowed by the schema"
)
else:
record[key] = None
elif prop.format == "date-time":
if type(row[key]) == str:
r = dateutil.parser.parse(row[key])
elif type(row[key]) == datetime.date:
Expand All @@ -207,24 +214,20 @@ def do_sync(config, state, stream):
day=row[key].day)
elif type(row[key]) == datetime.datetime:
r = row[key]
else:
raise ValueError(
"Record does not match datetime schema %s" %
row[key])
record[key] = r.isoformat()
elif prop.type[1] == "string":
record[key] = str(row[key])
elif prop.type[1] == "number" and row[key]:
elif prop.type[1] == "number":
record[key] = Decimal(row[key])
elif prop.type[1] == "integer" and row[key]:
elif prop.type[1] == "integer":
record[key] = int(row[key])
else:
record[key] = row[key]

if LEGACY_TIMESTAMP in properties.keys():
record[LEGACY_TIMESTAMP ] = int(round(time.time() * 1000))
record[LEGACY_TIMESTAMP] = int(round(time.time() * 1000))
if EXTRACT_TIMESTAMP in properties.keys():
record[EXTRACT_TIMESTAMP ] = extract_tstamp.isoformat()
record[EXTRACT_TIMESTAMP] = extract_tstamp.isoformat()

singer.write_record(stream.stream, record)

Expand Down

0 comments on commit c23108e

Please sign in to comment.