From 8d31e29e0e62a2bfe05e36f09e0fd00f9cf3081d Mon Sep 17 00:00:00 2001 From: Ivan Afonichkin <56842315+ivan-transferwise@users.noreply.github.com> Date: Tue, 7 Jan 2020 14:53:42 +0000 Subject: [PATCH] Tests for invalid and valid records from postgres-tap (#55) * Add optional validate_records option * fixed pylint * Introduce InvalidTableStructureException * Tests for invalid and valid records from postgres-tap * trigger build * do not raise InvalidTableStructureException in flush_records * fixed test * trigger build * removed extra tab Co-authored-by: Peter Kosztolanyi --- .../messages-pg-with-invalid-records.json | 4 ++++ .../messages-pg-with-valid-records.json | 4 ++++ tests/integration/test_target_snowflake.py | 19 +++++++++++++++++++ 3 files changed, 27 insertions(+) create mode 100644 tests/integration/resources/messages-pg-with-invalid-records.json create mode 100644 tests/integration/resources/messages-pg-with-valid-records.json diff --git a/tests/integration/resources/messages-pg-with-invalid-records.json b/tests/integration/resources/messages-pg-with-invalid-records.json new file mode 100644 index 00000000..ca812370 --- /dev/null +++ b/tests/integration/resources/messages-pg-with-invalid-records.json @@ -0,0 +1,4 @@ +{"type": "SCHEMA", "stream": "pg-invalid-records", "schema": {"definitions": {"sdc_recursive_boolean_array": {"items": {"$ref": "#/definitions/sdc_recursive_boolean_array"}, "type": ["null", "boolean", "array"]}, "sdc_recursive_integer_array": {"items": {"$ref": "#/definitions/sdc_recursive_integer_array"}, "type": ["null", "integer", "array"]}, "sdc_recursive_number_array": {"items": {"$ref": "#/definitions/sdc_recursive_number_array"}, "type": ["null", "number", "array"]}, "sdc_recursive_object_array": {"items": {"$ref": "#/definitions/sdc_recursive_object_array"}, "type": ["null", "object", "array"]}, "sdc_recursive_string_array": {"items": {"$ref": "#/definitions/sdc_recursive_string_array"}, "type": ["null", "string", "array"]}, "sdc_recursive_timestamp_array": {"format": "date-time", "items": {"$ref": "#/definitions/sdc_recursive_timestamp_array"}, "type": ["null", "string", "array"]}}, "properties": {"id": {"maximum": 9223372036854775807, "minimum": -9223372036854775808, "type": ["integer"]}, "results": {"type": ["null", "object"]}, "time_created": {"format": "date-time", "type": ["null", "string"]}}, "type": "object"}, "key_properties": ["id"], "bookmark_properties": ["id"]} +{"type": "STATE", "value": {"bookmarks": {"pg-invalid-records": {"replication_key": "id", "replication_key_value": 52799009, "version": 1, "last_replication_method": "INCREMENTAL"}}, "currently_syncing": "pg-invalid-records"}} +{"type": "ACTIVATE_VERSION", "stream": "pg-invalid-records", "version": 1} +{"type": "RECORD", "stream": "pg-invalid-records", "record": {"id": 52799009, "results": "[{\"rule\": \"XYZ\", \"reason\": \"XYZ\", \"actions\": []}, {\"rule\": \"XYZ\", \"reason\": \"XYZ\", \"actions\": []}, {\"rule\": \"XYZ\", \"reason\": \"XYZ\", \"actions\": []}, {\"rule\": \"XYZ\", \"reason\": \"XYZ\", \"actions\": []}, {\"rule\": \"XYZ\", \"reason\": \"XYZ\", \"actions\": []}, {\"rule\": \"XYZ\", \"reason\": \"XYZ\", \"actions\": []}]", "time_created": "2019-12-17T11:40:15.296955+00:00"}, "version": 1, "time_extracted": "2019-12-17T12:04:46.015519Z"} diff --git a/tests/integration/resources/messages-pg-with-valid-records.json b/tests/integration/resources/messages-pg-with-valid-records.json new file mode 100644 index 00000000..0afe3912 --- /dev/null +++ b/tests/integration/resources/messages-pg-with-valid-records.json @@ -0,0 +1,4 @@ +{"type": "SCHEMA", "stream": "pg-valid-records", "schema": {"definitions": {"sdc_recursive_boolean_array": {"items": {"$ref": "#/definitions/sdc_recursive_boolean_array"}, "type": ["null", "boolean", "array"]}, "sdc_recursive_integer_array": {"items": {"$ref": "#/definitions/sdc_recursive_integer_array"}, "type": ["null", "integer", "array"]}, "sdc_recursive_number_array": {"items": {"$ref": "#/definitions/sdc_recursive_number_array"}, "type": ["null", "number", "array"]}, "sdc_recursive_object_array": {"items": {"$ref": "#/definitions/sdc_recursive_object_array"}, "type": ["null", "object", "array"]}, "sdc_recursive_string_array": {"items": {"$ref": "#/definitions/sdc_recursive_string_array"}, "type": ["null", "string", "array"]}, "sdc_recursive_timestamp_array": {"format": "date-time", "items": {"$ref": "#/definitions/sdc_recursive_timestamp_array"}, "type": ["null", "string", "array"]}}, "properties": {"id": {"maximum": 9223372036854775807, "minimum": -9223372036854775808, "type": ["integer"]}, "results": {"type": ["null", "array", "object"]}, "time_created": {"format": "date-time", "type": ["null", "string"]}}, "type": "object"}, "key_properties": ["id"], "bookmark_properties": ["id"]} +{"type": "STATE", "value": {"bookmarks": {"pg-valid-records": {"replication_key": "id", "replication_key_value": 52799009, "version": 1, "last_replication_method": "INCREMENTAL"}}, "currently_syncing": "pg-valid-records"}} +{"type": "ACTIVATE_VERSION", "stream": "pg-valid-records", "version": 1} +{"type": "RECORD", "stream": "pg-valid-records", "record": {"id": 52799009, "results": [{"rule": "XYZ", "reason": "XYZ", "actions": []}, {"rule": "XYZ", "reason": "XYZ", "actions": []}, {"rule": "XYZ", "reason": "XYZ", "actions": []}, {"rule": "XYZ", "reason": "XYZ", "actions": []}, {"rule": "XYZ", "reason": "XYZ", "actions": []}], "time_created": "2019-12-17T11:40:15.296955+00:00"}, "version": 1, "time_extracted": "2019-12-17T19:12:12.006049Z"} diff --git a/tests/integration/test_target_snowflake.py b/tests/integration/test_target_snowflake.py index 87708e6b..b503bbe2 100644 --- a/tests/integration/test_target_snowflake.py +++ b/tests/integration/test_target_snowflake.py @@ -867,3 +867,22 @@ def test_record_validation(self): self.config['validate_records'] = False with assert_raises(ProgrammingError): self.persist_lines_with_cache(tap_lines) + + def test_pg_records_validation(self): + """Test validating records from postgres tap""" + tap_lines_invalid_records = test_utils.get_test_tap_lines('messages-pg-with-invalid-records.json') + + # Loading invalid records when record validation enabled should fail at ... + self.config['validate_records'] = True + with assert_raises(RecordValidationException): + self.persist_lines_with_cache(tap_lines_invalid_records) + + # Loading invalid records when record validation disabled, should pass without any exceptions + self.config['validate_records'] = False + self.persist_lines_with_cache(tap_lines_invalid_records) + + # Valid records should pass for both with and without validation + tap_lines_valid_records = test_utils.get_test_tap_lines('messages-pg-with-valid-records.json') + + self.config['validate_records'] = True + self.persist_lines_with_cache(tap_lines_valid_records)