From 31bd402b6a16c57876fd3e2252c8abd6dbadd845 Mon Sep 17 00:00:00 2001 From: Josh Lloyd Date: Thu, 18 Nov 2021 09:52:06 -0700 Subject: [PATCH 1/3] added more binary types --- .gitignore | 1 + tap_mysql/discover_utils.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index de67ba7..558db82 100644 --- a/.gitignore +++ b/.gitignore @@ -111,3 +111,4 @@ state.json # IDE .vscode +.idea diff --git a/tap_mysql/discover_utils.py b/tap_mysql/discover_utils.py index 481f1bd..d2c44dd 100644 --- a/tap_mysql/discover_utils.py +++ b/tap_mysql/discover_utils.py @@ -53,7 +53,7 @@ DATETIME_TYPES = {'datetime', 'timestamp', 'time', 'date'} -BINARY_TYPES = {'binary', 'varbinary'} +BINARY_TYPES = {'binary', 'varbinary', 'tinyblob', 'blob', 'mediumblob', 'longblob'} SPATIAL_TYPES = {'geometry', 'point', 'linestring', 'polygon', 'multipoint', 'multilinestring', From 6f4effc5e311b8082af072e014376ab5f019cd0d Mon Sep 17 00:00:00 2001 From: Josh Lloyd Date: Mon, 4 Apr 2022 16:42:29 -0600 Subject: [PATCH 2/3] added tests for blob and binary data types --- tests/integration/test_tap_mysql.py | 64 ++++++++++++++++++++++------- 1 file changed, 49 insertions(+), 15 deletions(-) diff --git a/tests/integration/test_tap_mysql.py b/tests/integration/test_tap_mysql.py index 0af047d..f23f921 100644 --- a/tests/integration/test_tap_mysql.py +++ b/tests/integration/test_tap_mysql.py @@ -70,7 +70,9 @@ def setUpClass(cls): c_multilinestring MULTILINESTRING, c_multipolygon MULTIPOLYGON, c_geometrycollection GEOMETRYCOLLECTION, - c_blob BLOB + c_blob BLOB, + c_binary BINARY (1), + c_varbinary VARBINARY (1) )''') catalog = test_utils.discover_catalog(conn, {}) @@ -118,7 +120,7 @@ def test_tinyint(self): maximum=127)) self.assertEqual(self.get_metadata_for_column('c_tinyint'), {'selected-by-default': True, - 'sql-datatype': 'tinyint(4)', + 'sql-datatype': 'tinyint', 'datatype': 'tinyint'}) def test_tinyint_1(self): @@ -132,11 +134,13 @@ def test_tinyint_1(self): def test_tinyint_1_unsigned(self): self.assertEqual(self.schema.properties['c_tinyint_1_unsigned'], - Schema(['null', 'boolean'], + Schema(['null', 'integer'], + minimum=0, + maximum=255, inclusion='available')) self.assertEqual(self.get_metadata_for_column('c_tinyint_1_unsigned'), {'selected-by-default': True, - 'sql-datatype': 'tinyint(1) unsigned', + 'sql-datatype': 'tinyint unsigned', 'datatype': 'tinyint'}) def test_smallint(self): @@ -147,7 +151,7 @@ def test_smallint(self): maximum=32767)) self.assertEqual(self.get_metadata_for_column('c_smallint'), {'selected-by-default': True, - 'sql-datatype': 'smallint(6)', + 'sql-datatype': 'smallint', 'datatype': 'smallint'}) def test_mediumint(self): @@ -158,7 +162,7 @@ def test_mediumint(self): maximum=8388607)) self.assertEqual(self.get_metadata_for_column('c_mediumint'), {'selected-by-default': True, - 'sql-datatype': 'mediumint(9)', + 'sql-datatype': 'mediumint', 'datatype': 'mediumint'}) def test_int(self): @@ -169,7 +173,7 @@ def test_int(self): maximum=2147483647)) self.assertEqual(self.get_metadata_for_column('c_int'), {'selected-by-default': True, - 'sql-datatype': 'int(11)', + 'sql-datatype': 'int', 'datatype': 'int'}) def test_bigint(self): @@ -180,7 +184,7 @@ def test_bigint(self): maximum=9223372036854775807)) self.assertEqual(self.get_metadata_for_column('c_bigint'), {'selected-by-default': True, - 'sql-datatype': 'bigint(20)', + 'sql-datatype': 'bigint', 'datatype': 'bigint'}) def test_bigint_unsigned(self): @@ -192,7 +196,7 @@ def test_bigint_unsigned(self): self.assertEqual(self.get_metadata_for_column('c_bigint_unsigned'), {'selected-by-default': True, - 'sql-datatype': 'bigint(20) unsigned', + 'sql-datatype': 'bigint unsigned', 'datatype': 'bigint'}) def test_float(self): @@ -247,7 +251,7 @@ def test_year(self): 'unsupported') self.assertEqual(self.get_metadata_for_column('c_year'), {'selected-by-default': False, - 'sql-datatype': 'year(4)', + 'sql-datatype': 'year', 'datatype': 'year'}) def test_pk(self): @@ -327,13 +331,43 @@ def test_multipolygon(self): def test_geometrycollection(self): self.assertEqual(self.schema.properties['c_geometrycollection'], - Schema(['null', 'object'], - format='spatial', - inclusion='available')) + Schema(None, + description='Unsupported column type geomcollection', + inclusion='unsupported')) self.assertEqual(self.get_metadata_for_column('c_geometrycollection'), + {'selected-by-default': False, + 'sql-datatype': 'geomcollection', + 'datatype': 'geomcollection'}) + + def test_binary(self): + self.assertEqual(self.schema.properties['c_blob'], + Schema(['null', 'string'], + format='binary', + inclusion='available')) + self.assertEqual(self.get_metadata_for_column('c_blob'), + {'selected-by-default': True, + 'sql-datatype': 'blob', + 'datatype': 'blob'}) + + def test_varbinary(self): + self.assertEqual(self.schema.properties['c_binary'], + Schema(['null', 'string'], + format='binary', + inclusion='available')) + self.assertEqual(self.get_metadata_for_column('c_binary'), + {'selected-by-default': True, + 'sql-datatype': 'binary(1)', + 'datatype': 'binary'}) + + def test_blob(self): + self.assertEqual(self.schema.properties['c_varbinary'], + Schema(['null', 'string'], + format='binary', + inclusion='available')) + self.assertEqual(self.get_metadata_for_column('c_varbinary'), {'selected-by-default': True, - 'sql-datatype': 'geometrycollection', - 'datatype': 'geometrycollection'}) + 'sql-datatype': 'varbinary(1)', + 'datatype': 'varbinary'}) class TestSelectsAppropriateColumns(unittest.TestCase): From fb85b2f36d2e8cc6068511498a805353faa76a83 Mon Sep 17 00:00:00 2001 From: Josh Lloyd Date: Tue, 6 Jun 2023 11:55:32 -0600 Subject: [PATCH 3/3] increased the connection timeout --- tap_mysql/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_mysql/connection.py b/tap_mysql/connection.py index 46ef001..a5c53b3 100644 --- a/tap_mysql/connection.py +++ b/tap_mysql/connection.py @@ -10,7 +10,7 @@ LOGGER = singer.get_logger('tap_mysql') -CONNECT_TIMEOUT_SECONDS = 30 +CONNECT_TIMEOUT_SECONDS = 300 # We need to hold onto this for self-signed SSL MATCH_HOSTNAME = ssl.match_hostname