From 3bcfa37b4b37ad613d3c18c75b99e966cfb0dbda Mon Sep 17 00:00:00 2001 From: Louis Pieterse Date: Wed, 8 May 2024 17:10:35 +0100 Subject: [PATCH] MariaDB sources prefer SSL --- CHANGELOG.md | 5 ++++ dev-project/docker-compose.yml | 21 ++++++++++++--- dev-project/my/ca-cert.pem | 19 +++++++++++++ dev-project/my/server-cert.pem | 18 +++++++++++++ dev-project/my/server-key.pem | 28 ++++++++++++++++++++ pipelinewise/fastsync/commons/tap_mysql.py | 16 ++++++++--- setup.py | 2 +- singer-connectors/tap-mysql/requirements.txt | 2 +- tests/db/tap_mysql_data.sql | 2 +- tests/db/tap_mysql_db.sh | 10 ++++--- tests/end_to_end/helpers/db.py | 1 + 11 files changed, 110 insertions(+), 14 deletions(-) create mode 100644 dev-project/my/ca-cert.pem create mode 100644 dev-project/my/server-cert.pem create mode 100644 dev-project/my/server-key.pem diff --git a/CHANGELOG.md b/CHANGELOG.md index d8c9f45b1..be957fc4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +0.60.0 (2024-05-10) +------------------- +- Bump `pipelinewise-tap-mysql`from `1.5.6` to `1.6.0` +- Attempt SSL for MariaDB and PG sources as Preferred connection method + 0.58.3 (2023-11-28) ------------------- - Bump `pipelinewise-tap-kafka`from `8.2.0` to `8.2.1` diff --git a/dev-project/docker-compose.yml b/dev-project/docker-compose.yml index cf4b60324..b27003e17 100644 --- a/dev-project/docker-compose.yml +++ b/dev-project/docker-compose.yml @@ -56,11 +56,26 @@ services: # MySQL service container used as test source database db_mysql_source: - image: mariadb:10.2.26 + image: mariadb:10.6.17 container_name: pipelinewise_dev_mysql_source ports: - ${TAP_MYSQL_PORT_ON_HOST}:${TAP_MYSQL_PORT} - command: --default-authentication-plugin=mysql_native_password --server-id=1 --log-bin=mysql-bin --binlog-format=ROW --binlog-do-db=${TAP_MYSQL_DB} --binlog-do-db=${TAP_MYSQL_REPLICA_DB} + volumes: + - ./my/server-cert.pem:/etc/my.cnf.d/certificates/server-cert.pem + - ./my/server-key.pem:/etc/my.cnf.d/certificates/server-key.pem + - ./my/ca-cert.pem:/etc/my.cnf.d/certificates/ca-cert.pem + command: > + --default-authentication-plugin=mysql_native_password + --server-id=1 + --log-bin=mysql-bin + --binlog-format=ROW + --binlog-do-db=${TAP_MYSQL_DB} + --binlog-do-db=${TAP_MYSQL_REPLICA_DB} + --tls_version=TLSv1.2 + --require_secure_transport=ON + --ssl_cert=/etc/my.cnf.d/certificates/server-cert.pem + --ssl_key=/etc/my.cnf.d/certificates/server-key.pem + --ssl_ca=/etc/my.cnf.d/certificates/ca-cert.pem environment: MYSQL_ROOT_PASSWORD: ${TAP_MYSQL_ROOT_PASSWORD} MYSQL_USER: ${TAP_MYSQL_USER} @@ -70,7 +85,7 @@ services: - pipelinewise_network db_mysql_source_replica: - image: mariadb:10.2.26 + image: mariadb:10.6.17 container_name: pipelinewise_dev_mysql_source_replica ports: - ${TAP_MYSQL_REPLICA_PORT_ON_HOST}:${TAP_MYSQL_REPLICA_PORT} diff --git a/dev-project/my/ca-cert.pem b/dev-project/my/ca-cert.pem new file mode 100644 index 000000000..8aff547ea --- /dev/null +++ b/dev-project/my/ca-cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDCzCCAfOgAwIBAgIUM6GLBTv83a3YJvi92Z2Uo+BQneQwDQYJKoZIhvcNAQEL +BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MCAXDTI0MDUwNzExMzkzN1oYDzMwMjMw +OTA4MTEzOTM3WjAUMRIwEAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQDB50ieR/FeYkAAUPVadABvo3BHtGmgTETk+E6ui0ip +pds+MW0tBCHV2jsstQFjf57y3dpN0DTBVSegPlZ2950J8yAVKvC53c8Szt2T+SrY +Akrlc09tTx8uOnHL9g9gP3UddREABrUaS6F1/UjAweS82JnlwCobFVqi/nCwm2Sf +Zp+CrzjbUIaHjYzO+gl95+g4RpqxENSCGx3U6iTA8hoCH+a2wPfttX7IcN1iBRQz +szLkWqA+gEsm42+eI/COMu/M3xsUnOAGusJ2oOuCrs3blCH/1ZOAB6rfe41Zkx9P +FS8FadpK5p8k1Sz0xsjV9ekfgDftXGXAReQDokLjzNg9AgMBAAGjUzBRMB0GA1Ud +DgQWBBTqN3S7ajqoO9AgljB5x/MXtppBlTAfBgNVHSMEGDAWgBTqN3S7ajqoO9Ag +ljB5x/MXtppBlTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBD +2sgXX7uP4L3LPff6UWnoLg8wS0pK8ifYsBQlF33kVmPA5RnDjabz5w6K/fag9lkw +qZlLiG976Ov7dg7Ic32sExo6QmTnmWzYHwZO0cI+QXMO1XUM63e+A5Y3/jGHxocY +6Ral9dy4tAbEGqPBCMYwVCl7LXcTgDHK9zSmzOkkpGgPp609Sd/bf/02kNDsuwOQ +rTT4JcTtmaXdZQo1T+Q8x6kc8MTvXa0jfuZ+vIfIHUkdsBTw7nfeYMXGzGbVlfvU +hxTKZeSxVrnfrL1CMMAxb21XP6TaeD4VIMgbW1yCV1BQX65w8apef58HlpShfKU0 +ClBg7sMUAlF8ViavUH9t +-----END CERTIFICATE----- diff --git a/dev-project/my/server-cert.pem b/dev-project/my/server-cert.pem new file mode 100644 index 000000000..2e0b1903a --- /dev/null +++ b/dev-project/my/server-cert.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC5TCCAc2gAwIBAgIBATANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDDAlsb2Nh +bGhvc3QwHhcNMjQwNTA3MTE0MDMwWhcNMjUwNTA3MTE0MDMwWjAUMRIwEAYDVQQD +DAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDRZTiB +ElhU/6zKo3WulHFsKjQwkSwwmDE1jfs8YT7Paxtl3izowWW1lf6/bLhFLQ3WcTI3 +BCAxyprWIAJPw1S0ie8zLOousx0tP9pt5L58UF6Y2cvGTi0lXMBkD8IpdVBZaDNV +jguo4qo9LK34Vs+PyTagQV+xBxxpUIVCWUVJBHoVzHfGSCJo/ZoUG1KtpQAUkWCN +sgQC7AhlsR/V0IEitvLnjx8P9dnVNRcHq7wI/156uMbhtn++K6Vh7xL47rnX85et +YEbjzb2S/66dc0c8IZvyEG84jDi94PYmVvM4OorOORkGfa2zbpRzHPYrpAe5CxUo +5R5S5jDyKjwz0JZnAgMBAAGjQjBAMB0GA1UdDgQWBBS4ha9z1pxAhTn9iVBkyjv+ +x4lPGjAfBgNVHSMEGDAWgBTqN3S7ajqoO9AgljB5x/MXtppBlTANBgkqhkiG9w0B +AQsFAAOCAQEACIWEkkS/YE0ozt7lXbLOtvN2IIAqkJN6tgWgT1P1QoNzBflYY3hW +Hb/Z48mnF8pHHllOcyILyM81FT5YNwurKHhoHj88Mquur2LuU1qwdKIPaQCgXyiU +F06gB3F6qoLyc+lzcXcH632/TQSaCeXwGPQYV1/CbR7ZssxrCG8drsFhmkPgQdfo +wbywXOLASkp60EtkqqEisKRm2KhaWoDyLGy4u9e5SDAlpYLgFSOQ/Swac5CTuM7c +xNaImXY8XB5za/OuG7vmRuvXjeCjmvJDBQbEvC6W0w5ZU+/dC6UMW180UMAwh8oR +j2jRFGGoDtFrYXTI2vHxydJRNpuOc+xwfw== +-----END CERTIFICATE----- diff --git a/dev-project/my/server-key.pem b/dev-project/my/server-key.pem new file mode 100644 index 000000000..356ea92e1 --- /dev/null +++ b/dev-project/my/server-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDRZTiBElhU/6zK +o3WulHFsKjQwkSwwmDE1jfs8YT7Paxtl3izowWW1lf6/bLhFLQ3WcTI3BCAxyprW +IAJPw1S0ie8zLOousx0tP9pt5L58UF6Y2cvGTi0lXMBkD8IpdVBZaDNVjguo4qo9 +LK34Vs+PyTagQV+xBxxpUIVCWUVJBHoVzHfGSCJo/ZoUG1KtpQAUkWCNsgQC7Ahl +sR/V0IEitvLnjx8P9dnVNRcHq7wI/156uMbhtn++K6Vh7xL47rnX85etYEbjzb2S +/66dc0c8IZvyEG84jDi94PYmVvM4OorOORkGfa2zbpRzHPYrpAe5CxUo5R5S5jDy +Kjwz0JZnAgMBAAECggEACoeD/V2OSovmxPPORZ6aS32dzTlnlvP4OYCvuvwK3AQ/ +RvKSoIulESn/OuH9LCFVWkyr4T9YXRiai5zlcNxaJI/xEso15m5ET9OvqKoBeB/U +eewtxoWE2KWe2Qk/0rTe+vf16N3xMSdgtx3Ji7zVyB2unQVcifrZD2Sopsr1ZHP5 +ZANM07D++IWnKaQHLs92RpnyZEvpfanUhZCoB5zLnpq0BRjXBMXeFc3hDNF6kml+ +YOYLijKVOm+WPqCFjDFqSK8YP9v4lqaOXFzAMYt8q1kHjVWe5/OzD9fOkiLbVWmw +29sY7ZZ3DIR5YTVRlKRXTd8btnBO+VDPNCyeFE9dEQKBgQDrDHr4aZ5d1KH4iaG0 +WhJZijDi6rKZxFIW6mjT8CE3w9b5fjb625tr2JSXhjAzqMkMGaHl0c8beDRNfrcv +YiPt0P551/rfT61VJSC6OhC0I30XaFeVJLhBkNDsOUvY4YkTPPyBagW3Y2MpVpFd +z8nJcVpb84iToJ5t0s8dp4mAtwKBgQDkD1zZ+nLLpavcokUKqoa4Pz3OchLQKDQ+ +WxHLn7UEW7Qm8tAcnyyltci1rGZBp7aNO0Oq3DxriPsivJb/Jjv7IifPG1TS2Prs +9eIEy4Dak4UGjIFUYbIYYbPCdpWclwMDAJjHQSl+N9hP1MCDRXKK25BwsF5ptfyu +a+zUmcKH0QKBgBC4ZI8DRGa8V4FMmRoxRhjkgbHCrE3P71798dMRw+spnUNGPTb/ +JEE5uOhGfWtPWG3p7OBiWm9Dz8SrfPB79L5YHaIhQRFdo6Do8TkOJF4MIDvoCkLr +x8eyO4pyUPjuokeiEPxLxHhVM5qfM9wOifYwgBMGcfK+fEantmAPqPJlAoGAHWwZ +94lHHBzQklNhAuJDJSwLhkHTSYWWT+Eh55pAGULXeBrlXHvwn6RXM5D9VEUV2Ryn +OzjUlq8Cb1XTSTGelWSvrBmDlPwnS0DhmurCQfC8Pb8MxF0iQ+D+9vyjiqM04UNH +cpVmjAEaaBozh/wCrgVrmg427/5pjfsgf8EihrECgYA62TsMmTcWvQOE8OSoInwT +rlaMn68+weFgQDHkHmxwSA4fnycbN4xfUBrnDffBSR/yrrM76laaYl05EegRhFSy +iwDL+vnkT7TvS6zcIk4VRWMrGNqXS7/xPeNxYr9Dsk2GlwqcH4DTtS3bLXIkYV4J +gTwIrfpIQVL/vwRcyP3Thg== +-----END PRIVATE KEY----- diff --git a/pipelinewise/fastsync/commons/tap_mysql.py b/pipelinewise/fastsync/commons/tap_mysql.py index 89b8c4389..cada792ac 100644 --- a/pipelinewise/fastsync/commons/tap_mysql.py +++ b/pipelinewise/fastsync/commons/tap_mysql.py @@ -122,10 +122,12 @@ def open_connections(self): self.conn: Connection = pymysql.connect( **conn_params, cursorclass=pymysql.cursors.DictCursor, + ssl={'': True} ) self.conn_unbuffered: Connection = pymysql.connect( **conn_params, cursorclass=pymysql.cursors.SSCursor, + ssl={'': True} ) # Set session variables by running a list of SQLs which is defined @@ -521,8 +523,11 @@ def __get_primary_server_uuid(self) -> str: Returns: server uuid """ - conn = pymysql.connect(**self.get_connection_parameters(prioritize_primary=True)[0], - cursorclass=pymysql.cursors.DictCursor) if self.is_replica else None + conn = pymysql.connect( + **self.get_connection_parameters(prioritize_primary=True)[0], + cursorclass=pymysql.cursors.DictCursor, + ssl={'': True} + ) if self.is_replica else None result = self.query('select @@server_uuid as server_uuid;', conn) @@ -537,8 +542,11 @@ def __get_primary_server_id(self) -> int: Returns: server uuid """ - conn = pymysql.connect(**self.get_connection_parameters(prioritize_primary=True)[0], - cursorclass=pymysql.cursors.DictCursor) if self.is_replica else None + conn = pymysql.connect( + **self.get_connection_parameters(prioritize_primary=True)[0], + cursorclass=pymysql.cursors.DictCursor, + ssl={'': True} + ) if self.is_replica else None result = self.query('select @@server_id as server_id;', conn) diff --git a/setup.py b/setup.py index 03bfc15c7..517ddcbd7 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup(name='pipelinewise', python_requires='>=3.7,<3.10', - version='0.58.3', + version='0.60.0', description='PipelineWise', long_description=LONG_DESCRIPTION, long_description_content_type='text/markdown', diff --git a/singer-connectors/tap-mysql/requirements.txt b/singer-connectors/tap-mysql/requirements.txt index f09a912c2..d9e4ff84a 100644 --- a/singer-connectors/tap-mysql/requirements.txt +++ b/singer-connectors/tap-mysql/requirements.txt @@ -1 +1 @@ -pipelinewise-tap-mysql==1.5.6 \ No newline at end of file +pipelinewise-tap-mysql==1.6.0 diff --git a/tests/db/tap_mysql_data.sql b/tests/db/tap_mysql_data.sql index 008547193..bd0213af4 100644 --- a/tests/db/tap_mysql_data.sql +++ b/tests/db/tap_mysql_data.sql @@ -52,7 +52,7 @@ Maatjies', 20, 'A', null, '09:16:10'), (8,'Liewe Maatjies', 10, null, '[{"key": "Value''s One", "actions": []},{"key": "Value\U00000027s Two", "actions": ' || '[]}]', '00:30:00'), (9, 'dolor sit amet', 19, 'C', '[]', '23:50:19'), - (10, 'sit amet', 100, 'E', '', '19:30:19') + (10, 'sit amet', 100, 'E', null, '19:30:19') ; /*!40000 ALTER TABLE `edgydata` ENABLE KEYS */; diff --git a/tests/db/tap_mysql_db.sh b/tests/db/tap_mysql_db.sh index d00d5a5e3..0fc1dfd5f 100755 --- a/tests/db/tap_mysql_db.sh +++ b/tests/db/tap_mysql_db.sh @@ -1,6 +1,6 @@ #!/bin/bash -e # -# Building a test MySQL database for integration testing of tap-mysql +# Building a test MySQL database for integration testing of tap-mysql # The sample database available at https://github.com/ikostan/RESTAURANT-DATABASE PWD="$(dirname "$0")" @@ -26,6 +26,7 @@ fi echo "SETTING UP MYSQL PRIMARY SERVER FOR REPLICATION" mysql --protocol TCP \ +--ssl \ --host ${TAP_MYSQL_HOST} \ --port ${TAP_MYSQL_PORT} \ --user root \ @@ -43,7 +44,7 @@ mysql --protocol TCP \ echo "GETTING MYSQL PRIMARY SERVER LOG INFO" -MASTER_LOG_STATUS=`mysql --protocol TCP --host ${TAP_MYSQL_HOST} --port ${TAP_MYSQL_PORT} --user root --password=${TAP_MYSQL_ROOT_PASSWORD} -e "SHOW MASTER STATUS;"` +MASTER_LOG_STATUS=`mysql --protocol TCP --ssl --host ${TAP_MYSQL_HOST} --port ${TAP_MYSQL_PORT} --user root --password=${TAP_MYSQL_ROOT_PASSWORD} -e "SHOW MASTER STATUS;"` CURRENT_LOG=`echo $MASTER_LOG_STATUS | awk '{print $5}'` CURRENT_POS=`echo $MASTER_LOG_STATUS | awk '{print $6}'` @@ -54,13 +55,13 @@ mysql --protocol TCP \ --port ${TAP_MYSQL_REPLICA_PORT} \ --user ${TAP_MYSQL_REPLICA_USER} \ --password=${TAP_MYSQL_REPLICA_PASSWORD} \ --e "STOP SLAVE; CHANGE MASTER TO MASTER_HOST='${TAP_MYSQL_HOST}',MASTER_USER='${TAP_MYSQL_USER}',MASTER_PASSWORD='${TAP_MYSQL_PASSWORD}',MASTER_LOG_FILE='${CURRENT_LOG}',MASTER_LOG_POS=${CURRENT_POS}; START SLAVE;" +-e "STOP SLAVE; CHANGE MASTER TO MASTER_SSL=1,MASTER_SSL_VERIFY_SERVER_CERT=0,MASTER_HOST='${TAP_MYSQL_HOST}',MASTER_USER='${TAP_MYSQL_USER}',MASTER_PASSWORD='${TAP_MYSQL_PASSWORD}',MASTER_LOG_FILE='${CURRENT_LOG}',MASTER_LOG_POS=${CURRENT_POS}; START SLAVE;" # Download the sample database and build it - echo "DUMPING DATA INTO PRIMARY MYSQL DATABASE" mysql --protocol TCP \ +--ssl \ --host ${TAP_MYSQL_HOST} \ --port ${TAP_MYSQL_PORT} \ --user ${TAP_MYSQL_USER} \ @@ -70,6 +71,7 @@ ${TAP_MYSQL_DB} < ${TEST_DB_SQL} echo "DUMPING DATA INTO PRIMARY MYSQL DATABASE2" mysql --protocol TCP \ +--ssl \ --host ${TAP_MYSQL_HOST} \ --port ${TAP_MYSQL_PORT} \ --user ${TAP_MYSQL_USER} \ diff --git a/tests/end_to_end/helpers/db.py b/tests/end_to_end/helpers/db.py index 700b58305..405f6e514 100644 --- a/tests/end_to_end/helpers/db.py +++ b/tests/end_to_end/helpers/db.py @@ -38,6 +38,7 @@ def run_query_mysql(query, host, port, user, password, database): database=database, charset='utf8mb4', cursorclass=pymysql.cursors.Cursor, + ssl={'': True} ) as cur: cur.execute(query) if cur.rowcount > 0: