MariaDB sources prefer SSL

CHANGELOG.md

@@ -1,3 +1,8 @@
+0.60.0 (2024-05-10)
+-------------------
+- Bump `pipelinewise-tap-mysql`from `1.5.6` to `1.6.0`
+- Attempt SSL for MariaDB and PG sources as Preferred connection method
+
 0.58.3 (2023-11-28)
 -------------------
 - Bump `pipelinewise-tap-kafka`from `8.2.0` to `8.2.1`

dev-project/docker-compose.yml

@@ -56,11 +56,26 @@ services:
 
   # MySQL service container used as test source database
   db_mysql_source:
-    image: mariadb:10.2.26
+    image: mariadb:10.6.17
     container_name: pipelinewise_dev_mysql_source
     ports:
       - ${TAP_MYSQL_PORT_ON_HOST}:${TAP_MYSQL_PORT}
-    command: --default-authentication-plugin=mysql_native_password --server-id=1 --log-bin=mysql-bin --binlog-format=ROW --binlog-do-db=${TAP_MYSQL_DB} --binlog-do-db=${TAP_MYSQL_REPLICA_DB}
+    volumes:
+      - ./my/server-cert.pem:/etc/my.cnf.d/certificates/server-cert.pem
+      - ./my/server-key.pem:/etc/my.cnf.d/certificates/server-key.pem
+      - ./my/ca-cert.pem:/etc/my.cnf.d/certificates/ca-cert.pem
+    command: >
+      --default-authentication-plugin=mysql_native_password
+      --server-id=1
+      --log-bin=mysql-bin
+      --binlog-format=ROW
+      --binlog-do-db=${TAP_MYSQL_DB}
+      --binlog-do-db=${TAP_MYSQL_REPLICA_DB}
+      --tls_version=TLSv1.2
+      --require_secure_transport=ON
+      --ssl_cert=/etc/my.cnf.d/certificates/server-cert.pem
+      --ssl_key=/etc/my.cnf.d/certificates/server-key.pem
+      --ssl_ca=/etc/my.cnf.d/certificates/ca-cert.pem
     environment:
       MYSQL_ROOT_PASSWORD: ${TAP_MYSQL_ROOT_PASSWORD}
       MYSQL_USER: ${TAP_MYSQL_USER}
@@ -70,7 +85,7 @@ services:
       - pipelinewise_network
 
   db_mysql_source_replica:
-    image: mariadb:10.2.26
+    image: mariadb:10.6.17
     container_name: pipelinewise_dev_mysql_source_replica
     ports:
       - ${TAP_MYSQL_REPLICA_PORT_ON_HOST}:${TAP_MYSQL_REPLICA_PORT}

dev-project/my/ca-cert.pem

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDCzCCAfOgAwIBAgIUM6GLBTv83a3YJvi92Z2Uo+BQneQwDQYJKoZIhvcNAQEL
+BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MCAXDTI0MDUwNzExMzkzN1oYDzMwMjMw
+OTA4MTEzOTM3WjAUMRIwEAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEB
+AQUAA4IBDwAwggEKAoIBAQDB50ieR/FeYkAAUPVadABvo3BHtGmgTETk+E6ui0ip
+pds+MW0tBCHV2jsstQFjf57y3dpN0DTBVSegPlZ2950J8yAVKvC53c8Szt2T+SrY
+Akrlc09tTx8uOnHL9g9gP3UddREABrUaS6F1/UjAweS82JnlwCobFVqi/nCwm2Sf
+Zp+CrzjbUIaHjYzO+gl95+g4RpqxENSCGx3U6iTA8hoCH+a2wPfttX7IcN1iBRQz
+szLkWqA+gEsm42+eI/COMu/M3xsUnOAGusJ2oOuCrs3blCH/1ZOAB6rfe41Zkx9P
+FS8FadpK5p8k1Sz0xsjV9ekfgDftXGXAReQDokLjzNg9AgMBAAGjUzBRMB0GA1Ud
+DgQWBBTqN3S7ajqoO9AgljB5x/MXtppBlTAfBgNVHSMEGDAWgBTqN3S7ajqoO9Ag
+ljB5x/MXtppBlTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBD
+2sgXX7uP4L3LPff6UWnoLg8wS0pK8ifYsBQlF33kVmPA5RnDjabz5w6K/fag9lkw
+qZlLiG976Ov7dg7Ic32sExo6QmTnmWzYHwZO0cI+QXMO1XUM63e+A5Y3/jGHxocY
+6Ral9dy4tAbEGqPBCMYwVCl7LXcTgDHK9zSmzOkkpGgPp609Sd/bf/02kNDsuwOQ
+rTT4JcTtmaXdZQo1T+Q8x6kc8MTvXa0jfuZ+vIfIHUkdsBTw7nfeYMXGzGbVlfvU
+hxTKZeSxVrnfrL1CMMAxb21XP6TaeD4VIMgbW1yCV1BQX65w8apef58HlpShfKU0
+ClBg7sMUAlF8ViavUH9t
+-----END CERTIFICATE-----

dev-project/my/server-cert.pem

@@ -0,0 +1,18 @@
+-----BEGIN CERTIFICATE-----
+MIIC5TCCAc2gAwIBAgIBATANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDDAlsb2Nh
+bGhvc3QwHhcNMjQwNTA3MTE0MDMwWhcNMjUwNTA3MTE0MDMwWjAUMRIwEAYDVQQD
+DAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDRZTiB
+ElhU/6zKo3WulHFsKjQwkSwwmDE1jfs8YT7Paxtl3izowWW1lf6/bLhFLQ3WcTI3
+BCAxyprWIAJPw1S0ie8zLOousx0tP9pt5L58UF6Y2cvGTi0lXMBkD8IpdVBZaDNV
+jguo4qo9LK34Vs+PyTagQV+xBxxpUIVCWUVJBHoVzHfGSCJo/ZoUG1KtpQAUkWCN
+sgQC7AhlsR/V0IEitvLnjx8P9dnVNRcHq7wI/156uMbhtn++K6Vh7xL47rnX85et
+YEbjzb2S/66dc0c8IZvyEG84jDi94PYmVvM4OorOORkGfa2zbpRzHPYrpAe5CxUo
+5R5S5jDyKjwz0JZnAgMBAAGjQjBAMB0GA1UdDgQWBBS4ha9z1pxAhTn9iVBkyjv+
+x4lPGjAfBgNVHSMEGDAWgBTqN3S7ajqoO9AgljB5x/MXtppBlTANBgkqhkiG9w0B
+AQsFAAOCAQEACIWEkkS/YE0ozt7lXbLOtvN2IIAqkJN6tgWgT1P1QoNzBflYY3hW
+Hb/Z48mnF8pHHllOcyILyM81FT5YNwurKHhoHj88Mquur2LuU1qwdKIPaQCgXyiU
+F06gB3F6qoLyc+lzcXcH632/TQSaCeXwGPQYV1/CbR7ZssxrCG8drsFhmkPgQdfo
+wbywXOLASkp60EtkqqEisKRm2KhaWoDyLGy4u9e5SDAlpYLgFSOQ/Swac5CTuM7c
+xNaImXY8XB5za/OuG7vmRuvXjeCjmvJDBQbEvC6W0w5ZU+/dC6UMW180UMAwh8oR
+j2jRFGGoDtFrYXTI2vHxydJRNpuOc+xwfw==
+-----END CERTIFICATE-----

dev-project/my/server-key.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDRZTiBElhU/6zK
+o3WulHFsKjQwkSwwmDE1jfs8YT7Paxtl3izowWW1lf6/bLhFLQ3WcTI3BCAxyprW
+IAJPw1S0ie8zLOousx0tP9pt5L58UF6Y2cvGTi0lXMBkD8IpdVBZaDNVjguo4qo9
+LK34Vs+PyTagQV+xBxxpUIVCWUVJBHoVzHfGSCJo/ZoUG1KtpQAUkWCNsgQC7Ahl
+sR/V0IEitvLnjx8P9dnVNRcHq7wI/156uMbhtn++K6Vh7xL47rnX85etYEbjzb2S
+/66dc0c8IZvyEG84jDi94PYmVvM4OorOORkGfa2zbpRzHPYrpAe5CxUo5R5S5jDy
+Kjwz0JZnAgMBAAECggEACoeD/V2OSovmxPPORZ6aS32dzTlnlvP4OYCvuvwK3AQ/
+RvKSoIulESn/OuH9LCFVWkyr4T9YXRiai5zlcNxaJI/xEso15m5ET9OvqKoBeB/U
+eewtxoWE2KWe2Qk/0rTe+vf16N3xMSdgtx3Ji7zVyB2unQVcifrZD2Sopsr1ZHP5
+ZANM07D++IWnKaQHLs92RpnyZEvpfanUhZCoB5zLnpq0BRjXBMXeFc3hDNF6kml+
+YOYLijKVOm+WPqCFjDFqSK8YP9v4lqaOXFzAMYt8q1kHjVWe5/OzD9fOkiLbVWmw
+29sY7ZZ3DIR5YTVRlKRXTd8btnBO+VDPNCyeFE9dEQKBgQDrDHr4aZ5d1KH4iaG0
+WhJZijDi6rKZxFIW6mjT8CE3w9b5fjb625tr2JSXhjAzqMkMGaHl0c8beDRNfrcv
+YiPt0P551/rfT61VJSC6OhC0I30XaFeVJLhBkNDsOUvY4YkTPPyBagW3Y2MpVpFd
+z8nJcVpb84iToJ5t0s8dp4mAtwKBgQDkD1zZ+nLLpavcokUKqoa4Pz3OchLQKDQ+
+WxHLn7UEW7Qm8tAcnyyltci1rGZBp7aNO0Oq3DxriPsivJb/Jjv7IifPG1TS2Prs
+9eIEy4Dak4UGjIFUYbIYYbPCdpWclwMDAJjHQSl+N9hP1MCDRXKK25BwsF5ptfyu
+a+zUmcKH0QKBgBC4ZI8DRGa8V4FMmRoxRhjkgbHCrE3P71798dMRw+spnUNGPTb/
+JEE5uOhGfWtPWG3p7OBiWm9Dz8SrfPB79L5YHaIhQRFdo6Do8TkOJF4MIDvoCkLr
+x8eyO4pyUPjuokeiEPxLxHhVM5qfM9wOifYwgBMGcfK+fEantmAPqPJlAoGAHWwZ
+94lHHBzQklNhAuJDJSwLhkHTSYWWT+Eh55pAGULXeBrlXHvwn6RXM5D9VEUV2Ryn
+OzjUlq8Cb1XTSTGelWSvrBmDlPwnS0DhmurCQfC8Pb8MxF0iQ+D+9vyjiqM04UNH
+cpVmjAEaaBozh/wCrgVrmg427/5pjfsgf8EihrECgYA62TsMmTcWvQOE8OSoInwT
+rlaMn68+weFgQDHkHmxwSA4fnycbN4xfUBrnDffBSR/yrrM76laaYl05EegRhFSy
+iwDL+vnkT7TvS6zcIk4VRWMrGNqXS7/xPeNxYr9Dsk2GlwqcH4DTtS3bLXIkYV4J
+gTwIrfpIQVL/vwRcyP3Thg==
+-----END PRIVATE KEY-----

pipelinewise/fastsync/commons/tap_mysql.py

@@ -122,10 +122,12 @@ def open_connections(self):
         self.conn: Connection = pymysql.connect(
             **conn_params,
             cursorclass=pymysql.cursors.DictCursor,
+            ssl={'': True}
         )
         self.conn_unbuffered: Connection = pymysql.connect(
             **conn_params,
             cursorclass=pymysql.cursors.SSCursor,
+            ssl={'': True}
         )
 
         # Set session variables by running a list of SQLs which is defined
@@ -521,8 +523,11 @@ def __get_primary_server_uuid(self) -> str:
 
         Returns: server uuid
         """
-        conn = pymysql.connect(**self.get_connection_parameters(prioritize_primary=True)[0],
-                               cursorclass=pymysql.cursors.DictCursor) if self.is_replica else None
+        conn = pymysql.connect(
+            **self.get_connection_parameters(prioritize_primary=True)[0],
+            cursorclass=pymysql.cursors.DictCursor,
+            ssl={'': True}
+        ) if self.is_replica else None
 
         result = self.query('select @@server_uuid as server_uuid;', conn)
 
@@ -537,8 +542,11 @@ def __get_primary_server_id(self) -> int:
 
         Returns: server uuid
         """
-        conn = pymysql.connect(**self.get_connection_parameters(prioritize_primary=True)[0],
-                               cursorclass=pymysql.cursors.DictCursor) if self.is_replica else None
+        conn = pymysql.connect(
+            **self.get_connection_parameters(prioritize_primary=True)[0],
+            cursorclass=pymysql.cursors.DictCursor,
+            ssl={'': True}
+        ) if self.is_replica else None
 
         result = self.query('select @@server_id as server_id;', conn)
 

setup.py

@@ -7,7 +7,7 @@
 
 setup(name='pipelinewise',
       python_requires='>=3.7,<3.10',
-      version='0.58.3',
+      version='0.60.0',
       description='PipelineWise',
       long_description=LONG_DESCRIPTION,
       long_description_content_type='text/markdown',

singer-connectors/tap-mysql/requirements.txt

@@ -1 +1 @@
-pipelinewise-tap-mysql==1.5.6
+pipelinewise-tap-mysql==1.6.0

tests/db/tap_mysql_data.sql

@@ -52,7 +52,7 @@ Maatjies', 20, 'A', null, '09:16:10'),
   (8,'Liewe	Maatjies', 10, null, '[{"key": "Value''s One", "actions": []},{"key": "Value\U00000027s Two", "actions": ' ||
                                  '[]}]', '00:30:00'),
   (9, 'dolor sit amet', 19, 'C', '[]', '23:50:19'),
-  (10, 'sit amet', 100, 'E', '', '19:30:19')
+  (10, 'sit amet', 100, 'E', null, '19:30:19')
 ;
 
 /*!40000 ALTER TABLE `edgydata` ENABLE KEYS */;

tests/db/tap_mysql_db.sh

@@ -1,6 +1,6 @@
 #!/bin/bash -e
 #
-# Building a test MySQL database for integration testing of tap-mysql 
+# Building a test MySQL database for integration testing of tap-mysql
 # The sample database available at https://github.com/ikostan/RESTAURANT-DATABASE
 PWD="$(dirname "$0")"
 
@@ -26,6 +26,7 @@ fi
 echo "SETTING UP MYSQL PRIMARY SERVER FOR REPLICATION"
 
 mysql --protocol TCP \
+--ssl \
 --host ${TAP_MYSQL_HOST} \
 --port ${TAP_MYSQL_PORT} \
 --user root \
@@ -43,7 +44,7 @@ mysql --protocol TCP \
 
 echo "GETTING MYSQL PRIMARY SERVER LOG INFO"
 
-MASTER_LOG_STATUS=`mysql --protocol TCP --host ${TAP_MYSQL_HOST} --port ${TAP_MYSQL_PORT} --user root --password=${TAP_MYSQL_ROOT_PASSWORD} -e "SHOW MASTER STATUS;"`
+MASTER_LOG_STATUS=`mysql --protocol TCP --ssl --host ${TAP_MYSQL_HOST} --port ${TAP_MYSQL_PORT} --user root --password=${TAP_MYSQL_ROOT_PASSWORD} -e "SHOW MASTER STATUS;"`
 CURRENT_LOG=`echo $MASTER_LOG_STATUS | awk '{print $5}'`
 CURRENT_POS=`echo $MASTER_LOG_STATUS | awk '{print $6}'`
 
@@ -54,13 +55,13 @@ mysql --protocol TCP \
 --port ${TAP_MYSQL_REPLICA_PORT} \
 --user ${TAP_MYSQL_REPLICA_USER} \
 --password=${TAP_MYSQL_REPLICA_PASSWORD} \
--e "STOP SLAVE; CHANGE MASTER TO MASTER_HOST='${TAP_MYSQL_HOST}',MASTER_USER='${TAP_MYSQL_USER}',MASTER_PASSWORD='${TAP_MYSQL_PASSWORD}',MASTER_LOG_FILE='${CURRENT_LOG}',MASTER_LOG_POS=${CURRENT_POS}; START SLAVE;"
+-e "STOP SLAVE; CHANGE MASTER TO MASTER_SSL=1,MASTER_SSL_VERIFY_SERVER_CERT=0,MASTER_HOST='${TAP_MYSQL_HOST}',MASTER_USER='${TAP_MYSQL_USER}',MASTER_PASSWORD='${TAP_MYSQL_PASSWORD}',MASTER_LOG_FILE='${CURRENT_LOG}',MASTER_LOG_POS=${CURRENT_POS}; START SLAVE;"
 
 # Download the sample database and build it
-
 echo "DUMPING DATA INTO PRIMARY MYSQL DATABASE"
 
 mysql --protocol TCP \
+--ssl \
 --host ${TAP_MYSQL_HOST} \
 --port ${TAP_MYSQL_PORT} \
 --user ${TAP_MYSQL_USER} \
@@ -70,6 +71,7 @@ ${TAP_MYSQL_DB} < ${TEST_DB_SQL}
 echo "DUMPING DATA INTO PRIMARY MYSQL DATABASE2"
 
 mysql --protocol TCP \
+--ssl \
 --host ${TAP_MYSQL_HOST} \
 --port ${TAP_MYSQL_PORT} \
 --user ${TAP_MYSQL_USER} \

tests/end_to_end/helpers/db.py

@@ -38,6 +38,7 @@ def run_query_mysql(query, host, port, user, password, database):
         database=database,
         charset='utf8mb4',
         cursorclass=pymysql.cursors.Cursor,
+        ssl={'': True}
     ) as cur:
         cur.execute(query)
         if cur.rowcount > 0: