Merge branch 'master' into dependabot/pip/jinja2-3.1.3

.github/CODEOWNERS

@@ -0,0 +1 @@
+* @transferwise/analytics-platform

.github/workflows/connectors.yml

@@ -12,20 +12,17 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  check:
+  test_install_connectors:
     runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        python-version: [3.7, 3.8, 3.9]
 
     steps:
       - name: Checking out repo
         uses: actions/checkout@v3
 
-      - name: Set up Python ${{ matrix.python-version }}
+      - name: Set up Python 3.8
         uses: actions/setup-python@v3
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: 3.8
 
       - name: Check all connectors are installable
         run: |

.github/workflows/e2e_tests.yml

@@ -48,7 +48,7 @@ jobs:
         if: steps.check.outcome == 'failure'
         run: |
           cp dev-project/.env.template dev-project/.env
-          docker-compose -f dev-project/docker-compose.yml up -d
+          docker compose -f dev-project/docker-compose.yml up -d
 
       - name: Wait for test containers to be ready
         timeout-minutes: 5
@@ -89,7 +89,7 @@ jobs:
         if: steps.check.outcome == 'failure'
         run: |
           cp dev-project/.env.template dev-project/.env
-          docker-compose -f dev-project/docker-compose.yml up -d
+          docker compose -f dev-project/docker-compose.yml up -d
 
       - name: Wait for test containers to be ready
         if: steps.check.outcome == 'failure'
@@ -139,7 +139,7 @@ jobs:
         if: steps.check.outcome == 'failure'
         run: |
           cp dev-project/.env.template dev-project/.env
-          docker-compose -f dev-project/docker-compose.yml up -d
+          docker compose -f dev-project/docker-compose.yml up -d
 
       - name: Wait for test containers to be ready
         if: steps.check.outcome == 'failure'
@@ -189,7 +189,7 @@ jobs:
         if: steps.check.outcome == 'failure'
         run: |
           cp dev-project/.env.template dev-project/.env
-          docker-compose -f dev-project/docker-compose.yml up -d
+          docker compose -f dev-project/docker-compose.yml up -d
 
       - name: Wait for test containers to be ready
         if: steps.check.outcome == 'failure'
@@ -239,7 +239,7 @@ jobs:
         if: steps.check.outcome == 'failure'
         run: |
           cp dev-project/.env.template dev-project/.env
-          docker-compose -f dev-project/docker-compose.yml up -d
+          docker compose -f dev-project/docker-compose.yml up -d
 
       - name: Wait for test containers to be ready
         if: steps.check.outcome == 'failure'
@@ -292,7 +292,7 @@ jobs:
         if: steps.check.outcome == 'failure'
         run: |
           cp dev-project/.env.template dev-project/.env
-          docker-compose -f dev-project/docker-compose.yml up -d
+          docker compose -f dev-project/docker-compose.yml up -d
 
       - name: Wait for test containers to be ready
         if: steps.check.outcome == 'failure'

.github/workflows/lint_unit_tests.yml

@@ -13,9 +13,6 @@ concurrency:
 jobs:
   lint_and_test:
     runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        python-version: ["3.7", "3.8", "3.9"]
 
     steps:
       - name: Checking out repo
@@ -29,11 +26,11 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh python
 
-      - name: Set up Python ${{ matrix.python-version }}
+      - name: Set up Python 3.8
         if: steps.check.outcome == 'failure'
         uses: actions/setup-python@v3
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: 3.8
 
       - name: Install dependencies
         if: steps.check.outcome == 'failure'

CHANGELOG.md

@@ -1,3 +1,16 @@
+0.60.2b1 (2024-06-12)
+---------------------
+- Bump `pymongo`from `4.7.3` to `3.13.0`
+
+0.60.1 (2024-06-12)
+-------------------
+- Bump `pymongo`from `3.12.3` to `4.7.3`
+
+0.60.0 (2024-05-10)
+-------------------
+- Bump `pipelinewise-tap-mysql`from `1.5.6` to `1.6.0`
+- Attempt SSL for MariaDB and PG sources as Preferred connection method
+
 0.58.3 (2023-11-28)
 -------------------
 - Bump `pipelinewise-tap-kafka`from `8.2.0` to `8.2.1`

dev-project/docker-compose.yml

@@ -1,8 +1,8 @@
-version: '3'
 services:
   ### Primary container with PipelineWise CLI
   pipelinewise:
-    image: python:3.8-slim-buster
+    platform: linux/amd64
+    image: ubuntu:20.04
     container_name: pipelinewise_dev
     working_dir: /opt/pipelinewise
     entrypoint: /opt/pipelinewise/dev-project/entrypoint.sh
@@ -34,8 +34,17 @@ services:
       context: ./pg
       dockerfile: Dockerfile
     container_name: pipelinewise_dev_postgres_source
-    # Making some logical decoding adjustments
-    command: -c "wal_level=logical" -c "max_replication_slots=5" -c "max_wal_senders=5"
+    volumes:
+      - ./pg/pg_hba.conf:/var/lib/postgresql/pg_hba.conf
+    # Making some logical decoding and ssl adjustments
+    command: >
+      -c "wal_level=logical"
+      -c "max_replication_slots=5"
+      -c "max_wal_senders=5"
+      -c "ssl=on"
+      -c "ssl_cert_file=/var/lib/postgresql/server.crt"
+      -c "ssl_key_file=/var/lib/postgresql/server.key"
+      -c "hba_file=/var/lib/postgresql/pg_hba.conf"
     ports:
       - ${TAP_POSTGRES_PORT_ON_HOST}:${TAP_POSTGRES_PORT}
     environment:
@@ -47,11 +56,26 @@ services:
 
   # MySQL service container used as test source database
   db_mysql_source:
-    image: mariadb:10.2.26
+    image: mariadb:10.6.17
     container_name: pipelinewise_dev_mysql_source
     ports:
       - ${TAP_MYSQL_PORT_ON_HOST}:${TAP_MYSQL_PORT}
-    command: --default-authentication-plugin=mysql_native_password --server-id=1 --log-bin=mysql-bin --binlog-format=ROW --binlog-do-db=${TAP_MYSQL_DB} --binlog-do-db=${TAP_MYSQL_REPLICA_DB}
+    volumes:
+      - ./my/server-cert.pem:/etc/my.cnf.d/certificates/server-cert.pem
+      - ./my/server-key.pem:/etc/my.cnf.d/certificates/server-key.pem
+      - ./my/ca-cert.pem:/etc/my.cnf.d/certificates/ca-cert.pem
+    command: >
+      --default-authentication-plugin=mysql_native_password
+      --server-id=1
+      --log-bin=mysql-bin
+      --binlog-format=ROW
+      --binlog-do-db=${TAP_MYSQL_DB}
+      --binlog-do-db=${TAP_MYSQL_REPLICA_DB}
+      --tls_version=TLSv1.2
+      --require_secure_transport=ON
+      --ssl_cert=/etc/my.cnf.d/certificates/server-cert.pem
+      --ssl_key=/etc/my.cnf.d/certificates/server-key.pem
+      --ssl_ca=/etc/my.cnf.d/certificates/ca-cert.pem
     environment:
       MYSQL_ROOT_PASSWORD: ${TAP_MYSQL_ROOT_PASSWORD}
       MYSQL_USER: ${TAP_MYSQL_USER}
@@ -61,7 +85,7 @@ services:
       - pipelinewise_network
 
   db_mysql_source_replica:
-    image: mariadb:10.2.26
+    image: mariadb:10.6.17
     container_name: pipelinewise_dev_mysql_source_replica
     ports:
       - ${TAP_MYSQL_REPLICA_PORT_ON_HOST}:${TAP_MYSQL_REPLICA_PORT}
@@ -76,7 +100,9 @@ services:
 
   db_mongo_source:
     container_name: "pipelinewise_dev_mongodb_source"
-    image: "mongo:4.2-bionic"
+    build:
+      context: ./mongo
+      dockerfile: Dockerfile
     ports:
       - ${TAP_MONGODB_PORT_ON_HOST}:${TAP_MONGODB_PORT}
     environment:
@@ -86,9 +112,15 @@ services:
       MONGO_USERNAME: ${TAP_MONGODB_USER}
       MONGO_PASSWORD: ${TAP_MONGODB_PASSWORD}
       MONGO_REPLICA_HOST: db_mongo_source
-    volumes:
-      - ./mongo/create_user.sh:/docker-entrypoint-initdb.d/create_user.sh
-    command: [mongod, --replSet, rs0]
+    command: >
+      mongod
+      --bind_ip_all
+      --replSet rs0
+      --keyFile /etc/ssl/replica.key
+      --tlsMode requireTLS
+      --tlsAllowConnectionsWithoutCertificates
+      --tlsCertificateKeyFile /etc/ssl/mongodb.pem
+      --tlsCAFile /etc/ssl/rootCA.pem
     networks:
       - pipelinewise_network
 

dev-project/entrypoint.sh

@@ -2,28 +2,34 @@
 
 set -e
 
-apt-get update
-apt-get install -y --no-install-recommends \
-  wget \
-  gnupg \
-  git
+apt update
 
-# Add Mongodb ppa
-wget -qO - https://www.mongodb.org/static/pgp/server-4.4.asc | apt-key add -
-echo "deb https://repo.mongodb.org/apt/debian buster/mongodb-org/4.4 main" | tee /etc/apt/sources.list.d/mongodb-org-4.4.list
+DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata
 
-apt-get update
-apt-get install -y --no-install-recommends \
+apt install -y --no-install-recommends \
+  wget \
+  gnupg \
+  git \
   alien \
   gettext-base \
   libaio1 \
   mariadb-client \
   mbuffer \
-  mongodb-database-tools \
-  mongodb-org-shell \
-  postgresql-client
+  postgresql-client \
+  python3.8 python3-pip python3-venv python3-dev
 
-rm -rf /var/lib/apt/lists/* \
+apt upgrade -y
+# rm -rf /var/lib/apt/lists/* \
+
+# Do a bunch of Mongo things
+wget -q --no-check-certificate https://downloads.mongodb.com/compass/mongodb-mongosh_2.2.9_amd64.deb
+apt install ./mongodb-mongosh_2.2.9_amd64.deb
+rm -f mongodb-mongosh_2.2.9_amd64.deb
+wget -q --no-check-certificate https://fastdl.mongodb.org/tools/db/mongodb-database-tools-ubuntu2004-x86_64-100.9.5.deb
+apt install ./mongodb-database-tools-ubuntu2004-x86_64-100.9.5.deb
+rm -f mongodb-database-tools-ubuntu2004-x86_64-100.9.5.deb
+
+dev-project/mongo/initiate-replica-set.sh
 
 # Install Oracle Instant Client required for tap-oracle
 # ORA_INSTACLIENT_URL=https://download.oracle.com/otn_software/linux/instantclient/193000/oracle-instantclient19.3-basiclite-19.3.0.0.0-1.x86_64.rpm
@@ -33,11 +39,8 @@ rm -rf /var/lib/apt/lists/* \
 # rm -f oracle-instantclient.rpm
 
 # Build test databases
-
 tests/db/tap_mysql_db.sh
 tests/db/tap_postgres_db.sh
-
-dev-project/mongo/init_rs.sh
 tests/db/tap_mongodb.sh
 tests/db/target_postgres.sh
 

dev-project/mongo/Dockerfile

@@ -0,0 +1,7 @@
+FROM mongo:5.0.26-focal
+
+COPY --chown=mongodb:root --chmod=400 rootCA.pem /etc/ssl/rootCA.pem
+COPY --chown=mongodb:root --chmod=400 mongodb.pem /etc/ssl/mongodb.pem
+COPY --chown=mongodb:root --chmod=400 replica.key /etc/ssl/replica.key
+
+COPY create-pipelinewise-user.sh /docker-entrypoint-initdb.d/create-pipelinewise-user.sh

dev-project/mongo/create-pipelinewise-user.sh

@@ -0,0 +1,10 @@
+echo 'CREATE MONGODB PIPELINEWISE USER'
+
+mongo --tls --tlsAllowInvalidCertificates -u $MONGO_INITDB_ROOT_USERNAME -p $MONGO_INITDB_ROOT_PASSWORD --authenticationDatabase admin admin <<EOF
+    db.getName();
+    db.createUser({
+        user: $(jq --arg 'user' $MONGO_USERNAME --null-input '$user'),
+        pwd: $(jq --arg 'pwd' $MONGO_PASSWORD --null-input '$pwd'),
+        roles: [{ role: 'readWrite', db: "$MONGO_INITDB_DATABASE" }]
+    });
+EOF

dev-project/mongo/initiate-replica-set.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -e
+
+echo
+echo 'INITIALIZING MONGODB REPLICASET'
+echo
+
+mongosh "mongodb://$TAP_MONGODB_ROOT_USER:$TAP_MONGODB_ROOT_PASSWORD@$TAP_MONGODB_HOST:$TAP_MONGODB_PORT/?tls=true&tlsAllowInvalidCertificates=true&authSource=admin&directConnection=true"<<EOF
+rs.initiate({_id: "rs0", members: [{_id: 0, host: '$TAP_MONGODB_HOST:$TAP_MONGODB_PORT'}]});
+EOF

dev-project/mongo/mongodb.pem

@@ -0,0 +1,46 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC8RDu+Q6GGIH0M
+IW/6RzDSxqbK5XSLyBSKhX+DNqY8QKvkh1NAhlob0MaDFWlf34gOS7YQ3/VDCxSi
+u/PadT28aCMJJqRko3u1OuhlAUuVBrFUA0mqQ+Xo7eXQiO7Xg17iF0vnMWB1eDSL
+LwDHLh+8fjFZdUGFiJvZW3rYNTRWYPf9BmL6HPfTlV6c8SeqAe05E++zh/2xdrbj
+h+mx0C8+XU4KluOX/4CSvVlIfQ/TZeuTc6GQMausxgavY52QAtV40CWp3LqHpXOM
+KNotbH1tc2YEwtUrCKS8ozOg5uw88VMan5Rua5VoAm1f3mPRSlIuttzHQ1Q8qi6c
+zKAGs9kfAgMBAAECggEABXkRqDXMxRqTrV3EJL6bZOfhIxEpjVi11ChjrRD9IyTu
+z6K+y6eRiLEjMYir+rKuVgVjF7l9pph0s0hgeI949HhYhbm/BK1F+YjqsZOoj/6q
+UAVqck3X/1NPH3v46YCdqlfqsnVagapgE1MR+Y2rNC5nnoX3RND/ACzAdPboUcHM
+p5w5KYSzUhAMDA7xwemQG0GCc0rxHQZ4Fx06Ka8j30PzjcdEdpo4tZqwTtEi5rHh
+uEtRAh8n4rE1p1UnekeF/YMrY+nNg0CVdagOvT/CZMtp2s+9Oe0EocbDap+lSSYc
+bD6Pb6kERpUGIzazUQSlpQN6n23jaoUXfGC9Z2CTmQKBgQD+Nap0GDqB4AdPB5Uy
+71Nwy9whGmjmr80mZRcISG7ze0NyujiLqcda1Bx6jDddTUYe41WTEivCwewM2wGg
+TWscR53kuVFsntqnXiRFHuy/6emvjp47SJHb9oP3QWwtVn6bxpw/O0M1fsdH059c
+EBGEf6nL2VALi3qpx5S+bJ2shwKBgQC9l6xzxAFjSH7W+ftvUVBPwAUfC8b0bsLJ
+0IWJV9N2yUiNe2QzcZwZrLQY0/OW6S9l6jppwt3Mimefxc6QhIvoQx0JO0YF/abN
+Zs9XIyuD1r1hgi3SqCLoBT7xp72V1SDI4hF8F9eDmIXq0WbvP1WasM28iL6nTU1C
+TPrn7IxsqQKBgE5vRomNlDfQZNDomQtfNpKeA+ye1nEfGyBWnpWqq7kqdjQm6x2E
+pAMNYUAalVaPaixLnnLDb4cnCGsrA8RvPrY6Dqis2Ue3d+WeMXi2Qvwv8y98xKus
+B30kAbG/mN4gSu5LiJgl8+XEmT47OHdSaeVE8qH4Ci+VuKI791QN9VRvAoGBAK8R
+MyjiwPVQp+8Sfl5YE3TQtunoEhr7e9qnNeeQta9mRhx+6BOowJ/knZiffrrHMAFH
+vBCU5AJPe0hJe2hvPFGIR3OSzCIUCsZ4PNUVoMTosY8sieuPXkwpK+oxHhsQ5mkY
+d1TDHzpgyKgAHLCnJwDkadXB32rVMpYaR0WJZFGhAoGBAPpjrHrmPRrCHd/y3Oix
+LbqrUaR0PxbcXNA2ExVgdOGNDC+e+ZjMV6pmm6niXyMfYwarr+aAJ8cU7POf50DW
+zAo42jcaiROwnO0U6gK6BBd7J+Yk2pOSxrLjSbqB8s3d1MVFgdBj3rA7QkZ5sDUc
+QY1lO3a1p9dCbnwn4Wmyr+q+
+-----END PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIIC+DCCAeCgAwIBAgIUVrJ6cs+6QRK8tTjOliaAJ87NGOswDQYJKoZIhvcNAQEL
+BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTI0MDUxNzEwMjQwNFoXDTI1MDky
+OTEwMjQwNFowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF
+AAOCAQ8AMIIBCgKCAQEAvEQ7vkOhhiB9DCFv+kcw0samyuV0i8gUioV/gzamPECr
+5IdTQIZaG9DGgxVpX9+IDku2EN/1QwsUorvz2nU9vGgjCSakZKN7tTroZQFLlQax
+VANJqkPl6O3l0Iju14Ne4hdL5zFgdXg0iy8Axy4fvH4xWXVBhYib2Vt62DU0VmD3
+/QZi+hz305VenPEnqgHtORPvs4f9sXa244fpsdAvPl1OCpbjl/+Akr1ZSH0P02Xr
+k3OhkDGrrMYGr2OdkALVeNAlqdy6h6VzjCjaLWx9bXNmBMLVKwikvKMzoObsPPFT
+Gp+UbmuVaAJtX95j0UpSLrbcx0NUPKounMygBrPZHwIDAQABo0IwQDAdBgNVHQ4E
+FgQU70ojmm8b0De4Qa45N/PVoxZNQwwwHwYDVR0jBBgwFoAUnnHpHxM13x8U9nSI
+HOmCJMamIR4wDQYJKoZIhvcNAQELBQADggEBAFdzTHva6pbkX/4dNJ4vy9oiYMuN
+iWYyqB/99KfPEHlU/RFiNePXfwL4BI18A7IAKQ5Dl0Vd1ub6X17SegA6+OMKDfu6
+DGmhgYHoBL4JXNd16LtOA9eHQtxynPTjhsSkuSz7i6eAumegK2EuZuw6fYHNXpmb
+0ZTsRlGWWsrZJFtL5P7RUici9ZheDBhjEs9yYNMOM/VFTn2OAN5MTIY79O28DWQj
+Uvx8DEw+YrxGLFqcDKjv7CKESti11PfAF67ERjQyURjqpeiVqO+D1SGzpdnXdisl
+c9/k/p4rWNZe4LX9XyA1oLJUcZNNl3XUtghGu93YUMquWeBH9C4aklIhV1s=
+-----END CERTIFICATE-----

dev-project/mongo/replica.key

@@ -0,0 +1,16 @@
+T2+O83yAELto/7Or19tC1xf8XGHIVYG5QUwtMDwCchC1Y0X/UgxnU34lsqoqtkjg
+4z2SPwFQ4N/aCr5Ah/dRrBTXhFv03P5UD0aRTcQxQQTv5ubLTO3fe5SFc3uhvKKk
+WiB9zlHElf21RVGXmnHHlNJQF2LeC8bRNIJtmCBIOYCy8SRt2NCecviME1B5k6Ly
+pcKOKttHDOKOhJXeS8YhpkPEbB0fuX8/vPcP8AdDlv6XbIe9MG4x06I/1bDuyl6V
+kp/VV1G6Hyy2w9q0ZV3sEgZay5wvzlWjcWaMJXpyjmQdQugGv59jQ23S0kToH7Pg
+S/HMVy1AhS/PSzZveYZwAB993jqxH/UYTs8CapeUpHG9WfJoJlJtX8yzK9HuSHY8
+ygCulhfXWIwhXP5Pv4cWM4DrFPYwLBBLXTyTOCCBEvrmFJB6ZggHbYgbzNK0Kvkj
+IvLceSDJpFiwlCQ9MKyLfZublxv0DyqI3TV7yvjJ/XwbqNNGj+6/CodvY0RbemPG
+4RmWLHsOx7sSt2AKoIM3LdNAZCq27hA/uEx8VzBXhz4qVraV50zz7yrtNRfu91At
+vBM+DKQdY4WjXqzTqCpVTK4hvggoIei0ea5R38pnG9tgU70FbhmM6waKmPbhTL1t
+ZKm7h8X2HcMfybTCs5OPhtTSL6GW0Ye6bW3GAuBz20tZP9LMt1AlgWGcmZNWyFSz
+ZD/eSi1DmiQi/FHbldlSpV5SXL1/5Gg21P0Tkfgb8uGlKWOG3ODLvgAXozdFkngr
+pSjo920r0QF+tCBfeJ8ckyq46Aprb/lCg9CwYeCGrQQrFwOx2w/lAzPIb6jmPvk/
+w1oT7JNWKpW82dCdmSXjZI/Sth0LIMKTuXGnXQ0NZGEqHCjUAMybHPMQ+uksukLA
+4+6PcDff0hqukrsJyYp3bkR70/F8kuz2TaHM4OEKDKwiyJW206u+YFV4l67wqpUn
+jXtRNIuXJPoVzmCaalcRDEkOnmCbTQIqeZu9+ndmNorJNEVl