Prefer SSL connection to Mongo sources

transferwise · Jun 19, 2024 · 6ce8362 · 6ce8362
1 parent 24577e9
commit 6ce8362
Show file tree

Hide file tree

Showing 9 changed files with 33 additions and 35 deletions.
diff --git a/.github/workflows/connectors.yml b/.github/workflows/connectors.yml
@@ -12,20 +12,17 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  check:
+  test_install_connectors:
     runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        python-version: [3.7, 3.8, 3.9]
 
     steps:
       - name: Checking out repo
         uses: actions/checkout@v3
 
-      - name: Set up Python ${{ matrix.python-version }}
+      - name: Set up Python 3.8
         uses: actions/setup-python@v3
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: 3.8
 
       - name: Check all connectors are installable
         run: |

diff --git a/.github/workflows/e2e_tests.yml b/.github/workflows/e2e_tests.yml
@@ -48,7 +48,7 @@ jobs:
         if: steps.check.outcome == 'failure'
         run: |
           cp dev-project/.env.template dev-project/.env
-          docker-compose -f dev-project/docker-compose.yml up -d
+          docker compose -f dev-project/docker-compose.yml up -d
 
       - name: Wait for test containers to be ready
         timeout-minutes: 5
@@ -89,7 +89,7 @@ jobs:
         if: steps.check.outcome == 'failure'
         run: |
           cp dev-project/.env.template dev-project/.env
-          docker-compose -f dev-project/docker-compose.yml up -d
+          docker compose -f dev-project/docker-compose.yml up -d
 
       - name: Wait for test containers to be ready
         if: steps.check.outcome == 'failure'
@@ -139,7 +139,7 @@ jobs:
         if: steps.check.outcome == 'failure'
         run: |
           cp dev-project/.env.template dev-project/.env
-          docker-compose -f dev-project/docker-compose.yml up -d
+          docker compose -f dev-project/docker-compose.yml up -d
 
       - name: Wait for test containers to be ready
         if: steps.check.outcome == 'failure'
@@ -189,7 +189,7 @@ jobs:
         if: steps.check.outcome == 'failure'
         run: |
           cp dev-project/.env.template dev-project/.env
-          docker-compose -f dev-project/docker-compose.yml up -d
+          docker compose -f dev-project/docker-compose.yml up -d
 
       - name: Wait for test containers to be ready
         if: steps.check.outcome == 'failure'
@@ -239,7 +239,7 @@ jobs:
         if: steps.check.outcome == 'failure'
         run: |
           cp dev-project/.env.template dev-project/.env
-          docker-compose -f dev-project/docker-compose.yml up -d
+          docker compose -f dev-project/docker-compose.yml up -d
 
       - name: Wait for test containers to be ready
         if: steps.check.outcome == 'failure'
@@ -292,7 +292,7 @@ jobs:
         if: steps.check.outcome == 'failure'
         run: |
           cp dev-project/.env.template dev-project/.env
-          docker-compose -f dev-project/docker-compose.yml up -d
+          docker compose -f dev-project/docker-compose.yml up -d
 
       - name: Wait for test containers to be ready
         if: steps.check.outcome == 'failure'

diff --git a/.github/workflows/lint_unit_tests.yml b/.github/workflows/lint_unit_tests.yml
@@ -13,9 +13,6 @@ concurrency:
 jobs:
   lint_and_test:
     runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        python-version: ["3.7", "3.8", "3.9"]
 
     steps:
       - name: Checking out repo
@@ -29,11 +26,11 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh python
 
-      - name: Set up Python ${{ matrix.python-version }}
+      - name: Set up Python 3.8
         if: steps.check.outcome == 'failure'
         uses: actions/setup-python@v3
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: 3.8
 
       - name: Install dependencies
         if: steps.check.outcome == 'failure'

diff --git a/dev-project/docker-compose.yml b/dev-project/docker-compose.yml
@@ -1,8 +1,8 @@
-version: '3'
 services:
   ### Primary container with PipelineWise CLI
   pipelinewise:
-    image: python:3.8-slim-buster
+    platform: linux/amd64
+    image: ubuntu:20.04
     container_name: pipelinewise_dev
     working_dir: /opt/pipelinewise
     entrypoint: /opt/pipelinewise/dev-project/entrypoint.sh

diff --git a/dev-project/entrypoint.sh b/dev-project/entrypoint.sh
@@ -3,6 +3,9 @@
 set -e
 
 apt update
+
+DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata
+
 apt install -y --no-install-recommends \
   wget \
   gnupg \
@@ -12,18 +15,20 @@ apt install -y --no-install-recommends \
   libaio1 \
   mariadb-client \
   mbuffer \
-  postgresql-client
+  postgresql-client \
+  python3.8 python3-pip python3-venv python3-dev
 
 apt upgrade -y
 # rm -rf /var/lib/apt/lists/* \
 
 # Do a bunch of Mongo things
-wget -q https://downloads.mongodb.com/compass/mongodb-mongosh_2.2.6_arm64.deb
-apt install ./mongodb-mongosh_2.2.6_arm64.deb
-rm -f mongodb-mongosh_2.2.6_arm64.deb
-wget -q https://fastdl.mongodb.org/tools/db/mongodb-database-tools-ubuntu2004-arm64-100.9.4.deb
-apt install ./mongodb-database-tools-ubuntu2004-arm64-100.9.4.deb
-rm -f mongodb-database-tools-ubuntu2004-arm64-100.9.4.deb
+wget -q --no-check-certificate https://downloads.mongodb.com/compass/mongodb-mongosh_2.2.9_amd64.deb
+apt install ./mongodb-mongosh_2.2.9_amd64.deb
+rm -f mongodb-mongosh_2.2.9_amd64.deb
+wget -q --no-check-certificate https://fastdl.mongodb.org/tools/db/mongodb-database-tools-ubuntu2004-x86_64-100.9.5.deb
+apt install ./mongodb-database-tools-ubuntu2004-x86_64-100.9.5.deb
+rm -f mongodb-database-tools-ubuntu2004-x86_64-100.9.5.deb
+
 dev-project/mongo/initiate-replica-set.sh
 
 # Install Oracle Instant Client required for tap-oracle

diff --git a/dev-project/mongo/Dockerfile b/dev-project/mongo/Dockerfile
@@ -1,5 +1,3 @@
-# FROM mongo:4.2-bionic
-# FROM mongo:5.0.22-focal
 FROM mongo:5.0.26-focal
 
 COPY --chown=mongodb:root --chmod=400 rootCA.pem /etc/ssl/rootCA.pem

diff --git a/setup.py b/setup.py
@@ -6,8 +6,8 @@
     LONG_DESCRIPTION = f.read()
 
 setup(name='pipelinewise',
-      python_requires='>=3.7,<3.10',
-      version='0.60.2b1',
+      python_requires='3.8.*',
+      version='0.62.0',
       description='PipelineWise',
       long_description=LONG_DESCRIPTION,
       long_description_content_type='text/markdown',
@@ -16,9 +16,7 @@
       classifiers=[
           'License :: OSI Approved :: Apache Software License',
           'Programming Language :: Python :: 3 :: Only',
-          'Programming Language :: Python :: 3.7',
           'Programming Language :: Python :: 3.8',
-          'Programming Language :: Python :: 3.9',
       ],
       install_requires=[
           'argparse==1.4.0',
@@ -33,7 +31,7 @@
           'google-cloud-bigquery==3.11.1',
           'pipelinewise-singer-python==1.*',
           'python-pidfile==3.0.0',
-          'pymongo==3.13.0',
+          'pymongo==4.7.*',
           'tzlocal>=2.0,<4.1',
           'slackclient==2.9.4',
           'sqlparse==0.4.4',

diff --git a/singer-connectors/tap-mongodb/requirements.txt b/singer-connectors/tap-mongodb/requirements.txt
@@ -1 +1 @@
-pipelinewise-tap-mongodb==1.4.0
+pipelinewise-tap-mongodb==1.5.0
diff --git a/tests/end_to_end/helpers/db.py b/tests/end_to_end/helpers/db.py
@@ -313,5 +313,8 @@ def get_mongodb_connection(
     Returns: Database instance with established connection
 
     """
-    connection_string = f"mongodb://{user}:{password}@{host}:{port}/{database}?authSource={auth_database}&tls=true&tlsAllowInvalidCertificates=true&directConnection=true"
+    connection_string = (
+        f'mongodb://{user}:{password}@{host}:{port}/{database}?authSource={auth_database}'
+        '&tls=true&tlsAllowInvalidCertificates=true&directConnection=true'
+    )
     return pymongo.MongoClient(connection_string)[database]
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		pipelinewise-tap-mongodb==1.4.0
		pipelinewise-tap-mongodb==1.5.0