Prefer SSL connection to Mongo sources

transferwise · Jun 19, 2024 · 65c68a5 · 65c68a5
1 parent 24577e9
commit 65c68a5
Show file tree

Hide file tree

Showing 8 changed files with 22 additions and 20 deletions.
diff --git a/.github/workflows/connectors.yml b/.github/workflows/connectors.yml
@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9]
+        python-version: [3.8]
 
     steps:
       - name: Checking out repo

diff --git a/.github/workflows/lint_unit_tests.yml b/.github/workflows/lint_unit_tests.yml
@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: ["3.7", "3.8", "3.9"]
+        python-version: ["3.8"]
 
     steps:
       - name: Checking out repo

diff --git a/dev-project/docker-compose.yml b/dev-project/docker-compose.yml
@@ -1,8 +1,7 @@
-version: '3'
 services:
   ### Primary container with PipelineWise CLI
   pipelinewise:
-    image: python:3.8-slim-buster
+    image: ubuntu:20.04
     container_name: pipelinewise_dev
     working_dir: /opt/pipelinewise
     entrypoint: /opt/pipelinewise/dev-project/entrypoint.sh

diff --git a/dev-project/entrypoint.sh b/dev-project/entrypoint.sh
@@ -3,6 +3,9 @@
 set -e
 
 apt update
+
+DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata
+
 apt install -y --no-install-recommends \
   wget \
   gnupg \
@@ -12,18 +15,19 @@ apt install -y --no-install-recommends \
   libaio1 \
   mariadb-client \
   mbuffer \
-  postgresql-client
+  postgresql-client \
+  python3.8 python3-pip python3-venv python3-dev
 
 apt upgrade -y
 # rm -rf /var/lib/apt/lists/* \
 
 # Do a bunch of Mongo things
-wget -q https://downloads.mongodb.com/compass/mongodb-mongosh_2.2.6_arm64.deb
-apt install ./mongodb-mongosh_2.2.6_arm64.deb
-rm -f mongodb-mongosh_2.2.6_arm64.deb
-wget -q https://fastdl.mongodb.org/tools/db/mongodb-database-tools-ubuntu2004-arm64-100.9.4.deb
-apt install ./mongodb-database-tools-ubuntu2004-arm64-100.9.4.deb
-rm -f mongodb-database-tools-ubuntu2004-arm64-100.9.4.deb
+wget -q --no-check-certificate https://downloads.mongodb.com/compass/mongodb-mongosh_2.2.9_arm64.deb
+apt install ./mongodb-mongosh_2.2.9_arm64.deb
+rm -f mongodb-mongosh_2.2.9_arm64.deb
+wget -q --no-check-certificate https://fastdl.mongodb.org/tools/db/mongodb-database-tools-ubuntu2004-arm64-100.9.5.deb
+apt install ./mongodb-database-tools-ubuntu2004-arm64-100.9.5.deb
+rm -f mongodb-database-tools-ubuntu2004-arm64-100.9.5.deb
 dev-project/mongo/initiate-replica-set.sh
 
 # Install Oracle Instant Client required for tap-oracle

diff --git a/dev-project/mongo/Dockerfile b/dev-project/mongo/Dockerfile
@@ -1,5 +1,3 @@
-# FROM mongo:4.2-bionic
-# FROM mongo:5.0.22-focal
 FROM mongo:5.0.26-focal
 
 COPY --chown=mongodb:root --chmod=400 rootCA.pem /etc/ssl/rootCA.pem

diff --git a/setup.py b/setup.py
@@ -6,8 +6,8 @@
     LONG_DESCRIPTION = f.read()
 
 setup(name='pipelinewise',
-      python_requires='>=3.7,<3.10',
-      version='0.60.2b1',
+      python_requires='>=3.8,<3.9',
+      version='0.62.0',
       description='PipelineWise',
       long_description=LONG_DESCRIPTION,
       long_description_content_type='text/markdown',
@@ -16,9 +16,7 @@
       classifiers=[
           'License :: OSI Approved :: Apache Software License',
           'Programming Language :: Python :: 3 :: Only',
-          'Programming Language :: Python :: 3.7',
           'Programming Language :: Python :: 3.8',
-          'Programming Language :: Python :: 3.9',
       ],
       install_requires=[
           'argparse==1.4.0',
@@ -33,7 +31,7 @@
           'google-cloud-bigquery==3.11.1',
           'pipelinewise-singer-python==1.*',
           'python-pidfile==3.0.0',
-          'pymongo==3.13.0',
+          'pymongo==4.7.*',
           'tzlocal>=2.0,<4.1',
           'slackclient==2.9.4',
           'sqlparse==0.4.4',

diff --git a/singer-connectors/tap-mongodb/requirements.txt b/singer-connectors/tap-mongodb/requirements.txt
@@ -1 +1 @@
-pipelinewise-tap-mongodb==1.4.0
+pipelinewise-tap-mongodb==1.5.0
diff --git a/tests/end_to_end/helpers/db.py b/tests/end_to_end/helpers/db.py
@@ -313,5 +313,8 @@ def get_mongodb_connection(
     Returns: Database instance with established connection
 
     """
-    connection_string = f"mongodb://{user}:{password}@{host}:{port}/{database}?authSource={auth_database}&tls=true&tlsAllowInvalidCertificates=true&directConnection=true"
+    connection_string = (
+        f'mongodb://{user}:{password}@{host}:{port}/{database}?authSource={auth_database}'
+        '&tls=true&tlsAllowInvalidCertificates=true&directConnection=true'
+    )
     return pymongo.MongoClient(connection_string)[database]
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		pipelinewise-tap-mongodb==1.4.0
		pipelinewise-tap-mongodb==1.5.0