diff --git a/.github/workflows/connectors.yml b/.github/workflows/connectors.yml index 577f70913..e66ef3164 100644 --- a/.github/workflows/connectors.yml +++ b/.github/workflows/connectors.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: [3.7, 3.8, 3.9] + python-version: [3.8] steps: - name: Checking out repo diff --git a/.github/workflows/lint_unit_tests.yml b/.github/workflows/lint_unit_tests.yml index fc817d160..637ff89ec 100644 --- a/.github/workflows/lint_unit_tests.yml +++ b/.github/workflows/lint_unit_tests.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: ["3.7", "3.8", "3.9"] + python-version: ["3.8"] steps: - name: Checking out repo diff --git a/dev-project/docker-compose.yml b/dev-project/docker-compose.yml index c18a55b4f..9a96a1567 100644 --- a/dev-project/docker-compose.yml +++ b/dev-project/docker-compose.yml @@ -1,8 +1,7 @@ -version: '3' services: ### Primary container with PipelineWise CLI pipelinewise: - image: python:3.8-slim-buster + image: ubuntu:20.04 container_name: pipelinewise_dev working_dir: /opt/pipelinewise entrypoint: /opt/pipelinewise/dev-project/entrypoint.sh diff --git a/dev-project/entrypoint.sh b/dev-project/entrypoint.sh index 32e3be878..dc3611380 100755 --- a/dev-project/entrypoint.sh +++ b/dev-project/entrypoint.sh @@ -3,6 +3,9 @@ set -e apt update + +DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata + apt install -y --no-install-recommends \ wget \ gnupg \ @@ -12,18 +15,19 @@ apt install -y --no-install-recommends \ libaio1 \ mariadb-client \ mbuffer \ - postgresql-client + postgresql-client \ + python3.8 python3-pip python3-venv python3-dev apt upgrade -y # rm -rf /var/lib/apt/lists/* \ # Do a bunch of Mongo things -wget -q https://downloads.mongodb.com/compass/mongodb-mongosh_2.2.6_arm64.deb -apt install ./mongodb-mongosh_2.2.6_arm64.deb -rm -f mongodb-mongosh_2.2.6_arm64.deb -wget -q https://fastdl.mongodb.org/tools/db/mongodb-database-tools-ubuntu2004-arm64-100.9.4.deb -apt install ./mongodb-database-tools-ubuntu2004-arm64-100.9.4.deb -rm -f mongodb-database-tools-ubuntu2004-arm64-100.9.4.deb +wget -q --no-check-certificate https://downloads.mongodb.com/compass/mongodb-mongosh_2.2.9_arm64.deb +apt install ./mongodb-mongosh_2.2.9_arm64.deb +rm -f mongodb-mongosh_2.2.9_arm64.deb +wget -q --no-check-certificate https://fastdl.mongodb.org/tools/db/mongodb-database-tools-ubuntu2004-arm64-100.9.5.deb +apt install ./mongodb-database-tools-ubuntu2004-arm64-100.9.5.deb +rm -f mongodb-database-tools-ubuntu2004-arm64-100.9.5.deb dev-project/mongo/initiate-replica-set.sh # Install Oracle Instant Client required for tap-oracle diff --git a/dev-project/mongo/Dockerfile b/dev-project/mongo/Dockerfile index b89d2668a..b33b1f838 100644 --- a/dev-project/mongo/Dockerfile +++ b/dev-project/mongo/Dockerfile @@ -1,5 +1,3 @@ -# FROM mongo:4.2-bionic -# FROM mongo:5.0.22-focal FROM mongo:5.0.26-focal COPY --chown=mongodb:root --chmod=400 rootCA.pem /etc/ssl/rootCA.pem diff --git a/setup.py b/setup.py index 2d52ad35f..38f8ac541 100644 --- a/setup.py +++ b/setup.py @@ -6,8 +6,8 @@ LONG_DESCRIPTION = f.read() setup(name='pipelinewise', - python_requires='>=3.7,<3.10', - version='0.60.2b1', + python_requires='>=3.8,<3.9', + version='0.62.0', description='PipelineWise', long_description=LONG_DESCRIPTION, long_description_content_type='text/markdown', @@ -16,9 +16,7 @@ classifiers=[ 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python :: 3 :: Only', - 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', ], install_requires=[ 'argparse==1.4.0', @@ -33,7 +31,7 @@ 'google-cloud-bigquery==3.11.1', 'pipelinewise-singer-python==1.*', 'python-pidfile==3.0.0', - 'pymongo==3.13.0', + 'pymongo==4.7.*', 'tzlocal>=2.0,<4.1', 'slackclient==2.9.4', 'sqlparse==0.4.4', diff --git a/singer-connectors/tap-mongodb/requirements.txt b/singer-connectors/tap-mongodb/requirements.txt index 8ce903d9d..6852783b9 100644 --- a/singer-connectors/tap-mongodb/requirements.txt +++ b/singer-connectors/tap-mongodb/requirements.txt @@ -1 +1 @@ -pipelinewise-tap-mongodb==1.4.0 \ No newline at end of file +pipelinewise-tap-mongodb==1.5.0 \ No newline at end of file diff --git a/tests/end_to_end/helpers/db.py b/tests/end_to_end/helpers/db.py index 529234359..5adf85199 100644 --- a/tests/end_to_end/helpers/db.py +++ b/tests/end_to_end/helpers/db.py @@ -313,5 +313,8 @@ def get_mongodb_connection( Returns: Database instance with established connection """ - connection_string = f"mongodb://{user}:{password}@{host}:{port}/{database}?authSource={auth_database}&tls=true&tlsAllowInvalidCertificates=true&directConnection=true" + connection_string = ( + f'mongodb://{user}:{password}@{host}:{port}/{database}?authSource={auth_database}' + '&tls=true&tlsAllowInvalidCertificates=true&directConnection=true' + ) return pymongo.MongoClient(connection_string)[database]