Update github workflow, support python12. support pg13, pg14, pg15, p…

…g16, pg17

.github/workflows/lint.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ "3.9", "3.10", "3.11" ]
+        python-version: [ "3.10", "3.11", "3.12" ]
     steps:
     - uses: actions/checkout@v3
 

.github/workflows/tests.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ "3.9", "3.10", "3.11" ]
+        python-version: [ "3.10", "3.11", "3.12" ]
     steps:
     - uses: actions/checkout@v3
 

Makefile

@@ -1,7 +1,7 @@
 
 PYTHON ?= python3
 PYTHON_SOURCE_DIRS = aiven_db_migrate/ test/
-PG_VERSIONS = 10 11 12 13 14
+PG_VERSIONS = 13 14 15 16 17
 
 generated = aiven_db_migrate/migrate/version.py
 
@@ -51,7 +51,7 @@ validate-style:
 
 .PHONY: test
 test: $(generated)
-	$(PYTHON) -m pytest -v -r test
+	$(PYTHON) -m pytest -vv -r test
 
 clean:
 	$(RM) aiven_db_migrate/migrate/version.py

README.md

@@ -13,7 +13,7 @@ such as `pg_dumpall`, not useful when migrating database to/from service provide
 
 Currently this tool supports only PostgreSQL but we aim to add support for other databases, such as MySQL.
 
-Requires Python 3.9 or newer.
+Requires Python 3.10 or newer.
 
 ## Usage
 

aiven_db_migrate/migrate/pgmigrate.py

@@ -11,7 +11,7 @@
 from copy import deepcopy
 from dataclasses import dataclass, field
 from datetime import datetime
-from distutils.version import LooseVersion
+from packaging.version import Version
 from pathlib import Path
 from psycopg2 import sql
 from psycopg2.extras import RealDictCursor
@@ -100,7 +100,7 @@ class PGCluster:
     conn_info: Dict[str, Any]
     _databases: Dict[str, PGDatabase]
     _params: Dict[str, str]
-    _version: Optional[LooseVersion]
+    _version: Optional[Version]
     _attributes: Optional[Dict[str, Any]]
     _pg_ext: Optional[List[PGExtension]]
     _pg_ext_whitelist: Optional[List[str]]
@@ -206,10 +206,10 @@ def params(self) -> Dict[str, str]:
         return self._params
 
     @property
-    def version(self) -> LooseVersion:
+    def version(self) -> Version:
         if self._version is None:
             # will make this work on ubuntu, for strings like '12.5 (Ubuntu 12.5-1.pgdg18.04+1)'
-            self._version = LooseVersion(self.params["server_version"].split(" ")[0])
+            self._version = Version(self.params["server_version"].split(" ")[0])
         return self._version
 
     @property
@@ -286,7 +286,7 @@ def pg_ext(self) -> List[PGExtension]:
         if self._pg_ext is None:
             # Starting from PotsgreSQL 13, extensions have a trusted flag that means
             # they can be created without being superuser.
-            trusted_field = ", extver.trusted" if self.version >= LooseVersion("13") else ""
+            trusted_field = ", extver.trusted" if self.version >= Version("13") else ""
             exts = self.c(
                 f"""
                 SELECT extver.name, extver.version, extver.superuser {trusted_field}
@@ -355,7 +355,7 @@ def pg_roles(self) -> Dict[str, PGRole]:
 
     @property
     def replication_available(self) -> bool:
-        return self.version >= "10"
+        return self.version >= Version("10")
 
     @property
     def replication_slots_count(self) -> int:
@@ -460,7 +460,7 @@ def create_publication(self, *, dbname: str, only_tables: Optional[List[str]] =
 
         pub_options: Union[List[str], str]
         pub_options = ["INSERT", "UPDATE", "DELETE"]
-        if self.version >= "11":
+        if self.version >= Version("11"):
             pub_options.append("TRUNCATE")
         pub_options = ",".join(pub_options)
         has_aiven_extras = self.has_aiven_extras(dbname=dbname)
@@ -1012,7 +1012,7 @@ def _check_pg_ext(self):
                             "Extension %r is installed in source and target database %r, source version: %r, "
                             "target version: %r", source_ext.name, dbname, source_ext.version, target_ext.version
                         )
-                        if LooseVersion(source_ext.version) <= LooseVersion(target_ext.version):
+                        if Version(source_ext.version) <= Version(target_ext.version):
                             continue
                         msg = (
                             f"Installed extension {source_ext.name!r} in target database {dbname!r} is older than "
@@ -1034,7 +1034,7 @@ def _check_pg_ext(self):
                     target_ext.version, source_ext.version
                 )
 
-                if LooseVersion(target_ext.version) < LooseVersion(source_ext.version):
+                if Version(target_ext.version) < Version(source_ext.version):
                     msg = (
                         f"Extension {target_ext.name!r} version available for installation in target is too old, "
                         f"source version: {source_ext.version}, target version: {target_ext.version}"
@@ -1220,7 +1220,7 @@ def _dump_schema(
 
         # PG 13 and older versions do not support `--extension` option.
         # The migration still succeeds with some unharmful error messages in the output.
-        if db and self.source.version >= LooseVersion("14"):
+        if db and self.source.version >= Version("14"):
             pg_dump_cmd.extend([f"--extension={ext}" for ext in self.filter_extensions(db)])
 
         if self.createdb:
@@ -1246,7 +1246,7 @@ def _dump_data(self, *, db: PGDatabase) -> PGMigrateStatus:
 
         # PG 13 and older versions do not support `--extension` option.
         # The migration still succeeds with some unharmful error messages in the output.
-        if self.source.version >= LooseVersion("14"):
+        if self.source.version >= Version("14"):
             pg_dump_cmd.extend([f"--extension={ext}" for ext in self.filter_extensions(db)])
 
         subtask: PGSubTask = self._pg_dump_pipe_psql(

aiven_db_migrate/migrate/pgutils.py

@@ -1,5 +1,5 @@
 # Copyright (c) 2020 Aiven, Helsinki, Finland. https://aiven.io/
-from distutils.version import LooseVersion
+from packaging.version import Version
 from pathlib import Path
 from typing import Any, Dict, Optional
 from urllib.parse import parse_qs, urlparse
@@ -19,8 +19,8 @@ def find_pgbin_dir(pgversion: str, *, max_pgversion: Optional[str] = None, usr_d
 
     Versions equal or above 10 only check the major version number: 10, 11, 12, 13...
     """
-    min_version = LooseVersion(pgversion).version
-    max_version = min_version if max_pgversion is None else LooseVersion(max_pgversion).version
+    min_version = list(Version(pgversion).release)
+    max_version = min_version if max_pgversion is None else list(Version(max_pgversion).release)
     max_version = max(max_version, min_version)
     max_parts = 1
     candidates = []
@@ -31,7 +31,7 @@ def find_pgbin_dir(pgversion: str, *, max_pgversion: Optional[str] = None, usr_d
                 match = re.search(pattern, path.name)
                 bin_path = path / "bin"
                 if match and bin_path.is_dir():
-                    candidate_version = LooseVersion(match.group(1)).version
+                    candidate_version = list(Version(match.group(1)).release)
                     if min_version[:max_parts] <= candidate_version[:max_parts] <= max_version[:max_parts]:
                         candidates.append((candidate_version, bin_path))
     candidates.sort()

aiven_db_migrate/migrate/version.py

@@ -0,0 +1 @@
+__version__ = "0.1.5-2-gfd83d7c"

ci/install.sh

@@ -13,10 +13,10 @@ echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main"
 wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
 apt-get -y update
 apt-get install -y \
-  postgresql-{10,11,12,13,14} \
-  postgresql-server-dev-{10,11,12,13,14} \
-  postgresql-{10,11,12,13,14}-postgis-3 \
-  postgresql-{10,11,12,13,14}-pgextwlist
+  postgresql-{13,14,15,16,17} \
+  postgresql-server-dev-{13,14,15,16,17} \
+  postgresql-{13,14,15,16,17}-postgis-3 \
+  postgresql-{13,14,15,16,17}-pgextwlist
 
 # Install aiven-extras, using the latest tag.
 git clone https://github.com/aiven/aiven-extras "${AIVEN_EXTRAS_TARGET}"
@@ -36,11 +36,11 @@ git -C "${AIVEN_PG_SECURITY_TARGET}" checkout ${AIVEN_PG_SECURITY_TAG}
 
 # maybe add a deb target to aiven-extras in the future, but for now, while hacky, this is (probably) terser and less intrusive
 
-for dest in "10" "11" "12" "13" "14"
+for dest in "13" "14" "15" "16" "17"
 do
     gcc -fPIC -I/usr/include/postgresql/${dest}/server \
-      -D_GNU_SOURCE -I/usr/include/libxml2  -I/usr/include -c -o standby_slots.o $AIVEN_EXTRAS_TARGET/src/standby_slots.c
-    gcc -fPIC -shared -o aiven_extras.so standby_slots.o -L/usr/lib/postgresql/${dest} \
+      -D_GNU_SOURCE -I/usr/include/libxml2  -I/usr/include -c -o aiven_extras.o $AIVEN_EXTRAS_TARGET/src/aiven_extras.c
+    gcc -fPIC -shared -o aiven_extras.so aiven_extras.o -L/usr/lib/postgresql/${dest} \
       -L/usr/lib64  -L/usr/lib64 -Wl,--as-needed -Wl,-rpath,/usr/lib/postgresql/${dest},--enable-new-dtags
 
     mkdir -p /usr/lib/postgresql/${dest}/lib/

mypy.ini

@@ -1,7 +1,7 @@
 # Global configuration
 
 [mypy]
-python_version = 3.9
+python_version = 3.10
 warn_redundant_casts = True
 
 # Module based overrides to disable errors on legacy code

setup.py

@@ -19,9 +19,10 @@ def get_version():
         ],
     },
     install_requires=[
-        "psycopg2"
+        "psycopg2",
+        "packaging",
     ],
-    python_requires=">=3.9",
+    python_requires=">=3.10",
     license="Apache 2.0",
     name="aiven-db-migrate",
     packages=find_packages(exclude=["test"]),
@@ -35,8 +36,8 @@ def get_version():
         "Intended Audience :: Developers",
         "Topic :: Software Development :: Libraries",
         "License :: OSI Approved :: Apache Software License",
-        "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
     ],
 )

test/conftest.py

@@ -7,8 +7,8 @@
 from aiven_db_migrate.migrate.pgmigrate import PGTarget
 from contextlib import contextmanager
 from copy import copy
-from distutils.version import LooseVersion
 from functools import partial, wraps
+from packaging.version import Version
 from pathlib import Path
 from psycopg2.extras import LogicalReplicationConnection, ReplicationCursor
 from test.utils import PGRunner, SUPPORTED_PG_VERSIONS
@@ -100,7 +100,7 @@ def generate_fixtures():
         source_name = f"{name_prefix}_source"
         inject_pg_fixture(name=source_name, pgversion=source, with_gatekeeper=False)
         for target in pg_target_versions:
-            if LooseVersion(source) > LooseVersion(target):
+            if Version(source) > Version(target):
                 continue
             name_prefix = "pg{}".format(target.replace(".", ""))
             target_name = f"{name_prefix}_target"

test/test_main.py

@@ -7,7 +7,7 @@
 
 
 @mock.patch.object(PGMigrate, "_check_aiven_pg_security_agent")
-@mock.patch.object(PGCluster, "params", new_callable=mock.PropertyMock, return_value={"server_version": "11.13"})
+@mock.patch.object(PGCluster, "params", new_callable=mock.PropertyMock, return_value={"server_version": "16.4"})
 @mock.patch.object(PGCluster, "databases", new_callable=mock.PropertyMock, return_value={})
 @mock.patch.object(PGCluster, "pg_lang", new_callable=mock.PropertyMock, return_value={})
 @mock.patch.object(PGMigrate, "migrate")

test/test_migrate_checks.py

@@ -17,9 +17,10 @@ def test_dbs_max_total_size_check(pg_source_and_target: Tuple[PGRunner, PGRunner
         source.create_db(dbname=dbname)
         target.create_db(dbname=dbname)
 
-    # This DB seems to be created outside the tests
-    dbnames.add("postgres")
+    # from >=PG15 there is a restriction non-super users in postgres db
+    # so let's just ignore it
 
+    default_filtered_dbs = ["postgres"]
     # Create few tables and insert some data
     tables = [f'table_{i}' for i in range(4)]
     for dbname in dbnames:
@@ -34,6 +35,7 @@ def test_dbs_max_total_size_check(pg_source_and_target: Tuple[PGRunner, PGRunner
         target_conn_info=target.conn_info(),
         createdb=False,
         verbose=True,
+        filtered_db=",".join(default_filtered_dbs),
     )
 
     with patch(
@@ -63,7 +65,7 @@ def test_dbs_max_total_size_check(pg_source_and_target: Tuple[PGRunner, PGRunner
         target_conn_info=target.conn_info(),
         createdb=False,
         verbose=True,
-        filtered_db=",".join(dbnames),
+        filtered_db=",".join(default_filtered_dbs + list(dbnames)),
     )
 
     with patch(

test/test_pg_cluster.py

@@ -1,7 +1,7 @@
 # Copyright (c) 2021 Aiven, Helsinki, Finland. https://aiven.io/
 from aiven_db_migrate.migrate.pgmigrate import PGCluster
-from distutils.version import LooseVersion
 from multiprocessing import Process
+from packaging.version import Version
 from test.utils import PGRunner
 
 import os
@@ -30,7 +30,7 @@ def test_trusted_extensions(pg_cluster: PGRunner):
     known_trusted = {"btree_gin", "btree_gist", "hstore", "intarray", "pgcrypto", "plpgsql", "unaccent"}
     known_untrusted = {"pg_buffercache", "pg_freespacemap", "pg_prewarm", "pg_stat_statements"}
     cluster = PGCluster(conn_info=pg_cluster.conn_info())
-    if cluster.version >= LooseVersion("13"):
+    if cluster.version >= Version("13"):
         for extension in cluster.pg_ext:
             assert isinstance(extension.trusted, bool)
             if extension.name in known_trusted:

test/test_pg_extensions.py

@@ -2,7 +2,7 @@
 
 from aiven_db_migrate.migrate.errors import PGMigrateValidationFailedError
 from aiven_db_migrate.migrate.pgmigrate import PGDatabase, PGExtension, PGMigrate
-from distutils.version import LooseVersion
+from packaging.version import Version
 from test.utils import PGRunner, random_string
 from typing import Tuple
 
@@ -34,7 +34,7 @@ def test_defaults(pg_source_and_target: Tuple[PGRunner, PGRunner], createdb: boo
     for dbname in dbnames:
         for ext1 in pg_mig.source.databases[dbname].pg_ext:
             ext2 = next(e for e in pg_mig.target.databases[dbname].pg_ext if e.name == ext1.name)
-            assert LooseVersion(ext2.version) >= LooseVersion(ext1.version)
+            assert Version(ext2.version) >= Version(ext1.version)
 
 
 @pytest.mark.parametrize("createdb", [True, False])
@@ -107,7 +107,7 @@ def test_extension_superuser(pg_source_and_target: Tuple[PGRunner, PGRunner], cr
 
     for ext1 in pg_mig.source.databases[dbname].pg_ext:
         ext2 = next(e for e in pg_mig.target.databases[dbname].pg_ext if e.name == ext1.name)
-        assert LooseVersion(ext2.version) >= LooseVersion(ext1.version)
+        assert Version(ext2.version) >= Version(ext1.version)
 
 
 @pytest.mark.parametrize("createdb", [True, False])
@@ -141,7 +141,7 @@ def test_extension_whitelist(pg_source_and_target: Tuple[PGRunner, PGRunner], cr
     for dbname in dbnames:
         for ext1 in pg_mig.source.databases[dbname].pg_ext:
             ext2 = next(e for e in pg_mig.target.databases[dbname].pg_ext if e.name == ext1.name)
-            assert LooseVersion(ext2.version) >= LooseVersion(ext1.version)
+            assert Version(ext2.version) >= Version(ext1.version)
 
 
 @pytest.mark.parametrize("createdb", [True, False])

test/utils.py

@@ -5,7 +5,7 @@
 from aiven_db_migrate.migrate.pgutils import find_pgbin_dir
 from contextlib import contextmanager
 from datetime import datetime
-from distutils.version import LooseVersion
+from packaging.version import Version
 from pathlib import Path
 from psycopg2._psycopg import connection
 from psycopg2.extras import RealDictCursor
@@ -21,7 +21,7 @@
 import threading
 import time
 
-SUPPORTED_PG_VERSIONS = ["10", "11", "12", "13", "14"]
+SUPPORTED_PG_VERSIONS = ["13", "14", "15", "16", "17"]
 
 
 def random_string(length=20):
@@ -453,7 +453,7 @@ def create_extension(self, *, extname: str, extversion: str = None, dbname: str,
         sql = f"CREATE EXTENSION IF NOT EXISTS {extname}"
         if extversion:
             sql += f" WITH VERSION '{extversion}'"
-        if LooseVersion(self.pgversion) > "9.5":
+        if Version(self.pgversion) > Version("9.5"):
             sql += " CASCADE"
         try:
             with self.cursor(username=self.superuser, dbname=dbname) as cur:
@@ -478,7 +478,7 @@ def create_extension(self, *, extname: str, extversion: str = None, dbname: str,
 
     def drop_extension(self, *, extname: str, dbname: str):
         sql = f"DROP EXTENSION IF EXISTS {extname}"
-        if LooseVersion(self.pgversion) > "9.5":
+        if Version(self.pgversion) > Version("9.5"):
             sql += " CASCADE"
         with self.cursor(username=self.superuser, dbname=dbname) as cur:
             cur.execute(sql)