#1225 Upgrade SQLAlchemy to v2

.talismanrc

@@ -1,41 +1,44 @@
 fileignoreconfig:
-- filename: tests/app/dao/notification_dao/test_notification_dao_performance_platform.py
-  checksum: b3ffec0e69481a73091ea8388eff348842a82ce00d7f8c9a65dbd8498ef1b220
-- filename: tests/app/dao/notification_dao/test_notification_dao_delete_notifications.py
-  checksum: a31c6f7abed822d20b6371c96aaa3f2fd19200a8a2f8dac8240233986df80e0b
-- filename: tests/app/notifications/test_rest.py
-  checksum: 22a54f4187f7dfe404e4eefb677ff376933aff2f053362662c6d5b0b2eaad891
-- filename: tests/app/service/test_api_key_endpoints.py
-  checksum: a8268dd19d4394733a974db319c5a2353aad455ff1bab189949fa1a9a757e166
+- filename: .github/workflows/dev-tests.yml
+  checksum: 0700d2eba3138fc34ef5bf2787ce5403cbd06797200922d35ce1f3f6ff8aaf74
+- filename: app/models.py
+  checksum: fe15ce236d203162933acdeb8c9488df52704b6ab18ac76faf3edfbcc80d54b0
+- filename: cd/application-deployment/perf/vaec-api-task-definition.json
+  checksum: b32eae40a0fb22d71ef585eaf56983ced8abfa24087f6d1d03e4a3e7b7d48cb0
+- filename: cd/application-deployment/perf/vaec-celery-task-definition.json
+  checksum: c7d10bd01c45d4f2199a0a89d6d94fa1c222d5770187051320a0a0e2a5ca5d30
 - filename: tests/app/conftest.py
   checksum: 96c0f09e85dc957d90da2728911d684744559fb08069c4d8763aa99329f26511
-- filename: tests/app/service/test_rest.py
-  checksum: c4331cbc7d3c935ceca4fa94c40034348f00369e50540b02a36dbf6d98bc4fe9
 - filename: tests/app/dao/notification_dao/test_notification_dao.py
   checksum: b32000cfe4245fb5080d1744789913d53b73d68fb0573b6c2e6eaf424ae8255d
+- filename: tests/app/dao/notification_dao/test_notification_dao_delete_notifications.py
+  checksum: a31c6f7abed822d20b6371c96aaa3f2fd19200a8a2f8dac8240233986df80e0b
+- filename: tests/app/dao/notification_dao/test_notification_dao_performance_platform.py
+  checksum: b3ffec0e69481a73091ea8388eff348842a82ce00d7f8c9a65dbd8498ef1b220
+- filename: tests/app/dao/test_api_key_dao.py
+  checksum: ef2a3ba24dc9debf0c48c0ebb0a83d9472abaadda7c6a19ac4fd637706913acd
+- filename: tests/app/dao/test_fact_notification_status_dao.py
+  checksum: 2cad8d3b4a53bad53c3adeb9c81968d11b21e75d729976120bdd66ca91031738
+- filename: tests/app/dao/test_fido2_keys_dao.py
+  checksum: d31637bf80436d874d1bd8291c62a0ada156fa71620c2299b0cee4e4c746eaeb
+- filename: tests/app/dao/test_services_dao.py
+  checksum: 3ea2f8d01afd32d1c1586488aa442835b45895f7ee270b0da49d3134f37c7685
+- filename: tests/app/db.py
+  checksum: df9a020fece4a10601c2e3200e663eed4bf327e2f0b16eac2d028aff56bca2ef
 - filename: tests/app/delivery/test_send_to_providers.py
   checksum: 261192636154bcebca11e0991525270a124cd57f8c06d2f515c8077762662e3e
 - filename: tests/app/notifications/test_process_notifications.py
   checksum: 4134fa90ed48750b6fc73ddd4cc0416ad21474296cdcc7ae75ada0df70e20a4c
-- filename: tests/app/notifications/test_validators.py
-  checksum: b81d29ae6ee3b88b429610261021069b388de1d50163c9f3176f7502c625a17a
 - filename: tests/app/notifications/test_receive_notification.py
   checksum: 37abf7a3f23fc8acf022930b4fad1c42cb344b575837dec602bdedad51360190
-- filename: tests/app/dao/test_services_dao.py
-  checksum: 3ea2f8d01afd32d1c1586488aa442835b45895f7ee270b0da49d3134f37c7685
-- filename: tests/app/dao/test_fact_notification_status_dao.py
-  checksum: 2cad8d3b4a53bad53c3adeb9c81968d11b21e75d729976120bdd66ca91031738
-- filename: tests/app/dao/test_fido2_keys_dao.py
-  checksum: d31637bf80436d874d1bd8291c62a0ada156fa71620c2299b0cee4e4c746eaeb
-- filename: tests/app/dao/test_api_key_dao.py
-  checksum: ef2a3ba24dc9debf0c48c0ebb0a83d9472abaadda7c6a19ac4fd637706913acd
+- filename: tests/app/notifications/test_rest.py
+  checksum: 22a54f4187f7dfe404e4eefb677ff376933aff2f053362662c6d5b0b2eaad891
+- filename: tests/app/notifications/test_validators.py
+  checksum: b81d29ae6ee3b88b429610261021069b388de1d50163c9f3176f7502c625a17a
+- filename: tests/app/service/test_api_key_endpoints.py
+  checksum: a8268dd19d4394733a974db319c5a2353aad455ff1bab189949fa1a9a757e166
+- filename: tests/app/service/test_rest.py
+  checksum: c4331cbc7d3c935ceca4fa94c40034348f00369e50540b02a36dbf6d98bc4fe9
 - filename: tests/app/v2/notifications/test_push_broadcast_notifications.py
   checksum: eed7b99992b43ff534736502d72a3b8f3f3cdbdd17c26be88f9dea9614321e38
-- filename: cd/application-deployment/perf/vaec-api-task-definition.json
-  checksum: b32eae40a0fb22d71ef585eaf56983ced8abfa24087f6d1d03e4a3e7b7d48cb0
-- filename: cd/application-deployment/perf/vaec-celery-task-definition.json
-  checksum: c7d10bd01c45d4f2199a0a89d6d94fa1c222d5770187051320a0a0e2a5ca5d30
-- filename: tests/app/db.py
-  checksum: df9a020fece4a10601c2e3200e663eed4bf327e2f0b16eac2d028aff56bca2ef
-- filename: .github/workflows/dev-tests.yml
-  checksum: 0700d2eba3138fc34ef5bf2787ce5403cbd06797200922d35ce1f3f6ff8aaf74
+version: "1.0"

Dockerfile

@@ -0,0 +1,26 @@
+# Python 3.10 is supported until October 2026.
+# Alpine Linux 3.19 is supported until 1 November 2025
+FROM python:3.10-alpine3.19
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    # https://flask.palletsprojects.com/en/2.2.x/config/?highlight=flask_debug#DEBUG
+    FLASK_DEBUG="true"
+
+RUN adduser -h /app -D vanotify
+WORKDIR /app
+
+
+RUN apk add --no-cache bash build-base postgresql-dev libffi-dev libmagic libcurl python3-dev openssl-dev curl-dev musl-dev rust cargo git \
+  && python -m pip install --upgrade pip \
+  && python -m pip install wheel \
+  && pip install "Cython<3.0" \
+  && pip install "pyyaml==6.0.0" --no-build-isolation \
+  && pip install --upgrade setuptools==65.5.1
+
+COPY --chown=vanotify requirements-app.txt .
+RUN pip install --no-cache-dir -r requirements-app.txt
+
+VOLUME /app
+
+USER vanotify
+CMD ["./scripts/run_app.sh"]

Makefile

@@ -35,11 +35,16 @@ check-dependencies: install-safety ## Scan dependencies for security vulnerabili
 
 	safety check -r requirements.txt --full-report -i 51668
 
+environment-vars: ## Export environment variables
+	echo "Exporting environment variables"
+	export $(grep -v '^#' .env | xargs)
+
 .PHONY:
 	help \
 	generate-version-file \
 	test \
 	test-requirements \
 	clean \
 	check-vulnerabilities \
-	check-dependencies
+	check-dependencies \
+	environment-vars

app/history_meta.py

@@ -18,10 +18,13 @@
 import datetime
 from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.orm import mapper, attributes, object_mapper
+from sqlalchemy.orm import registry, DeclarativeBase  # import map_imperatively
 from sqlalchemy.orm.properties import RelationshipProperty, ColumnProperty
 from sqlalchemy import Table, Column, ForeignKeyConstraint, Integer
 from sqlalchemy import util
 
+reg_mapper = registry()
+
 
 def col_references_table(
     col,
@@ -123,7 +126,7 @@ def _col_copy(col):
         bases = local_mapper.base_mapper.class_.__bases__
     versioned_cls = type.__new__(type, '%sHistory' % cls.__name__, bases, {})
 
-    m = mapper(
+    m = reg_mapper.map_imperatively(
         versioned_cls,
         table,
         inherits=super_history_mapper,
@@ -146,7 +149,7 @@ def map(
             *arg,
             **kw,
         ):
-            mp = mapper(cls, *arg, **kw)
+            mp = reg_mapper.map_imperatively(cls, *arg, **kw)
             _history_mapper(mp)
             return mp
 

app/model/user.py

@@ -59,7 +59,7 @@ def __init__(
     )
 
     services = db.relationship('Service', secondary='user_to_service', backref='users')
-    organisations = db.relationship('Organisation', secondary='user_to_organisation', backref='users')
+    # organisations = db.relationship('Organisation', secondary='user_to_organisation', backref='users')
 
     @property
     def password(self):

app/models.py

@@ -11,7 +11,7 @@
     check_hash,
     hashpw,
 )
-from app.history_meta import Versioned
+from app.history_meta import Versioned, reg_mapper
 from app.model import User, EMAIL_AUTH_TYPE
 from app.va.identifier import IdentifierType
 from flask import url_for, current_app
@@ -105,22 +105,36 @@ class ServiceUser(db.Model):
 
 user_to_organisation = db.Table(
     'user_to_organisation',
-    db.Model.metadata,
-    db.Column('user_id', UUID(as_uuid=True), db.ForeignKey('users.id')),
-    db.Column('organisation_id', UUID(as_uuid=True), db.ForeignKey('organisation.id')),
+    reg_mapper.metadata,
+    db.Column('user_id', UUID(as_uuid=True), db.ForeignKey('users.id'), primary_key=True),
+    db.Column('organisation_id', UUID(as_uuid=True), db.ForeignKey('organisation.id'), primary_key=True),
     UniqueConstraint('user_id', 'organisation_id', name='uix_user_to_organisation'),
 )
 
+
+class UserToOrganisation:
+    pass
+
+
+reg_mapper.map_imperatively(UserToOrganisation, user_to_organisation)
+
 user_folder_permissions = db.Table(
     'user_folder_permissions',
-    db.Model.metadata,
+    reg_mapper.metadata,
     db.Column('user_id', UUID(as_uuid=True), primary_key=True),
     db.Column('template_folder_id', UUID(as_uuid=True), db.ForeignKey('template_folder.id'), primary_key=True),
     db.Column('service_id', UUID(as_uuid=True), primary_key=True),
     db.ForeignKeyConstraint(['user_id', 'service_id'], ['user_to_service.user_id', 'user_to_service.service_id']),
     db.ForeignKeyConstraint(['template_folder_id', 'service_id'], ['template_folder.id', 'template_folder.service_id']),
 )
 
+
+class UserFolderPermissions:
+    pass
+
+
+reg_mapper.map_imperatively(UserFolderPermissions, user_folder_permissions)
+
 BRANDING_GOVUK = 'govuk'  # Deprecated outside migrations
 BRANDING_ORG = 'org'
 BRANDING_BOTH = 'both'
@@ -160,13 +174,20 @@ def serialize(self):
 
 service_email_branding = db.Table(
     'service_email_branding',
-    db.Model.metadata,
+    reg_mapper.metadata,
     # service_id is a primary key as you can only have one email branding per service
     db.Column('service_id', UUID(as_uuid=True), db.ForeignKey('services.id'), primary_key=True, nullable=False),
     db.Column('email_branding_id', UUID(as_uuid=True), db.ForeignKey('email_branding.id'), nullable=False),
 )
 
 
+class ServiceEmailBranding:
+    pass
+
+
+reg_mapper.map_imperatively(ServiceEmailBranding, service_email_branding)
+
+
 INTERNATIONAL_SMS_TYPE = 'international_sms'
 INBOUND_SMS_TYPE = 'inbound_sms'
 SCHEDULE_NOTIFICATIONS = 'schedule_notifications'
@@ -315,7 +336,7 @@ class ProviderDetails(db.Model):
     supports_international = db.Column(db.Boolean, nullable=False, default=False)
 
 
-class Service(db.Model, Versioned):
+class Service(db.Model):
     __tablename__ = 'services'
 
     id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
@@ -358,9 +379,9 @@ class Service(db.Model, Versioned):
 
     p2p_enabled = db.Column(db.Boolean, nullable=True, default=False)
 
-    email_branding = db.relationship(
-        'EmailBranding', secondary=service_email_branding, uselist=False, backref=db.backref('services', lazy='dynamic')
-    )
+    # email_branding = db.relationship(
+    #     'EmailBranding', secondary=service_email_branding, uselist=False, backref=db.backref('services', lazy='dynamic')
+    # )
 
     @classmethod
     def from_json(
@@ -649,7 +670,7 @@ def __repr__(self):
         return 'Recipient {} of type: {}'.format(self.recipient, self.recipient_type)
 
 
-class ServiceCallback(db.Model, Versioned):
+class ServiceCallback(db.Model):
     __tablename__ = 'service_callback'
 
     def __init__(
@@ -720,7 +741,7 @@ class ServiceCallbackChannel(db.Model):
     channel = db.Column(db.String, primary_key=True)
 
 
-class ApiKey(db.Model, Versioned):
+class ApiKey(db.Model):
     __tablename__ = 'api_keys'
 
     id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
@@ -780,13 +801,13 @@ class TemplateFolder(db.Model):
 
     service = db.relationship('Service', backref='all_template_folders')
     parent = db.relationship('TemplateFolder', remote_side=[id], backref='subfolders')
-    users = db.relationship(
-        'ServiceUser',
-        uselist=True,
-        backref=db.backref('folders', foreign_keys='user_folder_permissions.c.template_folder_id'),
-        secondary='user_folder_permissions',
-        primaryjoin='TemplateFolder.id == user_folder_permissions.c.template_folder_id',
-    )
+    # users = db.relationship(
+    #     'ServiceUser',
+    #     uselist=True,
+    #     backref=db.backref('folders', foreign_keys='user_folder_permissions.c.template_folder_id'),
+    #     secondary='user_folder_permissions',
+    #     primaryjoin='TemplateFolder.id == user_folder_permissions.c.template_folder_id',
+    # )
 
     __table_args__ = (UniqueConstraint('id', 'service_id', name='ix_id_service_id'), {})
 
@@ -818,12 +839,19 @@ def get_users_with_permission(self):
 
 template_folder_map = db.Table(
     'template_folder_map',
-    db.Model.metadata,
+    reg_mapper.metadata,
     # template_id is a primary key as a template can only belong in one folder
     db.Column('template_id', UUID(as_uuid=True), db.ForeignKey('templates.id'), primary_key=True, nullable=False),
     db.Column('template_folder_id', UUID(as_uuid=True), db.ForeignKey('template_folder.id'), nullable=False),
 )
 
+
+class TemplateFolderMap:
+    pass
+
+
+reg_mapper.map_imperatively(TemplateFolderMap, template_folder_map)
+
 PRECOMPILED_TEMPLATE_NAME = 'Pre-compiled PDF'
 
 
@@ -993,14 +1021,14 @@ class Template(TemplateBase):
     service = db.relationship('Service', backref='templates')
     version = db.Column(db.Integer, default=0, nullable=False)
 
-    folder = db.relationship(
-        'TemplateFolder',
-        secondary=template_folder_map,
-        uselist=False,
-        # eagerly load the folder whenever the template object is fetched
-        lazy='joined',
-        backref=db.backref('templates'),
-    )
+    # folder = db.relationship(
+    #     'TemplateFolder',
+    #     secondary=template_folder_map,
+    #     uselist=False,
+    #     # eagerly load the folder whenever the template object is fetched
+    #     lazy='joined',
+    #     backref=db.backref('templates'),
+    # )
 
     def get_link(self):
         # TODO: use "/v2/" route once available