Merge pull request #127 from ewels/database-improvements

Database improvements

docs/dev/migrations.md

@@ -0,0 +1,10 @@
+# Migrations (for developers)
+
+You need to generate a new migration whenever the database schema (ie any models class) changes. To generate a migration:
+
+```bash
+cd megaqc
+export FLASK_APP=wsgi.py
+flask db upgrade # Update to the latest migration
+flask db migrate
+```

docs/installation/migrations.md

@@ -1,11 +1,20 @@
 # Migrations
 
+## Introduction
+
 Migrations are updates to a database schema. This is relevant if, for example, you
 set up a MegaQC database (using `initdb`), and then a new version of MegaQC is released
 that needs new tables or columns.
 
+## When to migrate
+
 Every time a new version of MegaQC is released, you should ensure your database is
-up to date. Do so using the following commands:
+up to date. You don't need to run the migrations the first time you install MegaQC, because the `megaqc initdb` command
+replaces the need for migrations.
+
+## How to migrate
+
+To migrate, run the following commands:
 
 ```bash
 cd megaqc
@@ -17,3 +26,18 @@ Note: when you run these migrations, you **must** have the same environment as y
 to run MegaQC normally, which means the same value of `FLASK_DEBUG` and
 `MEGAQC_PRODUCTION` environment variables. Otherwise it will migrate the wrong database
 (or a non-existing one).
+
+## Stamping your database
+
+The complete migration history has only recently been added. This means that, if you were using MegaQC in the past when
+migrations were not included in the repo, your database won't know what version you're currently at.
+
+To fix this, first you need to work out which migration your database is up to. Browse through the files in
+`megaqc/migrations/versions`, starting from the oldest date (at the top of each file), until you find a change that
+wasn't present in your database. At this point, note the `revision` value at the top of the file, (e.g. `revision = "007c354223ec"`).
+
+Next, run the following command, replacing `<revision ID>` with the revision you noted above:
+
+```bash
+flask db stamp <revision ID>
+```

megaqc/database.py

@@ -6,6 +6,7 @@
 from builtins import object
 from copy import copy
 
+from flask_migrate import stamp
 from past.builtins import basestring
 from sqlalchemy import create_engine, inspect
 from sqlalchemy.engine.url import make_url
@@ -204,4 +205,8 @@ def init_db(url):
     """Initializes the database."""
     db.metadata.bind = engine
     db.metadata.create_all()
+
+    # Tell alembic that we're at the latest migration, since we just created everything from scratch
+    stamp()
+
     print("Initialized the database.")

megaqc/extensions.py

@@ -4,6 +4,8 @@
 
 Each extension is initialized in the app factory located in app.py.
 """
+from pathlib import Path
+
 from flask_caching import Cache
 from flask_debugtoolbar import DebugToolbarExtension
 from flask_login import LoginManager
@@ -21,5 +23,5 @@
 cache = Cache()
 debug_toolbar = DebugToolbarExtension()
 restful = Api(prefix="/rest_api/v1")
-migrate = Migrate()
+migrate = Migrate(directory=str(Path(__file__).parent / "migrations"))
 json_api = JsonApi()

megaqc/migrations/env.py

@@ -41,7 +41,9 @@ def run_migrations_offline():
     script output.
     """
     url = config.get_main_option("sqlalchemy.url")
-    context.configure(url=url, target_metadata=target_metadata, literal_binds=True)
+    context.configure(
+        url=url, target_metadata=target_metadata, literal_binds=True, compare_type=True
+    )
 
     with context.begin_transaction():
         context.run_migrations()
@@ -74,6 +76,7 @@ def process_revision_directives(context, revision, directives):
     with connectable.connect() as connection:
         context.configure(
             connection=connection,
+            compare_type=True,
             target_metadata=target_metadata,
             process_revision_directives=process_revision_directives,
             **current_app.extensions["migrate"].configure_args

megaqc/migrations/versions/007c354223ec_.py

@@ -0,0 +1,112 @@
+"""
+empty message.
+
+Revision ID: 007c354223ec
+Revises: e38ef2ac89ab
+Create Date: 2020-08-05 12:01:31.378972
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "007c354223ec"
+down_revision = "e38ef2ac89ab"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index("ix_report_report_hash", table_name="report")
+    op.create_index(
+        op.f("ix_report_report_hash"), "report", ["report_hash"], unique=True
+    )
+    op.drop_constraint("report_user_id_fkey", "report", type_="foreignkey")
+    op.create_foreign_key(
+        None, "report", "users", ["user_id"], ["user_id"], ondelete="SET NULL"
+    )
+    op.alter_column(
+        "report_meta", "report_id", existing_type=sa.INTEGER(), nullable=False
+    )
+    op.drop_constraint("report_meta_report_id_fkey", "report_meta", type_="foreignkey")
+    op.create_foreign_key(
+        None, "report_meta", "report", ["report_id"], ["report_id"], ondelete="CASCADE"
+    )
+    op.alter_column("sample", "report_id", existing_type=sa.INTEGER(), nullable=False)
+    op.drop_constraint("sample_report_id_fkey", "sample", type_="foreignkey")
+    op.create_foreign_key(
+        None, "sample", "report", ["report_id"], ["report_id"], ondelete="CASCADE"
+    )
+    op.alter_column(
+        "sample_data", "sample_data_type_id", existing_type=sa.INTEGER(), nullable=False
+    )
+    op.alter_column(
+        "sample_data", "sample_id", existing_type=sa.INTEGER(), nullable=False
+    )
+    op.drop_constraint("sample_data_sample_id_fkey", "sample_data", type_="foreignkey")
+    op.drop_constraint(
+        "sample_data_sample_data_type_id_fkey", "sample_data", type_="foreignkey"
+    )
+    op.create_foreign_key(
+        None, "sample_data", "sample", ["sample_id"], ["sample_id"], ondelete="CASCADE"
+    )
+    op.create_foreign_key(
+        None,
+        "sample_data",
+        "sample_data_type",
+        ["sample_data_type_id"],
+        ["sample_data_type_id"],
+        ondelete="CASCADE",
+    )
+    op.add_column("sample_data_type", sa.Column("schema", sa.Unicode(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("sample_data_type", "schema")
+    op.drop_constraint(None, "sample_data", type_="foreignkey")
+    op.drop_constraint(None, "sample_data", type_="foreignkey")
+    op.create_foreign_key(
+        "sample_data_sample_data_type_id_fkey",
+        "sample_data",
+        "sample_data_type",
+        ["sample_data_type_id"],
+        ["sample_data_type_id"],
+    )
+    op.create_foreign_key(
+        "sample_data_sample_id_fkey",
+        "sample_data",
+        "sample",
+        ["sample_id"],
+        ["sample_id"],
+    )
+    op.alter_column(
+        "sample_data", "sample_id", existing_type=sa.INTEGER(), nullable=True
+    )
+    op.alter_column(
+        "sample_data", "sample_data_type_id", existing_type=sa.INTEGER(), nullable=True
+    )
+    op.drop_constraint(None, "sample", type_="foreignkey")
+    op.create_foreign_key(
+        "sample_report_id_fkey", "sample", "report", ["report_id"], ["report_id"]
+    )
+    op.alter_column("sample", "report_id", existing_type=sa.INTEGER(), nullable=True)
+    op.drop_constraint(None, "report_meta", type_="foreignkey")
+    op.create_foreign_key(
+        "report_meta_report_id_fkey",
+        "report_meta",
+        "report",
+        ["report_id"],
+        ["report_id"],
+    )
+    op.alter_column(
+        "report_meta", "report_id", existing_type=sa.INTEGER(), nullable=True
+    )
+    op.drop_constraint(None, "report", type_="foreignkey")
+    op.create_foreign_key(
+        "report_user_id_fkey", "report", "users", ["user_id"], ["user_id"]
+    )
+    op.drop_index(op.f("ix_report_report_hash"), table_name="report")
+    op.create_index("ix_report_report_hash", "report", ["report_hash"], unique=False)
+    # ### end Alembic commands ###