Seperate integration and unit test in CI (#295)

exasol · Nov 27, 2024 · 38905d2 · 38905d2
1 parent e3953f7
commit 38905d2
Show file tree

Hide file tree

Showing 14 changed files with 133 additions and 63 deletions.
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
@@ -118,7 +118,7 @@ jobs:
           include-hidden-files: true
 
   Tests:
-    name: Tests (Python-${{ matrix.python-version }}, Exasol-${{ matrix.exasol-version}})
+    name: Unit-Tests (Python-${{ matrix.python-version }}, Exasol-${{ matrix.exasol-version}})
     needs: [ Documentation, Lint, Type-Check, Security]
     runs-on: ubuntu-latest
     env:
@@ -139,11 +139,11 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Run Tests and Collect Coverage
-        run: poetry run nox -s test:coverage -- -- --db-version ${{ matrix.exasol-version }}
+        run: poetry run nox -s test:unit -- -- --coverage --db-version ${{ matrix.exasol-version }}
 
       - name: Upload Artifacts
         uses: actions/[email protected]
         with:
-          name: coverage-python${{ matrix.python-version }}
+          name: coverage-python${{ matrix.python-version }}-fast
           path: .coverage
           include-hidden-files: true
diff --git a/.github/workflows/merge-gate.yml b/.github/workflows/merge-gate.yml
@@ -14,33 +14,17 @@ jobs:
 
   slow-checks:
     name: Slow
-    runs-on: ubuntu-latest
-
-    # Even though the environment "manual-approval" will be created automatically,
-    # it still needs to be configured to require interactive review.
-    # See project settings on GitHub (Settings / Environments / manual-approval).
-    environment: manual-approval
-
-    # Replace the steps below with the required actions 
-    # and/or add additional jobs if required
-    # Note: 
-    #   If you add additional jobs, make sure they are added as a requirement 
-    #   to the approve-merge job's input requirements (needs).
-    steps:
-      - name: Tests
-        run: |
-          echo "Slow tests ran successfully"
-
+    uses: ./.github/workflows/slow-checks.yml
 
   # This job ensures inputs have been executed successfully.
   approve-merge:
-    name: Allow Merge 
+    name: Allow Merge
     runs-on: ubuntu-latest
     # If you need additional jobs to be part of the merge gate, add them below
     needs: [ fast-checks, slow-checks ]
 
     # Each job requires a step, so we added this dummy step.
     steps:
-      - name: Approve 
+      - name: Approve
         run: |
           echo "Merge Approved"
diff --git a/.github/workflows/report.yml b/.github/workflows/report.yml
@@ -30,7 +30,8 @@ jobs:
       - name: Copy Artifacts into Root Folder
         working-directory: ./artifacts
         run: |
-          cp coverage-python3.9/.coverage ../
+          poetry run coverage combine --keep coverage-python3.9*/.coverage
+          cp .coverage ../
           cp lint-python3.9/.lint.txt ../
           cp security-python3.9/.security.json ../
 

diff --git a/.github/workflows/slow-checks.yml b/.github/workflows/slow-checks.yml
@@ -0,0 +1,43 @@
+name: Slow-Checks
+
+on:
+  workflow_call:
+    secrets:
+      ALTERNATIVE_GITHUB_TOKEN:
+        required: false
+
+jobs:
+
+  Tests:
+    name: Integration-Tests (Python-${{ matrix.python-version }}, Exasol-${{ matrix.exasol-version}})
+    runs-on: ubuntu-latest
+    # Even though the environment "manual-approval" will be created automatically,
+    # it still needs to be configured to require interactive review.
+    # See project settings on GitHub (Settings / Environments / manual-approval).
+    environment: manual-approval
+    env:
+      GITHUB_TOKEN: ${{ secrets.ALTERNATIVE_GITHUB_TOKEN ||  secrets.GITHUB_TOKEN }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [ "3.9", "3.10", "3.11", "3.12" ]
+        exasol-version: [ "7.1.9" ]
+
+    steps:
+      - name: SCM Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Python & Poetry Environment
+        uses: ./.github/actions/python-environment
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Run Tests and Collect Coverage
+        run: poetry run nox -s test:integration -- -- --coverage --db-version ${{ matrix.exasol-version }}
+
+      - name: Upload Artifacts
+        uses: actions/[email protected]
+        with:
+          name: coverage-python${{ matrix.python-version }}-slow
+          path: .coverage
+          include-hidden-files: true
diff --git a/exasol/toolbox/nox/_shared.py b/exasol/toolbox/nox/_shared.py
@@ -56,8 +56,12 @@ def _context_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(
         formatter_class=argparse.ArgumentDefaultsHelpFormatter
     )
-    parser.add_argument("--db-version")
-    parser.add_argument("--coverage", action="store_true")
+    parser.add_argument(
+        "--db-version", default="7.1.9", help="Specify the Exasol DB version to be used"
+    )
+    parser.add_argument(
+        "--coverage", action="store_true", help="Enable the collection of coverage data"
+    )
     return parser
 
 

diff --git a/exasol/toolbox/nox/_test.py b/exasol/toolbox/nox/_test.py
@@ -75,7 +75,7 @@ def _coverage(
 @nox.session(name="test:unit", python=False)
 def unit_tests(session: Session) -> None:
     """Runs all unit tests"""
-    context = _context(session, coverage=False)
+    context = _context(session)
     _unit_tests(session, PROJECT_CONFIG, context)
 
 
@@ -89,7 +89,7 @@ def integration_tests(session: Session) -> None:
         * pre_integration_tests_hook(session: Session, config: Config, context: MutableMapping[str, Any]) -> bool:
         * post_integration_tests_hook(session: Session, config: Config, context: MutableMapping[str, Any]) -> bool:
     """
-    context = _context(session, coverage=False)
+    context = _context(session)
     _integration_tests(session, PROJECT_CONFIG, context)
 
 

diff --git a/exasol/toolbox/sphinx/multiversion/main.py b/exasol/toolbox/sphinx/multiversion/main.py
@@ -231,7 +231,7 @@ def _create_parser():
     parser.add_argument(
         "--debug",
         action="store_true",
-        help="enable debug mode with increased log verbosity, etc."
+        help="enable debug mode with increased log verbosity, etc.",
     )
     return parser
 
@@ -286,13 +286,15 @@ def _main(args, argv):
     conffile = os.path.join(confdir, "conf.py")
 
     # Get git references
-    gitrefs = list(git.get_refs(
-        str(gitroot),
-        config.smv_tag_whitelist,
-        config.smv_branch_whitelist,
-        config.smv_remote_whitelist,
-        files=(sourcedir, conffile),
-    ))
+    gitrefs = list(
+        git.get_refs(
+            str(gitroot),
+            config.smv_tag_whitelist,
+            config.smv_branch_whitelist,
+            config.smv_remote_whitelist,
+            files=(sourcedir, conffile),
+        )
+    )
 
     # Order git refs
     if config.smv_prefer_remote_refs:
@@ -581,14 +583,18 @@ def _main(args, argv):
             with open(
                 os.path.join(args.outputdir, "index.html"), "w", encoding="utf-8"
             ) as f:
-                logger.debug("Picked up Git references: %s", [ref.name for ref in gitrefs])
+                logger.debug(
+                    "Picked up Git references: %s", [ref.name for ref in gitrefs]
+                )
                 tag_versions = [
                     ref.name
                     for ref in gitrefs
                     if re.match(config.smv_tag_whitelist, ref.name)
                 ]
                 tag_versions = sorted(
-                    tag_versions, key=lambda v: ExasolVersion.from_string(v), reverse=True
+                    tag_versions,
+                    key=lambda v: ExasolVersion.from_string(v),
+                    reverse=True,
                 )
                 branches = [
                     ref.name

diff --git a/exasol/toolbox/templates/github/workflows/checks.yml b/exasol/toolbox/templates/github/workflows/checks.yml
@@ -124,8 +124,8 @@ jobs:
           include-hidden-files: true
 
   Tests:
-    name: Tests (Python-${{ matrix.python-version }}, Exasol-${{ matrix.exasol-version}})
-    needs: [ Documentation, Lint, Type-Check, Security ]
+    name: Unit-Tests (Python-${{ matrix.python-version }}, Exasol-${{ matrix.exasol-version}})
+    needs: [ Documentation, Lint, Type-Check, Security]
     runs-on: ubuntu-latest
     env:
       GITHUB_TOKEN: ${{ secrets.ALTERNATIVE_GITHUB_TOKEN ||  secrets.GITHUB_TOKEN }}
@@ -144,13 +144,12 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Run Tests and Compute Coverage
-        run: poetry run nox -s test:coverage -- -- --db-version ${{ matrix.exasol-version }}
+      - name: Run Tests and Collect Coverage
+        run: poetry run nox -s test:unit -- -- --coverage --db-version ${{ matrix.exasol-version }}
 
       - name: Upload Artifacts
         uses: actions/[email protected]
         with:
-          name: coverage-python${{ matrix.python-version }}
+          name: coverage-python${{ matrix.python-version }}-fast
           path: .coverage
           include-hidden-files: true
-
diff --git a/exasol/toolbox/templates/github/workflows/merge-gate.yml b/exasol/toolbox/templates/github/workflows/merge-gate.yml
@@ -14,33 +14,17 @@ jobs:
 
   slow-checks:
     name: Slow
-    runs-on: ubuntu-latest
-
-    # Even though the environment "manual-approval" will be created automatically,
-    # it still needs to be configured to require interactive review.
-    # See project settings on GitHub (Settings / Environments / manual-approval).
-    environment: manual-approval
-
-    # Replace the steps below with the required actions 
-    # and/or add additional jobs if required
-    # Note: 
-    #   If you add additional jobs, make sure they are added as a requirement 
-    #   to the approve-merge job's input requirements (needs).
-    steps:
-      - name: Tests
-        run: |
-          echo "Slow tests ran successfully"
-
+    uses: ./.github/workflows/slow-checks.yml
 
   # This job ensures inputs have been executed successfully.
   approve-merge:
-    name: Allow Merge 
+    name: Allow Merge
     runs-on: ubuntu-latest
     # If you need additional jobs to be part of the merge gate, add them below
     needs: [ fast-checks, slow-checks ]
 
     # Each job requires a step, so we added this dummy step.
     steps:
-      - name: Approve 
+      - name: Approve
         run: |
           echo "Merge Approved"
diff --git a/exasol/toolbox/templates/github/workflows/report.yml b/exasol/toolbox/templates/github/workflows/report.yml
@@ -30,7 +30,8 @@ jobs:
       - name: Copy Artifacts into Root Folder
         working-directory: ./artifacts
         run: |
-          cp coverage-python3.9/.coverage ../
+          poetry run coverage combine --keep coverage-python3.9*/.coverage
+          cp .coverage ../
           cp lint-python3.9/.lint.txt ../
           cp security-python3.9/.security.json ../
 

diff --git a/exasol/toolbox/templates/github/workflows/slow-checks.yml b/exasol/toolbox/templates/github/workflows/slow-checks.yml
@@ -0,0 +1,43 @@
+name: Slow-Checks
+
+on:
+  workflow_call:
+    secrets:
+      ALTERNATIVE_GITHUB_TOKEN:
+        required: false
+
+jobs:
+
+  Tests:
+    name: Integration-Tests (Python-${{ matrix.python-version }}, Exasol-${{ matrix.exasol-version}})
+    runs-on: ubuntu-latest
+    # Even though the environment "manual-approval" will be created automatically,
+    # it still needs to be configured to require interactive review.
+    # See project settings on GitHub (Settings / Environments / manual-approval).
+    environment: manual-approval
+    env:
+      GITHUB_TOKEN: ${{ secrets.ALTERNATIVE_GITHUB_TOKEN ||  secrets.GITHUB_TOKEN }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [ "3.9", "3.10", "3.11", "3.12" ]
+        exasol-version: [ "7.1.9" ]
+
+    steps:
+      - name: SCM Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Python & Poetry Environment
+        uses: exasol/python-toolbox/.github/actions/[email protected]
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Run Tests and Collect Coverage
+        run: poetry run nox -s test:integration -- -- --coverage --db-version ${{ matrix.exasol-version }}
+
+      - name: Upload Artifacts
+        uses: actions/[email protected]
+        with:
+          name: coverage-python${{ matrix.python-version }}-slow
+          path: .coverage
+          include-hidden-files: true
diff --git a/test/integration/cli/workflow-install.t b/test/integration/cli/workflow-install.t
@@ -14,3 +14,4 @@ Check if all workflows have been installed
   merge-gate.yml
   pr-merge.yml
   report.yml
+  slow-checks.yml
diff --git a/test/integration/cli/workflow-list.t b/test/integration/cli/workflow-list.t
@@ -10,3 +10,4 @@ List all available workflows
   merge-gate
   pr-merge
   report
+  slow-checks
diff --git a/test/unit/template_test.py b/test/unit/template_test.py
@@ -18,6 +18,7 @@ def test_retrieve_workflow_templates():
         "merge-gate": "merge-gate.yml",
         "pr-merge": "pr-merge.yml",
         "report": "report.yml",
+        "slow-checks": "slow-checks.yml",
     }
     actual = template._templates(subpackage)
     actual = {name: path.name for name, path in actual.items()}
@@ -54,6 +55,7 @@ def test_retrieve_issue_templates():
                 "merge-gate": "merge-gate.yml",
                 "pr-merge": "pr-merge.yml",
                 "report": "report.yml",
+                "slow-checks": "slow-checks.yml",
             },
         ),
         (
@@ -105,6 +107,7 @@ def test_retrieve_templates(subpackage, expected):
                 "merge-gate.yml",
                 "pr-merge.yml",
                 "report.yml",
+                "slow-checks.yml",
             ],
         ),
     ],