Merge pull request #5435 from dfe-analytical-services/EES-5705-report…

…-robot-failures-from-final-test-report-not-merged-test-report

EES-5705 correcting failed test count in Slack / EES-5724 failing test suites early on publisher failure / EES-5522 breaking UI pipeline into templates

azure-pipelines-run-ui-test-suite.yml

@@ -0,0 +1,62 @@
+parameters:
+  - name: jobName
+    type: string
+  - name: displayName
+    type: string
+  - name: testFolder
+    type: string
+  - name: artifactName
+    type: string
+
+jobs:
+  - job: ${{ parameters.jobName }}
+    displayName: ${{ parameters.displayName }}
+    timeoutInMinutes: 160
+    cancelTimeoutInMinutes: 10
+    condition: succeededOrFailed()
+    pool: ees-ubuntu2204-large
+    workspace:
+      clean: all
+    steps:
+    - checkout: self
+      clean: true
+      fetchDepth: 5
+      fetchTags: false
+
+    - task: UsePythonVersion@0
+      displayName: Use Python 3.10
+      timeoutInMinutes: 5
+      inputs:
+        versionSpec: 3.10
+
+    - task: AzureKeyVault@2
+      displayName: Azure Key Vault - s101d01-kv-ees-01
+      inputs:
+        azureSubscription: $(SPN_NAME)
+        KeyVaultName: s101d01-kv-ees-01
+        SecretsFilter: ees-test-ADMIN-PASSWORD,ees-test-ANALYST-PASSWORD,ees-test-expiredinvite-password,ees-test-NOINVITE-PASSWORD,ees-test-PENDINGINVITE-PASSWORD,ees-alerts-slackapptoken
+        RunAsPreJob: true
+
+    - task: PythonScript@0
+      displayName: Run tests
+      condition: succeededOrFailed()
+      inputs:
+        scriptPath: tests/robot-tests/scripts/run_tests_pipeline.py
+        arguments: --admin-pass '"$(ees-test-ADMIN-PASSWORD)"' --analyst-pass '"$(ees-test-ANALYST-PASSWORD)"' --expiredinvite-pass '"$(ees-test-expiredinvite-password)"' --noinvite-pass '"$(ees-test-NOINVITE-PASSWORD)"' --pendinginvite-pass '"$(ees-test-PENDINGINVITE-PASSWORD)"' --env "dev" --file "${{ parameters.testFolder }}" --processes 4 --rerun-attempts 3
+        # The magic incantation '"$(variable)"'was added by Mark to resolve an issue with Analyst password that contained ampersands.
+        workingDirectory: tests/robot-tests
+      env:
+        SLACK_APP_TOKEN: $(ees-alerts-slackapptoken)
+
+    - task: PublishTestResults@2
+      displayName: Publish test results
+      inputs:
+        testResultsFiles: tests/robot-tests/test-results/xunit.xml
+        failTaskOnFailedTests: true
+
+    - task: PublishPipelineArtifact@1
+      displayName: Publish pipeline artifact
+      condition: succeededOrFailed()
+      inputs:
+        path: tests/robot-tests/test-results/
+        artifactName: ${{ parameters.artifactName }}

azure-pipelines-ui-tests.yml

@@ -17,261 +17,41 @@ resources:
     ref: refs/heads/dev
 
 jobs:
-- job: Public
-  displayName: Public suite - Robot UI tests
-  timeoutInMinutes: 160
-  cancelTimeoutInMinutes: 10
-  condition: succeededOrFailed()
-  pool: ees-ubuntu2204-large
-  workspace:
-    clean: all
-  steps:
-  - checkout: self
-    clean: true
-    fetchDepth: 5
-    fetchTags: false
+- template: azure-pipelines-run-ui-test-suite.yml
+  parameters:
+    jobName: Public
+    displayName: Public suite - Robot UI tests
+    testFolder: tests/general_public
+    artifactName: test-results-public
+
+- template: azure-pipelines-run-ui-test-suite.yml
+  parameters:
+    jobName: PublishAndAmend
+    displayName: Publish release and amend suites - Robot UI tests
+    testFolder: tests/admin_and_public_2
+    artifactName: test-results-admin-and-public-2
+
+- template: azure-pipelines-run-ui-test-suite.yml
+  parameters:
+    jobName: Admin
+    displayName: Admin suites - Robot UI tests
+    testFolder: tests/admin
+    artifactName: test-results-admin
+
+- template: azure-pipelines-run-ui-test-suite.yml
+  parameters:
+    jobName: AdminAndPublic
+    displayName: Admin & public suites - Robot UI tests
+    testFolder: tests/admin_and_public
+    artifactName: test-results-admin-public
+
+- template: azure-pipelines-run-ui-test-suite.yml
+  parameters:
+    jobName: PublicAPI
+    displayName: Public API suite - Robot UI tests
+    testFolder: tests/public_api
+    artifactName: test-results-admin-public-api
 
-  - task: UsePythonVersion@0
-    displayName: Use Python 3.10
-    timeoutInMinutes: 5
-    inputs:
-      versionSpec: 3.10
-
-  - task: AzureKeyVault@2
-    displayName: Azure Key Vault - s101d01-kv-ees-01
-    inputs:
-      azureSubscription: $(SPN_NAME)
-      KeyVaultName: s101d01-kv-ees-01
-      SecretsFilter: ees-alerts-slackapptoken
-      RunAsPreJob: true
-
-  - task: PythonScript@0
-    displayName: Public UI tests
-    inputs:
-      scriptPath: tests/robot-tests/scripts/run_tests_pipeline.py
-      arguments: --admin-pass "test" --analyst-pass "test" --expiredinvite-pass "test" --noinvite-pass "test" --pendinginvite-pass "test" --env "dev" --file "tests/general_public" --processes 4 --rerun-attempts 3
-      workingDirectory: tests/robot-tests
-    env:
-      SLACK_APP_TOKEN: $(ees-alerts-slackapptoken)
-
-  - task: PublishTestResults@2
-    displayName: Publish Test Results
-    condition: succeededOrFailed()
-    inputs:
-      testResultsFiles: tests/robot-tests/test-results/xunit.xml
-      failTaskOnFailedTests: true
-
-  - task: PublishPipelineArtifact@1
-    displayName: Publish Test Pipeline Artifact
-    condition: succeededOrFailed()
-    inputs:
-      path: tests/robot-tests/test-results/
-      artifactName: test-results-public
-
-- job: PublishAndAmend
-  displayName: Publish release and amend suites - Robot UI tests
-  timeoutInMinutes: 160
-  cancelTimeoutInMinutes: 10
-  condition: succeededOrFailed()
-  pool: ees-ubuntu2204-large
-  workspace:
-    clean: all
-  steps:
-  - checkout: self
-    clean: true
-    fetchDepth: 5
-    fetchTags: false
-
-  - task: UsePythonVersion@0
-    displayName: Use Python 3.10
-    timeoutInMinutes: 5
-    inputs:
-      versionSpec: 3.10
-
-  - task: AzureKeyVault@2
-    displayName: Azure Key Vault - s101d01-kv-ees-01
-    inputs:
-      azureSubscription: $(SPN_NAME)
-      KeyVaultName: s101d01-kv-ees-01
-      SecretsFilter: ees-test-ADMIN-PASSWORD,ees-test-ANALYST-PASSWORD,ees-test-expiredinvite-password,ees-test-NOINVITE-PASSWORD,ees-test-PENDINGINVITE-PASSWORD,ees-alerts-slackapptoken
-      RunAsPreJob: true
-
-  - task: PythonScript@0
-    displayName: Publish release and amend UI tests
-    condition: succeededOrFailed()
-    inputs:
-      scriptPath: tests/robot-tests/scripts/run_tests_pipeline.py
-      arguments: --admin-pass '"$(ees-test-ADMIN-PASSWORD)"' --analyst-pass '"$(ees-test-ANALYST-PASSWORD)"' --expiredinvite-pass '"$(ees-test-expiredinvite-password)"' --noinvite-pass '"$(ees-test-NOINVITE-PASSWORD)"' --pendinginvite-pass '"$(ees-test-PENDINGINVITE-PASSWORD)"' --env "dev" --file "tests/admin_and_public_2" --processes 4 --rerun-attempts 3
-    #  The magic incantation '"$(variable)"'was added by Mark to resolve an issue with Analyst password that contained ampersands.
-      workingDirectory: tests/robot-tests
-    env:
-      SLACK_APP_TOKEN: $(ees-alerts-slackapptoken)
-
-  - task: PublishTestResults@2
-    displayName: Publish Test Results
-    inputs:
-      testResultsFiles: tests/robot-tests/test-results/xunit.xml
-      failTaskOnFailedTests: true
-
-  - task: PublishPipelineArtifact@1
-    displayName: Publish Pipeline Artifact
-    condition: succeededOrFailed()
-    inputs:
-      path: tests/robot-tests/test-results/
-      artifactName: test-results-admin-and-public-2
-
-
-- job: Admin
-  displayName: Admin suites - Robot UI tests
-  timeoutInMinutes: 160
-  cancelTimeoutInMinutes: 10
-  condition: succeededOrFailed()
-  pool: ees-ubuntu2204-large
-  workspace:
-    clean: all
-  steps:
-  - checkout: self
-    clean: true
-    fetchDepth: 5
-    fetchTags: false
-
-  - task: UsePythonVersion@0
-    displayName: Use Python 3.10
-    timeoutInMinutes: 5
-    inputs:
-      versionSpec: 3.10
-
-  - task: AzureKeyVault@2
-    displayName: Azure Key Vault - s101d01-kv-ees-01
-    inputs:
-      azureSubscription: $(SPN_NAME)
-      KeyVaultName: s101d01-kv-ees-01
-      SecretsFilter: ees-test-ADMIN-PASSWORD,ees-test-ANALYST-PASSWORD,ees-test-expiredinvite-password,ees-test-NOINVITE-PASSWORD,ees-test-PENDINGINVITE-PASSWORD,ees-alerts-slackapptoken
-      RunAsPreJob: true
-
-  - task: PythonScript@0
-    displayName: Admin UI tests
-    condition: succeededOrFailed()
-    inputs:
-      scriptPath: tests/robot-tests/scripts/run_tests_pipeline.py
-      arguments: --admin-pass '"$(ees-test-ADMIN-PASSWORD)"' --analyst-pass '"$(ees-test-ANALYST-PASSWORD)"' --expiredinvite-pass '"$(ees-test-expiredinvite-password)"' --noinvite-pass '"$(ees-test-NOINVITE-PASSWORD)"' --pendinginvite-pass '"$(ees-test-PENDINGINVITE-PASSWORD)"' --env "dev" --file "tests/admin" --processes 4 --rerun-attempts 3
-      workingDirectory: tests/robot-tests
-    env:
-      SLACK_APP_TOKEN: $(ees-alerts-slackapptoken)
-
-  - task: PublishTestResults@2
-    displayName: Publish Test Results
-    condition: succeededOrFailed()
-    inputs:
-      testResultsFiles: tests/robot-tests/test-results/xunit.xml
-      failTaskOnFailedTests: true
-
-  - task: PublishPipelineArtifact@1
-    displayName: Publish Test Pipeline Artifact
-    condition: succeededOrFailed()
-    inputs:
-      path: tests/robot-tests/test-results/
-      artifactName: test-results-admin
-
-- job: AdminAndPublic
-  displayName: Admin & public suites - Robot UI tests
-  timeoutInMinutes: 160
-  cancelTimeoutInMinutes: 10
-  condition: succeededOrFailed()
-  pool: ees-ubuntu2204-large
-  workspace:
-    clean: all
-  steps:
-  - checkout: self
-    clean: true
-    fetchDepth: 5
-    fetchTags: false
-
-  - task: UsePythonVersion@0
-    displayName: Use Python 3.10
-    timeoutInMinutes: 5
-    inputs:
-      versionSpec: 3.10
-
-  - task: AzureKeyVault@2
-    displayName: Azure Key Vault - s101d01-kv-ees-01
-    inputs:
-      azureSubscription: $(SPN_NAME)
-      KeyVaultName: s101d01-kv-ees-01
-      SecretsFilter: ees-test-ADMIN-PASSWORD,ees-test-ANALYST-PASSWORD,ees-test-expiredinvite-password,ees-test-NOINVITE-PASSWORD,ees-test-PENDINGINVITE-PASSWORD,ees-alerts-slackapptoken
-      RunAsPreJob: true
-
-  - task: PythonScript@0
-    displayName: Admin public UI tests
-    inputs:
-      scriptPath: tests/robot-tests/scripts/run_tests_pipeline.py
-      arguments: --admin-pass '"$(ees-test-ADMIN-PASSWORD)"' --analyst-pass '"$(ees-test-ANALYST-PASSWORD)"' --expiredinvite-pass '"$(ees-test-expiredinvite-password)"' --noinvite-pass '"$(ees-test-NOINVITE-PASSWORD)"' --pendinginvite-pass '"$(ees-test-PENDINGINVITE-PASSWORD)"' --env "dev" --file "tests/admin_and_public" --processes 4 --rerun-attempts 3
-      workingDirectory: tests/robot-tests
-    env:
-      SLACK_APP_TOKEN: $(ees-alerts-slackapptoken)
-
-  - task: PublishTestResults@2
-    displayName: Publish Test Results
-    inputs:
-      testResultsFiles: tests/robot-tests/test-results/xunit.xml
-      failTaskOnFailedTests: true
-
-  - task: PublishPipelineArtifact@1
-    displayName: Publish Test Pipeline Artifact
-    condition: succeededOrFailed()
-    inputs:
-      path: tests/robot-tests/test-results/
-      artifactName: test-results-admin-public
-
-- job: PublicAPI
-  displayName: Public API suite - Robot UI tests
-  timeoutInMinutes: 160
-  cancelTimeoutInMinutes: 10
-  condition: succeededOrFailed()
-  pool: ees-ubuntu2204-large
-  workspace:
-    clean: all
-  steps:
-  - checkout: self
-    clean: true
-    fetchDepth: 5
-    fetchTags: false
-
-  - task: UsePythonVersion@0
-    displayName: Use Python 3.10
-    timeoutInMinutes: 5
-    inputs:
-      versionSpec: 3.10
-
-  - task: AzureKeyVault@2
-    displayName: Azure Key Vault - s101d01-kv-ees-01
-    inputs:
-      azureSubscription: $(SPN_NAME)
-      KeyVaultName: s101d01-kv-ees-01
-      SecretsFilter: ees-test-ADMIN-PASSWORD,ees-test-ANALYST-PASSWORD,ees-test-expiredinvite-password,ees-test-NOINVITE-PASSWORD,ees-test-PENDINGINVITE-PASSWORD,ees-alerts-slackapptoken
-      RunAsPreJob: true
-
-  - task: PythonScript@0
-    displayName: Public API UI tests
-    inputs:
-      scriptPath: tests/robot-tests/scripts/run_tests_pipeline.py
-      arguments: --admin-pass '"$(ees-test-ADMIN-PASSWORD)"' --analyst-pass '"$(ees-test-ANALYST-PASSWORD)"' --expiredinvite-pass '"$(ees-test-expiredinvite-password)"' --noinvite-pass '"$(ees-test-NOINVITE-PASSWORD)"' --pendinginvite-pass '"$(ees-test-PENDINGINVITE-PASSWORD)"' --env "dev" --file "tests/public_api" --processes 4 --rerun-attempts 3
-      workingDirectory: tests/robot-tests
-    env:
-      SLACK_APP_TOKEN: $(ees-alerts-slackapptoken)
-
-  - task: PublishTestResults@2
-    displayName: Publish Test Results
-    inputs:
-      testResultsFiles: tests/robot-tests/test-results/xunit.xml
-      failTaskOnFailedTests: true
-
-  - task: PublishPipelineArtifact@1
-    displayName: Publish Test Pipeline Artifact
-    condition: succeededOrFailed()
-    inputs:
-      path: tests/robot-tests/test-results/
-      artifactName: test-results-admin-public-api
 
         #- job: PublicPlaywrightUItest
         #  displayName: Public suite - Playwright UI tests

tests/robot-tests/report-modifiers/CheckForAtLeastOnePassingRunPrerebotModifier.py

@@ -16,9 +16,3 @@ def visit_test(self, test):
                 )
                 test.status = "PASS"
                 test.message = f'Marking test "{test}" as PASS because it passed in at least one of the test runs.  Previous message is {test.message}'
-            elif "SKIP" in test.message:
-                self.logger.info(
-                    f'CheckForAtLeastOnePassingRunPrerebotModifier - marking test "{test}" as SKIPPED because it was skipped in the latter run.'
-                )
-                test.status = "SKIP"
-                test.message = f'Marking test "{test}" as SKIP because it was skipped in the latter test run.  Previous message is {test.message}'

tests/robot-tests/run_tests.py

@@ -131,6 +131,8 @@ def run():
         # Run tests
         while test_run_index < max_run_attempts:
             try:
+                test_run_results_folder = f"{main_results_folder}{os.sep}run-{test_run_index + 1}"
+
                 # Ensure all SeleniumLibrary elements and keywords are updated to use a brand new
                 # Selenium instance for every test (re)run.
                 if test_run_index > 0:
@@ -142,7 +144,6 @@ def run():
                 _clear_files_before_next_test_run_attempt(rerunning_failed_suites)
 
                 # Create a folder to contain this test run attempt's outputs and reports.
-                test_run_results_folder = f"{main_results_folder}{os.sep}run-{test_run_index + 1}"
                 os.makedirs(test_run_results_folder)
 
                 if not Path(f"{main_results_folder}/downloads").exists():
@@ -204,6 +205,9 @@ def run():
             time.sleep(5)
             slack_service.send_test_report(args.env, args.tests, failing_suites, number_of_test_runs)
 
+        if len(failing_suites) > 0:
+            sys.exit(1)
+
     except Exception as ex:
         if args.enable_slack_notifications:
             slack_service = SlackService()