Merge pull request #4434 from dfe-analytical-services/EES-4117-direct…

…ors-cut EES-4117 - add "rerun-attempts" option to enable UI test retries on the pipeline
dfe-analytical-services · Nov 27, 2023 · 9703389 · 9703389
2 parents a8fd379 + 2085598
commit 9703389
Show file tree

Hide file tree

Showing 19 changed files with 814 additions and 610 deletions.
diff --git a/.gitignore b/.gitignore
@@ -40,6 +40,7 @@ src/explore-education-statistics-frontend/*.xml
 !/tests/robot-tests/.env.example
 /tests/robot-tests/IDENTITY_*
 /tests/robot-tests/.pabotsuitenames
+/tests/robot-tests/.failing_suites
 /tests/robot-tests/geckodriver.log
 /tests/robot-tests/debug.log
 **/*/jwt

diff --git a/azure-pipelines-ui-tests.dfe.yml b/azure-pipelines-ui-tests.dfe.yml
@@ -32,7 +32,6 @@ jobs:
 
   - task: UsePythonVersion@0
     displayName: Use Python 3.10
-  # retryCountOnTaskFailure: 2
     timeoutInMinutes: 5
     inputs:
       versionSpec: 3.10
@@ -49,19 +48,12 @@ jobs:
     displayName: Public UI tests
     inputs:
       scriptPath: tests/robot-tests/scripts/run_tests_pipeline.py
-      arguments: --admin-pass "test" --analyst-pass "test" --expiredinvite-pass "test" --env "dev" --file "tests/general_public/" --processes 2
+      arguments: --admin-pass "test" --analyst-pass "test" --expiredinvite-pass "test" --env "dev" --file "tests/general_public/" --processes 2 --rerun-attempts 5
       workingDirectory: tests/robot-tests
     env:
       SLACK_BOT_TOKEN: $(ees-test-SLACK-BOT-TOKEN)
       SLACK_TEST_REPORT_WEBHOOK_URL: $(ees-test-SLACK-TEST-REPORT-WEBHOOK-URL)
 
-  #- task: PythonScript@0
-  #  displayName: Public UI tests - rerun failed suites
-  #  condition: not(succeeded())
-  #  inputs:
-  #    scriptPath: tests/robot-tests/scripts/run_tests_pipeline.py
-  #    arguments: --admin-pass "test" --analyst-pass "test" --expiredinvite-pass "test" --env "dev" --file "tests/general_public/" --processes 2 --rerun-failed-suites
-  #    workingDirectory: tests/robot-tests
 
   - task: PublishTestResults@2
     displayName: Publish Test Results
@@ -92,7 +84,6 @@ jobs:
 
   - task: UsePythonVersion@0
     displayName: Use Python 3.10
-  # retryCountOnTaskFailure: 2
     timeoutInMinutes: 5
     inputs:
       versionSpec: 3.10
@@ -110,20 +101,13 @@ jobs:
     condition: succeededOrFailed()
     inputs:
       scriptPath: tests/robot-tests/scripts/run_tests_pipeline.py
-      arguments: --admin-pass '"$(ees-test-ADMIN-PASSWORD)"' --analyst-pass '"$(ees-test-ANALYST-PASSWORD)"' --expiredinvite-pass '"$(ees-test-expiredinvite-password)"' --env "dev" --file "tests/admin_and_public_2" --processes 2
+      arguments: --admin-pass '"$(ees-test-ADMIN-PASSWORD)"' --analyst-pass '"$(ees-test-ANALYST-PASSWORD)"' --expiredinvite-pass '"$(ees-test-expiredinvite-password)"' --env "dev" --file "tests/admin_and_public_2" --processes 2 --rerun-attempts 5
     #  The magic incantation '"$(variable)"'was added by Mark to resolve an issue with Analyst password that contained ampersands.
       workingDirectory: tests/robot-tests
     env:
       SLACK_BOT_TOKEN: $(ees-test-SLACK-BOT-TOKEN)
       SLACK_TEST_REPORT_WEBHOOK_URL: $(ees-test-SLACK-TEST-REPORT-WEBHOOK-URL)
 
-  #- task: PythonScript@0
-  #  displayName: 'Publish release and amend UI tests - rerun failed suites'
-  #  condition: not(succeeded())
-  #  inputs:
-  #    scriptPath: tests/robot-tests/scripts/run_tests_pipeline.py
-  #    arguments: --admin-pass '"$(ees-test-ADMIN-PASSWORD)"' --analyst-pass '"$(ees-test-ANALYST-PASSWORD)"' --expiredinvite-pass '"$(ees-test-expiredinvite-password)"' --env "dev" --file "tests/admin_and_public_2" --processes 2 --rerun-failed-suites
-  #    workingDirectory: tests/robot-tests
 
   - task: PublishTestResults@2
     displayName: Publish Test Results
@@ -153,7 +137,6 @@ jobs:
 
   - task: UsePythonVersion@0
     displayName: Use Python 3.10
-  # retryCountOnTaskFailure: 2
     timeoutInMinutes: 5
     inputs:
       versionSpec: 3.10
@@ -171,19 +154,12 @@ jobs:
     condition: succeededOrFailed()
     inputs:
       scriptPath: tests/robot-tests/scripts/run_tests_pipeline.py
-      arguments: --admin-pass  '"$(ees-test-ADMIN-PASSWORD)"' --analyst-pass  '"$(ees-test-ANALYST-PASSWORD)"' --expiredinvite-pass '"$(ees-test-expiredinvite-password)"' --env "dev" --file "tests/admin" --processes 2
+      arguments: --admin-pass  '"$(ees-test-ADMIN-PASSWORD)"' --analyst-pass  '"$(ees-test-ANALYST-PASSWORD)"' --expiredinvite-pass '"$(ees-test-expiredinvite-password)"' --env "dev" --file "tests/admin" --processes 2 --rerun-attempts 5
       workingDirectory: tests/robot-tests
     env:
       SLACK_BOT_TOKEN: $(ees-test-SLACK-BOT-TOKEN)
       SLACK_TEST_REPORT_WEBHOOK_URL: $(ees-test-SLACK-TEST-REPORT-WEBHOOK-URL)
 
-  #- task: PythonScript@0
-  #  displayName: Admin UI tests - rerun failed suites
-  #  condition: not(succeeded())
-  #  inputs:
-  #    scriptPath: tests/robot-tests/scripts/run_tests_pipeline.py
-  #    arguments: --admin-pass  '"$(ees-test-ADMIN-PASSWORD)"' --analyst-pass  '"$(ees-test-ANALYST-PASSWORD)"' --expiredinvite-pass '"$(ees-test-expiredinvite-password)"' --env "dev" --file "tests/admin" --processes 2 --rerun-failed-suites
-  #    workingDirectory: tests/robot-tests
 
   - task: PublishTestResults@2
     displayName: Publish Test Results
@@ -215,7 +191,6 @@ jobs:
 
   - task: UsePythonVersion@0
     displayName: Use Python 3.10
-  # retryCountOnTaskFailure: 2
     timeoutInMinutes: 5
     inputs:
       versionSpec: 3.10
@@ -232,19 +207,12 @@ jobs:
     displayName: Admin public UI tests
     inputs:
       scriptPath: tests/robot-tests/scripts/run_tests_pipeline.py
-      arguments: --admin-pass  '"$(ees-test-ADMIN-PASSWORD)"' --analyst-pass  '"$(ees-test-ANALYST-PASSWORD)"' --expiredinvite-pass '"$(ees-test-expiredinvite-password)"' --env "dev" --file "tests/admin_and_public" --processes 2
+      arguments: --admin-pass  '"$(ees-test-ADMIN-PASSWORD)"' --analyst-pass  '"$(ees-test-ANALYST-PASSWORD)"' --expiredinvite-pass '"$(ees-test-expiredinvite-password)"' --env "dev" --file "tests/admin_and_public" --processes 2 --rerun-attempts 5
       workingDirectory: tests/robot-tests
     env:
       SLACK_BOT_TOKEN: $(ees-test-SLACK-BOT-TOKEN)
       SLACK_TEST_REPORT_WEBHOOK_URL: $(ees-test-SLACK-TEST-REPORT-WEBHOOK-URL)
 
-  #- task: PythonScript@0
-  #  displayName: Admin public UI tests - rerun failed suites
-  #  condition: not(succeeded())
-  #  inputs:
-  #    scriptPath: tests/robot-tests/scripts/run_tests_pipeline.py
-  #    arguments: --admin-pass  '"$(ees-test-ADMIN-PASSWORD)"' --analyst-pass  '"$(ees-test-ANALYST-PASSWORD)"' --expiredinvite-pass '"$(ees-test-expiredinvite-password)"' --env "dev" --file "tests/admin_and_public" --processes 2 --rerun-failed-suites
-  #    workingDirectory: tests/robot-tests
 
   - task: PublishTestResults@2
     displayName: Publish Test Results

diff --git a/...oreEducationStatistics.Admin.Tests/Controllers/Api/Security/PermissionsControllerTests.cs b/...oreEducationStatistics.Admin.Tests/Controllers/Api/Security/PermissionsControllerTests.cs
@@ -12,10 +12,11 @@
 
 namespace GovUk.Education.ExploreEducationStatistics.Admin.Tests.Controllers.Api.Security;
 
+[Collection(CacheServiceTestFixture.CacheServiceTests)]
 public class PermissionsControllerTests : IClassFixture<TestApplicationFactory<TestStartup>>
 {
     private readonly WebApplicationFactory<TestStartup> _testApp;
-    
+
     public PermissionsControllerTests(TestApplicationFactory<TestStartup> testApp)
     {
         _testApp = testApp;
@@ -27,9 +28,9 @@ public async Task GetGlobalPermissions_AuthenticatedUser()
         var client = _testApp
             .SetUser(AuthenticatedUser())
             .CreateClient();
-        
+
         var response = await client.GetAsync("/api/permissions/access");
-        
+
         response.AssertOk(new GlobalPermissionsViewModel(
             CanAccessSystem: true,
             CanAccessAnalystPages: false,
@@ -39,16 +40,16 @@ public async Task GetGlobalPermissions_AuthenticatedUser()
             IsBauUser: false,
             IsApprover: false));
     }
-    
+
     [Fact]
     public async Task GetGlobalPermissions_BauUser()
     {
         var client = _testApp
             .SetUser(BauUser())
             .CreateClient();
-        
+
         var response = await client.GetAsync("/api/permissions/access");
-        
+
         response.AssertOk(new GlobalPermissionsViewModel(
             CanAccessSystem: true,
             CanAccessAnalystPages: true,
@@ -60,7 +61,7 @@ public async Task GetGlobalPermissions_BauUser()
             // individual Approver roles on Releases or Publications.
             IsApprover: false));
     }
-    
+
     [Fact]
     public async Task GetGlobalPermissions_AnalystUser_NotReleaseOrPublicationApprover()
     {
@@ -76,7 +77,7 @@ public async Task GetGlobalPermissions_AnalystUser_NotReleaseOrPublicationApprov
                     UserId = user.GetUserId(),
                     Role = ReleaseRole.Contributor
                 });
-                
+
                 // Add test data that gives the user access to a Publication without being an Approver.
                 context.UserPublicationRoles.Add(new UserPublicationRole
                 {
@@ -85,9 +86,9 @@ public async Task GetGlobalPermissions_AnalystUser_NotReleaseOrPublicationApprov
                 });
             })
             .CreateClient();
-        
+
         var response = await client.GetAsync("/api/permissions/access");
-        
+
         response.AssertOk(new GlobalPermissionsViewModel(
             CanAccessSystem: true,
             CanAccessAnalystPages: true,
@@ -98,7 +99,7 @@ public async Task GetGlobalPermissions_AnalystUser_NotReleaseOrPublicationApprov
             // Expect this to be false if the user isn't an approver of any kind
             IsApprover: false));
     }
-    
+
     [Fact]
     public async Task GetGlobalPermissions_AnalystUser_ReleaseApprover()
     {
@@ -115,9 +116,9 @@ public async Task GetGlobalPermissions_AnalystUser_ReleaseApprover()
                 });
             })
             .CreateClient();
-        
+
         var response = await client.GetAsync("/api/permissions/access");
-        
+
         response.AssertOk(new GlobalPermissionsViewModel(
             CanAccessSystem: true,
             CanAccessAnalystPages: true,
@@ -145,9 +146,9 @@ public async Task GetGlobalPermissions_AnalystUser_PublicationApprover()
                 });
             })
             .CreateClient();
-        
+
         var response = await client.GetAsync("/api/permissions/access");
-        
+
         response.AssertOk(new GlobalPermissionsViewModel(
             CanAccessSystem: true,
             CanAccessAnalystPages: true,
@@ -165,9 +166,9 @@ public async Task GetGlobalPermissions_PreReleaseUser()
         var client = _testApp
             .SetUser(PreReleaseUser())
             .CreateClient();
-        
+
         var response = await client.GetAsync("/api/permissions/access");
-        
+
         response.AssertOk(new GlobalPermissionsViewModel(
             CanAccessSystem: true,
             CanAccessAnalystPages: false,
@@ -177,7 +178,7 @@ public async Task GetGlobalPermissions_PreReleaseUser()
             IsBauUser: false,
             IsApprover: false));
     }
-    
+
     [Fact]
     public async Task GetGlobalPermissions_UnauthenticatedUser()
     {

diff --git a/...vUk.Education.ExploreEducationStatistics.Common.Tests/Fixtures/CacheServiceTestFixture.cs b/...vUk.Education.ExploreEducationStatistics.Common.Tests/Fixtures/CacheServiceTestFixture.cs
@@ -12,7 +12,7 @@ namespace GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures
     /// </summary>
     public class CacheServiceTestFixture : IDisposable
     {
-        protected const string CacheServiceTests = "Cache service tests";
+        public const string CacheServiceTests = "Cache service tests";
 
         protected static readonly Mock<IBlobCacheService> BlobCacheService = new(MockBehavior.Strict);
         protected static readonly Mock<IBlobCacheService> PublicBlobCacheService = new(MockBehavior.Strict);