mrc-4333 e2e tests

.github/workflows/frontend-test-and-build.yml → .github/workflows/frontend-test-and-build-and-e2e.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [ 20.x ]
+        node-version: [ 22.x ]
         java-version: [ 17 ]
     steps:
       - name: Checkout repository
@@ -56,10 +56,10 @@ jobs:
         run: npm run lint --prefix=app
       - name: Build app
         run: npm run build --prefix=app
-      - name: Test app
+      - name: Run unit tests
         run: npm test --prefix=app -- --coverage
-      - name: Run dependencies
-        run: ./scripts/run-dependencies && ./api/scripts/run && ./api/scripts/smoke-test
+      - name: Run dependencies and API
+        run: ./scripts/run-dependencies && ./api/scripts/run basicauth && ./api/scripts/smoke-test
       - name: Upload coverage to codecov
         uses: codecov/codecov-action@v3
         with:
@@ -68,6 +68,27 @@ jobs:
       - name: Build image
         working-directory: app
         run: ./scripts/build-and-push
-      - name: Smoke test
+      - name: Run front end and smoke test
         working-directory: app
         run: ./scripts/run && ./scripts/smoke-test
+      - name: Create super user
+        run: ./scripts/basic-create-super-user
+      - name: Install playwright
+        working-directory: app
+        run: npx playwright install --with-deps
+      - name: Run app
+        working-directory: app
+        run: npm start &
+      - name: E2E tests
+        working-directory: app
+        run: npm run test:e2e
+      - name: Delete auth.json before E2E test-results upload
+        if: ${{ failure() }}
+        run: rm -rf app/test-results/auth.setup.ts-authenticate-setup/auth.json
+      - name: Upload E2E test reports
+        uses: actions/upload-artifact@v4
+        if: ${{ failure() }}
+        with:
+          name: e2e test results
+          path: |
+            app/test-results/

.gitignore

@@ -1,3 +1,6 @@
 .idea
 .vscode
-packit.iml
+.token
+packit.iml
+app/test-results
+app/playwright-report

README.md

@@ -14,6 +14,8 @@ The application can be used through the browser as a portal by researchers, to m
 
 Packit can also be used programmatically through its `/outpack` route, which forwards requests to [outpack server](https://github.com/mrc-ide/outpack_server). 
 
+Developing packit requires Node 22 or above.
+
 ## Quick start
 
 To run the whole app (default is github auth):
@@ -60,3 +62,62 @@ See [app/README.md](https://github.com/mrc-ide/packit/blob/main/app/README.md)
 ## Authentication in Packit
 
 See [docs/auth.md](docs/auth.md)
+
+## e2e Tests
+
+Playwright tests are in `./app/e2e`. There are three npm scripts defined in `app/package.json`:
+- `test:e2e` - tests against `http://localhost:3000`. Can use either github or basic auth (assumes default superuser credentials).
+- `test:e2e-dev` - tests against `https://reside-dev.packit-dev.dide.ic.ac.uk/` excluding demo dataset and state mutating tests
+- `test:e2e-prod` - tests against `https://reside.packit.dide.ic.ac.uk/` excluding demo dataset and state mutating tests
+
+To run any other variant, you can run `npx playwright test`, providing any required environment variables as detailed
+below and `--grep` or `--grep-invert` arg to include or exclude tagged tests as required. 
+
+The packit base url to use should be provided in the `PACKIT_E2E_BASE_URL` env var. If this is not set, playwright
+will use `http://localhost:3000`. If you're testing with a base url which is not a domain root, ensure that you 
+append a "/". In this case it's also important to use `./` as the base route in the Playwright tests, not `/`.
+
+
+### Authentication
+Authentication in the e2e tests is done in the setup project `auth.setup.ts` which is a dependency of all other projects
+as defined in `playwright.config.ts`. We require the packit user who authenticates to have admin permissions. 
+
+The setup project first checks the packit api's `/auth/config` endpoint to determine if basic or github auth is required. 
+
+If basic auth, the env vars `PACKIT_E2E_BASIC_USER` and `PACKIT_E2E_BASIC_PASSOWRD` must be set. These will be used
+to authenticate via the login page.
+
+If github auth, the setup code checks for the optional `GITHUB_ACCESS_TOKEN`. If this is not set, it uses oauth device
+flow to obtain a github token (this is an interactive process so we won't be able to use this on CI). The github token
+is then posted to packit api's login endpoint to obtain a packit token, which is then passed to packit's redirect endpoint
+login in the browser. 
+
+For both auth methods, we then write out page context to a temporary location which is then picked up by dependent tests
+so they start off in an authenticated state. 
+
+### Tags
+
+We want to have e2e tests which test all packit features, but also to have a suite of tests which we can run against any
+packit server to test that its deployment has succeeded. In order to support detailed tests of all features, we want to 
+have some tests which expect our standard demo dataset (available on localhost and perhaps on some dev instances), but
+to exclude those tests when running generic tests on servers where demo packets are not present. 
+
+We also want to have some tests which change the state of the server e.g. by running packets or changing user permissions. 
+These should never be run on production servers. (We do not have any real tests of this type yet, but we have one tagged placeholder.)
+
+We indicate these types of tests using tags:
+- `@demoPackets` indicates that the test specifically expects the demo dataset.
+- `@stateMutate` indicates that the test will change the server state
+
+When running playwright, you can use `--grep` or `--grep-invert` to include or exclude particular tags. The npm 
+scripts `test:e2e-dev` and `test:e2e-prod` use `--grep-invert '@demoPackets|@stateMutate'` to exclude these tags. 
+
+As an additional guardrail, we want to make sure that `@stateMutate` tests are never run against production servers. For this
+reason we have a custom test fixture `tagCheckFixture.ts` which all our e2e tests must use. It checks if any tests tagged
+with `@stateMutate` are running when base url is not on `localhost` or `packit-dev.dide.ic.ac.uk`, and will throw an error
+if that is the case. It will also warn if such tests are being run on dev, as this might be undesirable. It also warns
+if `@demoPackets` tests are being run on non-localhost as this may explain any test failures. 
+
+By custom, we suffix generic tests with `.generic.spec.ts` and demo datasets tests with `.demo.spec.ts`, but this 
+naming convention is not interpreted by any test matchers or fixtures. 
+

app/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:18
+FROM node:22
 
 COPY package.json /app/package.json
 COPY package-lock.json /app/package-lock.json

app/README.md

@@ -3,7 +3,7 @@ Interface is built with [React library](https://reactjs.org)
 
 ## Requirements
 
-Node 18.
+Node 22.
 
 ## Available Scripts
 

app/e2e/auth.setup.ts

@@ -0,0 +1,94 @@
+import { test as setup, expect } from "./tagCheckFixture";
+import { Page } from "@playwright/test";
+
+const authMethodIsBasic = async (apiURL: string) => {
+  const response = await fetch(`${apiURL}/auth/config`);
+  if (!response.ok) {
+    throw Error(`Unable to get auth type from api. Status was ${response.status}`);
+  }
+  const json = await response.json();
+  return json.enableBasicLogin;
+};
+
+const getBasicCredentials = () => {
+  const user = process.env.PACKIT_E2E_BASIC_USER;
+  const password = process.env.PACKIT_E2E_BASIC_PASSWORD;
+
+  if (!user || !password) {
+    throw Error(
+      "This packitserver uses Basic auth. Please set environment variables " +
+        "PACKIT_E2E_BASIC_USER and PACKIT_E2E_BASIC_PASSWORD"
+    );
+  }
+
+  return [user, password];
+};
+
+const doBasicLogin = async (user: string, password: string, page: Page) => {
+  await page.goto("./");
+  await page.getByLabel("Email").fill(user);
+  await page.getByLabel("Password").fill(password);
+  await page.getByRole("button", { name: /Log in/i }).click();
+};
+
+const doGithubLogin = async (page: Page, apiURL: string) => {
+  // github PAT may be set in env, otherwise login to github interactively
+  let githubToken = process.env.GITHUB_ACCESS_TOKEN;
+
+  if (!githubToken) {
+    const aod = await import("@octokit/auth-oauth-device"); // This package requires dynamic import with our ts config
+    const auth = aod.createOAuthDeviceAuth({
+      clientType: "oauth-app",
+      clientId: "Ov23liUrbkR0qUtAO1zu", // Packit Oauth App
+      scopes: ["read:org"],
+      onVerification(verification) {
+        console.log("Open %s", verification.verification_uri);
+        console.log("Enter code: %s", verification.user_code);
+      }
+    });
+
+    const tokenAuthentication = await auth({
+      type: "oauth"
+    });
+    githubToken = tokenAuthentication.token;
+  }
+
+  const packitResponse = await fetch(`${apiURL}/auth/login/api`, {
+    method: "POST",
+    headers: {
+      Accept: "application/json",
+      "Content-Type": "application/json"
+    },
+    body: JSON.stringify({ token: githubToken })
+  });
+  if (!packitResponse.ok) {
+    throw Error(`Unable to get token from api. Status was ${packitResponse.status}`);
+  }
+  const json = await packitResponse.json();
+  const packitToken = json.token;
+
+  // Use redirect endpoint to login
+  await page.goto(`./redirect?token=${packitToken}`);
+};
+
+// Define "setup" as a dependency for any test project which requires prior authentication
+setup("authenticate", async ({ page, baseURL }, testInfo) => {
+  console.log(`Authenticating with ${baseURL}`);
+  // If baseURL is default localhost (used by CI),  we assume that we're running locally with api on
+  // port 8080, otherwise that api is accessible from baseURL/api
+  const apiURL = baseURL === "http://localhost:3000/" ? "http://localhost:8080" : `${baseURL}/packit/api`;
+  const basicAuth = await authMethodIsBasic(apiURL);
+  if (basicAuth) {
+    const [basicUser, basicPassword] = getBasicCredentials();
+    await doBasicLogin(basicUser, basicPassword, page); // get credentials interactively
+  } else {
+    await doGithubLogin(page, apiURL);
+  }
+
+  // Check login has succeeded - admin user should have user access role
+  await expect(page.locator("body")).toHaveText(/Manage Access/);
+
+  // write out context to tmp location to be picked up by dependent tests
+  const authFile = testInfo.outputPath("auth.json");
+  await page.context().storageState({ path: authFile });
+});

app/e2e/globalSetup.ts

@@ -0,0 +1,23 @@
+import { FullConfig } from "@playwright/test";
+import { glob, readFile } from "fs/promises";
+import { join } from "path";
+
+// Ensure that all test files use the tagCheckFixture rather than playwright's default test fixture
+const checkTestImports = async (config: FullConfig) => {
+  console.log("Checking test imports...");
+  const { rootDir } = config;
+  const importRegex = /import\s+{[^}]*\btest\b[^}]*}\s+from\s+".\/tagCheckFixture";/;
+
+  // Use a generic glob here which should pick up every test file - this is
+  // simpler than checking testMatch for every project, which could be Regex or glob, or an
+  // array of either.
+  for await (const file of glob("**/*.@(spec|test).?(c|m)[jt]s?(x)", { cwd: rootDir })) {
+    const filePath = join(rootDir, file);
+    const content = await readFile(filePath, "utf-8");
+    if (!importRegex.test(content)) {
+      throw new Error(`Test file ${file} does not import "test" from "tagCheckFixture".`);
+    }
+  }
+};
+
+export default checkTestImports;

app/e2e/index.spec.ts

@@ -0,0 +1,63 @@
+import { test, expect } from "./tagCheckFixture";
+import { Locator } from "@playwright/test";
+import {
+  getBreadcrumbLocator,
+  getContentLocator,
+  getReadableIdString,
+  navigateToFirstPacketGroup,
+  navigateToFirstPacketGroupLatestPacket,
+  packetGroupNameFromListItem
+} from "./utils";
+
+test.describe("Index page", () => {
+  let content: Locator;
+  let packetGroups: Locator;
+
+  test.beforeEach(async ({ page }) => {
+    await page.goto("./");
+    content = await getContentLocator(page);
+    packetGroups = await content.getByRole("listitem");
+    expect(await packetGroups.count()).toBeGreaterThan(0);
+    // wait for list items to not be skeletal
+    await expect(await packetGroups.first().getByRole("heading")).toBeVisible();
+  });
+
+  test("can view packet group list", async () => {
+    (await packetGroups.all()).forEach((packetGroup) => {
+      expect(packetGroup.getByRole("heading")).toBeEnabled(); // packet group name
+      expect(packetGroup.getByRole("link", { name: "Latest" })).toBeEnabled();
+      expect(packetGroup.getByText(/^\d+ packets?$/)).toBeVisible(); // packet count
+      expect(packetGroup.getByText(/^Updated \d+ (second|minute|hour|day)s? ago$/)).toBeVisible(); // updated label
+    });
+  });
+
+  test("can filter packet groups", async ({ page }) => {
+    const firstPacketGroup = await packetGroups.first();
+    const firstPacketGroupName = await packetGroupNameFromListItem(firstPacketGroup);
+    const filterInput = await page.getByPlaceholder("Filter packet groups...");
+    await filterInput.fill(firstPacketGroupName);
+    // wait for reset-filter button to become visible
+    await expect(await content.getByLabel("reset filter")).toBeVisible();
+    const filteredGroups = await content.getByRole("listitem");
+    // expect to have at least one packet group remaining, and expect all to have filter term  as a substring
+    expect(await filteredGroups.count()).toBeGreaterThan(0);
+    for (const packetGroup of await filteredGroups.all()) {
+      expect(await packetGroupNameFromListItem(packetGroup)).toContain(firstPacketGroupName);
+    }
+  });
+
+  test("can navigate from packet group name link to packet group page", async ({ page }) => {
+    const firstPacketGroupName = await navigateToFirstPacketGroup(content);
+    const displayName = getReadableIdString(firstPacketGroupName);
+    // wait for packet group name to be visible in breadcrumb
+    await expect(await getBreadcrumbLocator(page)).toHaveText(`home${displayName}`);
+  });
+
+  test("can navigate from latest packet link to packet page", async ({ page }) => {
+    const { packetGroupName, packetId } = await navigateToFirstPacketGroupLatestPacket(content);
+    // wait for packet group name and latest packet id to be visible in breadcrumb
+    const displayPacketId = getReadableIdString(packetId);
+    const displayPacketGroupName = getReadableIdString(packetGroupName);
+    await expect(await getBreadcrumbLocator(page)).toHaveText(`home${displayPacketGroupName}${displayPacketId}`);
+  });
+});