wip: documenting end to end tests

Author: robinbourianes-kalisio

docs/.vitepress/config.mjs

@@ -59,6 +59,7 @@ function getGuidesSidebar () {
         { text: 'Testing your app', link: '/guides/development/test' },
         { text: 'Configure your app', link: '/guides/development/configure' },
         { text: 'Deploy your app', link: '/guides/development/deploy' },
+        { text: 'Automating tests', link: '/guides/development/automated_tests' },
         { text: 'Document your app', link: '/guides/development/documentation' },
         { text: 'Managing app flavors', link: '/guides/development/flavors' },
         { text: 'Use our CLI', link: '/guides/development/kli' },

docs/.vitepress/public/images/run-tests-steps.svg

@@ -0,0 +1,22 @@
+# Automating tests
+
+We use the CI system to automate running tests we defined in the [Testing your app](./test) section (API and client) .
+
+## API (or backend) tests
+
+These tests are directly run by the `run_tests` **CI job** (defined in the [CI workflow](https://github.com/kalisio/skeleton/blob/master/.github/workflows/main.yaml)). They run on the CI system after each push to the repo. The basically run the tests you defined in the [API tests](./test#api) section.
+
+![Run tests steps](./../../.vitepress/public/images/run-tests-steps.svg)
+
+There's also an `additional_tests` **CI job** whose purpose is to run the same set of API tests, but using different combinations of `Node.js` and `MongoDB` versions.
+
+## Client (or frontend) tests
+
+These tests are not directly run by the CI system. Instead we build a container image using the CI system. That container image embeds everything needed to run client test (most likely [Puppeteer](https://pptr.dev/) along with the actual tests and some reference screenshots). That container is then run *on the infrastructure hosting the app*, once a day (but it can also be run locally on a developper workstation).
+
+The **CI job** responsible for building the container image is `build_e2e_tests`. It uses the `e2e-tests.Dockerfile` as recipe for the image, and embeds the `run_e2e_tests.sh` **CI script**, which is used as container command.
+
+The `run_e2e_tests.sh` script is used in the container image to run the frontend tests. It's purpose is to:
+* run the frontend tests (as defined in the [Client tests](./tests#client) section)
+* upload all test artefacts on an S3 storage
+* generate a report from those tests (can be through a slack notification or as a Markdown file in a git repository)

docs/guides/development/automated_tests.md

@@ -4,25 +4,16 @@
 ARG DEBIAN_VERSION=bookworm
 ARG NODE_VERSION=20
 
-
 ## Use a builder
 ##
 
 FROM node:${NODE_VERSION}-${DEBIAN_VERSION}-slim as builder
 LABEL maintainer="contact@kalisio.xyz"
 
-COPY . /opt/kalisio 
+COPY . /opt/kalisio
 
 # Setting environment variables
 ARG APP
-ARG NODE_APP_INSTANCE
-ARG SUBDOMAIN
-ARG HEADLESS
-
-ENV APP=$APP
-ENV NODE_APP_INSTANCE=$NODE_APP_INSTANCE
-ENV SUBDOMAIN=$SUBDOMAIN
-ENV HEADLESS=$HEADLESS
 
 # Development environment setup
 WORKDIR /opt/kalisio/
@@ -43,18 +34,17 @@ LABEL maintainer="contact@kalisio.xyz"
 ARG APP
 ARG NODE_APP_INSTANCE
 ARG SUBDOMAIN
-ARG HEADLESS
 
 ENV APP=$APP
 ENV NODE_APP_INSTANCE=$NODE_APP_INSTANCE
 ENV SUBDOMAIN=$SUBDOMAIN
-ENV HEADLESS=$HEADLESS
+ENV HEADLESS=true
 
 # Setup Puppeteer & Rclone and installation of the necessary packages
 # https://github.com/puppeteer/puppeteer/blob/main/docs/troubleshooting.md#running-puppeteer-in-docker
 RUN export DEBIAN_FRONTEND=noninteractive \
   && apt-get update \
-  && apt-get install -y wget gnupg curl zip unzip rclone \
+  && apt-get install -y wget gnupg curl zip rclone git \
   && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \
   && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \
   && apt-get update \
@@ -93,10 +83,11 @@ RUN mkdir -p /home/node/.local/bin/ \
   && touch /home/node/.local/bin/cc-test-reporter \
   && chmod 755 /home/node/.local/bin/cc-test-reporter
 
-# Grant execute permissions
+# Make sure runner is properly setup
 WORKDIR /opt/kalisio/$APP/scripts
-RUN chmod +x run_e2e_test.sh
+RUN ./init_runner.sh run_e2e_tests
 
 # Run tests
 WORKDIR /opt/kalisio/$APP
-CMD ["bash", "-c", "/opt/kalisio/$APP/scripts/run_e2e_test.sh $APP"]
+ENV PATH="$PATH:/home/node/.local/bin"
+CMD ["bash", "-c", "/opt/kalisio/$APP/scripts/run_e2e_test.sh $APP"]

e2e-tests.Dockerfile

@@ -1,9 +1,18 @@
-# But ignore these anyway
+# This is the dockerignore used by e2e-tests.Dockerfile, it filters out
+# everything we don't want in the 'COPY . /opt/kalisio' step where
+# '.' is the workspace folder.
+# Since we expect the workspace to be clean, we don't allowlist
+# specific folders, we copy everyhting but the following :
+
+# Ignore all these
+development/*
 **/node_modules
 **/npm-debug.log
 **/*Dockerfile
 **/docker-compose*
 **/*dockerignore
 **/.git
 **/.gitignore
-**/*.dec.*
+**/.quasar
+**/dist
+**/logs

e2e-tests.Dockerfile.dockerignore

@@ -2,6 +2,11 @@
 set -euo pipefail
 # set -x
 
+# This script is used to build the end to end tests container and to eventually push the produced image on a remote
+# container repository.
+# It must be run in a proper 'CI workspace' to find everything it expects (cf setup_workspace.sh script).
+# By default it only build the image container, it won't push it, you need '-p' for that
+
 THIS_FILE=$(readlink -f "${BASH_SOURCE[0]}")
 THIS_DIR=$(dirname "$THIS_FILE")
 ROOT_DIR=$(dirname "$THIS_DIR")
@@ -13,6 +18,7 @@ WORKSPACE_DIR="$(dirname "$ROOT_DIR")"
 ##
 
 PUBLISH=false
+CI_STEP_NAME="Build e2e tests"
 while getopts "pr:" option; do
     case $option in
         p) # define to publish container to registry
@@ -31,4 +37,38 @@ done
 ## Build e2e tests
 ##
 
-build_e2e_tests "$ROOT_DIR" "$PUBLISH"
+# This uses the 'name' field in the app package.json, which must be in lowercase.
+# TODO: you may need to adjust the second parameter to point it to the $KLI_BASE
+init_app_infos "$ROOT_DIR" "$WORKSPACE_DIR/development/workspace/apps"
+
+APP=$(get_app_name)
+VERSION=$(get_app_version)
+FLAVOR=$(get_app_flavor)
+
+# This loads credentials for the target container repository
+# TODO: you may adjust these files to use the ones in your associated 'development' repository
+load_env_files "$WORKSPACE_DIR/development/common/kalisio_harbor.enc.env"
+load_value_files "$WORKSPACE_DIR/development/common/KALISIO_HARBOR_PASSWORD.enc.value"
+
+# TODO: you may change the path for the container image, here it'll be pushed in a 'kalisio' group
+IMAGE_NAME="kalisio/$APP-e2e-tests"
+IMAGE_TAG="$VERSION-$FLAVOR"
+# TODO: you may change the subdomain on which the app to be tested is hosted
+SUBDOMAIN="dev.kalisio.xyz"
+
+case "$FLAVOR" in
+     "prod")
+         SUBDOMAIN="kalisio.com"
+         ;;
+     "test")
+         SUBDOMAIN="test.kalisio.xyz"
+         ;;
+     *)
+         ;;
+esac
+
+# TODO: the environment variables to use here may not be the same (container registry credentials)
+build_e2e_tests \
+    "$ROOT_DIR" "$SUBDOMAIN" "$PUBLISH" \
+    "$KALISIO_HARBOR_URL" "$KALISIO_HARBOR_USERNAME" "$KALISIO_HARBOR_PASSWORD" \
+    "$IMAGE_NAME" "$IMAGE_TAG"

scripts/build_docs.sh

@@ -2,6 +2,10 @@
 set -uo pipefail
 # set -x
 
+# This script is not directly run by the CI system. Instead it is embedded in the end to end test
+# container and used as the container command.
+# It requires the app name to be the first argument
+
 APP=$1
 
 THIS_FILE=$(readlink -f "${BASH_SOURCE[0]}")
@@ -10,8 +14,24 @@ ROOT_DIR=$(dirname "$THIS_DIR")
 
 . "$THIS_DIR/kash/kash.sh"
 
-## Run e2e tests
-##
+# TODO: depending on where you want your end to end tests report to be published, you'll need to adjust here
+
+# This is to publish the report as a slack channel message
+# We expect the following to be defined as environment variables:
+# - S3_BUCKET
+# - RCLONE_CONF
+# - SLACK_WEBHOOK
+run_and_publish_e2e_tests_to_slack \
+    "$ROOT_DIR" "$APP" \
+    "$S3_BUCKET" "$RCLONE_CONF" \
+    "$SLACK_WEBHOOK"
 
-# SLACK_WEBHOOK is set upon Kubernetes container startup
-run_e2e_tests "$ROOT_DIR" "$APP" "$SLACK_WEBHOOK"
+# This is to publish as a markdown report in a git repository
+# We expect the following to be defined as environment variables:
+# - S3_BUCKET
+# - RCLONE_CONF
+# - TARGET_GIT_REPO_URL
+run_and_publish_e2e_tests_to_git_repo \
+    "$ROOT_DIR" "$APP" \
+    "$S3_BUCKET" "$RCLONE_CONF" \
+    "$TARGET_GIT_REPO_URL" "reports"