test(perf): new CLI for running and comparing perf tests (#1389)

* test(perf): new perf testing CLI

* test(perf): added --processors option to explain command

* test(perf): added --tests option to explain command

* refactor(perf): optimize reading results

* test(perf): use proxy agent when validating metrics availability

.eslintrc.js

@@ -48,13 +48,14 @@ module.exports = {
   },
   overrides: [
     {
-      files: ['__performance__/**'],
+      files: ['__performance__/scripts/**'],
       globals: {
         __ENV: true,
       },
       rules: {
         // These scripts are not run directly, and the k6 package is a placeholder
         'import/no-unresolved': ['error', { ignore: ['k6'] }],
+        'import/extensions': ['error', 'ignorePackages', { js: 'always' }],
       },
     },
     {
@@ -126,6 +127,7 @@ module.exports = {
       files: [
         'scripts/**',
         'one-app-statics/**',
+        '__performance__/bin/**',
       ],
       rules: {
         // these scripts should be used only during development
@@ -134,6 +136,10 @@ module.exports = {
         'no-console': 'off',
         'global-require': 'off',
         'import/no-dynamic-require': 'off',
+        'unicorn/no-process-exit': 'off',
+        'no-await-in-loop': 'off',
+        'no-restricted-syntax': 'off',
+        'no-plusplus': 'off',
       },
     },
     {

.gitignore

@@ -23,6 +23,7 @@ bundle.integrity.manifest.json
 test-results
 .jest-cache
 .clinic
+__performance__/results
 
 # sample prod setup
 prod-sample/nginx/origin-statics

__mocks__/chalk.js → __mocks__/colorette.js

@@ -15,17 +15,15 @@
  */
 
 function buildTag(name) {
-  const tag = (txt) => `<${name}>${txt}</${name}>`;
-  tag.bgRed = (txt) => tag(buildTag('bgRed')(txt));
-  return tag;
+  return (txt = '') => `<${name}>${txt}</${name}>`;
 }
 
+const tags = Object.keys(jest.requireActual('colorette')).reduce((acc, tag) => ({
+  ...acc,
+  [tag]: buildTag(tag),
+}), {});
+
 module.exports = {
-  dim: buildTag('dim'),
-  black: buildTag('black'),
-  blue: buildTag('blue'),
-  gray: buildTag('gray'),
-  green: buildTag('green'),
-  red: buildTag('red'),
-  yellow: buildTag('yellow'),
+  ...tags,
+  createColors: () => tags,
 };

__performance__/README.md

@@ -10,58 +10,77 @@ the [prod-sample](../prod-sample/readme.md) or a chosen one-app server.
 * [prometheus](https://prometheus.io/docs/introduction/overview/) - Monitoring toolkit, scrapes metrics exposed by one-app
 * [Grafana](https://grafana.com/docs/grafana/latest/) - Visualization and analytics software used to monitor the performance tests, visualizes metrics from influxdb and prometheus.
 
-
 ## Available Tests
 
 Each type of performance test has a single corresponding file, sharing the same name, which is used by k6 to run the test.
 
 ### Smoke
 
+```sh
+npm run perf -- test --type=smoke
+```
+
 > Smoke test is a regular load test, configured for minimal load. You want to run a smoke test as a sanity check every time you write a new script or modify an existing script. - [k6](https://k6.io/docs/test-types/smoke-testing)
 
-TARGET_URL is used to set the absolute test path.
+`--target` is used to set the absolute test path.
 
 ### Load
 
+```sh
+npm run perf -- test --type=load
+```
+
 > Load Testing is primarily concerned with assessing the current performance of your system in terms of concurrent users or requests per second. - [k6](https://k6.io/docs/test-types/load-testing)
 
-TARGET_URL is used to set the absolute test path.
+`--target` is used to set the absolute test path.
 
 ### Soak
 
+```sh
+npm run perf -- test --type=soak
+```
+
 > Soak testing is concerned with reliability over a long time. [k6](https://k6.io/docs/test-types/soak-testing)
 
-TARGET_URL is used to set the absolute test path.
+`--target` is used to set the absolute test path.
 
 ### Stress
 
+```sh
+npm run perf -- test --type=stress
+```
+
 > The purpose of stress testing is to assess the availability and stability of the system under heavy load. - [k6](https://k6.io/docs/test-types/stress-testing)
 
-`TARGET_BASE_URL` is used to set the base test path. It is recommended to only run this against test environments which use the `prod-sample` modules.
+`--target` is used to set the base test path. It is recommended to only run this against test environments which use the `prod-sample` modules.
 
 The following paths will be expected to be available:
-- /success
-- /healthy-frank
-- /demo/ssr-frank
-- /demo/ssr-frank
-- /demo/needy-frank?api=https://fast.api.frank/posts
-- /demo/needy-frank?api=https://slow.api.frank/posts
+
+* /success
+* /healthy-frank
+* /demo/ssr-frank
+* /demo/ssr-frank
+* /demo/needy-frank?api=<https://fast.api.frank/posts>
+* /demo/needy-frank?api=<https://slow.api.frank/posts>
 
 ### Spike
 
-> Spike test is a variation of a stress test, but it does not gradually increase the load, instead it spikes to extreme load over a very short window of time. While a stress test allows the SUT (System Under Test) to gradually scale up its infrastructure, a spike test does not. [k6](https://k6.io/docs/test-types/stress-testing#spike-testing)
+```sh
+npm run perf -- test --type=spike
+```
 
+> Spike test is a variation of a stress test, but it does not gradually increase the load, instead it spikes to extreme load over a very short window of time. While a stress test allows the SUT (System Under Test) to gradually scale up its infrastructure, a spike test does not. [k6](https://k6.io/docs/test-types/stress-testing#spike-testing)
 
-`TARGET_BASE_URL` is used to set the base test path. It is recommended to only run this against test environments which use the `prod-sample` modules.
+`--target` is used to set the base test path. It is recommended to only run this against test environments which use the `prod-sample` modules.
 
 The following paths will be expected to be available:
-- /success
-- /healthy-frank
-- /demo/ssr-frank
-- /demo/ssr-frank
-- /demo/needy-frank?api=https://fast.api.frank/posts
-- /demo/needy-frank?api=https://slow.api.frank/posts
 
+* /success
+* /healthy-frank
+* /demo/ssr-frank
+* /demo/ssr-frank
+* /demo/needy-frank?api=<https://fast.api.frank/posts>
+* /demo/needy-frank?api=<https://slow.api.frank/posts>
 
 ## Running against prod-sample
 
@@ -80,54 +99,41 @@ Wait for `🌎 One App server listening on port 8443` to appear in the logs to i
 After prod-sample has successfully started you will need to start the performance test suite infrastructure in a new terminal window, navigate to `one-app` and run the following:
 
 ```bash
-npm run monitor:performance
+npm run perf -- monitor
 ```
 
 You can now view the [Grafana metrics on localhost:3030](http://localhost:3030/d/tDGvrq7Mz/one-app-performance)
 
-> `npm run monitor:performance` will also allow you to monitor the development build of one-app.
+> `npm run perf -- monitor` will also allow you to monitor the development build of one-app.
 
 ### Run the smoke test
 
-In another new window from within `one-app/__performance__` run the following:
+In another new window run the following:
 
 ```bash
-docker compose -f docker-compose.yml -f docker-compose.prod-sample.yml run k6 run --insecure-skip-tls-verify /scripts/smoke.js
+npm run perf -- test --type=smoke
 ```
 
-Each file under `__perforamance__/scripts` correlates to a different test:
+Each file under `__performance__/scripts` correlates to a different test:
 
 Smoke - `/scripts/smoke.js`
 Stress - `/scripts/stress.js`
 Load - `/scripts/load.js`
 Soak - `/scripts/soak.js`
 
-To run a test other than the smoke test replace `/scripts/smoke.js` with the corresponding
-file path outlined above.
+To run a test other than the smoke test replace `smoke` with the corresponding
+desired type outlined above.
 
 Once the test has finished running you can use Grafana to share a snapshot
 
 ## Running against another one-app server
 
 ### Setup
 
-If you want to run against a remote server you will need to update `targets` within  `__performance__/prometheus/targets.json` to point to where the one-app metrics are exposed. If you are running against a locally running version you should not need to change this.
-
-```json
-[
-  {
-    "targets": ["my-one-app-domain:3005"],
-    "labels": {
-      "group": "one-app-metrics"
-    }
-  }
-]
-```
-
-Next you can start the performance test suite infrastructure, navigate to `one-app` and run the following:
+If you want to run against a remote server you will need to include the target host when running monitoring services.
 
 ```bash
-npm run monitor:performance
+npm run perf -- monitor --target=my-one-app-domain:3005
 ```
 
 You can now view the [Grafana metrics on localhost:3030](http://localhost:3030/d/tDGvrq7Mz/one-app-performance)
@@ -137,9 +143,8 @@ You can now view the [Grafana metrics on localhost:3030](http://localhost:3030/d
 In another new window from within `one-app/__performance__` run the following:
 
 ```bash
-docker compose run -e TARGET_URL=http://host.docker.internal:3000/success k6 run /scripts/smoke.js
+npm run perf -- test --type=smoke --target=http://my-one-app-domain/success
 ```
 
-Replace the `TARGET_URL` value with your intended target domain.
-Replace `/scripts/smoke.js` with your chosen performance test script.
-For the spike and stress tests set `TARGET_BASE_URL` instead of `TARGET_URL`.
+Replace the `--target` value with your intended target URL.
+Replace `smoke` with your chosen performance test script.

__performance__/bin/commands/clean.js

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2024 American Express Travel Related Services Company, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+const { spawnSync } = require('node:child_process');
+const path = require('node:path');
+
+module.exports.command = 'clean';
+
+module.exports.describe = 'Clean up performance test results';
+
+module.exports.handler = function clean(argv) {
+  spawnSync('git', ['clean', argv.gitArgs, path.join('__performance__', 'results')], { stdio: 'inherit' });
+};
+
+module.exports.builder = (yargs) => yargs
+  .option('dry-run', {
+    alias: 'n',
+    description: 'List files that would be deleted without deleting them',
+    type: 'boolean',
+  })
+  .option('force', {
+    alias: 'f',
+    description: 'Force deletion of files',
+    type: 'boolean',
+  })
+  .option('quiet', {
+    alias: 'q',
+    description: 'Suppress output',
+    type: 'boolean',
+  })
+  .conflicts('dry-run', 'force')
+  .check((argv) => {
+    if (!argv.dryRun && !argv.force) {
+      throw new Error('You must specify either --dry-run or --force');
+    }
+    // eslint-disable-next-line no-param-reassign -- yargs API
+    argv.gitArgs = Object.entries(argv)
+      .filter(([key, value]) => /^\w$/.test(key) && value === true)
+      .reduce((acc, [key]) => acc + key, '-xd');
+    return true;
+  });