From d203d9a4ce5e1b8a89a88635fc0f24c56715a5ae Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Thu, 30 Jan 2025 19:01:13 -0500
Subject: [PATCH] add test profiling utility to analyse speedup opportunities +
 define make run_test function to centralize definition

---
 Makefile             | 42 ++++++++++++++++++++++++------------------
 requirements-dev.txt |  1 +
 2 files changed, 25 insertions(+), 18 deletions(-)

diff --git a/Makefile b/Makefile
index e7ee8cbf1..bb51fd744 100644
--- a/Makefile
+++ b/Makefile
@@ -381,7 +381,14 @@ ifeq ($(filter $(TEST_VERBOSITY),"--capture"),)
     TEST_VERBOSITY := $(TEST_VERBOSITY) --capture tee-sys
   endif
 endif
+
+TEST_PROFILE ?= true
+ifneq ($(filter "$(TEST_PROFILE)","true"),)
+  override TEST_PROFILE_ARGS = --profile --profile-svg --pstats-dir "$(REPORTS_DIR)/profiling" --durations 100
+endif
+
 TEST_XARGS ?=
+override TEST_XARGS := $(TEST_VERBOSITY) $(TEST_PROFILE_ARGS) $(TEST_XARGS)
 
 # autogen tests variants with pre-install of dependencies using the '-only' target references
 TESTS := unit func cli workflow online offline no-tb14 spec coverage
@@ -389,6 +396,14 @@ TESTS := $(addprefix test-, $(TESTS))
 
 $(TESTS): test-%: install-dev test-%-only
 
+define run_test =
+	$(eval $@_PYTEST_JUNIT = --junitxml "$(REPORTS_DIR)/test-results.xml")
+	$(eval $@_PYTEST_CASE = $(1))
+	$(eval $@_PYTEST_CMD = pytest tests ${$@_PYTEST_CASE} $(TEST_XARGS) ${$@__PYTEST_JUNIT})
+	@echo ">" '${$@_PYTEST_CMD}'
+	@bash -c '${$@_PYTEST_CMD}'
+endef
+
 .PHONY: test
 test: clean-test test-all   ## alias for 'test-all' target
 
@@ -398,57 +413,48 @@ test-all: install-dev test-only		## run all tests (including long running tests)
 .PHONY: test-only
 test-only: mkdir-reports			## run all tests but without prior validation of installed dependencies
 	@echo "Running all tests (including slow and online tests)..."
-	@bash -c '$(CONDA_CMD) pytest tests $(TEST_VERBOSITY) $(TEST_XARGS) \
-		--junitxml "$(REPORTS_DIR)/test-results.xml"'
+	@$(call run_test,)
 
 .PHONY: test-unit-only
 test-unit-only: mkdir-reports 		## run unit tests (skip long running and online tests)
 	@echo "Running unit tests (skip slow and online tests)..."
-	@bash -c '$(CONDA_CMD) pytest tests $(TEST_VERBOSITY) $(TEST_XARGS) \
-		-m "not slow and not online and not functional" --junitxml "$(REPORTS_DIR)/test-results.xml"'
+	@$(call run_test,-m "not slow and not online and not functional")
 
 .PHONY: test-func-only
 test-func-only: mkdir-reports   	## run functional tests (online and usage specific)
 	@echo "Running functional tests..."
-	@bash -c '$(CONDA_CMD) pytest tests $(TEST_VERBOSITY) $(TEST_XARGS) \
-		-m "functional" --junitxml "$(REPORTS_DIR)/test-results.xml"'
+	@$(call run_test,-m "functional")
 
 .PHONY: test-cli-only
 test-cli-only: mkdir-reports   		## run WeaverClient and CLI tests
 	@echo "Running CLI tests..."
-	@bash -c '$(CONDA_CMD) pytest tests $(TEST_VERBOSITY) $(TEST_XARGS) \
-		-m "cli" --junitxml "$(REPORTS_DIR)/test-results.xml"'
+	@$(call run_test,-m "cli")
 
 .PHONY: test-workflow-only
 test-workflow-only:	mkdir-reports	## run EMS workflow End-2-End tests
 	@echo "Running workflow tests..."
-	@bash -c '$(CONDA_CMD) pytest tests $(TEST_VERBOSITY) $(TEST_XARGS) \
-		-m "workflow" --junitxml "$(REPORTS_DIR)/test-results.xml"'
+	@$(call run_test,-m "workflow")
 
 .PHONY: test-online-only
 test-online-only: mkdir-reports  	## run online tests (running instance required)
 	@echo "Running online tests (running instance required)..."
-	@bash -c '$(CONDA_CMD) pytest tests $(TEST_VERBOSITY) $(TEST_XARGS) \
-		-m "online" --junitxml "$(REPORTS_DIR)/test-results.xml"'
+	@$(call run_test,-m "online")
 
 .PHONY: test-offline-only
 test-offline-only: mkdir-reports  	## run offline tests (not marked as online)
 	@echo "Running offline tests (not marked as online)..."
-	@bash -c '$(CONDA_CMD) pytest tests $(TEST_VERBOSITY) $(TEST_XARGS) \
-		-m "not online" --junitxml "$(REPORTS_DIR)/test-results.xml"'
+	@$(call run_test,-m "not online")
 
 .PHONY: test-no-tb14-only
 test-no-tb14-only: mkdir-reports  	## run all tests except ones marked for 'Testbed-14'
 	@echo "Running all tests except ones marked for 'Testbed-14'..."
-	@bash -c '$(CONDA_CMD) pytest tests $(TEST_VERBOSITY) $(TEST_XARGS) \
-		-m "not testbed14" --junitxml "$(REPORTS_DIR)/test-results.xml"'
+	@$(call run_test,-m "not testbed14")
 
 .PHONY: test-spec-only
 test-spec-only:	mkdir-reports  ## run tests with custom specification (pytest format) [make SPEC='<spec>' test-spec]
 	@echo "Running custom tests from input specification..."
 	@[ "${SPEC}" ] || ( echo ">> 'SPEC' is not set"; exit 1 )
-	@bash -c '$(CONDA_CMD) pytest tests $(TEST_VERBOSITY) $(TEST_XARGS) \
-		-k "${SPEC}" --junitxml "$(REPORTS_DIR)/test-results.xml"'
+	@$(call run_test,-k "${SPEC}")
 
 .PHONY: test-smoke
 test-smoke: docker-test     ## alias to 'docker-test' executing smoke test of built docker images
diff --git a/requirements-dev.txt b/requirements-dev.txt
index 778515da3..0a8913fc8 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -26,6 +26,7 @@ path!=16.12.0,!=17.0.0  # patch pytest-shutil (https://github.com/man-group/pyte
 pluggy>=0.7
 pytest
 pytest-httpserver>=1.0.7  # support werkzeug>=3
+pytest-profiling
 pytest-server-fixtures
 #pytest-rerunfailures
 pytest-retry