Add my homebaked test analyzer

ScriptingEngineTester/build.gradle

@@ -13,7 +13,8 @@ dependencies {
 	api project(':Shared')
 	implementation 'org.junit.jupiter:junit-jupiter-api:5.9.0'
 	implementation 'org.junit.platform:junit-platform-engine:1.9.0'
-    testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.9.0'
+	testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.9.0'
+	implementation 'com.google.code.gson:gson:2.10.1'
 }
 
 test {

ScriptingEngineTester/src/main/java/org/openzen/scriptingenginetester/ScriptingEngineTestExecutor.java

@@ -3,6 +3,8 @@
 import org.junit.platform.engine.ExecutionRequest;
 import org.junit.platform.engine.TestDescriptor;
 import org.junit.platform.engine.TestExecutionResult;
+import org.openzen.scriptingenginetester.watcher.ZenCodeTestResults;
+import org.openzen.scriptingenginetester.watcher.ZenCodeTestWatcher;
 
 import java.util.Optional;
 
@@ -40,8 +42,10 @@ private void executeCase(ExecutionRequest request, TestableScriptingEngine engin
 			TestOutput output = engine.run(descriptor.getTest());
 			descriptor.getTest().validate(output);
 			request.getEngineExecutionListener().executionFinished(descriptor, TestExecutionResult.successful());
+			ZenCodeTestResults.getInstance().testSuccessful(ZenCodeTestWatcher.getTestName(descriptor));
 		} catch (AssertionError|RuntimeException ex) {
 			request.getEngineExecutionListener().executionFinished(descriptor, TestExecutionResult.failed(ex));
+			ZenCodeTestResults.getInstance().testFailed(ZenCodeTestWatcher.getTestName(descriptor));
 		}
 	}
 }

ScriptingEngineTester/src/main/java/org/openzen/scriptingenginetester/watcher/TestResultStatus.java

@@ -0,0 +1,5 @@
+package org.openzen.scriptingenginetester.watcher;
+
+public enum TestResultStatus {
+	SUCCESSFUL, ABORTED, FAILED, DISABLED;
+}

ScriptingEngineTester/src/main/java/org/openzen/scriptingenginetester/watcher/ZenCodeTestResults.java

@@ -0,0 +1,103 @@
+package org.openzen.scriptingenginetester.watcher;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonObject;
+
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.TreeMap;
+
+
+public class ZenCodeTestResults {
+
+	public static final ZenCodeTestResults INSTANCE = new ZenCodeTestResults();
+	public static final String TEST_SEPARATOR = "ñ";
+
+	private final JsonObject results = new JsonObject();
+
+	private ZenCodeTestResults() {}
+
+	public static ZenCodeTestResults getInstance() {
+		return INSTANCE;
+	}
+
+	public void testSuccessful(String testName) {
+		saveResult(testName, TestResultStatus.SUCCESSFUL);
+	}
+
+	public void testDisabled(String testName) {
+		saveResult(testName, TestResultStatus.DISABLED);
+	}
+
+	public void testAborted(String testName) {
+		saveResult(testName, TestResultStatus.ABORTED);
+	}
+
+	public void testFailed(String testName) {
+		saveResult(testName, TestResultStatus.FAILED);
+	}
+
+	private void addChild(JsonObject object, String[] names, int position, TestResultStatus status) {
+		String parentName = names[position - 1];
+		if (position == names.length - 1) {
+			if (object.has(parentName)) {
+				object.get(parentName).getAsJsonObject().addProperty(names[position], status.name());
+			}
+			else {
+				JsonObject child = new JsonObject();
+				child.addProperty(names[position], status.name());
+				object.add(parentName, child);
+			}
+		}
+		else {
+			JsonObject child = new JsonObject();
+			addChild(child, names, position + 1, status);
+			if (object.has(parentName)) {
+				JsonObject parentObject = object.getAsJsonObject(parentName);
+				if (parentObject.has(names[position])) {
+					parentObject.getAsJsonArray(names[position]).add(child.getAsJsonObject(names[position]));
+				}
+				else {
+					JsonArray subTests = new JsonArray();
+					subTests.add(child.getAsJsonObject(names[position]));
+					object.get(parentName).getAsJsonObject().add(names[position], subTests);
+				}
+			}
+			else {
+				JsonObject parentObject = new JsonObject();
+				JsonArray nestedTest = new JsonArray();
+				nestedTest.add(child.getAsJsonObject(names[position]));
+				parentObject.add(names[position], nestedTest);
+				object.add(parentName, parentObject);
+			}
+		}
+	}
+
+	private void saveResult(String name, TestResultStatus status) {
+		String[] testNames = name.split(TEST_SEPARATOR);
+		if (testNames.length > 1) addChild(results, testNames, 1, status);
+	}
+
+	public void saveToDisk() {
+		Gson gson = new GsonBuilder().setPrettyPrinting().create();
+		try {
+			FileWriter writer = new FileWriter("results.json");
+			String json = gson.toJson(results);
+			//Sort alphabetically
+			TreeMap<String, Object> map = gson.fromJson(json, TreeMap.class);
+			String sortedJson = gson.toJson(map);
+			writer.write(sortedJson);
+			writer.flush();
+			writer.close();
+		} catch (IOException e) {
+			throw new RuntimeException(e);
+		}
+	}
+
+
+
+}
+
+

ScriptingEngineTester/src/main/java/org/openzen/scriptingenginetester/watcher/ZenCodeTestWatcher.java

@@ -0,0 +1,75 @@
+package org.openzen.scriptingenginetester.watcher;
+
+import org.junit.jupiter.api.extension.AfterAllCallback;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.TestWatcher;
+import org.junit.platform.engine.TestDescriptor;
+
+import java.util.*;
+
+public class ZenCodeTestWatcher implements TestWatcher, AfterAllCallback {
+
+	private static final String JUNIT_EXECUTOR = "JUnit Jupiter";
+
+	@Override
+	public void testSuccessful(ExtensionContext context) {
+		ZenCodeTestResults.getInstance().testSuccessful(getTestName(context));
+	}
+
+	@Override
+	public void testDisabled(ExtensionContext context, Optional<String> reason) {
+		ZenCodeTestResults.getInstance().testDisabled(getTestName(context));
+	}
+
+	@Override
+	public void testAborted(ExtensionContext context, Throwable cause) {
+
+		ZenCodeTestResults.getInstance().testAborted(getTestName(context));
+	}
+
+	@Override
+	public void testFailed(ExtensionContext context, Throwable cause) {
+		ZenCodeTestResults.getInstance().testFailed(getTestName(context));
+	}
+
+	public static String getTestName(TestDescriptor descriptor) {
+		StringBuilder builder = new StringBuilder();
+		builder.append(descriptor.getDisplayName());
+		while(hasValidParent(descriptor)) {
+			TestDescriptor parent = descriptor.getParent().get();
+			builder.insert(0, parent.getDisplayName() + ZenCodeTestResults.TEST_SEPARATOR);
+			descriptor = parent;
+		}
+		return builder.toString();
+	}
+
+	private static boolean hasValidParent(TestDescriptor descriptor) {
+		return descriptor.getParent().isPresent() && !"JavaTestingEngineTest".equals(descriptor.getParent().get().getDisplayName());
+	}
+
+	private String getTestName(ExtensionContext context) {
+		StringBuilder builder = new StringBuilder();
+		builder.append(context.getDisplayName());
+		while (hasValidParent(context)) {
+			ExtensionContext parent = context.getParent().get();
+			builder.insert(0, parent.getDisplayName() + ZenCodeTestResults.TEST_SEPARATOR);
+			context = parent;
+		}
+		return builder.toString();
+	}
+
+	private boolean hasValidParent(ExtensionContext context) {
+		return context.getParent().isPresent() && !JUNIT_EXECUTOR.equals(context.getParent().get().getDisplayName());
+	}
+
+	/**
+	 * Callback that is invoked once <em>after</em> all tests in the current
+	 * container.
+	 *
+	 * @param context the current extension context; never {@code null}
+	 */
+	@Override
+	public void afterAll(ExtensionContext context) throws Exception {
+		ZenCodeTestResults.getInstance().saveToDisk();
+	}
+}

ScriptingExample/build.gradle

@@ -1,3 +1,5 @@
+import java.nio.file.Files
+
 // Note: "common.gradle" in the root project contains additional initialization
 //   for this project. This initialization is applied in the "build.gradle"
 //   of the root project.
@@ -25,6 +27,128 @@ processResources {
 	from files(evaluationDependsOn(":StdLibs").tasks.getByName("zipItUp").outputs)
 }
 
+tasks.register("savePreviousResults").configure {
+	it.doFirst {
+		def results = file("results.json")
+		if (results.exists()) {
+			def previous = file("previous_results.json")
+			if (previous.exists()) previous.delete()
+			Files.copy(results.toPath(), previous.toPath())
+		}
+	}
+}
+
+def printSeparator(message) {
+	project.logger.lifecycle("======================================")
+	project.logger.lifecycle(" ${message} ")
+	project.logger.lifecycle("======================================")
+}
+
+def countTests(map) {
+	int children = 0;
+
+	map.each { key, value ->
+		if (value instanceof Map) {
+			children += countTests(value)
+		}
+		else children++
+	}
+
+	return children
+}
+
+def exploreNewMap(map, indentLevel) {
+	map.each { key, value ->
+		def s = "|" + String.join("", Collections.nCopies(indentLevel, "---"))
+		project.logger.lifecycle(s + key)
+		if (value instanceof Map) {
+			exploreNewMap(value, indentLevel + 1)
+		}
+	}
+}
+
+def getEntryName(key, parentName) {
+	if (parentName.empty || parentName.blank) return key
+	return parentName + "-" + key
+}
+
+def analiseMap(map, reference, parentName) {
+	int count = 0
+	map.each { key, value ->
+		def oldValue = reference[key]
+		if (oldValue == null) {
+			if (value instanceof Map) {
+				def internalCount = countTests(value)
+				count += internalCount
+				project.logger.lifecycle("Entry${key} did not exist previously and has ${internalCount} childTests!")
+				exploreNewMap(value, 1)
+			}
+			else {
+				project.logger.lifecycle("Entry${getEntryName(key, parentName)} did not exist previously!")
+				count += 1
+			}
+		}
+		else if (value instanceof Map) {
+			count += analiseMap(value, oldValue, parentName + " " + key)
+		}
+		else if (value instanceof List) {
+			value.eachWithIndex { it, index ->
+				count += analiseMap(it, oldValue[index], parentName + " " + key)
+			}
+		}
+		else {
+			//value is a result
+			if (oldValue instanceof String) {
+				if (oldValue != value) {
+					if (oldValue == "SUCCESSFUL") {
+						count--
+					}
+					else if(value == "SUCCESSFUL") {
+						count++
+					}
+					project.logger.lifecycle("Test:${parentName}-${key} changed from: ${oldValue} to ${value}")
+				}
+			}
+		}
+	}
+	return count
+}
+
+tasks.register("analyseResults").configure {
+	it.doFirst {
+		if (file("results.json").exists() && file("previous_results.json").exists()) {
+			printSeparator("Starting ZenCode Test Difference")
+			def newJson = new groovy.json.JsonSlurper().parseText(file("results.json").text)
+			def oldJson = new groovy.json.JsonSlurper().parseText(file("previous_results.json").text)
+			def newTests = countTests(newJson)
+			def oldTests = countTests(oldJson)
+			def successes = analiseMap(newJson, oldJson, "")
+			if (newTests != oldTests) {
+				project.logger.lifecycle("There are ${newTests - oldTests} more tests!")
+			}
+			if (successes > 0) {
+				project.logger.lifecycle("There are ${successes} more passing tests than earlier!")
+			}
+			else if (successes < 0) {
+				def normalisedSuccesses = -1 * successes
+				project.logger.log(LogLevel.ERROR, "There are ${normalisedSuccesses} LESS passing tests than earlier!")
+				if (newTests == oldTests)
+					project.logger.log(LogLevel.ERROR, "This is very likely a symptom of a regression, or tests have been deleted.")
+			}
+			else {
+				project.logger.lifecycle("No changes have been observed with the test suite.")
+			}
+			printSeparator("Finished ZenCode Test Difference")
+			//Check extra success counter and see if it was positive + regressions.
+		}
+		else project.logger.log(LogLevel.ERROR, "No previous tests were found")
+	}
+}
+
+test.dependsOn(tasks.named("savePreviousResults"))
+
+test.finalizedBy(tasks.named("analyseResults"))
+
 test {
 	useJUnitPlatform()
 	testLogging {