Skip to content

Commit e9b637b

Browse files
Witixin1512Witixin1512
committed
Add my homebaked test analyzer
1 parent 4ac76c2 commit e9b637b

File tree

9 files changed

+3894
-1
lines changed

9 files changed

+3894
-1
lines changed

‎ScriptingEngineTester/build.gradle

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@ dependencies {
1313
api project(':Shared')
1414
implementation 'org.junit.jupiter:junit-jupiter-api:5.9.0'
1515
implementation 'org.junit.platform:junit-platform-engine:1.9.0'
16-
testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.9.0'
16+
testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.9.0'
17+
implementation 'com.google.code.gson:gson:2.10.1'
1718
}
1819

1920
test {

‎ScriptingEngineTester/src/main/java/org/openzen/scriptingenginetester/ScriptingEngineTestExecutor.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
import org.junit.platform.engine.ExecutionRequest;
44
import org.junit.platform.engine.TestDescriptor;
55
import org.junit.platform.engine.TestExecutionResult;
6+
import org.openzen.scriptingenginetester.watcher.ZenCodeTestResults;
7+
import org.openzen.scriptingenginetester.watcher.ZenCodeTestWatcher;
68

79
import java.util.Optional;
810

@@ -40,8 +42,10 @@ private void executeCase(ExecutionRequest request, TestableScriptingEngine engin
4042
TestOutput output = engine.run(descriptor.getTest());
4143
descriptor.getTest().validate(output);
4244
request.getEngineExecutionListener().executionFinished(descriptor, TestExecutionResult.successful());
45+
ZenCodeTestResults.getInstance().testSuccessful(ZenCodeTestWatcher.getTestName(descriptor));
4346
} catch (AssertionError|RuntimeException ex) {
4447
request.getEngineExecutionListener().executionFinished(descriptor, TestExecutionResult.failed(ex));
48+
ZenCodeTestResults.getInstance().testFailed(ZenCodeTestWatcher.getTestName(descriptor));
4549
}
4650
}
4751
}

‎ScriptingEngineTester/src/main/java/org/openzen/scriptingenginetester/watcher/TestResultStatus.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
package org.openzen.scriptingenginetester.watcher;
2+
3+
public enum TestResultStatus {
4+
SUCCESSFUL, ABORTED, FAILED, DISABLED;
5+
}

‎ScriptingEngineTester/src/main/java/org/openzen/scriptingenginetester/watcher/ZenCodeTestResults.java

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
package org.openzen.scriptingenginetester.watcher;
2+
3+
import com.google.gson.Gson;
4+
import com.google.gson.GsonBuilder;
5+
import com.google.gson.JsonArray;
6+
import com.google.gson.JsonObject;
7+
8+
import java.io.FileWriter;
9+
import java.io.IOException;
10+
import java.util.TreeMap;
11+
12+
13+
public class ZenCodeTestResults {
14+
15+
public static final ZenCodeTestResults INSTANCE = new ZenCodeTestResults();
16+
public static final String TEST_SEPARATOR = "ñ";
17+
18+
private final JsonObject results = new JsonObject();
19+
20+
private ZenCodeTestResults() {}
21+
22+
public static ZenCodeTestResults getInstance() {
23+
return INSTANCE;
24+
}
25+
26+
public void testSuccessful(String testName) {
27+
saveResult(testName, TestResultStatus.SUCCESSFUL);
28+
}
29+
30+
public void testDisabled(String testName) {
31+
saveResult(testName, TestResultStatus.DISABLED);
32+
}
33+
34+
public void testAborted(String testName) {
35+
saveResult(testName, TestResultStatus.ABORTED);
36+
}
37+
38+
public void testFailed(String testName) {
39+
saveResult(testName, TestResultStatus.FAILED);
40+
}
41+
42+
private void addChild(JsonObject object, String[] names, int position, TestResultStatus status) {
43+
String parentName = names[position - 1];
44+
if (position == names.length - 1) {
45+
if (object.has(parentName)) {
46+
object.get(parentName).getAsJsonObject().addProperty(names[position], status.name());
47+
}
48+
else {
49+
JsonObject child = new JsonObject();
50+
child.addProperty(names[position], status.name());
51+
object.add(parentName, child);
52+
}
53+
}
54+
else {
55+
JsonObject child = new JsonObject();
56+
addChild(child, names, position + 1, status);
57+
if (object.has(parentName)) {
58+
JsonObject parentObject = object.getAsJsonObject(parentName);
59+
if (parentObject.has(names[position])) {
60+
parentObject.getAsJsonArray(names[position]).add(child.getAsJsonObject(names[position]));
61+
}
62+
else {
63+
JsonArray subTests = new JsonArray();
64+
subTests.add(child.getAsJsonObject(names[position]));
65+
object.get(parentName).getAsJsonObject().add(names[position], subTests);
66+
}
67+
}
68+
else {
69+
JsonObject parentObject = new JsonObject();
70+
JsonArray nestedTest = new JsonArray();
71+
nestedTest.add(child.getAsJsonObject(names[position]));
72+
parentObject.add(names[position], nestedTest);
73+
object.add(parentName, parentObject);
74+
}
75+
}
76+
}
77+
78+
private void saveResult(String name, TestResultStatus status) {
79+
String[] testNames = name.split(TEST_SEPARATOR);
80+
if (testNames.length > 1) addChild(results, testNames, 1, status);
81+
}
82+
83+
public void saveToDisk() {
84+
Gson gson = new GsonBuilder().setPrettyPrinting().create();
85+
try {
86+
FileWriter writer = new FileWriter("results.json");
87+
String json = gson.toJson(results);
88+
//Sort alphabetically
89+
TreeMap<String, Object> map = gson.fromJson(json, TreeMap.class);
90+
String sortedJson = gson.toJson(map);
91+
writer.write(sortedJson);
92+
writer.flush();
93+
writer.close();
94+
} catch (IOException e) {
95+
throw new RuntimeException(e);
96+
}
97+
}
98+
99+
100+
101+
}
102+
103+

‎ScriptingEngineTester/src/main/java/org/openzen/scriptingenginetester/watcher/ZenCodeTestWatcher.java

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
package org.openzen.scriptingenginetester.watcher;
2+
3+
import org.junit.jupiter.api.extension.AfterAllCallback;
4+
import org.junit.jupiter.api.extension.ExtensionContext;
5+
import org.junit.jupiter.api.extension.TestWatcher;
6+
import org.junit.platform.engine.TestDescriptor;
7+
8+
import java.util.*;
9+
10+
public class ZenCodeTestWatcher implements TestWatcher, AfterAllCallback {
11+
12+
private static final String JUNIT_EXECUTOR = "JUnit Jupiter";
13+
14+
@Override
15+
public void testSuccessful(ExtensionContext context) {
16+
ZenCodeTestResults.getInstance().testSuccessful(getTestName(context));
17+
}
18+
19+
@Override
20+
public void testDisabled(ExtensionContext context, Optional<String> reason) {
21+
ZenCodeTestResults.getInstance().testDisabled(getTestName(context));
22+
}
23+
24+
@Override
25+
public void testAborted(ExtensionContext context, Throwable cause) {
26+
27+
ZenCodeTestResults.getInstance().testAborted(getTestName(context));
28+
}
29+
30+
@Override
31+
public void testFailed(ExtensionContext context, Throwable cause) {
32+
ZenCodeTestResults.getInstance().testFailed(getTestName(context));
33+
}
34+
35+
public static String getTestName(TestDescriptor descriptor) {
36+
StringBuilder builder = new StringBuilder();
37+
builder.append(descriptor.getDisplayName());
38+
while(hasValidParent(descriptor)) {
39+
TestDescriptor parent = descriptor.getParent().get();
40+
builder.insert(0, parent.getDisplayName() + ZenCodeTestResults.TEST_SEPARATOR);
41+
descriptor = parent;
42+
}
43+
return builder.toString();
44+
}
45+
46+
private static boolean hasValidParent(TestDescriptor descriptor) {
47+
return descriptor.getParent().isPresent() && !"JavaTestingEngineTest".equals(descriptor.getParent().get().getDisplayName());
48+
}
49+
50+
private String getTestName(ExtensionContext context) {
51+
StringBuilder builder = new StringBuilder();
52+
builder.append(context.getDisplayName());
53+
while (hasValidParent(context)) {
54+
ExtensionContext parent = context.getParent().get();
55+
builder.insert(0, parent.getDisplayName() + ZenCodeTestResults.TEST_SEPARATOR);
56+
context = parent;
57+
}
58+
return builder.toString();
59+
}
60+
61+
private boolean hasValidParent(ExtensionContext context) {
62+
return context.getParent().isPresent() && !JUNIT_EXECUTOR.equals(context.getParent().get().getDisplayName());
63+
}
64+
65+
/**
66+
* Callback that is invoked once <em>after</em> all tests in the current
67+
* container.
68+
*
69+
* @param context the current extension context; never {@code null}
70+
*/
71+
@Override
72+
public void afterAll(ExtensionContext context) throws Exception {
73+
ZenCodeTestResults.getInstance().saveToDisk();
74+
}
75+
}

‎ScriptingExample/build.gradle

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
import java.nio.file.Files
2+
13
// Note: "common.gradle" in the root project contains additional initialization
24
// for this project. This initialization is applied in the "build.gradle"
35
// of the root project.
@@ -25,6 +27,128 @@ processResources {
2527
from files(evaluationDependsOn(":StdLibs").tasks.getByName("zipItUp").outputs)
2628
}
2729

30+
tasks.register("savePreviousResults").configure {
31+
it.doFirst {
32+
def results = file("results.json")
33+
if (results.exists()) {
34+
def previous = file("previous_results.json")
35+
if (previous.exists()) previous.delete()
36+
Files.copy(results.toPath(), previous.toPath())
37+
}
38+
}
39+
}
40+
41+
def printSeparator(message) {
42+
project.logger.lifecycle("======================================")
43+
project.logger.lifecycle(" ${message} ")
44+
project.logger.lifecycle("======================================")
45+
}
46+
47+
def countTests(map) {
48+
int children = 0;
49+
50+
map.each { key, value ->
51+
if (value instanceof Map) {
52+
children += countTests(value)
53+
}
54+
else children++
55+
}
56+
57+
return children
58+
}
59+
60+
def exploreNewMap(map, indentLevel) {
61+
map.each { key, value ->
62+
def s = "|" + String.join("", Collections.nCopies(indentLevel, "---"))
63+
project.logger.lifecycle(s + key)
64+
if (value instanceof Map) {
65+
exploreNewMap(value, indentLevel + 1)
66+
}
67+
}
68+
}
69+
70+
def getEntryName(key, parentName) {
71+
if (parentName.empty || parentName.blank) return key
72+
return parentName + "-" + key
73+
}
74+
75+
def analiseMap(map, reference, parentName) {
76+
int count = 0
77+
map.each { key, value ->
78+
def oldValue = reference[key]
79+
if (oldValue == null) {
80+
if (value instanceof Map) {
81+
def internalCount = countTests(value)
82+
count += internalCount
83+
project.logger.lifecycle("Entry${key} did not exist previously and has ${internalCount} childTests!")
84+
exploreNewMap(value, 1)
85+
}
86+
else {
87+
project.logger.lifecycle("Entry${getEntryName(key, parentName)} did not exist previously!")
88+
count += 1
89+
}
90+
}
91+
else if (value instanceof Map) {
92+
count += analiseMap(value, oldValue, parentName + " " + key)
93+
}
94+
else if (value instanceof List) {
95+
value.eachWithIndex { it, index ->
96+
count += analiseMap(it, oldValue[index], parentName + " " + key)
97+
}
98+
}
99+
else {
100+
//value is a result
101+
if (oldValue instanceof String) {
102+
if (oldValue != value) {
103+
if (oldValue == "SUCCESSFUL") {
104+
count--
105+
}
106+
else if(value == "SUCCESSFUL") {
107+
count++
108+
}
109+
project.logger.lifecycle("Test:${parentName}-${key} changed from: ${oldValue} to ${value}")
110+
}
111+
}
112+
}
113+
}
114+
return count
115+
}
116+
117+
tasks.register("analyseResults").configure {
118+
it.doFirst {
119+
if (file("results.json").exists() && file("previous_results.json").exists()) {
120+
printSeparator("Starting ZenCode Test Difference")
121+
def newJson = new groovy.json.JsonSlurper().parseText(file("results.json").text)
122+
def oldJson = new groovy.json.JsonSlurper().parseText(file("previous_results.json").text)
123+
def newTests = countTests(newJson)
124+
def oldTests = countTests(oldJson)
125+
def successes = analiseMap(newJson, oldJson, "")
126+
if (newTests != oldTests) {
127+
project.logger.lifecycle("There are ${newTests - oldTests} more tests!")
128+
}
129+
if (successes > 0) {
130+
project.logger.lifecycle("There are ${successes} more passing tests than earlier!")
131+
}
132+
else if (successes < 0) {
133+
def normalisedSuccesses = -1 * successes
134+
project.logger.log(LogLevel.ERROR, "There are ${normalisedSuccesses} LESS passing tests than earlier!")
135+
if (newTests == oldTests)
136+
project.logger.log(LogLevel.ERROR, "This is very likely a symptom of a regression, or tests have been deleted.")
137+
}
138+
else {
139+
project.logger.lifecycle("No changes have been observed with the test suite.")
140+
}
141+
printSeparator("Finished ZenCode Test Difference")
142+
//Check extra success counter and see if it was positive + regressions.
143+
}
144+
else project.logger.log(LogLevel.ERROR, "No previous tests were found")
145+
}
146+
}
147+
148+
test.dependsOn(tasks.named("savePreviousResults"))
149+
150+
test.finalizedBy(tasks.named("analyseResults"))
151+
28152
test {
29153
useJUnitPlatform()
30154
testLogging {

0 commit comments

Comments
 (0)