[TEST] use system rules (#123)

* pom: add dependency to System Rules

* refactor: call direclty System.exit()

* refactor: update binaries

* test: use System Rules to expect a System.exit(0)

* ci: remove system property

.github/workflows/mvn-build-action.yml

@@ -18,6 +18,6 @@ jobs:
       - name: Prepare resources binaries
         run: ./src/compile-and-clean-test-projects.sh
       - name: Export MAVEN_HOME, run tests and collect coverage
-        run: export MAVEN_HOME=`mvn --version | grep 'Maven home' | sed -e 's/Maven h.* //'` && echo "🎉 ${MAVEN_HOME}" && mvn test -DnoExitForTesting=true jacoco:report
+        run: export MAVEN_HOME=`mvn --version | grep 'Maven home' | sed -e 's/Maven h.* //'` && echo "🎉 ${MAVEN_HOME}" && mvn test jacoco:report
       - name: Report coverage to Codecov
         uses: codecov/[email protected]

pom.xml

@@ -183,6 +183,12 @@
             <version>0.8</version>
         </dependency>
 
+        <dependency>
+            <groupId>com.github.stefanbirkner</groupId>
+            <artifactId>system-rules</artifactId>
+            <version>1.19.0</version>
+        </dependency>
+
     </dependencies>
 
     <build>

src/main/java/eu/stamp_project/testrunner/runner/JUnit4Runner.java

@@ -33,7 +33,7 @@ public static void main(String[] args) {
                 JUnit4Runner.class.getClassLoader()
         );
         jUnit4TestResult.save();
-        ConstantsHelper.exit();
+        System.exit(0);
     }
 
     /**

src/main/java/eu/stamp_project/testrunner/runner/JUnit5Runner.java

@@ -37,7 +37,7 @@ public static void main(String args[]) {
                 JUnit5Runner.class.getClassLoader()
         );
         jUnit5TestResult.save();
-        ConstantsHelper.exit();
+        System.exit(0);
     }
 
     /**

src/main/java/eu/stamp_project/testrunner/runner/coverage/JUnit4JacocoRunner.java

@@ -63,7 +63,7 @@ public static void main(String[] args) {
                 ).save();
             }
         }
-        ConstantsHelper.exit();
+        System.exit(0);
     }
 
     @Override

src/main/java/eu/stamp_project/testrunner/runner/coverage/JUnit4JacocoRunnerCoveredResultPerTestMethod.java

@@ -54,7 +54,7 @@ public static void main(String[] args) {
 				options.getFullQualifiedNameOfTestClassesToRun(),
 				options.getTestMethodNamesToRun()
 		).save();
-		ConstantsHelper.exit();
+		System.exit(0);
 	}
 
 }

src/main/java/eu/stamp_project/testrunner/runner/coverage/JUnit4JacocoRunnerPerTestMethod.java

@@ -59,6 +59,6 @@ public static void main(String[] args) {
                 options.getFullQualifiedNameOfTestClassesToRun(),
                 options.getTestMethodNamesToRun()
         ).save();
-        ConstantsHelper.exit();
+        System.exit(0);
     }
 }

src/main/java/eu/stamp_project/testrunner/runner/coverage/JUnit4OnlineJacocoRunner.java

@@ -48,6 +48,6 @@ public static void main(String[] args) {
 				options.getFullQualifiedNameOfTestClassesToRun(),
 				options.getTestMethodNamesToRun()
 		).save();
-		ConstantsHelper.exit();
+		System.exit(0);
 	}
 }

src/main/java/eu/stamp_project/testrunner/runner/coverage/JUnit5JacocoRunner.java

@@ -63,7 +63,7 @@ public static void main(String[] args) {
                 ).save();
             }
         }
-        ConstantsHelper.exit();
+        System.exit(0);
     }
 
     @Override

src/main/java/eu/stamp_project/testrunner/runner/coverage/JUnit5JacocoRunnerCoveredResultPerTestMethod.java

@@ -54,7 +54,7 @@ public static void main(String[] args) {
 				options.getFullQualifiedNameOfTestClassesToRun(),
 				options.getTestMethodNamesToRun()
 		).save();
-		ConstantsHelper.exit();
+		System.exit(0);
 	}
 
 }

src/main/java/eu/stamp_project/testrunner/runner/coverage/JUnit5JacocoRunnerPerTestMethod.java

@@ -58,6 +58,6 @@ public static void main(String[] args) {
                 options.getFullQualifiedNameOfTestClassesToRun(),
                 options.getTestMethodNamesToRun()
         ).save();
-        ConstantsHelper.exit();
+        System.exit(0);
     }
 }

src/main/java/eu/stamp_project/testrunner/runner/coverage/JUnit5OnlineJacocoRunner.java

@@ -51,6 +51,6 @@ public static void main(String[] args) {
 				options.getFullQualifiedNameOfTestClassesToRun(),
 				options.getTestMethodNamesToRun()
 		).save();
-		ConstantsHelper.exit();
+		System.exit(0);
 	}
 }

src/main/java/eu/stamp_project/testrunner/utils/ConstantsHelper.java

@@ -13,12 +13,6 @@
  */
 public class ConstantsHelper {
 
-    public static void exit() {
-        if (!"true".equals(System.getProperty("noExitForTesting"))) {
-            System.exit(0);
-        }
-    }
-
     public enum MutationEngine {
         GREGOR(Collections.singletonList("ALL")),
         DESCARTES(Arrays.asList(

src/test/java/eu/stamp_project/testrunner/AbstractTest.java

@@ -5,6 +5,10 @@
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.contrib.java.lang.system.ExpectedSystemExit;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TestRule;
 import spoon.Launcher;
 
 import java.io.File;
@@ -23,6 +27,11 @@
  * on 19/12/17
  */
 public class AbstractTest {
+
+    public final ExpectedSystemExit exit = ExpectedSystemExit.none();
+    @Rule
+    public TestRule allRules = RuleChain.emptyRuleChain().around(exit);
+
     public static String MAVEN_HOME;
 
     public static String SOURCE_PROJECT_CLASSES;

src/test/java/eu/stamp_project/testrunner/listener/CoverageTest.java

@@ -5,7 +5,11 @@
 import eu.stamp_project.testrunner.runner.ParserOptions;
 import eu.stamp_project.testrunner.runner.coverage.JUnit4JacocoRunner;
 import eu.stamp_project.testrunner.utils.ConstantsHelper;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.contrib.java.lang.system.ExpectedSystemExit;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TestRule;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
@@ -19,6 +23,7 @@ public void compareTwoCoverages() throws Exception {
             Using the api to compute the coverage on a test class
          */
 
+        exit.expectSystemExitWithStatus(0);
         JUnit4JacocoRunner.main(new String[]{
                         ParserOptions.FLAG_pathToCompiledClassesOfTheProject, SOURCE_PROJECT_CLASSES,
                         ParserOptions.FLAG_pathToCompiledTestClassesOfTheProject, TEST_PROJECT_CLASSES,
@@ -28,6 +33,7 @@ public void compareTwoCoverages() throws Exception {
         );
         final Coverage test4Coverage = CoverageImpl.load();
 
+        exit.expectSystemExitWithStatus(0);
         JUnit4JacocoRunner.main(new String[]{
                         ParserOptions.FLAG_pathToCompiledClassesOfTheProject, SOURCE_PROJECT_CLASSES,
                         ParserOptions.FLAG_pathToCompiledTestClassesOfTheProject, TEST_PROJECT_CLASSES,

src/test/java/eu/stamp_project/testrunner/runner/coverage/JUnit4JacocoRunnerCoveredResultPerTestMethodTest.java

@@ -23,6 +23,7 @@ public void testWithoutNewJvmOnTestClassParametrized() throws Exception {
             Using the api to compute the coverage on a test class
          */
 
+		exit.expectSystemExitWithStatus(0);
 		JUnit4JacocoRunnerCoveredResultPerTestMethod.main(new String[]{
 						ParserOptions.FLAG_pathToCompiledClassesOfTheProject, SOURCE_PROJECT_CLASSES,
 						ParserOptions.FLAG_pathToCompiledTestClassesOfTheProject, TEST_PROJECT_CLASSES,
@@ -55,6 +56,7 @@ public void testWithoutNewJvmOnTestCases() throws Exception {
             Using the api to compute the coverage on test cases
          */
 
+		exit.expectSystemExitWithStatus(0);
 		JUnit4JacocoRunnerCoveredResultPerTestMethod.main(new String[]{
 						ParserOptions.FLAG_pathToCompiledClassesOfTheProject, SOURCE_PROJECT_CLASSES,
 						ParserOptions.FLAG_pathToCompiledTestClassesOfTheProject, TEST_PROJECT_CLASSES,
@@ -86,6 +88,7 @@ public void testWithoutNewJvmOnTestClassAll() throws Exception {
             Using the api to compute the coverage on test cases
          */
 
+		exit.expectSystemExitWithStatus(0);
 		JUnit4JacocoRunnerCoveredResultPerTestMethod.main(new String[]{
 						ParserOptions.FLAG_pathToCompiledClassesOfTheProject, SOURCE_PROJECT_CLASSES,
 						ParserOptions.FLAG_pathToCompiledTestClassesOfTheProject, TEST_PROJECT_CLASSES,

src/test/java/eu/stamp_project/testrunner/runner/coverage/JUnit5JacocoRunnerCoveredResultPerTestMethodTest.java

@@ -21,6 +21,7 @@ public void testWithoutNewJvmOnTestClassParametrized() throws Exception {
             Using the api to compute the coverage on a test class
          */
 
+		exit.expectSystemExitWithStatus(0);
 		JUnit5JacocoRunnerCoveredResultPerTestMethod.main(new String[]{
 						ParserOptions.FLAG_pathToCompiledClassesOfTheProject, SOURCE_PROJECT_CLASSES,
 						ParserOptions.FLAG_pathToCompiledTestClassesOfTheProject, TEST_PROJECT_CLASSES,
@@ -41,6 +42,7 @@ public void testWithoutNewJvmOnTestCases() throws Exception {
             Using the api to compute the coverage on test cases
          */
 
+		exit.expectSystemExitWithStatus(0);
 		JUnit5JacocoRunnerCoveredResultPerTestMethod.main(new String[]{
 						ParserOptions.FLAG_pathToCompiledClassesOfTheProject, SOURCE_PROJECT_CLASSES,
 						ParserOptions.FLAG_pathToCompiledTestClassesOfTheProject, TEST_PROJECT_CLASSES,
@@ -70,6 +72,7 @@ public void testWithoutNewJvmOnTestClassAll() throws Exception {
             Using the api to compute the coverage on test cases
          */
 
+		exit.expectSystemExitWithStatus(0);
 		JUnit5JacocoRunnerCoveredResultPerTestMethod.main(new String[]{
 						ParserOptions.FLAG_pathToCompiledClassesOfTheProject, SOURCE_PROJECT_CLASSES,
 						ParserOptions.FLAG_pathToCompiledTestClassesOfTheProject, TEST_PROJECT_CLASSES,

src/test/java/eu/stamp_project/testrunner/runner/coverage/JUnit5JacocoRunnerTest.java

@@ -27,6 +27,7 @@ public void testWithoutNewJvmOnTestClass() throws Exception {
             Using the api to compute the coverage on a test class
          */
 
+        exit.expectSystemExitWithStatus(0);
         JUnit5JacocoRunner.main(new String[]{
                         ParserOptions.FLAG_pathToCompiledClassesOfTheProject, SOURCE_PROJECT_CLASSES,
                         ParserOptions.FLAG_pathToCompiledTestClassesOfTheProject, TEST_PROJECT_CLASSES,
@@ -56,6 +57,7 @@ public void testWithoutNewJvmOnTestCases() throws Exception {
             Using the api to compute the coverage on test cases
          */
 
+        exit.expectSystemExitWithStatus(0);
         JUnit5JacocoRunner.main(new String[]{
                         ParserOptions.FLAG_pathToCompiledClassesOfTheProject, SOURCE_PROJECT_CLASSES,
                         ParserOptions.FLAG_pathToCompiledTestClassesOfTheProject, TEST_PROJECT_CLASSES,
@@ -76,6 +78,7 @@ public void testWithoutNewJvmOnTestCasesOnParametrized() throws Exception {
             Using the api to compute the coverage on test cases
          */
 
+        exit.expectSystemExitWithStatus(0);
         JUnit5JacocoRunner.main(new String[]{
                         ParserOptions.FLAG_pathToCompiledClassesOfTheProject, SOURCE_PROJECT_CLASSES,
                         ParserOptions.FLAG_pathToCompiledTestClassesOfTheProject, TEST_PROJECT_CLASSES,
@@ -101,6 +104,7 @@ public void testWithoutNewJvmOnTestCasesOnParametrized() throws Exception {
     @Test
     public void testMethodDetailedCoverageDetail() throws Exception {
 
+        exit.expectSystemExitWithStatus(0);
         JUnit5JacocoRunner.main(new String[]{
                         ParserOptions.FLAG_pathToCompiledClassesOfTheProject, SOURCE_PROJECT_CLASSES,
                         ParserOptions.FLAG_pathToCompiledTestClassesOfTheProject, TEST_PROJECT_CLASSES,

src/test/java/eu/stamp_project/testrunner/runner/coverage/JacocoRunnerPerTestMethodTest.java

@@ -1,11 +1,10 @@
 package eu.stamp_project.testrunner.runner.coverage;
 
 import eu.stamp_project.testrunner.AbstractTest;
-import eu.stamp_project.testrunner.listener.Coverage;
-import eu.stamp_project.testrunner.listener.impl.CoverageImpl;
 import eu.stamp_project.testrunner.listener.impl.CoveragePerTestMethodImpl;
 import eu.stamp_project.testrunner.runner.ParserOptions;
 import org.junit.Test;
+import org.junit.contrib.java.lang.system.ExpectedSystemExit;
 
 import static org.junit.Assert.assertEquals;
 
@@ -23,6 +22,7 @@ public void testWithoutNewJvmOnTestClassParametrized() throws Exception {
             Using the api to compute the coverage on a test class
          */
 
+        exit.expectSystemExitWithStatus(0);
         JUnit4JacocoRunnerPerTestMethod.main(new String[]{
                         ParserOptions.FLAG_pathToCompiledClassesOfTheProject, SOURCE_PROJECT_CLASSES,
                         ParserOptions.FLAG_pathToCompiledTestClassesOfTheProject, TEST_PROJECT_CLASSES,
@@ -43,6 +43,7 @@ public void testWithoutNewJvmOnTestCases() throws Exception {
             Using the api to compute the coverage on test cases
          */
 
+        exit.expectSystemExitWithStatus(0);
         JUnit4JacocoRunnerPerTestMethod.main(new String[]{
                         ParserOptions.FLAG_pathToCompiledClassesOfTheProject, SOURCE_PROJECT_CLASSES,
                         ParserOptions.FLAG_pathToCompiledTestClassesOfTheProject, TEST_PROJECT_CLASSES,
@@ -67,6 +68,7 @@ public void testWithoutNewJvmOnTestClassAll() throws Exception {
             Using the api to compute the coverage on test cases
          */
 
+        exit.expectSystemExitWithStatus(0);
         JUnit4JacocoRunnerPerTestMethod.main(new String[]{
                         ParserOptions.FLAG_pathToCompiledClassesOfTheProject, SOURCE_PROJECT_CLASSES,
                         ParserOptions.FLAG_pathToCompiledTestClassesOfTheProject, TEST_PROJECT_CLASSES,

src/test/java/eu/stamp_project/testrunner/runner/coverage/JacocoRunnerTest.java

@@ -25,6 +25,7 @@ public void testWithoutNewJvmOnTestClass() throws Exception {
             Using the api to compute the coverage on a test class
          */
 
+        exit.expectSystemExitWithStatus(0);
         JUnit4JacocoRunner.main(new String[]{
                         ParserOptions.FLAG_pathToCompiledClassesOfTheProject, SOURCE_PROJECT_CLASSES,
                         ParserOptions.FLAG_pathToCompiledTestClassesOfTheProject, TEST_PROJECT_CLASSES,
@@ -45,6 +46,7 @@ public void testWithoutNewJvmOnTestCases() throws Exception {
             Using the api to compute the coverage on test cases
          */
 
+        exit.expectSystemExitWithStatus(0);
         JUnit4JacocoRunner.main(new String[]{
                         ParserOptions.FLAG_pathToCompiledClassesOfTheProject, SOURCE_PROJECT_CLASSES,
                         ParserOptions.FLAG_pathToCompiledTestClassesOfTheProject, TEST_PROJECT_CLASSES,