refactor: simplifies angelic value analysis (SpoonLabs#175)

Author: monperrus

.travis_defects4j_math.sh

@@ -1,11 +1,18 @@
 #!/usr/bin/env bash
-# executing all tests in Defects4jEvaluationTest
+# executing all tests of Commons Math in Defects4J
+
+# fails if any command fails
+set -e
+
 cd nopol
-mvn -q versions:set -DnewVersion=TRAVIS
-# creating target/nopol-TRAVIS-jar-with-dependencies.jar
-mvn -q clean package -DskipTests
 
-env NOPOL_EVAL_DEFECTS4J=1 mvn -q test -Dtest="Defects4jEvaluationMathTest"
+NOPOL_EVAL_DEFECTS4J=1 mvn test -Dtest="fr.inria.lille.repair.nopol.Defects4jEvaluationMathTest#test_Math42"
+
+NOPOL_EVAL_DEFECTS4J=1 mvn test -Dtest="fr.inria.lille.repair.nopol.Defects4jEvaluationMathTest#test_Math49"
+
+NOPOL_EVAL_DEFECTS4J=1 mvn test -Dtest="fr.inria.lille.repair.nopol.Defects4jEvaluationMathTest#test_Math69"
 
+# NOPOL_EVAL_DEFECTS4J=1 mvn test -Dtest="fr.inria.lille.repair.nopol.Defects4jEvaluationMathTest#test_Math32"
 
+NOPOL_EVAL_DEFECTS4J=1 mvn test -Dtest="fr.inria.lille.repair.nopol.Defects4jEvaluationMathTest#test_Math33"
 

nopol/src/main/java/fr/inria/lille/commons/synthesis/ConstraintBasedSynthesis.java

@@ -19,6 +19,7 @@
 
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 
@@ -83,7 +84,7 @@ protected Expression<?> outputExpressionFor(Class<?> outputClass) {
     }
 
     private <T> Collection<Map<String, Object>> synthesisInputValues(Collection<Specification<T>> specifications, Expression<?> outputExpression) {
-        logCollection(logger(), "Specifications:", specifications);
+        // logCollection(logger(), "Specifications:", new LinkedList<>(specifications).subList(0, Math.max(Math.min(20, specifications.size()-1), specifications.size()-1)));
         Collection<Map<String, Object>> synthesisInputs = MetaList.newLinkedList();
         for (Specification<T> specification : specifications) {
             Map<String, Object> newMap = specification.inputs();

nopol/src/main/java/fr/inria/lille/repair/nopol/synth/ConstraintModelBuilder.java

@@ -38,7 +38,6 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.Iterator;
 import java.util.List;
 
 /**
@@ -47,7 +46,6 @@
 public final class ConstraintModelBuilder implements InstrumentedProgram<Boolean> {
 
     private final Logger logger = LoggerFactory.getLogger(this.getClass());
-    private boolean viablePatch;
     private final ClassLoader classLoader;
     private RuntimeValues<Boolean> runtimeValues;
     private SourceLocation sourceLocation;
@@ -86,7 +84,7 @@ public Collection<Specification<Boolean>> collectSpecifications(URL[] classpath,
         // come back to default mode
         AngelicExecution.disable();
 
-        if (!determineViability(firstResult, secondResult)) {
+        if (!isSynthesisPossible(firstResult, secondResult)) {
             return Collections.emptyList();
         }
         /* to collect information for passing tests */
@@ -140,26 +138,31 @@ public void testRunStarted(Description description)
 
 
 
-    private boolean determineViability(final Result firstResult, final Result secondResult) {
-        Collection<Description> firstFailures = TestSuiteExecution.collectDescription(firstResult.getFailures());
-        Collection<Description> secondFailures = TestSuiteExecution.collectDescription(secondResult.getFailures());
-        firstFailures.retainAll(secondFailures);
-        viablePatch = firstFailures.isEmpty();
-        int nbFirstSuccess = firstResult.getRunCount() - firstResult.getFailureCount();
-        int nbSecondSuccess = secondResult.getRunCount() - secondResult.getFailureCount();
-        if (!viablePatch || (nbFirstSuccess == 0 && nbSecondSuccess == 0)) {
-            logger.debug("Failing test(s): {}\n{}", sourceLocation, firstFailures);
+    private boolean isSynthesisPossible(final Result firstResultWithFalse, final Result secondResultWithTrue) {
+    	// this method is a key optimization in Nopol
+		// it enables to not start the synthesis when we are sure that it is not possible
+		// only based on the observed test outcomes with angelic values
+
+        Collection<Description> testFailuresWithFalse = TestSuiteExecution.collectDescription(firstResultWithFalse.getFailures());
+        Collection<Description> testFailuresWithTrue = TestSuiteExecution.collectDescription(secondResultWithTrue.getFailures());
+
+        // contract: all test failures must either in testFailuresWithFalse or in secondFailures
+		// removes from testFailuresWithFalse all of its elements that are not contained in testFailuresWithTrue
+		// consequently, it remains the elements that are always failing
+		Collection<Description> testsThatAreAlwasyFailing = new ArrayList<>(testFailuresWithFalse);
+		testsThatAreAlwasyFailing.retainAll(testFailuresWithTrue);
+
+        boolean synthesisIsPossible = testsThatAreAlwasyFailing.isEmpty();
+
+        // some logging
+        int nbFirstSuccess = firstResultWithFalse.getRunCount() - firstResultWithFalse.getFailureCount();
+        int nbSecondSuccess = secondResultWithTrue.getRunCount() - secondResultWithTrue.getFailureCount();
+        if (!synthesisIsPossible || (nbFirstSuccess == 0 && nbSecondSuccess == 0)) {
             Logger testsOutput = LoggerFactory.getLogger("tests.output");
-            testsOutput.debug("First set: \n{}", firstResult.getFailures());
-            testsOutput.debug("Second set: \n{}", secondResult.getFailures());
+            testsOutput.debug("Failing tests with false: \n{}", firstResultWithFalse.getFailures());
+            testsOutput.debug("Failing tests with true: \n{}", secondResultWithTrue.getFailures());
         }
-        return viablePatch;
+        return synthesisIsPossible;
     }
 
-    /**
-     * @see InstrumentedProgram#isAViablePatch()
-     */
-    public boolean isAViablePatch() {
-        return viablePatch;
-    }
 }

nopol/src/main/java/fr/inria/lille/repair/nopol/synth/InstrumentedProgram.java

@@ -22,5 +22,4 @@ public interface InstrumentedProgram<T> {
      */
     Collection<Specification<T>> collectSpecifications(final URL[] classpath, List<TestResult> testClasses, final Collection<TestCase> failures);
 
-    boolean isAViablePatch();
 }

nopol/src/main/java/fr/inria/lille/repair/nopol/synth/SMTNopolSynthesizer.java

@@ -78,12 +78,6 @@ public List<Patch> findAngelicValuesAndBuildPatch(URL[] classpath, List<TestResu
 		}
 
 
-		// and it should be a viable patch, ie. fix the bug
-		if (!instrumentedProgram.isAViablePatch()) {
-			LoggerFactory.getLogger(this.getClass()).info("Changing only this statement does not solve the bug. {}", sourceLocation);
-			return Collections.EMPTY_LIST;
-		}
-
 		nopolResult.incrementNbAngelicValues(sourceLocation, conditionalProcessor);
 		nbStatementsWithAngelicValue++;
 

nopol/src/test/java/fr/inria/lille/repair/nopol/Defects4jEvaluationMathTest.java

@@ -12,18 +12,20 @@
 
 public class Defects4jEvaluationMathTest {
 
-	@Test(timeout = FIVE_MINUTES_TIMEOUT *2)
+	@Test(timeout = FIFTEEN_MINUTES_TIMEOUT)
 	public void test_Math32() throws Exception {
 		// On Travis 454sec
+		// Sep 30: 2018 timeout on Travis with ten minutes (600 seconds), moving to 15 minutes
 		if (!testShouldBeRun()) { return; }
 		NopolContext nopolContext = nopolConfigFor("Math32");
 		NopolResult result = new NoPol(nopolContext).build();
 		assertEquals(1, result.getPatches().size());
 	}
 
-	@Test(timeout = FIVE_MINUTES_TIMEOUT *2)
+	@Test(timeout = FIFTEEN_MINUTES_TIMEOUT)
 	public void test_Math33() throws Exception {
 		// on Travis: 458sec
+		// Sep 30: 2018 timeout on Travis with ten minutes (600 seconds), moving to 15 minutes
 		if (!testShouldBeRun()) { return; }
 		NopolContext nopolContext = nopolConfigFor("Math33");
 		NopolResult result = new NoPol(nopolContext).build();
@@ -57,14 +59,14 @@ public void test_Math49() throws Exception {
 		assertEquals(1, result.getPatches().size());
 	}
 
-	@Test(timeout = FIFTEEN_MINUTES_TIMEOUT)
-	public void test_Math50() throws Exception {
-		// on Travis: 8 minutes, sometimes more than 10 minutes
-		if (!testShouldBeRun()) { return; }
-		NopolContext nopolContext = nopolConfigFor("Math50");
-		NopolResult result = new NoPol(nopolContext).build();
-		assertEquals(1, result.getPatches().size());
-	}
+//	@Test(timeout = FIFTEEN_MINUTES_TIMEOUT)
+//	public void test_Math50() throws Exception {
+//		// on Travis: 8 minutes, sometimes more than 10 minutes
+//		if (!testShouldBeRun()) { return; }
+//		NopolContext nopolContext = nopolConfigFor("Math50");
+//		NopolResult result = new NoPol(nopolContext).build();
+//		assertEquals(1, result.getPatches().size());
+//	}
 
 //	@Test(timeout = FIVE_MINUTES_TIMEOUT *2)
 //	public void test_Math57() throws Exception {
@@ -84,15 +86,15 @@ public void test_Math50() throws Exception {
 //		assertEquals(1, result.getPatches().size());
 //	}
 
-//	@Test(timeout = FIVE_MINUTES_TIMEOUT *2)
-//	public void test_Math69() throws Exception {
-//		// on Travis: 22sec
-//		if (!testShouldBeRun()) { return; }
-//		NopolContext nopolContext = nopolConfigFor("Math69");
-//		NopolResult result = new NoPol(nopolContext).build();
-//		assertEquals(1, result.getPatches().size());
-//	}
-//
+		@Test(timeout = FIVE_MINUTES_TIMEOUT *2)
+		public void test_Math69() throws Exception {
+			// on Travis: 22sec
+			if (!testShouldBeRun()) { return; }
+			NopolContext nopolContext = nopolConfigFor("Math69");
+			NopolResult result = new NoPol(nopolContext).build();
+			assertEquals(1, result.getPatches().size());
+		}
+
 //	@Test(timeout = FIVE_MINUTES_TIMEOUT *2)
 //	public void test_Math71() throws Exception {
 //		if (!testShouldBeRun()) { return; }