Merge pull request MarathonLabs#758 from MarathonLabs/feature/executi…

…on-strategy feat(core): execution strategy
agoda-com · Mar 6, 2023 · 6e87c35 · 6e87c35
2 parents 48c3c9f + 07b25ea
commit 6e87c35
Show file tree

Hide file tree

Showing 52 changed files with 1,681 additions and 1,046 deletions.
diff --git a/.idea/runConfigurations/android_app.xml b/.idea/runConfigurations/android_app.xml
diff --git a/.idea/runConfigurations/android_library.xml b/.idea/runConfigurations/android_library.xml
diff --git a/configuration/src/main/kotlin/com/malinskiy/marathon/config/Configuration.kt b/configuration/src/main/kotlin/com/malinskiy/marathon/config/Configuration.kt
@@ -2,6 +2,7 @@ package com.malinskiy.marathon.config
 
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize
 import com.malinskiy.marathon.config.strategy.BatchingStrategyConfiguration
+import com.malinskiy.marathon.config.strategy.ExecutionStrategyConfiguration
 import com.malinskiy.marathon.config.strategy.FlakinessStrategyConfiguration
 import com.malinskiy.marathon.config.strategy.PoolingStrategyConfiguration
 import com.malinskiy.marathon.config.strategy.RetryStrategyConfiguration
@@ -18,7 +19,7 @@ private const val DEFAULT_DEVICE_INITIALIZATION_TIMEOUT_MILLIS = 180_000L
 data class Configuration private constructor(
     val name: String,
     val outputDir: File,
-    
+
     val outputConfiguration: OutputConfiguration,
 
     val analyticsConfiguration: AnalyticsConfiguration,
@@ -31,8 +32,8 @@ data class Configuration private constructor(
     val filteringConfiguration: FilteringConfiguration,
 
     val ignoreFailures: Boolean,
+    val executionStrategy: ExecutionStrategyConfiguration,
     val isCodeCoverageEnabled: Boolean,
-    val strictMode: Boolean,
     val uncompletedTestRetryQuota: Int,
 
     val testClassRegexes: Collection<Regex>,
@@ -65,7 +66,7 @@ data class Configuration private constructor(
             "filtering" to filteringConfiguration.toString(),
             "ignoreFailures" to ignoreFailures.toString(),
             "isCodeCoverageEnabled" to isCodeCoverageEnabled.toString(),
-            "strictMode" to strictMode.toString(),
+            "executionStrategy" to executionStrategy.toString(),
             "testClassRegexes" to testClassRegexes.toString(),
             "includeSerialRegexes" to includeSerialRegexes.toString(),
             "excludeSerialRegexes" to excludeSerialRegexes.toString(),
@@ -96,7 +97,7 @@ data class Configuration private constructor(
         if (filteringConfiguration != other.filteringConfiguration) return false
         if (ignoreFailures != other.ignoreFailures) return false
         if (isCodeCoverageEnabled != other.isCodeCoverageEnabled) return false
-        if (strictMode != other.strictMode) return false
+        if (executionStrategy != other.executionStrategy) return false
         if (uncompletedTestRetryQuota != other.uncompletedTestRetryQuota) return false
         //For testing we need to compare configuration instances. Unfortunately Regex equality is broken so need to map it to String
         if (testClassRegexes.map { it.pattern } != other.testClassRegexes.map { it.pattern }) return false
@@ -127,7 +128,7 @@ data class Configuration private constructor(
         result = 31 * result + filteringConfiguration.hashCode()
         result = 31 * result + ignoreFailures.hashCode()
         result = 31 * result + isCodeCoverageEnabled.hashCode()
-        result = 31 * result + strictMode.hashCode()
+        result = 31 * result + executionStrategy.hashCode()
         result = 31 * result + uncompletedTestRetryQuota
         result = 31 * result + testClassRegexes.hashCode()
         result = 31 * result + includeSerialRegexes.hashCode()
@@ -141,38 +142,39 @@ data class Configuration private constructor(
         result = 31 * result + deviceInitializationTimeoutMillis.hashCode()
         return result
     }
+
      data class Builder(
-        val name: String,
-        val outputDir: File,
-        var analyticsConfiguration: AnalyticsConfiguration = AnalyticsConfiguration.DisabledAnalytics,
-        var poolingStrategy: PoolingStrategyConfiguration = PoolingStrategyConfiguration.OmniPoolingStrategyConfiguration,
-        var shardingStrategy: ShardingStrategyConfiguration = ShardingStrategyConfiguration.ParallelShardingStrategyConfiguration,
-        var sortingStrategy: SortingStrategyConfiguration = SortingStrategyConfiguration.NoSortingStrategyConfiguration,
-        var batchingStrategy: BatchingStrategyConfiguration = BatchingStrategyConfiguration.IsolateBatchingStrategyConfiguration,
-        var flakinessStrategy: FlakinessStrategyConfiguration = FlakinessStrategyConfiguration.IgnoreFlakinessStrategyConfiguration,
-        var retryStrategy: RetryStrategyConfiguration = RetryStrategyConfiguration.NoRetryStrategyConfiguration,
-        var filteringConfiguration: FilteringConfiguration = FilteringConfiguration(emptyList(), emptyList()),
-
-        var ignoreFailures: Boolean = false,
-        var isCodeCoverageEnabled: Boolean = false,
-        var strictMode: Boolean = false,
-        var uncompletedTestRetryQuota: Int = Integer.MAX_VALUE,
-
-        var testClassRegexes: Collection<Regex> = listOf(Regex("^((?!Abstract).)*Test[s]*$")),
-        var includeSerialRegexes: Collection<Regex> = emptyList(),
-        var excludeSerialRegexes: Collection<Regex> = emptyList(),
-
-        var testBatchTimeoutMillis: Long = DEFAULT_BATCH_EXECUTION_TIMEOUT_MILLIS,
-        var testOutputTimeoutMillis: Long = DEFAULT_OUTPUT_TIMEOUT_MILLIS,
-        var debug: Boolean = true,
-
-        var screenRecordingPolicy: ScreenRecordingPolicy = ScreenRecordingPolicy.ON_FAILURE,
-
-        var analyticsTracking: Boolean = false,
-        var deviceInitializationTimeoutMillis: Long = DEFAULT_DEVICE_INITIALIZATION_TIMEOUT_MILLIS,
-
-        var outputConfiguration: OutputConfiguration = OutputConfiguration(),
-        var vendorConfiguration: VendorConfiguration = VendorConfiguration.EmptyVendorConfiguration(),
+         val name: String,
+         val outputDir: File,
+         var analyticsConfiguration: AnalyticsConfiguration = AnalyticsConfiguration.DisabledAnalytics,
+         var poolingStrategy: PoolingStrategyConfiguration = PoolingStrategyConfiguration.OmniPoolingStrategyConfiguration,
+         var shardingStrategy: ShardingStrategyConfiguration = ShardingStrategyConfiguration.ParallelShardingStrategyConfiguration,
+         var sortingStrategy: SortingStrategyConfiguration = SortingStrategyConfiguration.NoSortingStrategyConfiguration,
+         var batchingStrategy: BatchingStrategyConfiguration = BatchingStrategyConfiguration.IsolateBatchingStrategyConfiguration,
+         var flakinessStrategy: FlakinessStrategyConfiguration = FlakinessStrategyConfiguration.IgnoreFlakinessStrategyConfiguration,
+         var retryStrategy: RetryStrategyConfiguration = RetryStrategyConfiguration.NoRetryStrategyConfiguration,
+         var filteringConfiguration: FilteringConfiguration = FilteringConfiguration(emptyList(), emptyList()),
+
+         var ignoreFailures: Boolean = false,
+         var isCodeCoverageEnabled: Boolean = false,
+         var executionStrategy: ExecutionStrategyConfiguration = ExecutionStrategyConfiguration(),
+         var uncompletedTestRetryQuota: Int = Integer.MAX_VALUE,
+
+         var testClassRegexes: Collection<Regex> = listOf(Regex("^((?!Abstract).)*Test[s]*$")),
+         var includeSerialRegexes: Collection<Regex> = emptyList(),
+         var excludeSerialRegexes: Collection<Regex> = emptyList(),
+
+         var testBatchTimeoutMillis: Long = DEFAULT_BATCH_EXECUTION_TIMEOUT_MILLIS,
+         var testOutputTimeoutMillis: Long = DEFAULT_OUTPUT_TIMEOUT_MILLIS,
+         var debug: Boolean = true,
+
+         var screenRecordingPolicy: ScreenRecordingPolicy = ScreenRecordingPolicy.ON_FAILURE,
+
+         var analyticsTracking: Boolean = false,
+         var deviceInitializationTimeoutMillis: Long = DEFAULT_DEVICE_INITIALIZATION_TIMEOUT_MILLIS,
+
+         var outputConfiguration: OutputConfiguration = OutputConfiguration(),
+         var vendorConfiguration: VendorConfiguration = VendorConfiguration.EmptyVendorConfiguration(),
     ) {
         fun build(): Configuration {
             return Configuration(
@@ -189,7 +191,7 @@ data class Configuration private constructor(
                 filteringConfiguration = filteringConfiguration,
                 ignoreFailures = ignoreFailures,
                 isCodeCoverageEnabled = isCodeCoverageEnabled,
-                strictMode = strictMode,
+                executionStrategy = executionStrategy,
                 uncompletedTestRetryQuota = uncompletedTestRetryQuota,
                 testClassRegexes = testClassRegexes,
                 includeSerialRegexes = includeSerialRegexes,

diff --git a/configuration/src/main/kotlin/com/malinskiy/marathon/config/LogicalConfigurationValidator.kt b/configuration/src/main/kotlin/com/malinskiy/marathon/config/LogicalConfigurationValidator.kt
@@ -1,7 +1,9 @@
 package com.malinskiy.marathon.config
 
 import com.malinskiy.marathon.config.exceptions.ConfigurationException
+import com.malinskiy.marathon.config.strategy.ExecutionMode
 import com.malinskiy.marathon.config.strategy.FlakinessStrategyConfiguration
+import com.malinskiy.marathon.config.strategy.RetryStrategyConfiguration
 import com.malinskiy.marathon.config.strategy.ShardingStrategyConfiguration
 import com.malinskiy.marathon.config.vendor.VendorConfiguration
 
@@ -28,5 +30,27 @@ class LogicalConfigurationValidator : ConfigurationValidator {
 
             else -> Unit
         }
+
+        when(configuration.executionStrategy.mode) {
+            ExecutionMode.ANY_SUCCESS -> {
+                if (configuration.shardingStrategy !is ShardingStrategyConfiguration.ParallelShardingStrategyConfiguration) {
+                    throw ConfigurationException(
+                        "Configuration is invalid: can't use complex sharding and any success execution strategy at the same time. Consult documentation for the any success execution logic"
+                    )
+                }
+            }
+            ExecutionMode.ALL_SUCCESS -> {
+                if (configuration.flakinessStrategy !is FlakinessStrategyConfiguration.IgnoreFlakinessStrategyConfiguration) {
+                    throw ConfigurationException(
+                        "Configuration is invalid: can't use complex flakiness strategy and all success execution strategy at the same time. Consult documentation for the all success execution logic"
+                    )
+                }
+                if (configuration.retryStrategy !is RetryStrategyConfiguration.NoRetryStrategyConfiguration) {
+                    throw ConfigurationException(
+                        "Configuration is invalid: can't use complex retry strategy and all success execution strategy at the same time. Consult documentation for the all success execution logic"
+                    )
+                }
+            }
+        }
     }
 }
diff --git a/.../src/main/kotlin/com/malinskiy/marathon/config/strategy/ExecutionStrategyConfiguration.kt b/.../src/main/kotlin/com/malinskiy/marathon/config/strategy/ExecutionStrategyConfiguration.kt
@@ -0,0 +1,31 @@
+package com.malinskiy.marathon.config.strategy
+
+import com.fasterxml.jackson.annotation.JsonProperty
+
+/**
+ * @property fast fail-fast or success-fast depending on the value of [mode]. This doesn't affect the result
+ *  of the run, it only saves compute time
+ */
+data class ExecutionStrategyConfiguration(
+    @JsonProperty("mode") val mode: ExecutionMode = ExecutionMode.ANY_SUCCESS,
+    @JsonProperty("fast") val fast: Boolean = true,
+)
+
+/**
+ * @property ANY_SUCCESS test passes if any of executions is passing
+ *  this mode works only if there is no complex sharding strategy applied
+ *  
+ *  Why: it doesn't make sense when user asks for N number of tests
+ *  to run explicitly, and we pass on the first one. Sharding used to verify probability of passing with an explicit boundary for precision
+ * @property ALL_SUCCESS test passes if and only if all the executions are passing
+ *  this mode works only if there are no retries, i.e. no complex flakiness strategy, no retry strategy
+ *  
+ *  Why: when adding retries to tests with retry+flakiness strategies users want to trade-off cost for reliability, i.e. add more retries
+ *  and pass if one of them passes, so retries only make sense for the [ANY_SUCCESS] mode. When we use [ALL_SUCCESS] mode it means user
+ *  wants to verify each test with a number of tries (they are not retries per se) and pass only if all of them succeed. This is the case
+ *  when fixing a flaky test or adding a new test, and we want to have a signal that the test is fixed/not flaky.
+ */
+enum class ExecutionMode {
+    @JsonProperty("ANY_SUCCESS") ANY_SUCCESS,
+    @JsonProperty("ALL_SUCCESS") ALL_SUCCESS,
+}
diff --git a/...n/src/test/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactoryTest.kt b/...n/src/test/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactoryTest.kt
@@ -163,7 +163,6 @@ class ConfigurationFactoryTest {
         configuration.excludeSerialRegexes.joinToString(separator = "") { it.pattern } shouldBeEqualTo """emulator-5002""".toRegex().pattern
         configuration.ignoreFailures shouldBeEqualTo false
         configuration.isCodeCoverageEnabled shouldBeEqualTo false
-        configuration.strictMode shouldBeEqualTo true
         configuration.testBatchTimeoutMillis shouldBeEqualTo 20_000
         configuration.testOutputTimeoutMillis shouldBeEqualTo 30_000
         configuration.debug shouldBeEqualTo true

diff --git a/configuration/src/test/resources/fixture/config/sample_1.yaml b/configuration/src/test/resources/fixture/config/sample_1.yaml
@@ -68,7 +68,9 @@ excludeSerialRegexes:
   - "emulator-5002"
 ignoreFailures: false
 isCodeCoverageEnabled: false
-strictMode: true
+executionStrategy:
+  mode: ANY_SUCCESS
+  fast: true
 testBatchTimeoutMillis: 20000
 testOutputTimeoutMillis: 30000
 debug: true

diff --git a/configuration/src/test/resources/fixture/config/sample_1_rp.yaml b/configuration/src/test/resources/fixture/config/sample_1_rp.yaml
@@ -66,7 +66,9 @@ excludeSerialRegexes:
   - "emulator-5002"
 ignoreFailures: false
 isCodeCoverageEnabled: false
-strictMode: true
+executionStrategy:
+  mode: ANY_SUCCESS
+  fast: true
 testBatchTimeoutMillis: 20000
 testOutputTimeoutMillis: 30000
 debug: true

diff --git a/core/src/main/kotlin/com/malinskiy/marathon/Marathon.kt b/core/src/main/kotlin/com/malinskiy/marathon/Marathon.kt
@@ -19,7 +19,6 @@ import com.malinskiy.marathon.execution.TestParser
 import com.malinskiy.marathon.execution.TestShard
 import com.malinskiy.marathon.execution.bundle.TestBundleIdentifier
 import com.malinskiy.marathon.execution.command.parse.MarathonTestParseCommand
-import com.malinskiy.marathon.execution.progress.ProgressReporter
 import com.malinskiy.marathon.execution.withRetry
 import com.malinskiy.marathon.extension.toFlakinessStrategy
 import com.malinskiy.marathon.extension.toShardingStrategy
@@ -34,7 +33,6 @@ import com.malinskiy.marathon.usageanalytics.UsageAnalytics
 import com.malinskiy.marathon.usageanalytics.tracker.Event
 import kotlinx.coroutines.NonCancellable
 import kotlinx.coroutines.async
-import kotlinx.coroutines.delay
 import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.withContext
 import kotlinx.coroutines.withTimeoutOrNull
@@ -52,7 +50,6 @@ class Marathon(
     private val logConfigurator: MarathonLogConfigurator,
     private val tracker: TrackerInternal,
     private val analytics: Analytics,
-    private val progressReporter: ProgressReporter,
     private val track: Track,
     private val timer: Timer,
     private val marathonTestParseCommand: MarathonTestParseCommand
@@ -142,7 +139,6 @@ class Marathon(
             analytics,
             configuration,
             shard,
-            progressReporter,
             track,
             timer,
             testBundleIdentifier,
@@ -155,12 +151,13 @@ class Marathon(
         }
         configuration.outputDir.mkdirs()
 
-        if (parsedFilteredTests.isNotEmpty()) {
+        val result = if (parsedFilteredTests.isNotEmpty()) {
             scheduler.execute()
+        } else {
+            true
         }
 
         onFinish(analytics, deviceProvider)
-        val result = progressReporter.aggregateResult()
 
         stopKoin()
         return result

diff --git a/core/src/main/kotlin/com/malinskiy/marathon/device/Device.kt b/core/src/main/kotlin/com/malinskiy/marathon/device/Device.kt
@@ -2,7 +2,6 @@ package com.malinskiy.marathon.device
 
 import com.malinskiy.marathon.config.Configuration
 import com.malinskiy.marathon.execution.TestBatchResults
-import com.malinskiy.marathon.execution.progress.ProgressReporter
 import com.malinskiy.marathon.test.TestBatch
 import kotlinx.coroutines.CompletableDeferred
 import mu.KLogger
@@ -45,7 +44,6 @@ interface Device {
         devicePoolId: DevicePoolId,
         testBatch: TestBatch,
         deferred: CompletableDeferred<TestBatchResults>,
-        progressReporter: ProgressReporter
     )
 
     /**

diff --git a/core/src/main/kotlin/com/malinskiy/marathon/di/Modules.kt b/core/src/main/kotlin/com/malinskiy/marathon/di/Modules.kt
@@ -7,7 +7,6 @@ import com.malinskiy.marathon.analytics.external.AnalyticsFactory
 import com.malinskiy.marathon.analytics.internal.pub.Track
 import com.malinskiy.marathon.config.Configuration
 import com.malinskiy.marathon.execution.command.parse.MarathonTestParseCommand
-import com.malinskiy.marathon.execution.progress.ProgressReporter
 import com.malinskiy.marathon.io.FileManager
 import com.malinskiy.marathon.json.FileSerializer
 import com.malinskiy.marathon.time.SystemTimer
@@ -37,12 +36,11 @@ val coreModule = module {
     }
     single<Clock> { Clock.systemDefaultZone() }
     single<Timer> { SystemTimer(get()) }
-    single { ProgressReporter(get()) }
     single {
         val configuration = get<Configuration>()
         MarathonTestParseCommand(configuration.outputDir)
     }
-    single { Marathon(get(), get(), get(), get(), get(), get(), get(), get(), get(), get(), get()) }
+    single { Marathon(get(), get(), get(), get(), get(), get(), get(), get(), get(), get()) }
 }
 
 fun marathonStartKoin(configuration: Configuration, modules: List<Module>): KoinApplication {