Add basic support for execution multiple repititions

not tested so far

theodolite-quarkus/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt

@@ -16,17 +16,13 @@ class AnalysisExecutor(private val slo: BenchmarkExecution.Slo) {
         offset = Duration.ofHours(slo.offset.toLong())
     )
 
-    fun analyze(load: LoadDimension, res: Resource, executionDuration: Duration): Boolean {
+    fun analyze(load: LoadDimension, res: Resource, executionIntervals: List<Pair<Instant, Instant>>): Boolean {
         var result = false
-
+        val exporter = CsvExporter()
+        val prometheusData = executionIntervals.map { interval -> fetcher.fetchMetric( start = interval.first, end = interval.second, query = "sum by(group)(kafka_consumergroup_group_lag >= 0)") }
+        var repetitionCounter = 0
+        prometheusData.forEach{ data -> exporter.toCsv(name = "${load.get()}_${res.get()}_${slo.sloType}_rep_${repetitionCounter++}", prom = data) }
         try {
-            val prometheusData = fetcher.fetchMetric(
-                start = Instant.now().minus(executionDuration),
-                end = Instant.now(),
-                query = "sum by(group)(kafka_consumergroup_group_lag >= 0)"
-            )
-
-            CsvExporter().toCsv(name = "${load.get()}_${res.get()}_${slo.sloType}", prom = prometheusData)
 
             val sloChecker = SloCheckerFactory().create(
                 slotype = slo.sloType,
@@ -35,10 +31,7 @@ class AnalysisExecutor(private val slo: BenchmarkExecution.Slo) {
                 warmup = slo.warmup
             )
 
-            result = sloChecker.evaluate(
-                start = Instant.now().minus(executionDuration),
-                end = Instant.now(), fetchedData = prometheusData
-            )
+            result = sloChecker.evaluate(prometheusData)
 
         } catch (e: Exception) {
             logger.error { "Evaluation failed for resource: ${res.get()} and load: ${load.get()} error: $e" }

theodolite-quarkus/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt

@@ -19,13 +19,15 @@ class ExternalSloChecker(
 
     private val logger = KotlinLogging.logger {}
 
-    override fun evaluate(start: Instant, end: Instant, fetchedData: PrometheusResponse): Boolean {
+    override fun evaluate(fetchedData: List<PrometheusResponse>): Boolean {
         var counter = 0
-        val data =
-            Gson().toJson(mapOf("total_lag" to fetchedData.data?.result, "threshold" to threshold, "warmup" to warmup))
+        var requestData = fetchedData.map { entry -> Gson().toJson(mapOf("total_lag" to entry.data?.result)) }.toMutableList()
+        requestData.add(mapOf("threshold" to threshold).toString())
+        requestData.add(mapOf("warmup" to warmup).toString())
+
 
         while (counter < RETRIES) {
-            val result = post(externalSlopeURL, data = data, timeout = TIMEOUT)
+            val result = post(externalSlopeURL, data = requestData, timeout = TIMEOUT)
             if (result.statusCode != 200) {
                 counter++
                 logger.error { "Could not reach external slope analysis" }

theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SloChecker.kt

@@ -4,5 +4,5 @@ import theodolite.util.PrometheusResponse
 import java.time.Instant
 
 interface SloChecker {
-    fun evaluate(start: Instant, end: Instant, fetchedData: PrometheusResponse): Boolean
+    fun evaluate(fetchedData: List<PrometheusResponse>): Boolean
 }

theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt

@@ -25,7 +25,8 @@ abstract class BenchmarkExecutor(
     val results: Results,
     val executionDuration: Duration,
     configurationOverrides: List<ConfigurationOverride?>,
-    val slo: BenchmarkExecution.Slo
+    val slo: BenchmarkExecution.Slo,
+    val repetitions: Int
 ) {
 
     var run: AtomicBoolean = AtomicBoolean(true)

theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt

@@ -5,11 +5,9 @@ import mu.KotlinLogging
 import theodolite.benchmark.Benchmark
 import theodolite.benchmark.BenchmarkExecution
 import theodolite.evaluation.AnalysisExecutor
-import theodolite.util.ConfigurationOverride
-import theodolite.util.LoadDimension
-import theodolite.util.Resource
-import theodolite.util.Results
+import theodolite.util.*
 import java.time.Duration
+import java.time.Instant
 
 private val logger = KotlinLogging.logger {}
 
@@ -19,27 +17,40 @@ class BenchmarkExecutorImpl(
     results: Results,
     executionDuration: Duration,
     private val configurationOverrides: List<ConfigurationOverride?>,
-    slo: BenchmarkExecution.Slo
-) : BenchmarkExecutor(benchmark, results, executionDuration, configurationOverrides, slo) {
+    slo: BenchmarkExecution.Slo,
+    repetitions: Int
+) : BenchmarkExecutor(benchmark, results, executionDuration, configurationOverrides, slo, repetitions) {
     override fun runExperiment(load: LoadDimension, res: Resource): Boolean {
         var result = false
-        val benchmarkDeployment = benchmark.buildDeployment(load, res, this.configurationOverrides)
+        val executionIntervals: MutableList<Pair<Instant, Instant>> = ArrayList(repetitions)
+        executionIntervals
+        for (i in 1.rangeTo(repetitions)) {
+            if (this.run.get()) {
+                executionIntervals.add(runSingleExperiment(load,res))
+            }
+        }
+
+        if (this.run.get()) {
+            result =
+                AnalysisExecutor(slo = slo).analyze(load = load, res = res, executionIntervals = executionIntervals)
+            this.results.setResult(Pair(load, res), result)
+        }
+        return result
+    }
 
+    private fun runSingleExperiment(load: LoadDimension, res: Resource): Pair<Instant, Instant> {
+        val benchmarkDeployment = benchmark.buildDeployment(load, res, this.configurationOverrides)
+        val from = Instant.now()
         try {
             benchmarkDeployment.setup()
             this.waitAndLog()
-        } catch(e: Exception) {
+        } catch (e: Exception) {
             logger.error { "Error while setup experiment." }
             logger.error { "Error is: $e" }
             this.run.set(false)
         }
-
-        if (this.run.get()) {
-            result =
-                AnalysisExecutor(slo = slo).analyze(load = load, res = res, executionDuration = executionDuration)
-            this.results.setResult(Pair(load, res), result)
-        }
+        val to = Instant.now()
         benchmarkDeployment.teardown()
-        return result
+        return Pair(from,to)
     }
 }

theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt

@@ -41,7 +41,8 @@ class TheodoliteExecutor(
                 results,
                 executionDuration,
                 config.configOverrides,
-                config.slos[0]
+                config.slos[0],
+                config.execution.repetitions
             )
 
         return Config(

theodolite-quarkus/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt

@@ -19,7 +19,8 @@ class TestBenchmarkExecutorImpl(
         results,
         executionDuration = Duration.ofSeconds(1),
         configurationOverrides = emptyList(),
-        slo = slo
+        slo = slo,
+        repetitions = 1
     ) {
 
     override fun runExperiment(load: LoadDimension, res: Resource): Boolean {