diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt
index ce63f23a31840b8e47cb15b71a026d35e438e953..bb2dc7ed81dbc0b4cb31963aba4f7ca2dd420ca3 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt
@@ -16,17 +16,13 @@ class AnalysisExecutor(private val slo: BenchmarkExecution.Slo) {
offset = Duration.ofHours(slo.offset.toLong())
)
- fun analyze(load: LoadDimension, res: Resource, executionDuration: Duration): Boolean {
+ fun analyze(load: LoadDimension, res: Resource, executionIntervals: List<Pair<Instant, Instant>>): Boolean {
var result = false
-
+ val exporter = CsvExporter()
+ val prometheusData = executionIntervals.map { interval -> fetcher.fetchMetric( start = interval.first, end = interval.second, query = "sum by(group)(kafka_consumergroup_group_lag >= 0)") }
+ var repetitionCounter = 0
+ prometheusData.forEach{ data -> exporter.toCsv(name = "${load.get()}_${res.get()}_${slo.sloType}_rep_${repetitionCounter++}", prom = data) }
try {
- val prometheusData = fetcher.fetchMetric(
- start = Instant.now().minus(executionDuration),
- end = Instant.now(),
- query = "sum by(group)(kafka_consumergroup_group_lag >= 0)"
- )
-
- CsvExporter().toCsv(name = "${load.get()}_${res.get()}_${slo.sloType}", prom = prometheusData)
val sloChecker = SloCheckerFactory().create(
slotype = slo.sloType,
@@ -35,10 +31,7 @@ class AnalysisExecutor(private val slo: BenchmarkExecution.Slo) {
warmup = slo.warmup
)
- result = sloChecker.evaluate(
- start = Instant.now().minus(executionDuration),
- end = Instant.now(), fetchedData = prometheusData
- )
+ result = sloChecker.evaluate(prometheusData)
} catch (e: Exception) {
logger.error { "Evaluation failed for resource: ${res.get()} and load: ${load.get()} error: $e" }
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt
index e65116c0a6b562c0e05714d09ab5a9b528249a05..74a121e41369dc952ef954c5c39878b9d738f2f6 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt
@@ -19,13 +19,15 @@ class ExternalSloChecker(
private val logger = KotlinLogging.logger {}
- override fun evaluate(start: Instant, end: Instant, fetchedData: PrometheusResponse): Boolean {
+ override fun evaluate(fetchedData: List<PrometheusResponse>): Boolean {
var counter = 0
- val data =
- Gson().toJson(mapOf("total_lag" to fetchedData.data?.result, "threshold" to threshold, "warmup" to warmup))
+ var requestData = fetchedData.map { entry -> Gson().toJson(mapOf("total_lag" to entry.data?.result)) }.toMutableList()
+ requestData.add(mapOf("threshold" to threshold).toString())
+ requestData.add(mapOf("warmup" to warmup).toString())
+
while (counter < RETRIES) {
- val result = post(externalSlopeURL, data = data, timeout = TIMEOUT)
+ val result = post(externalSlopeURL, data = requestData, timeout = TIMEOUT)
if (result.statusCode != 200) {
counter++
logger.error { "Could not reach external slope analysis" }
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SloChecker.kt b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SloChecker.kt
index 66ea1d201f7b48e09c3acb4365436caae637e6fa..d3948533d21ef195007e481de34416b578b52526 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SloChecker.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SloChecker.kt
@@ -4,5 +4,5 @@ import theodolite.util.PrometheusResponse
import java.time.Instant
interface SloChecker {
- fun evaluate(start: Instant, end: Instant, fetchedData: PrometheusResponse): Boolean
+ fun evaluate(fetchedData: List<PrometheusResponse>): Boolean
}
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt
index 9332af89810d24f87016fd02ce56d35a5ab07ca5..f76dc72df1d9dbfaff219dae85d9797ac45ac71e 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt
@@ -25,7 +25,8 @@ abstract class BenchmarkExecutor(
val results: Results,
val executionDuration: Duration,
configurationOverrides: List<ConfigurationOverride?>,
- val slo: BenchmarkExecution.Slo
+ val slo: BenchmarkExecution.Slo,
+ val repetitions: Int
) {
var run: AtomicBoolean = AtomicBoolean(true)
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt
index 027b9328336adcfc66299c94335a895d48a7181f..9750558fe739b8dbbeee0cd50181cd7346202b16 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt
@@ -5,11 +5,9 @@ import mu.KotlinLogging
import theodolite.benchmark.Benchmark
import theodolite.benchmark.BenchmarkExecution
import theodolite.evaluation.AnalysisExecutor
-import theodolite.util.ConfigurationOverride
-import theodolite.util.LoadDimension
-import theodolite.util.Resource
-import theodolite.util.Results
+import theodolite.util.*
import java.time.Duration
+import java.time.Instant
private val logger = KotlinLogging.logger {}
@@ -19,27 +17,40 @@ class BenchmarkExecutorImpl(
results: Results,
executionDuration: Duration,
private val configurationOverrides: List<ConfigurationOverride?>,
- slo: BenchmarkExecution.Slo
-) : BenchmarkExecutor(benchmark, results, executionDuration, configurationOverrides, slo) {
+ slo: BenchmarkExecution.Slo,
+ repetitions: Int
+) : BenchmarkExecutor(benchmark, results, executionDuration, configurationOverrides, slo, repetitions) {
override fun runExperiment(load: LoadDimension, res: Resource): Boolean {
var result = false
- val benchmarkDeployment = benchmark.buildDeployment(load, res, this.configurationOverrides)
+ val executionIntervals: MutableList<Pair<Instant, Instant>> = ArrayList(repetitions)
+ executionIntervals
+ for (i in 1.rangeTo(repetitions)) {
+ if (this.run.get()) {
+ executionIntervals.add(runSingleExperiment(load,res))
+ }
+ }
+
+ if (this.run.get()) {
+ result =
+ AnalysisExecutor(slo = slo).analyze(load = load, res = res, executionIntervals = executionIntervals)
+ this.results.setResult(Pair(load, res), result)
+ }
+ return result
+ }
+ private fun runSingleExperiment(load: LoadDimension, res: Resource): Pair<Instant, Instant> {
+ val benchmarkDeployment = benchmark.buildDeployment(load, res, this.configurationOverrides)
+ val from = Instant.now()
try {
benchmarkDeployment.setup()
this.waitAndLog()
- } catch(e: Exception) {
+ } catch (e: Exception) {
logger.error { "Error while setup experiment." }
logger.error { "Error is: $e" }
this.run.set(false)
}
-
- if (this.run.get()) {
- result =
- AnalysisExecutor(slo = slo).analyze(load = load, res = res, executionDuration = executionDuration)
- this.results.setResult(Pair(load, res), result)
- }
+ val to = Instant.now()
benchmarkDeployment.teardown()
- return result
+ return Pair(from,to)
}
}
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt
index 3d7718214e15488ad8c0d7c1cea7441474653571..9e1e8e1d1e217bd36b7ad6b5a07bf4d4084baac0 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt
@@ -41,7 +41,8 @@ class TheodoliteExecutor(
results,
executionDuration,
config.configOverrides,
- config.slos[0]
+ config.slos[0],
+ config.execution.repetitions
)
return Config(
diff --git a/theodolite-quarkus/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt b/theodolite-quarkus/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt
index 2de07d48dbe59313506745e6c97b2ae9a5ed114e..8930c4bda2adf0c2ac350ef9d406f97bfd7deb31 100644
--- a/theodolite-quarkus/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt
+++ b/theodolite-quarkus/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt
@@ -19,7 +19,8 @@ class TestBenchmarkExecutorImpl(
results,
executionDuration = Duration.ofSeconds(1),
configurationOverrides = emptyList(),
- slo = slo
+ slo = slo,
+ repetitions = 1
) {
override fun runExperiment(load: LoadDimension, res: Resource): Boolean {