From 0d2bbce86507d7ad16d8e6692b682a645025c3db Mon Sep 17 00:00:00 2001
From: "stu126940@mail.uni-kiel.de" <stu126940@mail.uni-kiel.de>
Date: Tue, 13 Apr 2021 19:01:05 +0200
Subject: [PATCH] Add basic support for execution multiple repititions
not tested so far
---
.../theodolite/evaluation/AnalysisExecutor.kt | 19 +++------
.../evaluation/ExternalSloChecker.kt | 10 +++--
.../theodolite/evaluation/SloChecker.kt | 2 +-
.../theodolite/execution/BenchmarkExecutor.kt | 3 +-
.../execution/BenchmarkExecutorImpl.kt | 41 ++++++++++++-------
.../execution/TheodoliteExecutor.kt | 3 +-
.../theodolite/TestBenchmarkExecutorImpl.kt | 3 +-
7 files changed, 45 insertions(+), 36 deletions(-)
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt
index ce63f23a3..bb2dc7ed8 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt
@@ -16,17 +16,13 @@ class AnalysisExecutor(private val slo: BenchmarkExecution.Slo) {
offset = Duration.ofHours(slo.offset.toLong())
)
- fun analyze(load: LoadDimension, res: Resource, executionDuration: Duration): Boolean {
+ fun analyze(load: LoadDimension, res: Resource, executionIntervals: List<Pair<Instant, Instant>>): Boolean {
var result = false
-
+ val exporter = CsvExporter()
+ val prometheusData = executionIntervals.map { interval -> fetcher.fetchMetric( start = interval.first, end = interval.second, query = "sum by(group)(kafka_consumergroup_group_lag >= 0)") }
+ var repetitionCounter = 0
+ prometheusData.forEach{ data -> exporter.toCsv(name = "${load.get()}_${res.get()}_${slo.sloType}_rep_${repetitionCounter++}", prom = data) }
try {
- val prometheusData = fetcher.fetchMetric(
- start = Instant.now().minus(executionDuration),
- end = Instant.now(),
- query = "sum by(group)(kafka_consumergroup_group_lag >= 0)"
- )
-
- CsvExporter().toCsv(name = "${load.get()}_${res.get()}_${slo.sloType}", prom = prometheusData)
val sloChecker = SloCheckerFactory().create(
slotype = slo.sloType,
@@ -35,10 +31,7 @@ class AnalysisExecutor(private val slo: BenchmarkExecution.Slo) {
warmup = slo.warmup
)
- result = sloChecker.evaluate(
- start = Instant.now().minus(executionDuration),
- end = Instant.now(), fetchedData = prometheusData
- )
+ result = sloChecker.evaluate(prometheusData)
} catch (e: Exception) {
logger.error { "Evaluation failed for resource: ${res.get()} and load: ${load.get()} error: $e" }
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt
index e65116c0a..74a121e41 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt
@@ -19,13 +19,15 @@ class ExternalSloChecker(
private val logger = KotlinLogging.logger {}
- override fun evaluate(start: Instant, end: Instant, fetchedData: PrometheusResponse): Boolean {
+ override fun evaluate(fetchedData: List<PrometheusResponse>): Boolean {
var counter = 0
- val data =
- Gson().toJson(mapOf("total_lag" to fetchedData.data?.result, "threshold" to threshold, "warmup" to warmup))
+ var requestData = fetchedData.map { entry -> Gson().toJson(mapOf("total_lag" to entry.data?.result)) }.toMutableList()
+ requestData.add(mapOf("threshold" to threshold).toString())
+ requestData.add(mapOf("warmup" to warmup).toString())
+
while (counter < RETRIES) {
- val result = post(externalSlopeURL, data = data, timeout = TIMEOUT)
+ val result = post(externalSlopeURL, data = requestData, timeout = TIMEOUT)
if (result.statusCode != 200) {
counter++
logger.error { "Could not reach external slope analysis" }
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SloChecker.kt b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SloChecker.kt
index 66ea1d201..d3948533d 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SloChecker.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SloChecker.kt
@@ -4,5 +4,5 @@ import theodolite.util.PrometheusResponse
import java.time.Instant
interface SloChecker {
- fun evaluate(start: Instant, end: Instant, fetchedData: PrometheusResponse): Boolean
+ fun evaluate(fetchedData: List<PrometheusResponse>): Boolean
}
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt
index 9332af898..f76dc72df 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt
@@ -25,7 +25,8 @@ abstract class BenchmarkExecutor(
val results: Results,
val executionDuration: Duration,
configurationOverrides: List<ConfigurationOverride?>,
- val slo: BenchmarkExecution.Slo
+ val slo: BenchmarkExecution.Slo,
+ val repetitions: Int
) {
var run: AtomicBoolean = AtomicBoolean(true)
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt
index 027b93283..9750558fe 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt
@@ -5,11 +5,9 @@ import mu.KotlinLogging
import theodolite.benchmark.Benchmark
import theodolite.benchmark.BenchmarkExecution
import theodolite.evaluation.AnalysisExecutor
-import theodolite.util.ConfigurationOverride
-import theodolite.util.LoadDimension
-import theodolite.util.Resource
-import theodolite.util.Results
+import theodolite.util.*
import java.time.Duration
+import java.time.Instant
private val logger = KotlinLogging.logger {}
@@ -19,27 +17,40 @@ class BenchmarkExecutorImpl(
results: Results,
executionDuration: Duration,
private val configurationOverrides: List<ConfigurationOverride?>,
- slo: BenchmarkExecution.Slo
-) : BenchmarkExecutor(benchmark, results, executionDuration, configurationOverrides, slo) {
+ slo: BenchmarkExecution.Slo,
+ repetitions: Int
+) : BenchmarkExecutor(benchmark, results, executionDuration, configurationOverrides, slo, repetitions) {
override fun runExperiment(load: LoadDimension, res: Resource): Boolean {
var result = false
- val benchmarkDeployment = benchmark.buildDeployment(load, res, this.configurationOverrides)
+ val executionIntervals: MutableList<Pair<Instant, Instant>> = ArrayList(repetitions)
+ executionIntervals
+ for (i in 1.rangeTo(repetitions)) {
+ if (this.run.get()) {
+ executionIntervals.add(runSingleExperiment(load,res))
+ }
+ }
+
+ if (this.run.get()) {
+ result =
+ AnalysisExecutor(slo = slo).analyze(load = load, res = res, executionIntervals = executionIntervals)
+ this.results.setResult(Pair(load, res), result)
+ }
+ return result
+ }
+ private fun runSingleExperiment(load: LoadDimension, res: Resource): Pair<Instant, Instant> {
+ val benchmarkDeployment = benchmark.buildDeployment(load, res, this.configurationOverrides)
+ val from = Instant.now()
try {
benchmarkDeployment.setup()
this.waitAndLog()
- } catch(e: Exception) {
+ } catch (e: Exception) {
logger.error { "Error while setup experiment." }
logger.error { "Error is: $e" }
this.run.set(false)
}
-
- if (this.run.get()) {
- result =
- AnalysisExecutor(slo = slo).analyze(load = load, res = res, executionDuration = executionDuration)
- this.results.setResult(Pair(load, res), result)
- }
+ val to = Instant.now()
benchmarkDeployment.teardown()
- return result
+ return Pair(from,to)
}
}
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt
index 3d7718214..9e1e8e1d1 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt
@@ -41,7 +41,8 @@ class TheodoliteExecutor(
results,
executionDuration,
config.configOverrides,
- config.slos[0]
+ config.slos[0],
+ config.execution.repetitions
)
return Config(
diff --git a/theodolite-quarkus/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt b/theodolite-quarkus/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt
index 2de07d48d..8930c4bda 100644
--- a/theodolite-quarkus/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt
+++ b/theodolite-quarkus/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt
@@ -19,7 +19,8 @@ class TestBenchmarkExecutorImpl(
results,
executionDuration = Duration.ofSeconds(1),
configurationOverrides = emptyList(),
- slo = slo
+ slo = slo,
+ repetitions = 1
) {
override fun runExperiment(load: LoadDimension, res: Resource): Boolean {
--
GitLab