diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/Benchmark.kt b/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/Benchmark.kt
index 8a52987c497e08eb7337934ee50c7b2753ca1fa2..df92ed111206bb58f50a589a612d6f598575244b 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/Benchmark.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/Benchmark.kt
@@ -10,6 +10,6 @@ interface Benchmark {
fun buildDeployment(
load: LoadDimension,
res: Resource,
- configurationOverrides: List<ConfigurationOverride>
+ configurationOverrides: List<ConfigurationOverride?>
): BenchmarkDeployment
}
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt b/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt
index 14bb9e49a13a70fa38cfbc3669d54cc35c8a0a2b..d90cb18ccba1967572677a5ae23be9d6c426d943 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt
@@ -12,7 +12,7 @@ class BenchmarkExecution {
lateinit var resources: ResourceDefinition
lateinit var slos: List<Slo>
lateinit var execution: Execution
- lateinit var configOverrides: List<ConfigurationOverride>
+ lateinit var configOverrides: List<ConfigurationOverride?>
@RegisterForReflection
class Execution {
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt b/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt
index 3975dd110eac7e7616d9228dd35b298d43c1e1ad..dea845ec4c9209c603641a57112acf52815430a7 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt
@@ -43,7 +43,7 @@ class KubernetesBenchmark : Benchmark {
override fun buildDeployment(
load: LoadDimension,
res: Resource,
- configurationOverrides: List<ConfigurationOverride>
+ configurationOverrides: List<ConfigurationOverride?>
): BenchmarkDeployment {
val resources = loadKubernetesResources(this.appResource + this.loadGenResource)
val patcherFactory = PatcherFactory()
@@ -53,7 +53,7 @@ class KubernetesBenchmark : Benchmark {
res.getType().forEach{ patcherDefinition -> patcherFactory.createPatcher(patcherDefinition, resources).patch(res.get().toString()) }
// Patch the given overrides
- configurationOverrides.forEach { override -> patcherFactory.createPatcher(override.patcher, resources).patch(override.value) }
+ configurationOverrides.forEach { override -> override?.let { patcherFactory.createPatcher(it.patcher, resources).patch(override.value) } }
return KubernetesBenchmarkDeployment(
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt
new file mode 100644
index 0000000000000000000000000000000000000000..2910d84991c2c37051b4b053c0c024344c0b3ff0
--- /dev/null
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt
@@ -0,0 +1,48 @@
+package theodolite.evaluation
+
+import mu.KotlinLogging
+import theodolite.benchmark.BenchmarkExecution
+import theodolite.util.LoadDimension
+import theodolite.util.Resource
+import java.time.Duration
+import java.time.Instant
+
+private val logger = KotlinLogging.logger {}
+
+class AnalysisExecutor(private val slo: BenchmarkExecution.Slo) {
+
+ private val fetcher = MetricFetcher(
+ prometheusURL = slo.prometheusUrl,
+ offset = Duration.ofHours(slo.offset.toLong())
+ )
+
+ fun analyse(load: LoadDimension, res: Resource, executionDuration: Duration): Boolean {
+ var result = false
+
+ try {
+ val prometheusData = fetcher.fetchMetric(
+ start = Instant.now().minus(executionDuration),
+ end = Instant.now(),
+ query = "sum by(group)(kafka_consumergroup_group_lag >= 0)"
+ )
+
+ CsvExporter().toCsv(name = "${load.get()}_${res.get()}_${slo.sloType}", prom = prometheusData)
+
+ val sloChecker = SloCheckerFactory().create(
+ slotype = slo.sloType,
+ externalSlopeURL = slo.externalSloUrl,
+ threshold = slo.threshold,
+ warmup = slo.warmup
+ )
+
+ result = sloChecker.evaluate(
+ start = Instant.now().minus(executionDuration),
+ end = Instant.now(), fetchedData = prometheusData
+ )
+
+ } catch (e: Exception) {
+ logger.error { "Evaluation failed for resource: ${res.get()} and load: ${load.get()} error: $e" }
+ }
+ return result
+ }
+}
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/CsvExporter.kt b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/CsvExporter.kt
index 929a7914f5cf8501212f020bdc54cc19c43bcd7c..e2f536af6dba838b2b4027d6cfafb032ebd3d04d 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/CsvExporter.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/CsvExporter.kt
@@ -1,43 +1,45 @@
package theodolite.evaluation
+import mu.KotlinLogging
import theodolite.util.PrometheusResponse
import java.io.File
import java.io.PrintWriter
+import java.util.*
+
+private val logger = KotlinLogging.logger {}
class CsvExporter {
/**
* Uses the PrintWriter to transform a PrometheusResponse to Csv
*/
- fun toCsv(name : String,prom: PrometheusResponse){
- val x = toArray(prom)
- val csvOutputFile: File = File(name+".csv")
+ fun toCsv(name: String, prom: PrometheusResponse) {
+ val responseArray = promResponseToList(prom)
+ val csvOutputFile = File("$name.csv")
PrintWriter(csvOutputFile).use { pw ->
- pw.println(listOf("name","time","value").joinToString())
- x.forEach{
+ pw.println(listOf("name", "time", "value").joinToString())
+ responseArray.forEach {
pw.println(it.joinToString())
}
}
+ logger.info { "Wrote csv file: $name to ${csvOutputFile.absolutePath}" }
}
/**
* Converts a PrometheusResponse into a List of List of Strings
*/
- private fun toArray(prom : PrometheusResponse): MutableList<List<String>> {
-
+ private fun promResponseToList(prom: PrometheusResponse): List<List<String>> {
val name = prom.data?.result?.get(0)?.metric?.group.toString()
val values = prom.data?.result?.get(0)?.values
val dataList = mutableListOf<List<String>>()
if (values != null) {
- for (x in values){
+ for (x in values) {
val y = x as List<*>
-
- dataList.add(listOf(name,"${y[0]}","${y[1]}"))
+ dataList.add(listOf(name, "${y[0]}", "${y[1]}"))
}
}
-
- return dataList
+ return Collections.unmodifiableList(dataList)
}
}
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt
index 2de8e2dc9c03ec5449c9f04585622d6730644aa2..e65116c0a6b562c0e05714d09ab5a9b528249a05 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt
@@ -2,16 +2,14 @@ package theodolite.evaluation
import com.google.gson.Gson
import khttp.post
+import mu.KotlinLogging
+import theodolite.util.PrometheusResponse
import java.net.ConnectException
-import java.time.Duration
import java.time.Instant
class ExternalSloChecker(
- private val prometheusURL: String,
- private val query: String,
private val externalSlopeURL: String,
private val threshold: Int,
- private val offset: Duration,
private val warmup: Int
) :
SloChecker {
@@ -19,10 +17,10 @@ class ExternalSloChecker(
private val RETRIES = 2
private val TIMEOUT = 60.0
- override fun evaluate(start: Instant, end: Instant): Boolean {
+ private val logger = KotlinLogging.logger {}
+
+ override fun evaluate(start: Instant, end: Instant, fetchedData: PrometheusResponse): Boolean {
var counter = 0
- val metricFetcher = MetricFetcher(prometheusURL = prometheusURL, offset = offset)
- val fetchedData = metricFetcher.fetchMetric(start, end, query)
val data =
Gson().toJson(mapOf("total_lag" to fetchedData.data?.result, "threshold" to threshold, "warmup" to warmup))
@@ -30,6 +28,7 @@ class ExternalSloChecker(
val result = post(externalSlopeURL, data = data, timeout = TIMEOUT)
if (result.statusCode != 200) {
counter++
+ logger.error { "Could not reach external slope analysis" }
} else {
return result.text.toBoolean()
}
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/MetricFetcher.kt b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/MetricFetcher.kt
index 7dbaf568c3452e7ae565002ae00e5314502f8930..19a8bbe9ba0bdd8a694eb37b9db42de6fdf3d620 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/MetricFetcher.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/MetricFetcher.kt
@@ -37,7 +37,7 @@ class MetricFetcher(private val prometheusURL: String, private val offset: Durat
} else {
val values = parseValues(response)
if (values.data?.result.isNullOrEmpty()) {
- logger.error { "Empty query result: $values" }
+ logger.error { "Empty query result: $values between $start and $end for querry $query" }
throw NoSuchFieldException()
}
return parseValues(response)
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SloChecker.kt b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SloChecker.kt
index 53ed1b7fa02681f97b121f93d690c0654f961a94..66ea1d201f7b48e09c3acb4365436caae637e6fa 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SloChecker.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SloChecker.kt
@@ -1,7 +1,8 @@
package theodolite.evaluation
+import theodolite.util.PrometheusResponse
import java.time.Instant
interface SloChecker {
- fun evaluate(start: Instant, end: Instant): Boolean
+ fun evaluate(start: Instant, end: Instant, fetchedData: PrometheusResponse): Boolean
}
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SloCheckerFactory.kt b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SloCheckerFactory.kt
index 2170ef7b6abdb74499d05ac623c7892ac36b72d9..50b7b0aec3c5d48146d4f9423b06fe62f55e3c56 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SloCheckerFactory.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SloCheckerFactory.kt
@@ -6,21 +6,15 @@ class SloCheckerFactory {
fun create(
slotype: String,
- prometheusURL: String,
- query: String,
externalSlopeURL: String,
threshold: Int,
- offset: Duration,
warmup: Int
): SloChecker {
return when (slotype) {
"lag trend" -> ExternalSloChecker(
- prometheusURL = prometheusURL,
- query = query,
externalSlopeURL = externalSlopeURL,
threshold = threshold,
- offset = offset,
warmup = warmup
)
else -> throw IllegalArgumentException("Slotype $slotype not found.")
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt
index d3c2fdcbc0274066e62dd2dfe01fd2a8cf940f13..f18fc1cbbe989b41b8786630f6ee2dd8ffe174d3 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt
@@ -23,7 +23,7 @@ abstract class BenchmarkExecutor(
val benchmark: Benchmark,
val results: Results,
val executionDuration: Duration,
- configurationOverrides: List<ConfigurationOverride>,
+ configurationOverrides: List<ConfigurationOverride?>,
val slo: BenchmarkExecution.Slo
) {
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt
index 72b479c97ba93d46b8ed00e481af978241c3b37f..74c4ca3971daa2ac535a8f40c7448651cd3d6f7d 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt
@@ -4,13 +4,12 @@ import io.quarkus.runtime.annotations.RegisterForReflection
import mu.KotlinLogging
import theodolite.benchmark.Benchmark
import theodolite.benchmark.BenchmarkExecution
-import theodolite.evaluation.SloCheckerFactory
+import theodolite.evaluation.AnalysisExecutor
import theodolite.util.ConfigurationOverride
import theodolite.util.LoadDimension
import theodolite.util.Resource
import theodolite.util.Results
import java.time.Duration
-import java.time.Instant
private val logger = KotlinLogging.logger {}
@@ -19,33 +18,17 @@ class BenchmarkExecutorImpl(
benchmark: Benchmark,
results: Results,
executionDuration: Duration,
- private val configurationOverrides: List<ConfigurationOverride>,
+ private val configurationOverrides: List<ConfigurationOverride?>,
slo: BenchmarkExecution.Slo
) : BenchmarkExecutor(benchmark, results, executionDuration, configurationOverrides, slo) {
- //TODO ADD SHUTDOWN HOOK HERE
override fun runExperiment(load: LoadDimension, res: Resource): Boolean {
val benchmarkDeployment = benchmark.buildDeployment(load, res, this.configurationOverrides)
benchmarkDeployment.setup()
this.waitAndLog()
- var result = false
- try {
- result = SloCheckerFactory().create(
- slotype = slo.sloType,
- prometheusURL = slo.prometheusUrl,
- query = "sum by(group)(kafka_consumergroup_group_lag >= 0)",
- externalSlopeURL = slo.externalSloUrl,
- threshold = slo.threshold,
- offset = Duration.ofHours(slo.offset.toLong()),
- warmup = slo.warmup
- )
- .evaluate(
- Instant.now().minus(executionDuration),
- Instant.now()
- )
- } catch (e: Exception) {
- logger.error { "Evaluation failed for resource: ${res.get()} and load: ${load.get()} error: $e" }
- }
+ val result = AnalysisExecutor(slo = slo).analyse(load = load, res = res, executionDuration = executionDuration)
+
+ benchmarkDeployment.teardown()
benchmarkDeployment.teardown()
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/Shutdown.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/Shutdown.kt
new file mode 100644
index 0000000000000000000000000000000000000000..9ca7e70f6ecd5ed30120940be8de965c73f79a6a
--- /dev/null
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/Shutdown.kt
@@ -0,0 +1,27 @@
+package theodolite.execution
+
+import mu.KotlinLogging
+import theodolite.benchmark.BenchmarkExecution
+import theodolite.benchmark.KubernetesBenchmark
+import theodolite.util.LoadDimension
+import theodolite.util.Resource
+
+private val logger = KotlinLogging.logger {}
+
+class Shutdown(private val benchmarkExecution: BenchmarkExecution, private val benchmark: KubernetesBenchmark) :
+ Thread() {
+
+ override fun run() {
+ // Build Configuration to teardown
+ logger.info { "Received shutdown signal -> Shutting down" }
+ val deployment =
+ benchmark.buildDeployment(
+ load = LoadDimension(0, emptyList()),
+ res = Resource(0, emptyList()),
+ configurationOverrides = benchmarkExecution.configOverrides
+ )
+ logger.info { "Teardown the everything deployed" }
+ deployment.teardown()
+ logger.info { "Teardown completed" }
+ }
+}
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt
index 89a3fb4fe5e0f81aa12aa566c9dbb2630c9b9bfe..689c07fc6419f8d8a63c2c0fe5f2d5961e15e374 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt
@@ -21,8 +21,13 @@ class TheodoliteExecutor(
val strategyFactory = StrategyFactory()
val executionDuration = Duration.ofSeconds(config.execution.duration)
- val resourcePatcherDefinition = PatcherDefinitionFactory().createPatcherDefinition(config.resources.resourceType, this.kubernetesBenchmark.resourceTypes)
- val loadDimensionPatcherDefinition = PatcherDefinitionFactory().createPatcherDefinition(config.load.loadType, this.kubernetesBenchmark.loadTypes)
+
+ val resourcePatcherDefinition = PatcherDefinitionFactory().createPatcherDefinition(
+ config.resources.resourceType,
+ this.kubernetesBenchmark.resourceTypes
+ )
+ val loadDimensionPatcherDefinition =
+ PatcherDefinitionFactory().createPatcherDefinition(config.load.loadType, this.kubernetesBenchmark.loadTypes)
val executor =
BenchmarkExecutorImpl(
@@ -35,7 +40,12 @@ class TheodoliteExecutor(
return Config(
loads = config.load.loadValues.map { load -> LoadDimension(load, loadDimensionPatcherDefinition) },
- resources = config.resources.resourceValues.map { resource -> Resource(resource, resourcePatcherDefinition) },
+ resources = config.resources.resourceValues.map { resource ->
+ Resource(
+ resource,
+ resourcePatcherDefinition
+ )
+ },
compositeStrategy = CompositeStrategy(
benchmarkExecutor = executor,
searchStrategy = strategyFactory.createSearchStrategy(executor, config.execution.strategy),
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteYamlExecutor.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteYamlExecutor.kt
index 976b711bd75287adc6ef5718315628c779768a22..4b983d64d884e4359a5da7d389a5b3c745f04a7a 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteYamlExecutor.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteYamlExecutor.kt
@@ -32,9 +32,13 @@ object TheodoliteYamlExecutor {
parser.parse(path = benchmarkPath, E = KubernetesBenchmark::class.java)!!
benchmark.path = appResource
+ val shutdown = Shutdown(benchmarkExecution, benchmark)
+ Runtime.getRuntime().addShutdownHook(shutdown)
+
val executor = TheodoliteExecutor(benchmarkExecution, benchmark)
executor.run()
logger.info { "Theodolite finished" }
+ Runtime.getRuntime().removeShutdownHook(shutdown)
exitProcess(0)
}
}
diff --git a/theodolite-quarkus/src/test/kotlin/theodolite/TestBenchmark.kt b/theodolite-quarkus/src/test/kotlin/theodolite/TestBenchmark.kt
index 487b583188e8e5741900615108e2b2fa913353df..6f476278d08eacfc9857c1e5431636e5a219f26c 100644
--- a/theodolite-quarkus/src/test/kotlin/theodolite/TestBenchmark.kt
+++ b/theodolite-quarkus/src/test/kotlin/theodolite/TestBenchmark.kt
@@ -11,7 +11,7 @@ class TestBenchmark : Benchmark {
override fun buildDeployment(
load: LoadDimension,
res: Resource,
- configurationOverrides: List<ConfigurationOverride>
+ configurationOverrides: List<ConfigurationOverride?>
): BenchmarkDeployment {
return TestBenchmarkDeployment()
}