diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/Benchmark.kt b/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/Benchmark.kt
index 8c15fa1dc60d2d72a5b30508ea4570b8449f817b..fb343a0fa594c3f0d46125ca0debc7d0a6c223d8 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/Benchmark.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/Benchmark.kt
@@ -8,6 +8,6 @@ interface Benchmark {
fun buildDeployment(
load: LoadDimension,
res: Resource,
- configurationOverrides: List<ConfigurationOverride>
+ configurationOverrides: List<ConfigurationOverride?>
): BenchmarkDeployment
}
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt b/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt
index 21d9d53cf0b268d72eb9ec0c6135e844bc7c7ca3..417d5cea2a7df667bea3a7066ceab98341a0f37c 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt
@@ -14,7 +14,7 @@ class BenchmarkExecution : CustomResource(){
lateinit var resources: ResourceDefinition
lateinit var slos: List<Slo>
lateinit var execution: Execution
- lateinit var configOverrides: List<ConfigurationOverride>
+ lateinit var configOverrides: List<ConfigurationOverride?>
@JsonDeserialize
class Execution : KubernetesResource {
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt b/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt
index 754cd086612d771247ef8c72a261f0b3d19d53de..61224e61f43fb385442c18e4e7824f1559bb857f 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt
@@ -42,7 +42,7 @@ class KubernetesBenchmark : Benchmark , CustomResource() {
override fun buildDeployment(
load: LoadDimension,
res: Resource,
- configurationOverrides: List<ConfigurationOverride>
+ configurationOverrides: List<ConfigurationOverride?>
): BenchmarkDeployment {
val resources = loadKubernetesResources(this.appResource + this.loadGenResource)
val patcherFactory = PatcherFactory()
@@ -52,7 +52,7 @@ class KubernetesBenchmark : Benchmark , CustomResource() {
res.getType().forEach{ patcherDefinition -> patcherFactory.createPatcher(patcherDefinition, resources).patch(res.get().toString()) }
// Patch the given overrides
- configurationOverrides.forEach { override -> patcherFactory.createPatcher(override.patcher, resources).patch(override.value) }
+ configurationOverrides.forEach { override -> override?.let { patcherFactory.createPatcher(it.patcher, resources).patch(override.value) } }
return KubernetesBenchmarkDeployment(
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt
new file mode 100644
index 0000000000000000000000000000000000000000..2910d84991c2c37051b4b053c0c024344c0b3ff0
--- /dev/null
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt
@@ -0,0 +1,48 @@
+package theodolite.evaluation
+
+import mu.KotlinLogging
+import theodolite.benchmark.BenchmarkExecution
+import theodolite.util.LoadDimension
+import theodolite.util.Resource
+import java.time.Duration
+import java.time.Instant
+
+private val logger = KotlinLogging.logger {}
+
+class AnalysisExecutor(private val slo: BenchmarkExecution.Slo) {
+
+ private val fetcher = MetricFetcher(
+ prometheusURL = slo.prometheusUrl,
+ offset = Duration.ofHours(slo.offset.toLong())
+ )
+
+ fun analyse(load: LoadDimension, res: Resource, executionDuration: Duration): Boolean {
+ var result = false
+
+ try {
+ val prometheusData = fetcher.fetchMetric(
+ start = Instant.now().minus(executionDuration),
+ end = Instant.now(),
+ query = "sum by(group)(kafka_consumergroup_group_lag >= 0)"
+ )
+
+ CsvExporter().toCsv(name = "${load.get()}_${res.get()}_${slo.sloType}", prom = prometheusData)
+
+ val sloChecker = SloCheckerFactory().create(
+ slotype = slo.sloType,
+ externalSlopeURL = slo.externalSloUrl,
+ threshold = slo.threshold,
+ warmup = slo.warmup
+ )
+
+ result = sloChecker.evaluate(
+ start = Instant.now().minus(executionDuration),
+ end = Instant.now(), fetchedData = prometheusData
+ )
+
+ } catch (e: Exception) {
+ logger.error { "Evaluation failed for resource: ${res.get()} and load: ${load.get()} error: $e" }
+ }
+ return result
+ }
+}
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/CsvExporter.kt b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/CsvExporter.kt
new file mode 100644
index 0000000000000000000000000000000000000000..e2f536af6dba838b2b4027d6cfafb032ebd3d04d
--- /dev/null
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/CsvExporter.kt
@@ -0,0 +1,45 @@
+package theodolite.evaluation
+
+import mu.KotlinLogging
+import theodolite.util.PrometheusResponse
+import java.io.File
+import java.io.PrintWriter
+import java.util.*
+
+private val logger = KotlinLogging.logger {}
+
+class CsvExporter {
+
+ /**
+ * Uses the PrintWriter to transform a PrometheusResponse to Csv
+ */
+ fun toCsv(name: String, prom: PrometheusResponse) {
+ val responseArray = promResponseToList(prom)
+ val csvOutputFile = File("$name.csv")
+
+ PrintWriter(csvOutputFile).use { pw ->
+ pw.println(listOf("name", "time", "value").joinToString())
+ responseArray.forEach {
+ pw.println(it.joinToString())
+ }
+ }
+ logger.info { "Wrote csv file: $name to ${csvOutputFile.absolutePath}" }
+ }
+
+ /**
+ * Converts a PrometheusResponse into a List of List of Strings
+ */
+ private fun promResponseToList(prom: PrometheusResponse): List<List<String>> {
+ val name = prom.data?.result?.get(0)?.metric?.group.toString()
+ val values = prom.data?.result?.get(0)?.values
+ val dataList = mutableListOf<List<String>>()
+
+ if (values != null) {
+ for (x in values) {
+ val y = x as List<*>
+ dataList.add(listOf(name, "${y[0]}", "${y[1]}"))
+ }
+ }
+ return Collections.unmodifiableList(dataList)
+ }
+}
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt
index 2de8e2dc9c03ec5449c9f04585622d6730644aa2..e65116c0a6b562c0e05714d09ab5a9b528249a05 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt
@@ -2,16 +2,14 @@ package theodolite.evaluation
import com.google.gson.Gson
import khttp.post
+import mu.KotlinLogging
+import theodolite.util.PrometheusResponse
import java.net.ConnectException
-import java.time.Duration
import java.time.Instant
class ExternalSloChecker(
- private val prometheusURL: String,
- private val query: String,
private val externalSlopeURL: String,
private val threshold: Int,
- private val offset: Duration,
private val warmup: Int
) :
SloChecker {
@@ -19,10 +17,10 @@ class ExternalSloChecker(
private val RETRIES = 2
private val TIMEOUT = 60.0
- override fun evaluate(start: Instant, end: Instant): Boolean {
+ private val logger = KotlinLogging.logger {}
+
+ override fun evaluate(start: Instant, end: Instant, fetchedData: PrometheusResponse): Boolean {
var counter = 0
- val metricFetcher = MetricFetcher(prometheusURL = prometheusURL, offset = offset)
- val fetchedData = metricFetcher.fetchMetric(start, end, query)
val data =
Gson().toJson(mapOf("total_lag" to fetchedData.data?.result, "threshold" to threshold, "warmup" to warmup))
@@ -30,6 +28,7 @@ class ExternalSloChecker(
val result = post(externalSlopeURL, data = data, timeout = TIMEOUT)
if (result.statusCode != 200) {
counter++
+ logger.error { "Could not reach external slope analysis" }
} else {
return result.text.toBoolean()
}
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/MetricFetcher.kt b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/MetricFetcher.kt
index 7dbaf568c3452e7ae565002ae00e5314502f8930..19a8bbe9ba0bdd8a694eb37b9db42de6fdf3d620 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/MetricFetcher.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/MetricFetcher.kt
@@ -37,7 +37,7 @@ class MetricFetcher(private val prometheusURL: String, private val offset: Durat
} else {
val values = parseValues(response)
if (values.data?.result.isNullOrEmpty()) {
- logger.error { "Empty query result: $values" }
+ logger.error { "Empty query result: $values between $start and $end for querry $query" }
throw NoSuchFieldException()
}
return parseValues(response)
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SloChecker.kt b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SloChecker.kt
index 53ed1b7fa02681f97b121f93d690c0654f961a94..66ea1d201f7b48e09c3acb4365436caae637e6fa 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SloChecker.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SloChecker.kt
@@ -1,7 +1,8 @@
package theodolite.evaluation
+import theodolite.util.PrometheusResponse
import java.time.Instant
interface SloChecker {
- fun evaluate(start: Instant, end: Instant): Boolean
+ fun evaluate(start: Instant, end: Instant, fetchedData: PrometheusResponse): Boolean
}
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SloCheckerFactory.kt b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SloCheckerFactory.kt
index 2170ef7b6abdb74499d05ac623c7892ac36b72d9..50b7b0aec3c5d48146d4f9423b06fe62f55e3c56 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SloCheckerFactory.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SloCheckerFactory.kt
@@ -6,21 +6,15 @@ class SloCheckerFactory {
fun create(
slotype: String,
- prometheusURL: String,
- query: String,
externalSlopeURL: String,
threshold: Int,
- offset: Duration,
warmup: Int
): SloChecker {
return when (slotype) {
"lag trend" -> ExternalSloChecker(
- prometheusURL = prometheusURL,
- query = query,
externalSlopeURL = externalSlopeURL,
threshold = threshold,
- offset = offset,
warmup = warmup
)
else -> throw IllegalArgumentException("Slotype $slotype not found.")
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt
index d3c2fdcbc0274066e62dd2dfe01fd2a8cf940f13..f18fc1cbbe989b41b8786630f6ee2dd8ffe174d3 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt
@@ -23,7 +23,7 @@ abstract class BenchmarkExecutor(
val benchmark: Benchmark,
val results: Results,
val executionDuration: Duration,
- configurationOverrides: List<ConfigurationOverride>,
+ configurationOverrides: List<ConfigurationOverride?>,
val slo: BenchmarkExecution.Slo
) {
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt
index ddfaf8e9b4a97a8185cc3f086bbc773776d4e38b..e8b7a1a26790aeaf8caf7903a5d98479d7d3e21e 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt
@@ -3,13 +3,12 @@ package theodolite.execution
import mu.KotlinLogging
import theodolite.benchmark.Benchmark
import theodolite.benchmark.BenchmarkExecution
-import theodolite.evaluation.SloCheckerFactory
+import theodolite.evaluation.AnalysisExecutor
import theodolite.util.ConfigurationOverride
import theodolite.util.LoadDimension
import theodolite.util.Resource
import theodolite.util.Results
import java.time.Duration
-import java.time.Instant
private val logger = KotlinLogging.logger {}
@@ -17,33 +16,17 @@ class BenchmarkExecutorImpl(
benchmark: Benchmark,
results: Results,
executionDuration: Duration,
- private val configurationOverrides: List<ConfigurationOverride>,
+ private val configurationOverrides: List<ConfigurationOverride?>,
slo: BenchmarkExecution.Slo
) : BenchmarkExecutor(benchmark, results, executionDuration, configurationOverrides, slo) {
override fun runExperiment(load: LoadDimension, res: Resource): Boolean {
val benchmarkDeployment = benchmark.buildDeployment(load, res, this.configurationOverrides)
benchmarkDeployment.setup()
this.waitAndLog()
- benchmarkDeployment.teardown()
- var result = false
- try {
- result = SloCheckerFactory().create(
- slotype = slo.sloType,
- prometheusURL = slo.prometheusUrl,
- query = "sum by(group)(kafka_consumergroup_group_lag >= 0)",
- externalSlopeURL = slo.externalSloUrl,
- threshold = slo.threshold,
- offset = Duration.ofHours(slo.offset.toLong()),
- warmup = slo.warmup
- )
- .evaluate(
- Instant.now().minus(executionDuration),
- Instant.now()
- )
- } catch (e: Exception) {
- logger.error { "Evaluation failed for resource: ${res.get()} and load: ${load.get()} error: $e" }
- }
+ val result = AnalysisExecutor(slo = slo).analyse(load = load, res = res, executionDuration = executionDuration)
+
+ benchmarkDeployment.teardown()
this.results.setResult(Pair(load, res), result)
return result
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteYamlExecutor.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteYamlExecutor.kt
index d5f73338718af26c49ff86d63c16a4fa5a903646..490d37c3297908bc657fcfb96f98fe8a036a6ea7 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteYamlExecutor.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteYamlExecutor.kt
@@ -22,11 +22,13 @@ object TheodoliteYamlExecutor {
val benchmark =
parser.parse("./../../../resources/main/yaml/BenchmarkType.yaml", KubernetesBenchmark::class.java)!!
- Runtime.getRuntime().addShutdownHook(Shutdown(benchmarkExecution, benchmark))
+ val shutdown = Shutdown(benchmarkExecution, benchmark)
+ Runtime.getRuntime().addShutdownHook(shutdown)
val executor = TheodoliteExecutor(benchmarkExecution, benchmark)
executor.run()
logger.info { "Theodolite finished" }
+ Runtime.getRuntime().removeShutdownHook(shutdown)
exitProcess(0)
}
}
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/k8s/K8sCustomResourceWrapper.kt b/theodolite-quarkus/src/main/kotlin/theodolite/k8s/K8sCustomResourceWrapper.kt
index a20018dd4ec050a6a20adde3f0e4d5b595a61585..8b3ac08265d5caa75f602b18e279e36efd24e187 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/k8s/K8sCustomResourceWrapper.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/k8s/K8sCustomResourceWrapper.kt
@@ -4,11 +4,26 @@ import io.fabric8.kubernetes.client.CustomResource
import io.fabric8.kubernetes.client.NamespacedKubernetesClient
import io.fabric8.kubernetes.client.dsl.base.CustomResourceDefinitionContext
+/**
+* Fabric8 handles custom resources as plain HashMaps. These need to be handled differently than normal
+* Kubernetes resources. The K8sCustomResourceWrapper class provides a wrapper to deploy and delete
+* custom resources in a uniform way.
+*
+* @property map custom resource as plain hashmap
+* @constructor Create empty K8s custom resource wrapper
+*/
class K8sCustomResourceWrapper(private val map : Map<String,String>) : CustomResource() {
+ /**
+ * Deploy a custom resource
+ *
+ * @param client a namespaced Kubernetes client which are used to deploy the CR object.
+ */
fun deploy(client : NamespacedKubernetesClient){
val kind = this.map["kind"]
+ // Search the CustomResourceDefinition to which the CR Object belongs.
+ // This should be exactly one if the CRD is registered for Kubernetes, zero otherwise.
val crds = client.apiextensions().v1beta1().customResourceDefinitions().list()
crds.items
.filter { crd -> crd.toString().contains("kind=$kind") }
@@ -18,6 +33,11 @@ class K8sCustomResourceWrapper(private val map : Map<String,String>) : CustomRes
}
}
+ /**
+ * Delete a custom resource
+ *
+ * @param client a namespaced Kubernetes client which are used to delete the CR object.
+ */
fun delete(client : NamespacedKubernetesClient){
val kind = this.map["kind"]
val metadata = this.map["metadata"] as HashMap<String,String>
diff --git a/theodolite-quarkus/src/main/resources/yaml/BenchmarkExecution.yaml b/theodolite-quarkus/src/main/resources/yaml/BenchmarkExecution.yaml
index d058ac3da5081de633525faf7bc3f1b94f790de5..99c4ea236b7fe7a0208feda3327d4acd9ce4d77e 100644
--- a/theodolite-quarkus/src/main/resources/yaml/BenchmarkExecution.yaml
+++ b/theodolite-quarkus/src/main/resources/yaml/BenchmarkExecution.yaml
@@ -15,7 +15,6 @@ resources:
slos:
- sloType: "lag trend"
threshold: 1000
- #prometheusUrl: "http://localhost:32656"
prometheusUrl: "http://prometheus-operated:9090"
externalSloUrl: "http://localhost:80/evaluate-slope"
offset: 0
diff --git a/theodolite-quarkus/src/test/kotlin/theodolite/TestBenchmark.kt b/theodolite-quarkus/src/test/kotlin/theodolite/TestBenchmark.kt
index 487b583188e8e5741900615108e2b2fa913353df..6f476278d08eacfc9857c1e5431636e5a219f26c 100644
--- a/theodolite-quarkus/src/test/kotlin/theodolite/TestBenchmark.kt
+++ b/theodolite-quarkus/src/test/kotlin/theodolite/TestBenchmark.kt
@@ -11,7 +11,7 @@ class TestBenchmark : Benchmark {
override fun buildDeployment(
load: LoadDimension,
res: Resource,
- configurationOverrides: List<ConfigurationOverride>
+ configurationOverrides: List<ConfigurationOverride?>
): BenchmarkDeployment {
return TestBenchmarkDeployment()
}