From 3d000edc6f7b9509660be6126aa0a906fd434b0d Mon Sep 17 00:00:00 2001
From: Marcel Becker <stu117960@mail.uni-kiel.de>
Date: Sun, 27 Feb 2022 14:31:53 +0100
Subject: [PATCH] Moved Main SLO Definition from Execution to Benchmark yaml
---
theodolite/crd/crd-benchmark.yaml | 27 ++++++++++++++++++-
theodolite/crd/crd-execution.yaml | 10 ++++---
.../benchmark/BenchmarkExecution.kt | 8 +++---
.../benchmark/KubernetesBenchmark.kt | 21 +++++++++++++++
.../theodolite/evaluation/AnalysisExecutor.kt | 6 ++---
.../theodolite/evaluation/SloConfigHandler.kt | 5 ++--
.../theodolite/execution/BenchmarkExecutor.kt | 4 +--
.../execution/BenchmarkExecutorImpl.kt | 22 +++++++--------
.../kotlin/theodolite/execution/SloFactory.kt | 25 +++++++++++++++++
.../execution/TheodoliteExecutor.kt | 5 +++-
.../theodolite/CompositeStrategyTest.kt | 8 +++---
.../InitialGuessSearchStrategyTest.kt | 8 +++---
.../theodolite/TestBenchmarkExecutorImpl.kt | 16 +++++------
13 files changed, 123 insertions(+), 42 deletions(-)
create mode 100644 theodolite/src/main/kotlin/theodolite/execution/SloFactory.kt
diff --git a/theodolite/crd/crd-benchmark.yaml b/theodolite/crd/crd-benchmark.yaml
index c901e6136..7d24a2b97 100644
--- a/theodolite/crd/crd-benchmark.yaml
+++ b/theodolite/crd/crd-benchmark.yaml
@@ -20,7 +20,7 @@ spec:
properties:
spec:
type: object
- required: ["sut", "loadGenerator", "resourceTypes", "loadTypes"]
+ required: ["sut", "loadGenerator", "resourceTypes", "loadTypes", "slos"]
properties:
name:
description: This field exists only for technical reasons and should not be set by the user. The value of the field will be overwritten.
@@ -425,6 +425,31 @@ spec:
additionalProperties: true
x-kubernetes-map-type: "granular"
default: {}
+ slos: # def of service level objectives
+ description: List of resource values for the specified resource type.
+ type: array
+ items:
+ type: object
+ required: ["name", "sloType", "prometheusUrl", "offset"]
+ properties:
+ name:
+ description: The name of the SLO.
+ type: string
+ sloType:
+ description: The type of the SLO. It must match 'lag trend'.
+ type: string
+ prometheusUrl:
+ description: Connection string for Promehteus.
+ type: string
+ offset:
+ description: Hours by which the start and end timestamp will be shifted (for different timezones).
+ type: integer
+ properties:
+ description: (Optional) SLO specific additional arguments.
+ type: object
+ additionalProperties: true
+ x-kubernetes-map-type: "granular"
+ default: { }
kafkaConfig:
description: Contains the Kafka configuration.
type: object
diff --git a/theodolite/crd/crd-execution.yaml b/theodolite/crd/crd-execution.yaml
index 92a8ca18d..bc73b84c3 100644
--- a/theodolite/crd/crd-execution.yaml
+++ b/theodolite/crd/crd-execution.yaml
@@ -20,7 +20,7 @@ spec:
properties:
spec:
type: object
- required: ["benchmark", "load", "resources", "slos", "execution", "configOverrides"]
+ required: ["benchmark", "load", "resources", "execution", "configOverrides"]
properties:
name:
description: This field exists only for technical reasons and should not be set by the user. The value of the field will be overwritten.
@@ -56,12 +56,16 @@ spec:
items:
type: integer
slos: # def of service level objectives
- description: List of resource values for the specified resource type.
+ description: List of SLOs with their properties, which differ from the benchmark definition.
type: array
items:
type: object
- required: ["sloType", "prometheusUrl", "offset"]
+ required: ["name"]
properties:
+ name:
+ description: The name of the SLO. It must match a SLO specified in the Benchmark.
+ type: string
+ # TODO Do we need to keep it here or just move to the Benchmark? Does it make sense to override the type at this point?
sloType:
description: The type of the SLO. It must match 'lag trend'.
type: string
diff --git a/theodolite/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt b/theodolite/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt
index f2dda487d..1b295116e 100644
--- a/theodolite/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt
+++ b/theodolite/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt
@@ -30,7 +30,7 @@ class BenchmarkExecution : KubernetesResource {
lateinit var benchmark: String
lateinit var load: LoadDefinition
lateinit var resources: ResourceDefinition
- lateinit var slos: List<Slo>
+ lateinit var slos: List<KubernetesBenchmark.Slo>
lateinit var execution: Execution
lateinit var configOverrides: MutableList<ConfigurationOverride?>
@@ -49,6 +49,7 @@ class BenchmarkExecution : KubernetesResource {
var afterTeardownDelay = 5L
}
+ //TODO: use new SLO class since the values do not need to be set (just optional)
/**
* Measurable metric.
* [sloType] determines the type of the metric.
@@ -58,14 +59,15 @@ class BenchmarkExecution : KubernetesResource {
* The [warmup] determines after which time the metric should be evaluated to avoid starting interferences.
* The [warmup] time unit depends on the Slo: for the lag trend it is in seconds.
*/
- @JsonDeserialize
+ /* @JsonDeserialize
@RegisterForReflection
class Slo : KubernetesResource {
+ lateinit var name: String
lateinit var sloType: String
lateinit var prometheusUrl: String
var offset by Delegates.notNull<Int>()
lateinit var properties: MutableMap<String, String>
- }
+ }*/
/**
* Represents a Load that should be created and checked.
diff --git a/theodolite/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt b/theodolite/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt
index d42c2ea3c..62dd30c67 100644
--- a/theodolite/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt
+++ b/theodolite/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt
@@ -10,6 +10,7 @@ import theodolite.k8s.K8sManager
import theodolite.k8s.resourceLoader.K8sResourceLoader
import theodolite.patcher.PatcherFactory
import theodolite.util.*
+import kotlin.properties.Delegates
private val logger = KotlinLogging.logger {}
@@ -39,12 +40,32 @@ class KubernetesBenchmark : KubernetesResource, Benchmark {
lateinit var name: String
lateinit var resourceTypes: List<TypeName>
lateinit var loadTypes: List<TypeName>
+ lateinit var slos: List<Slo>
var kafkaConfig: KafkaConfig? = null
lateinit var infrastructure: Resources
lateinit var sut: Resources
lateinit var loadGenerator: Resources
private var namespace = System.getenv("NAMESPACE") ?: DEFAULT_NAMESPACE
+ /**
+ * Measurable metric.
+ * [sloType] determines the type of the metric.
+ * It is evaluated using the [theodolite.evaluation.ExternalSloChecker] by data measured by Prometheus.
+ * The evaluation checks if a [threshold] is reached or not.
+ * [offset] determines the shift in hours by which the start and end timestamps should be shifted.
+ * The [warmup] determines after which time the metric should be evaluated to avoid starting interferences.
+ * The [warmup] time unit depends on the Slo: for the lag trend it is in seconds.
+ */
+ @JsonDeserialize
+ @RegisterForReflection
+ class Slo : KubernetesResource {
+ lateinit var name: String
+ lateinit var sloType: String
+ lateinit var prometheusUrl: String
+ var offset by Delegates.notNull<Int>()
+ lateinit var properties: MutableMap<String, String>
+ }
+
@Transient
private var client: NamespacedKubernetesClient = DefaultKubernetesClient().inNamespace(namespace)
diff --git a/theodolite/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt b/theodolite/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt
index be3e48be4..9ac78b78f 100644
--- a/theodolite/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt
+++ b/theodolite/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt
@@ -1,6 +1,6 @@
package theodolite.evaluation
-import theodolite.benchmark.BenchmarkExecution
+import theodolite.benchmark.KubernetesBenchmark
import theodolite.util.EvaluationFailedException
import theodolite.util.IOHandler
import theodolite.util.LoadDimension
@@ -16,8 +16,8 @@ import java.util.regex.Pattern
* @param slo Slo that is used for the analysis.
*/
class AnalysisExecutor(
- private val slo: BenchmarkExecution.Slo,
- private val executionId: Int
+ private val slo: KubernetesBenchmark.Slo,
+ private val executionId: Int
) {
private val fetcher = MetricFetcher(
diff --git a/theodolite/src/main/kotlin/theodolite/evaluation/SloConfigHandler.kt b/theodolite/src/main/kotlin/theodolite/evaluation/SloConfigHandler.kt
index 924305660..75ae8672e 100644
--- a/theodolite/src/main/kotlin/theodolite/evaluation/SloConfigHandler.kt
+++ b/theodolite/src/main/kotlin/theodolite/evaluation/SloConfigHandler.kt
@@ -1,16 +1,17 @@
package theodolite.evaluation
-import theodolite.benchmark.BenchmarkExecution
+import theodolite.benchmark.KubernetesBenchmark
import theodolite.util.InvalidPatcherConfigurationException
import javax.enterprise.context.ApplicationScoped
private const val CONSUMER_LAG_QUERY = "sum by(consumergroup) (kafka_consumergroup_lag >= 0)"
private const val DROPPED_RECORDS_QUERY = "sum by(job) (kafka_streams_stream_task_metrics_dropped_records_total>=0)"
+//TODO: slo.sloType.toLowerCase() is deprecated
@ApplicationScoped
class SloConfigHandler {
companion object {
- fun getQueryString(slo: BenchmarkExecution.Slo): String {
+ fun getQueryString(slo: KubernetesBenchmark.Slo): String {
return when (slo.sloType.toLowerCase()) {
SloTypes.GENERIC.value -> slo.properties["promQLQuery"] ?: throw IllegalArgumentException("promQLQuery expected")
SloTypes.LAG_TREND.value, SloTypes.LAG_TREND_RATIO.value -> CONSUMER_LAG_QUERY
diff --git a/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt b/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt
index 3238f447b..95b2abeaa 100644
--- a/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt
+++ b/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt
@@ -2,7 +2,7 @@ package theodolite.execution
import mu.KotlinLogging
import theodolite.benchmark.Benchmark
-import theodolite.benchmark.BenchmarkExecution
+import theodolite.benchmark.KubernetesBenchmark
import theodolite.util.ConfigurationOverride
import theodolite.util.LoadDimension
import theodolite.util.Resource
@@ -25,7 +25,7 @@ abstract class BenchmarkExecutor(
val results: Results,
val executionDuration: Duration,
val configurationOverrides: List<ConfigurationOverride?>,
- val slos: List<BenchmarkExecution.Slo>,
+ val slos: List<KubernetesBenchmark.Slo>,
val repetitions: Int,
val executionId: Int,
val loadGenerationDelay: Long,
diff --git a/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt b/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt
index 2e938be3a..415fd1976 100644
--- a/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt
+++ b/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt
@@ -3,7 +3,7 @@ package theodolite.execution
import io.quarkus.runtime.annotations.RegisterForReflection
import mu.KotlinLogging
import theodolite.benchmark.Benchmark
-import theodolite.benchmark.BenchmarkExecution
+import theodolite.benchmark.KubernetesBenchmark
import theodolite.evaluation.AnalysisExecutor
import theodolite.execution.operator.EventCreator
import theodolite.util.*
@@ -14,16 +14,16 @@ private val logger = KotlinLogging.logger {}
@RegisterForReflection
class BenchmarkExecutorImpl(
- benchmark: Benchmark,
- results: Results,
- executionDuration: Duration,
- configurationOverrides: List<ConfigurationOverride?>,
- slos: List<BenchmarkExecution.Slo>,
- repetitions: Int,
- executionId: Int,
- loadGenerationDelay: Long,
- afterTeardownDelay: Long,
- executionName: String
+ benchmark: Benchmark,
+ results: Results,
+ executionDuration: Duration,
+ configurationOverrides: List<ConfigurationOverride?>,
+ slos: List<KubernetesBenchmark.Slo>,
+ repetitions: Int,
+ executionId: Int,
+ loadGenerationDelay: Long,
+ afterTeardownDelay: Long,
+ executionName: String
) : BenchmarkExecutor(
benchmark,
results,
diff --git a/theodolite/src/main/kotlin/theodolite/execution/SloFactory.kt b/theodolite/src/main/kotlin/theodolite/execution/SloFactory.kt
new file mode 100644
index 000000000..89b815d0f
--- /dev/null
+++ b/theodolite/src/main/kotlin/theodolite/execution/SloFactory.kt
@@ -0,0 +1,25 @@
+package theodolite.execution
+
+import theodolite.benchmark.BenchmarkExecution
+import theodolite.benchmark.KubernetesBenchmark
+
+class SloFactory {
+
+ fun createSlos(execution: BenchmarkExecution, benchmark: KubernetesBenchmark): List<KubernetesBenchmark.Slo> {
+ var benchmarkSlos = benchmark.slos
+ var executionSlos = execution.slos
+ //TODO: test if we can actually overwrite entries of the objects
+ for(executionSlo in executionSlos) {
+ for(benchmarkSlo in benchmarkSlos) {
+ if(executionSlo.name == benchmarkSlo.name) {
+ benchmarkSlo.offset = executionSlo.offset ?: benchmarkSlo.offset
+ benchmarkSlo.prometheusUrl = executionSlo.prometheusUrl ?: benchmarkSlo.prometheusUrl
+ for(executionProperty in executionSlo.properties) {
+ benchmarkSlo.properties[executionProperty.key] = executionProperty.value
+ }
+ }
+ }
+ }
+ return benchmarkSlos
+ }
+}
\ No newline at end of file
diff --git a/theodolite/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt b/theodolite/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt
index 8596576e0..6f6227fb7 100644
--- a/theodolite/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt
+++ b/theodolite/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt
@@ -55,13 +55,16 @@ class TheodoliteExecutor(
this.kubernetesBenchmark.loadTypes
)
+ val slos = SloFactory().createSlos(this.config, this.kubernetesBenchmark)
+
+ // TODO not config.slos!, maybe change naming here cause its kinda missleading due to the Config.kt file
executor =
BenchmarkExecutorImpl(
benchmark = kubernetesBenchmark,
results = results,
executionDuration = executionDuration,
configurationOverrides = config.configOverrides,
- slos = config.slos,
+ slos = slos,
repetitions = config.execution.repetitions,
executionId = config.executionId,
loadGenerationDelay = config.execution.loadGenerationDelay,
diff --git a/theodolite/src/test/kotlin/theodolite/CompositeStrategyTest.kt b/theodolite/src/test/kotlin/theodolite/CompositeStrategyTest.kt
index 580d9e747..1d8823e28 100644
--- a/theodolite/src/test/kotlin/theodolite/CompositeStrategyTest.kt
+++ b/theodolite/src/test/kotlin/theodolite/CompositeStrategyTest.kt
@@ -3,7 +3,7 @@ package theodolite
import io.quarkus.test.junit.QuarkusTest
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Test
-import theodolite.benchmark.BenchmarkExecution
+import theodolite.benchmark.KubernetesBenchmark
import theodolite.strategies.restriction.LowerBoundRestriction
import theodolite.strategies.searchstrategy.BinarySearch
import theodolite.strategies.searchstrategy.CompositeStrategy
@@ -30,7 +30,7 @@ class CompositeStrategyTest {
val mockResources: List<Resource> = (0..6).map { number -> Resource(number, emptyList()) }
val results = Results()
val benchmark = TestBenchmark()
- val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo()
+ val sloChecker: KubernetesBenchmark.Slo = KubernetesBenchmark.Slo()
val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, listOf(sloChecker), 0, 0, 5)
val linearSearch = LinearSearch(benchmarkExecutor)
val lowerBoundRestriction = LowerBoundRestriction(results)
@@ -63,7 +63,7 @@ class CompositeStrategyTest {
val mockResources: List<Resource> = (0..6).map { number -> Resource(number, emptyList()) }
val results = Results()
val benchmark = TestBenchmark()
- val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo()
+ val sloChecker: KubernetesBenchmark.Slo = KubernetesBenchmark.Slo()
val benchmarkExecutorImpl =
TestBenchmarkExecutorImpl(mockResults, benchmark, results, listOf(sloChecker), 0, 0, 0)
val binarySearch = BinarySearch(benchmarkExecutorImpl)
@@ -97,7 +97,7 @@ class CompositeStrategyTest {
val mockResources: List<Resource> = (0..7).map { number -> Resource(number, emptyList()) }
val results = Results()
val benchmark = TestBenchmark()
- val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo()
+ val sloChecker: KubernetesBenchmark.Slo = KubernetesBenchmark.Slo()
val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, listOf(sloChecker), 0, 0, 0)
val binarySearch = BinarySearch(benchmarkExecutor)
val lowerBoundRestriction = LowerBoundRestriction(results)
diff --git a/theodolite/src/test/kotlin/theodolite/InitialGuessSearchStrategyTest.kt b/theodolite/src/test/kotlin/theodolite/InitialGuessSearchStrategyTest.kt
index 1af6f548b..81f210762 100644
--- a/theodolite/src/test/kotlin/theodolite/InitialGuessSearchStrategyTest.kt
+++ b/theodolite/src/test/kotlin/theodolite/InitialGuessSearchStrategyTest.kt
@@ -3,12 +3,12 @@ package theodolite
import io.quarkus.test.junit.QuarkusTest
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Test
-import theodolite.benchmark.BenchmarkExecution
import theodolite.strategies.searchstrategy.InitialGuessSearchStrategy
import theodolite.util.LoadDimension
import theodolite.util.Resource
import theodolite.util.Results
import mu.KotlinLogging
+import theodolite.benchmark.KubernetesBenchmark
import theodolite.strategies.searchstrategy.PrevResourceMinGuess
private val logger = KotlinLogging.logger {}
@@ -32,7 +32,7 @@ class InitialGuessSearchStrategyTest {
val results = Results()
val benchmark = TestBenchmark()
val guessStrategy = PrevResourceMinGuess()
- val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo()
+ val sloChecker: KubernetesBenchmark.Slo = KubernetesBenchmark.Slo()
val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, listOf(sloChecker), 0, 0, 5)
val strategy = InitialGuessSearchStrategy(benchmarkExecutor,guessStrategy, results)
@@ -70,7 +70,7 @@ class InitialGuessSearchStrategyTest {
val results = Results()
val benchmark = TestBenchmark()
val guessStrategy = PrevResourceMinGuess()
- val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo()
+ val sloChecker: KubernetesBenchmark.Slo = KubernetesBenchmark.Slo()
val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, listOf(sloChecker), 0, 0, 5)
val strategy = InitialGuessSearchStrategy(benchmarkExecutor,guessStrategy, results)
@@ -108,7 +108,7 @@ class InitialGuessSearchStrategyTest {
val results = Results()
val benchmark = TestBenchmark()
val guessStrategy = PrevResourceMinGuess()
- val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo()
+ val sloChecker: KubernetesBenchmark.Slo = KubernetesBenchmark.Slo()
val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, listOf(sloChecker), 0, 0, 5)
val strategy = InitialGuessSearchStrategy(benchmarkExecutor, guessStrategy, results)
diff --git a/theodolite/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt b/theodolite/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt
index 2efddc48c..72fdc4a11 100644
--- a/theodolite/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt
+++ b/theodolite/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt
@@ -1,7 +1,7 @@
package theodolite
import theodolite.benchmark.Benchmark
-import theodolite.benchmark.BenchmarkExecution
+import theodolite.benchmark.KubernetesBenchmark
import theodolite.execution.BenchmarkExecutor
import theodolite.util.LoadDimension
import theodolite.util.Resource
@@ -9,13 +9,13 @@ import theodolite.util.Results
import java.time.Duration
class TestBenchmarkExecutorImpl(
- private val mockResults: Array<Array<Boolean>>,
- benchmark: Benchmark,
- results: Results,
- slo: List<BenchmarkExecution.Slo>,
- executionId: Int,
- loadGenerationDelay: Long,
- afterTeardownDelay: Long
+ private val mockResults: Array<Array<Boolean>>,
+ benchmark: Benchmark,
+ results: Results,
+ slo: List<KubernetesBenchmark.Slo>,
+ executionId: Int,
+ loadGenerationDelay: Long,
+ afterTeardownDelay: Long
) :
BenchmarkExecutor(
benchmark,
--
GitLab