diff --git a/docs/api-reference/crds.md b/docs/api-reference/crds.md
index 996ea1a3a4486ca27031f3d29ad64e018f66f5fb..f0962d47400dfb85f8f9a2b695fec49cc45fe6e8 100644
--- a/docs/api-reference/crds.md
+++ b/docs/api-reference/crds.md
@@ -114,6 +114,13 @@ Resource Types:
A list of resource types that can be scaled for this `benchmark` resource. For each resource type the concrete values are defined in the `execution` object.<br/>
</td>
<td>true</td>
+ </tr><tr>
+ <td><b><a href="#benchmarkspecslosindex">slos</a></b></td>
+ <td>[]object</td>
+ <td>
+ List of resource values for the specified resource type.<br/>
+ </td>
+ <td>true</td>
</tr><tr>
<td><b><a href="#benchmarkspecsut">sut</a></b></td>
<td>object</td>
@@ -736,6 +743,63 @@ The fileSystem resourceSet loads the Kubernetes manifests from the filesystem.
</table>
+### benchmark.spec.slos[index]
+<sup><sup>[↩ Parent](#benchmarkspec)</sup></sup>
+
+
+
+
+
+<table>
+ <thead>
+ <tr>
+ <th>Name</th>
+ <th>Type</th>
+ <th>Description</th>
+ <th>Required</th>
+ </tr>
+ </thead>
+ <tbody><tr>
+ <td><b>name</b></td>
+ <td>string</td>
+ <td>
+ The name of the SLO.<br/>
+ </td>
+ <td>true</td>
+ </tr><tr>
+ <td><b>offset</b></td>
+ <td>integer</td>
+ <td>
+ Hours by which the start and end timestamp will be shifted (for different timezones).<br/>
+ </td>
+ <td>true</td>
+ </tr><tr>
+ <td><b>prometheusUrl</b></td>
+ <td>string</td>
+ <td>
+ Connection string for Promehteus.<br/>
+ </td>
+ <td>true</td>
+ </tr><tr>
+ <td><b>sloType</b></td>
+ <td>string</td>
+ <td>
+ The type of the SLO. It must match 'lag trend'.<br/>
+ </td>
+ <td>true</td>
+ </tr><tr>
+ <td><b>properties</b></td>
+ <td>map[string]string</td>
+ <td>
+ (Optional) SLO specific additional arguments.<br/>
+ <br/>
+ <i>Default</i>: map[]<br/>
+ </td>
+ <td>false</td>
+ </tr></tbody>
+</table>
+
+
### benchmark.spec.sut
<sup><sup>[↩ Parent](#benchmarkspec)</sup></sup>
@@ -1787,13 +1851,6 @@ Contains the Kafka configuration.
Specifies the scaling resource that is benchmarked.<br/>
</td>
<td>true</td>
- </tr><tr>
- <td><b><a href="#executionspecslosindex">slos</a></b></td>
- <td>[]object</td>
- <td>
- List of resource values for the specified resource type.<br/>
- </td>
- <td>true</td>
</tr><tr>
<td><b>name</b></td>
<td>string</td>
@@ -1803,6 +1860,13 @@ Contains the Kafka configuration.
<i>Default</i>: <br/>
</td>
<td>false</td>
+ </tr><tr>
+ <td><b><a href="#executionspecslosindex">slos</a></b></td>
+ <td>[]object</td>
+ <td>
+ List of SLOs with their properties, which differ from the benchmark definition.<br/>
+ </td>
+ <td>false</td>
</tr></tbody>
</table>
@@ -2076,24 +2140,10 @@ Specifies the scaling resource that is benchmarked.
</tr>
</thead>
<tbody><tr>
- <td><b>offset</b></td>
- <td>integer</td>
- <td>
- Hours by which the start and end timestamp will be shifted (for different timezones).<br/>
- </td>
- <td>true</td>
- </tr><tr>
- <td><b>prometheusUrl</b></td>
- <td>string</td>
- <td>
- Connection string for Promehteus.<br/>
- </td>
- <td>true</td>
- </tr><tr>
- <td><b>sloType</b></td>
+ <td><b>name</b></td>
<td>string</td>
<td>
- The type of the SLO. It must match 'lag trend'.<br/>
+ The name of the SLO. It must match a SLO specified in the Benchmark.<br/>
</td>
<td>true</td>
</tr><tr>
@@ -2104,7 +2154,7 @@ Specifies the scaling resource that is benchmarked.
<br/>
<i>Default</i>: map[]<br/>
</td>
- <td>false</td>
+ <td>true</td>
</tr></tbody>
</table>
diff --git a/theodolite/crd/crd-benchmark.yaml b/theodolite/crd/crd-benchmark.yaml
index d2418ee005e2c0168254a9423b9c383ace2d3ca7..d73ec0105c7f97d0e425307fc4f21fd84d1e9b46 100644
--- a/theodolite/crd/crd-benchmark.yaml
+++ b/theodolite/crd/crd-benchmark.yaml
@@ -20,7 +20,7 @@ spec:
properties:
spec:
type: object
- required: ["sut", "loadGenerator", "resourceTypes", "loadTypes"]
+ required: ["sut", "loadGenerator", "resourceTypes", "loadTypes", "slos"]
properties:
name:
description: This field exists only for technical reasons and should not be set by the user. The value of the field will be overwritten.
@@ -429,6 +429,31 @@ spec:
additionalProperties: true
x-kubernetes-map-type: "granular"
default: {}
+ slos: # def of service level objectives
+ description: List of resource values for the specified resource type.
+ type: array
+ items:
+ type: object
+ required: ["name", "sloType", "prometheusUrl", "offset"]
+ properties:
+ name:
+ description: The name of the SLO.
+ type: string
+ sloType:
+ description: The type of the SLO. It must match 'lag trend'.
+ type: string
+ prometheusUrl:
+ description: Connection string for Promehteus.
+ type: string
+ offset:
+ description: Hours by which the start and end timestamp will be shifted (for different timezones).
+ type: integer
+ properties:
+ description: (Optional) SLO specific additional arguments.
+ type: object
+ additionalProperties: true
+ x-kubernetes-map-type: "granular"
+ default: { }
kafkaConfig:
description: Contains the Kafka configuration.
type: object
diff --git a/theodolite/crd/crd-execution.yaml b/theodolite/crd/crd-execution.yaml
index 0240fc6ee05f5a573b1f738d931d78c3bb9b5859..0db251864bac3e6f2541534eec8895297b21cf39 100644
--- a/theodolite/crd/crd-execution.yaml
+++ b/theodolite/crd/crd-execution.yaml
@@ -20,7 +20,7 @@ spec:
properties:
spec:
type: object
- required: ["benchmark", "loads", "resources", "slos", "execution", "configOverrides"]
+ required: ["benchmark", "loads", "resources", "execution", "configOverrides"]
properties:
name:
description: This field exists only for technical reasons and should not be set by the user. The value of the field will be overwritten.
@@ -56,21 +56,15 @@ spec:
items:
type: integer
slos: # def of service level objectives
- description: List of resource values for the specified resource type.
+ description: List of SLOs with their properties, which differ from the benchmark definition.
type: array
items:
type: object
- required: ["sloType", "prometheusUrl", "offset"]
+ required: ["name", "properties"]
properties:
- sloType:
- description: The type of the SLO. It must match 'lag trend'.
+ name:
+ description: The name of the SLO. It must match a SLO specified in the Benchmark.
type: string
- prometheusUrl:
- description: Connection string for Promehteus.
- type: string
- offset:
- description: Hours by which the start and end timestamp will be shifted (for different timezones).
- type: integer
properties:
description: (Optional) SLO specific additional arguments.
type: object
diff --git a/theodolite/examples/operator/example-benchmark.yaml b/theodolite/examples/operator/example-benchmark.yaml
index 62920091e831ff914fb67e85a67cd3f1d98995ab..be7116c46f8222eeba0f3bffd086f7cc2b6ee227 100644
--- a/theodolite/examples/operator/example-benchmark.yaml
+++ b/theodolite/examples/operator/example-benchmark.yaml
@@ -33,6 +33,15 @@ spec:
resource: "uc1-load-generator-deployment.yaml"
properties:
loadGenMaxRecords: "150000"
+ slos:
+ - name: "lag trend"
+ sloType: "lag trend"
+ prometheusUrl: "http://prometheus-operated:9090"
+ offset: 0
+ properties:
+ threshold: 3000
+ externalSloUrl: "http://localhost:80/evaluate-slope"
+ warmup: 60 # in seconds
kafkaConfig:
bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092"
topics:
diff --git a/theodolite/examples/operator/example-execution.yaml b/theodolite/examples/operator/example-execution.yaml
index b94aa9b2656f02f1b3f7fbf87edeceaa923e1ab8..87e6275a8409177394dd313f2f1f0436e2162577 100644
--- a/theodolite/examples/operator/example-execution.yaml
+++ b/theodolite/examples/operator/example-execution.yaml
@@ -11,13 +11,9 @@ spec:
resourceType: "Instances"
resourceValues: [1, 2, 3, 4, 5]
slos:
- - sloType: "lag trend"
- prometheusUrl: "http://prometheus-operated:9090"
- offset: 0
+ - name: "lag trend"
properties:
threshold: 2000
- externalSloUrl: "http://localhost:80/evaluate-slope"
- warmup: 60 # in seconds
execution:
strategy:
name: "RestrictionSearch"
diff --git a/theodolite/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt b/theodolite/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt
index 1010a8a344442b24f818d73a400a799153ed8947..13c98d7013fc3c6f24066419604d42c576db2c94 100644
--- a/theodolite/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt
+++ b/theodolite/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt
@@ -14,7 +14,7 @@ import kotlin.properties.Delegates
* - The [benchmark] that should be executed.
* - The [loads]s that should be checked in the benchmark.
* - The [resources] that should be checked in the benchmark.
- * - A list of [slos] that are used for the evaluation of the experiments.
+ * - The [slos] further restrict the Benchmark SLOs for the evaluation of the experiments.
* - An [execution] that encapsulates: the strategy, the duration, and the restrictions
* for the execution of the benchmark.
* - [configOverrides] additional configurations.
@@ -30,7 +30,7 @@ class BenchmarkExecution : KubernetesResource {
lateinit var benchmark: String
lateinit var loads: LoadDefinition
lateinit var resources: ResourceDefinition
- lateinit var slos: List<Slo>
+ lateinit var slos: List<SloConfiguration>
lateinit var execution: Execution
lateinit var configOverrides: MutableList<ConfigurationOverride?>
@@ -65,21 +65,13 @@ class BenchmarkExecution : KubernetesResource {
}
/**
- * Measurable metric.
- * [sloType] determines the type of the metric.
- * It is evaluated using the [theodolite.evaluation.ExternalSloChecker] by data measured by Prometheus.
- * The evaluation checks if a [threshold] is reached or not.
- * [offset] determines the shift in hours by which the start and end timestamps should be shifted.
- * The [warmup] determines after which time the metric should be evaluated to avoid starting interferences.
- * The [warmup] time unit depends on the Slo: for the lag trend it is in seconds.
+ * Further SLO configurations for the SLOs specified in the Benchmark.
*/
@JsonDeserialize
@RegisterForReflection
- class Slo : KubernetesResource {
- lateinit var sloType: String
- lateinit var prometheusUrl: String
- var offset by Delegates.notNull<Int>()
- lateinit var properties: MutableMap<String, String>
+ class SloConfiguration : KubernetesResource {
+ lateinit var name: String
+ var properties: MutableMap<String, String>? = null
}
/**
diff --git a/theodolite/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt b/theodolite/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt
index 599f723621db2b86c9163af9351ac896f82b2b86..d40d91961266098702171c7c3d040db87b2c6f9f 100644
--- a/theodolite/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt
+++ b/theodolite/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt
@@ -42,6 +42,7 @@ class KubernetesBenchmark : KubernetesResource, Benchmark {
var waitForResourcesEnabled = false
lateinit var resourceTypes: List<TypeName>
lateinit var loadTypes: List<TypeName>
+ lateinit var slos: MutableList<Slo>
var kafkaConfig: KafkaConfig? = null
lateinit var infrastructure: Resources
lateinit var sut: Resources
diff --git a/theodolite/src/main/kotlin/theodolite/benchmark/Slo.kt b/theodolite/src/main/kotlin/theodolite/benchmark/Slo.kt
new file mode 100644
index 0000000000000000000000000000000000000000..c9aac4a1b68692669f0db577003856b964ade4ec
--- /dev/null
+++ b/theodolite/src/main/kotlin/theodolite/benchmark/Slo.kt
@@ -0,0 +1,25 @@
+package theodolite.benchmark
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize
+import io.fabric8.kubernetes.api.model.KubernetesResource
+import io.quarkus.runtime.annotations.RegisterForReflection
+import kotlin.properties.Delegates
+
+/**
+ * Measurable metric.
+ * [sloType] determines the type of the metric.
+ * It is evaluated using the [theodolite.evaluation.ExternalSloChecker] by data measured by Prometheus.
+ * The evaluation checks if a [threshold] is reached or not.
+ * [offset] determines the shift in hours by which the start and end timestamps should be shifted.
+ * The [warmup] determines after which time the metric should be evaluated to avoid starting interferences.
+ * The [warmup] time unit depends on the Slo: for the lag trend it is in seconds.
+ */
+@JsonDeserialize
+@RegisterForReflection
+class Slo : KubernetesResource {
+ lateinit var name: String
+ lateinit var sloType: String
+ lateinit var prometheusUrl: String
+ var offset by Delegates.notNull<Int>()
+ lateinit var properties: MutableMap<String, String>
+}
\ No newline at end of file
diff --git a/theodolite/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt b/theodolite/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt
index 9e460fd0befafef8a644de870a5b33ccdfcf2029..7948058f0716a29712c360b5f90362dcedce2d7f 100644
--- a/theodolite/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt
+++ b/theodolite/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt
@@ -1,6 +1,6 @@
package theodolite.evaluation
-import theodolite.benchmark.BenchmarkExecution
+import theodolite.benchmark.Slo
import theodolite.strategies.Metric
import theodolite.util.EvaluationFailedException
import theodolite.util.IOHandler
@@ -15,8 +15,8 @@ import java.util.regex.Pattern
* @param slo Slo that is used for the analysis.
*/
class AnalysisExecutor(
- private val slo: BenchmarkExecution.Slo,
- private val executionId: Int
+ private val slo: Slo,
+ private val executionId: Int
) {
private val fetcher = MetricFetcher(
diff --git a/theodolite/src/main/kotlin/theodolite/evaluation/SloConfigHandler.kt b/theodolite/src/main/kotlin/theodolite/evaluation/SloConfigHandler.kt
index 089f40dc6b5ef7d8ac4b063cae68e5e9621d1f50..ee892109dc1fc03a9c270151944c6d90fbacf45e 100644
--- a/theodolite/src/main/kotlin/theodolite/evaluation/SloConfigHandler.kt
+++ b/theodolite/src/main/kotlin/theodolite/evaluation/SloConfigHandler.kt
@@ -1,6 +1,6 @@
package theodolite.evaluation
-import theodolite.benchmark.BenchmarkExecution
+import theodolite.benchmark.Slo
import theodolite.util.InvalidPatcherConfigurationException
import javax.enterprise.context.ApplicationScoped
@@ -11,7 +11,7 @@ private const val DEFAULT_DROPPED_RECORDS_QUERY = "sum by(job) (kafka_streams_st
@ApplicationScoped
class SloConfigHandler {
companion object {
- fun getQueryString(slo: BenchmarkExecution.Slo): String {
+ fun getQueryString(slo: Slo): String {
return when (slo.sloType.lowercase()) {
SloTypes.GENERIC.value -> slo.properties["promQLQuery"] ?: throw IllegalArgumentException("promQLQuery expected")
SloTypes.LAG_TREND.value, SloTypes.LAG_TREND_RATIO.value -> slo.properties["promQLQuery"] ?:
diff --git a/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt b/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt
index 55233246fedcd5163f5d3c954d32bd2056b2f31d..9ae267f42ca3f8420dbd507b0b92e92bf49a31f5 100644
--- a/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt
+++ b/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt
@@ -2,7 +2,7 @@ package theodolite.execution
import mu.KotlinLogging
import theodolite.benchmark.Benchmark
-import theodolite.benchmark.BenchmarkExecution
+import theodolite.benchmark.Slo
import theodolite.util.ConfigurationOverride
import theodolite.util.PatcherDefinition
import theodolite.util.Results
@@ -24,7 +24,7 @@ abstract class BenchmarkExecutor(
val results: Results,
val executionDuration: Duration,
val configurationOverrides: List<ConfigurationOverride?>,
- val slos: List<BenchmarkExecution.Slo>,
+ val slos: List<Slo>,
val repetitions: Int,
val executionId: Int,
val loadGenerationDelay: Long,
diff --git a/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt b/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt
index e2b6c0930764e282b92905066a488f76d036db4c..c49a40a531df50eb58f21d68cf2d8be13b284374 100644
--- a/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt
+++ b/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt
@@ -3,7 +3,7 @@ package theodolite.execution
import io.quarkus.runtime.annotations.RegisterForReflection
import mu.KotlinLogging
import theodolite.benchmark.Benchmark
-import theodolite.benchmark.BenchmarkExecution
+import theodolite.benchmark.Slo
import theodolite.evaluation.AnalysisExecutor
import theodolite.execution.operator.EventCreator
import theodolite.util.*
@@ -18,7 +18,7 @@ class BenchmarkExecutorImpl(
results: Results,
executionDuration: Duration,
configurationOverrides: List<ConfigurationOverride?>,
- slos: List<BenchmarkExecution.Slo>,
+ slos: List<Slo>,
repetitions: Int,
executionId: Int,
loadGenerationDelay: Long,
diff --git a/theodolite/src/main/kotlin/theodolite/execution/SloFactory.kt b/theodolite/src/main/kotlin/theodolite/execution/SloFactory.kt
new file mode 100644
index 0000000000000000000000000000000000000000..b990828fa1db09532767b9f9255aa53e9c9e894a
--- /dev/null
+++ b/theodolite/src/main/kotlin/theodolite/execution/SloFactory.kt
@@ -0,0 +1,24 @@
+package theodolite.execution
+
+import theodolite.benchmark.BenchmarkExecution
+import theodolite.benchmark.KubernetesBenchmark
+import theodolite.benchmark.Slo
+
+class SloFactory {
+
+ fun createSlos(execution: BenchmarkExecution, benchmark: KubernetesBenchmark): List<Slo> {
+ var benchmarkSlos = benchmark.slos
+ var executionSlos = execution.slos
+
+ for(executionSlo in executionSlos) {
+ for(i in 0 until benchmarkSlos.size) {
+ if(executionSlo.name == benchmarkSlos[i].name && executionSlo.properties != null) {
+ for (executionProperty in executionSlo.properties!!) {
+ benchmarkSlos[i].properties[executionProperty.key] = executionProperty.value
+ }
+ }
+ }
+ }
+ return benchmarkSlos
+ }
+}
\ No newline at end of file
diff --git a/theodolite/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt b/theodolite/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt
index 3879e67239c9ba7140114ee605a2e71de3409c98..82b406264fe7a07da1759936f4f2667150a7dee1 100644
--- a/theodolite/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt
+++ b/theodolite/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt
@@ -16,12 +16,12 @@ private val logger = KotlinLogging.logger {}
/**
* The Theodolite executor runs all the experiments defined with the given execution and benchmark configuration.
*
- * @property config Configuration of a execution
+ * @property benchmarkExecution Configuration of a execution
* @property kubernetesBenchmark Configuration of a benchmark
* @constructor Create empty Theodolite executor
*/
class TheodoliteExecutor(
- private val config: BenchmarkExecution,
+ private val benchmarkExecution: BenchmarkExecution,
private val kubernetesBenchmark: KubernetesBenchmark
) {
/**
@@ -38,67 +38,69 @@ class TheodoliteExecutor(
* The [searchStrategy] is configured and able to find the minimum required resource for the given load.
*/
private fun buildConfig(): Config {
- val results = Results(Metric.from(config.execution.metric))
+ val results = Results(Metric.from(benchmarkExecution.execution.metric))
val strategyFactory = StrategyFactory()
- val executionDuration = Duration.ofSeconds(config.execution.duration)
+ val executionDuration = Duration.ofSeconds(benchmarkExecution.execution.duration)
val resourcePatcherDefinition =
PatcherDefinitionFactory().createPatcherDefinition(
- config.resources.resourceType,
+ benchmarkExecution.resources.resourceType,
this.kubernetesBenchmark.resourceTypes
)
val loadDimensionPatcherDefinition =
PatcherDefinitionFactory().createPatcherDefinition(
- config.loads.loadType,
+ benchmarkExecution.loads.loadType,
this.kubernetesBenchmark.loadTypes
)
+ val slos = SloFactory().createSlos(this.benchmarkExecution, this.kubernetesBenchmark)
+
executor =
BenchmarkExecutorImpl(
benchmark = kubernetesBenchmark,
results = results,
executionDuration = executionDuration,
- configurationOverrides = config.configOverrides,
- slos = config.slos,
- repetitions = config.execution.repetitions,
- executionId = config.executionId,
- loadGenerationDelay = config.execution.loadGenerationDelay,
- afterTeardownDelay = config.execution.afterTeardownDelay,
- executionName = config.name,
+ configurationOverrides = benchmarkExecution.configOverrides,
+ slos = slos,
+ repetitions = benchmarkExecution.execution.repetitions,
+ executionId = benchmarkExecution.executionId,
+ loadGenerationDelay = benchmarkExecution.execution.loadGenerationDelay,
+ afterTeardownDelay = benchmarkExecution.execution.afterTeardownDelay,
+ executionName = benchmarkExecution.name,
loadPatcherDefinitions = loadDimensionPatcherDefinition,
resourcePatcherDefinitions = resourcePatcherDefinition
)
- if (config.loads.loadValues != config.loads.loadValues.sorted()) {
- config.loads.loadValues = config.loads.loadValues.sorted()
+ if (benchmarkExecution.loads.loadValues != benchmarkExecution.loads.loadValues.sorted()) {
+ benchmarkExecution.loads.loadValues = benchmarkExecution.loads.loadValues.sorted()
logger.info {
- "Load values are not sorted correctly, Theodolite sorts them in ascending order." +
- "New order is: ${config.loads.loadValues}"
+ "Load values are not sorted correctly. Theodolite sorts them in ascending order." +
+ "New order is: ${benchmarkExecution.loads.loadValues}."
}
}
- if (config.resources.resourceValues != config.resources.resourceValues.sorted()) {
- config.resources.resourceValues = config.resources.resourceValues.sorted()
+ if (benchmarkExecution.resources.resourceValues != benchmarkExecution.resources.resourceValues.sorted()) {
+ benchmarkExecution.resources.resourceValues = benchmarkExecution.resources.resourceValues.sorted()
logger.info {
- "Load values are not sorted correctly, Theodolite sorts them in ascending order." +
- "New order is: ${config.resources.resourceValues}"
+ "Load values are not sorted correctly. Theodolite sorts them in ascending order." +
+ "New order is: ${benchmarkExecution.resources.resourceValues}."
}
}
return Config(
- loads = config.loads.loadValues,
+ loads = benchmarkExecution.loads.loadValues,
loadPatcherDefinitions = loadDimensionPatcherDefinition,
- resources = config.resources.resourceValues,
+ resources = benchmarkExecution.resources.resourceValues,
resourcePatcherDefinitions = resourcePatcherDefinition,
- searchStrategy = strategyFactory.createSearchStrategy(executor, config.execution.strategy, results),
- metric = Metric.from(config.execution.metric)
+ searchStrategy = strategyFactory.createSearchStrategy(executor, benchmarkExecution.execution.strategy, results),
+ metric = Metric.from(benchmarkExecution.execution.metric)
)
}
fun getExecution(): BenchmarkExecution {
- return this.config
+ return this.benchmarkExecution
}
/**
@@ -110,11 +112,11 @@ class TheodoliteExecutor(
val ioHandler = IOHandler()
val resultsFolder = ioHandler.getResultFolderURL()
- this.config.executionId = getAndIncrementExecutionID(resultsFolder + "expID.txt")
- ioHandler.writeToJSONFile(this.config, "${resultsFolder}exp${this.config.executionId}-execution-configuration")
+ this.benchmarkExecution.executionId = getAndIncrementExecutionID(resultsFolder + "expID.txt")
+ ioHandler.writeToJSONFile(this.benchmarkExecution, "${resultsFolder}exp${this.benchmarkExecution.executionId}-execution-configuration")
ioHandler.writeToJSONFile(
kubernetesBenchmark,
- "${resultsFolder}exp${this.config.executionId}-benchmark-configuration"
+ "${resultsFolder}exp${this.benchmarkExecution.executionId}-benchmark-configuration"
)
val config = buildConfig()
@@ -126,19 +128,19 @@ class TheodoliteExecutor(
} finally {
ioHandler.writeToJSONFile(
config.searchStrategy.benchmarkExecutor.results,
- "${resultsFolder}exp${this.config.executionId}-result"
+ "${resultsFolder}exp${this.benchmarkExecution.executionId}-result"
)
// Create expXYZ_demand.csv file or expXYZ_capacity.csv depending on metric
when(config.metric) {
Metric.DEMAND ->
ioHandler.writeToCSVFile(
- "${resultsFolder}exp${this.config.executionId}_demand",
+ "${resultsFolder}exp${this.benchmarkExecution.executionId}_demand",
calculateMetric(config.loads, config.searchStrategy.benchmarkExecutor.results),
listOf("load","resources")
)
Metric.CAPACITY ->
ioHandler.writeToCSVFile(
- "${resultsFolder}exp${this.config.executionId}_capacity",
+ "${resultsFolder}exp${this.benchmarkExecution.executionId}_capacity",
calculateMetric(config.resources, config.searchStrategy.benchmarkExecutor.results),
listOf("resource", "loads")
)
diff --git a/theodolite/src/test/kotlin/theodolite/InitialGuessSearchStrategyTest.kt b/theodolite/src/test/kotlin/theodolite/InitialGuessSearchStrategyTest.kt
index de7a1f8f170bb4c25aeb328d3f68e3a027712a62..dbe55ff36f591a45df3fd9898419befe5a5fdeb7 100644
--- a/theodolite/src/test/kotlin/theodolite/InitialGuessSearchStrategyTest.kt
+++ b/theodolite/src/test/kotlin/theodolite/InitialGuessSearchStrategyTest.kt
@@ -3,11 +3,11 @@ package theodolite
import io.quarkus.test.junit.QuarkusTest
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Test
-import theodolite.benchmark.BenchmarkExecution
import theodolite.strategies.searchstrategy.InitialGuessSearchStrategy
import theodolite.strategies.Metric
import theodolite.util.Results
import mu.KotlinLogging
+import theodolite.benchmark.Slo
import theodolite.strategies.searchstrategy.PrevInstanceOptGuess
private val logger = KotlinLogging.logger {}
@@ -31,7 +31,7 @@ class InitialGuessSearchStrategyTest {
val results = Results(Metric.from("demand"))
val benchmark = TestBenchmark()
val guessStrategy = PrevInstanceOptGuess()
- val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo()
+ val sloChecker = Slo()
val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, listOf(sloChecker), 0, 0, 5)
val strategy = InitialGuessSearchStrategy(benchmarkExecutor,guessStrategy, results)
@@ -69,7 +69,7 @@ class InitialGuessSearchStrategyTest {
val results = Results(Metric.from("demand"))
val benchmark = TestBenchmark()
val guessStrategy = PrevInstanceOptGuess()
- val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo()
+ val sloChecker = Slo()
val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, listOf(sloChecker), 0, 0, 5)
val strategy = InitialGuessSearchStrategy(benchmarkExecutor,guessStrategy, results)
@@ -107,7 +107,7 @@ class InitialGuessSearchStrategyTest {
val results = Results(Metric.from("demand"))
val benchmark = TestBenchmark()
val guessStrategy = PrevInstanceOptGuess()
- val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo()
+ val sloChecker = Slo()
val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, listOf(sloChecker), 0, 0, 5)
val strategy = InitialGuessSearchStrategy(benchmarkExecutor, guessStrategy, results)
diff --git a/theodolite/src/test/kotlin/theodolite/RestrictionSearchTest.kt b/theodolite/src/test/kotlin/theodolite/RestrictionSearchTest.kt
index 1e973852d8448d0bff3410d5bb84bc5543be497a..23fa99c6d1775f291949a9068399f5bcf6e5179a 100644
--- a/theodolite/src/test/kotlin/theodolite/RestrictionSearchTest.kt
+++ b/theodolite/src/test/kotlin/theodolite/RestrictionSearchTest.kt
@@ -3,7 +3,7 @@ package theodolite
import io.quarkus.test.junit.QuarkusTest
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Test
-import theodolite.benchmark.BenchmarkExecution
+import theodolite.benchmark.Slo
import theodolite.strategies.Metric
import theodolite.strategies.restriction.LowerBoundRestriction
import theodolite.strategies.searchstrategy.BinarySearch
@@ -31,7 +31,7 @@ class RestrictionSearchTest {
val mockResources: List<Int> = (0..6).toList()
val results = Results(Metric.from("demand"))
val benchmark = TestBenchmark()
- val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo()
+ val sloChecker = Slo()
val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, listOf(sloChecker), 0, 0, 5)
val linearSearch = LinearSearch(benchmarkExecutor)
val lowerBoundRestriction = LowerBoundRestriction(results)
@@ -64,7 +64,7 @@ class RestrictionSearchTest {
val mockResources: List<Int> = (0..6).toList()
val results = Results(Metric.from("demand"))
val benchmark = TestBenchmark()
- val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo()
+ val sloChecker = Slo()
val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, listOf(sloChecker), 0, 0, 5)
val fullSearch = FullSearch(benchmarkExecutor)
val lowerBoundRestriction = LowerBoundRestriction(results)
@@ -97,7 +97,7 @@ class RestrictionSearchTest {
val mockResources: List<Int> = (0..6).toList()
val results = Results(Metric.from("demand"))
val benchmark = TestBenchmark()
- val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo()
+ val sloChecker = Slo()
val benchmarkExecutorImpl =
TestBenchmarkExecutorImpl(mockResults, benchmark, results, listOf(sloChecker), 0, 0, 0)
val binarySearch = BinarySearch(benchmarkExecutorImpl)
@@ -130,7 +130,7 @@ class RestrictionSearchTest {
val mockResources: List<Int> = (0..7).toList()
val results = Results(Metric.from("demand"))
val benchmark = TestBenchmark()
- val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo()
+ val sloChecker = Slo()
val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, listOf(sloChecker), 0, 0, 0)
val binarySearch = BinarySearch(benchmarkExecutor)
val lowerBoundRestriction = LowerBoundRestriction(results)
diff --git a/theodolite/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt b/theodolite/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt
index a5346896f6d20b6d9c0a828d2d3798c70c0861c4..97197f3bad066235634869c8c37de4bc5c570f8b 100644
--- a/theodolite/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt
+++ b/theodolite/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt
@@ -1,19 +1,19 @@
package theodolite
import theodolite.benchmark.Benchmark
-import theodolite.benchmark.BenchmarkExecution
+import theodolite.benchmark.Slo
import theodolite.execution.BenchmarkExecutor
import theodolite.util.Results
import java.time.Duration
class TestBenchmarkExecutorImpl(
- private val mockResults: Array<Array<Boolean>>,
- benchmark: Benchmark,
- results: Results,
- slo: List<BenchmarkExecution.Slo>,
- executionId: Int,
- loadGenerationDelay: Long,
- afterTeardownDelay: Long
+ private val mockResults: Array<Array<Boolean>>,
+ benchmark: Benchmark,
+ results: Results,
+ slo: List<Slo>,
+ executionId: Int,
+ loadGenerationDelay: Long,
+ afterTeardownDelay: Long
) :
BenchmarkExecutor(
benchmark,
diff --git a/theodolite/src/test/kotlin/theodolite/execution/operator/BenchmarkCRDummy.kt b/theodolite/src/test/kotlin/theodolite/execution/operator/BenchmarkCRDummy.kt
index d6841429166d1549e84ad27887fbf0cba86b174d..152191bc271552dfb50c022c678a023ce0eb65cd 100644
--- a/theodolite/src/test/kotlin/theodolite/execution/operator/BenchmarkCRDummy.kt
+++ b/theodolite/src/test/kotlin/theodolite/execution/operator/BenchmarkCRDummy.kt
@@ -44,6 +44,7 @@ class BenchmarkCRDummy(name: String) {
benchmark.resourceTypes = emptyList()
benchmark.loadTypes = emptyList()
+ benchmark.slos = mutableListOf()
benchmark.kafkaConfig = kafkaConfig
benchmark.name = benchmarkCR.metadata.name
}
diff --git a/theodolite/src/test/kotlin/theodolite/execution/operator/SloFactoryTest.kt b/theodolite/src/test/kotlin/theodolite/execution/operator/SloFactoryTest.kt
new file mode 100644
index 0000000000000000000000000000000000000000..c3dcd1b9529b6f24bd5b0deda920ba3e3ebb2978
--- /dev/null
+++ b/theodolite/src/test/kotlin/theodolite/execution/operator/SloFactoryTest.kt
@@ -0,0 +1,72 @@
+package theodolite.execution.operator
+
+import io.quarkus.test.junit.QuarkusTest
+import org.junit.jupiter.api.Test
+import theodolite.benchmark.BenchmarkExecution
+import theodolite.benchmark.BenchmarkExecution.SloConfiguration
+import theodolite.benchmark.KubernetesBenchmark
+import theodolite.benchmark.Slo
+import theodolite.execution.SloFactory
+import org.junit.jupiter.api.Assertions.*
+
+@QuarkusTest
+internal class SloFactoryTest {
+
+ @Test
+ fun overwriteSloTest() {
+
+ val benchmark = KubernetesBenchmark()
+ val execution = BenchmarkExecution()
+
+ // Define Benchmark SLOs
+ val slo = Slo()
+ slo.name="test"
+ slo.sloType="lag trend"
+ slo.prometheusUrl="test.de"
+ slo.offset=0
+
+ val benchmarkSloProperties = mutableMapOf<String, String>()
+ benchmarkSloProperties["threshold"] = "2000"
+ benchmarkSloProperties["externalSloUrl"] = "http://localhost:80/evaluate-slope"
+ benchmarkSloProperties["warmup"] = "60"
+
+ slo.properties=benchmarkSloProperties
+
+ benchmark.slos = mutableListOf(slo)
+
+
+ // Define Execution SLOs, benchmark SLO values for these properties should be overwritten
+ val sloConfig = SloConfiguration()
+ sloConfig.name = "test"
+
+ val executionSloProperties = mutableMapOf<String, String>()
+ // overwriting properties 'threshold' and 'warmup' and adding property 'extensionTest'
+ executionSloProperties["threshold"] = "3000"
+ executionSloProperties["warmup"] = "80"
+ executionSloProperties["extensionTest"] = "extended"
+
+ sloConfig.properties = executionSloProperties
+
+ // SLO has 'name' that isn't defined in the benchmark, therefore it will be ignored by the SloFactory
+ val sloConfig2 = SloConfiguration()
+ sloConfig2.name = "test2"
+ sloConfig2.properties = executionSloProperties
+
+ execution.slos = listOf(sloConfig, sloConfig2)
+
+ val sloFactory = SloFactory()
+ val combinedSlos = sloFactory.createSlos(execution,benchmark)
+
+ assertEquals(1, combinedSlos.size)
+ assertEquals("test", combinedSlos[0].name)
+ assertEquals("lag trend", combinedSlos[0].sloType)
+ assertEquals("test.de", combinedSlos[0].prometheusUrl)
+ assertEquals(0, combinedSlos[0].offset)
+
+ assertEquals(4, combinedSlos[0].properties.size)
+ assertEquals("3000", combinedSlos[0].properties["threshold"])
+ assertEquals("http://localhost:80/evaluate-slope", combinedSlos[0].properties["externalSloUrl"])
+ assertEquals("80", combinedSlos[0].properties["warmup"])
+ assertEquals("extended", combinedSlos[0].properties["extensionTest"])
+ }
+}
\ No newline at end of file
diff --git a/theodolite/src/test/resources/k8s-resource-files/test-benchmark.yaml b/theodolite/src/test/resources/k8s-resource-files/test-benchmark.yaml
index 102a6a249ab06301396eaf375e7bd2590b334b22..ad26f6c658c72231887a8e3cd4c5dc17cc787641 100644
--- a/theodolite/src/test/resources/k8s-resource-files/test-benchmark.yaml
+++ b/theodolite/src/test/resources/k8s-resource-files/test-benchmark.yaml
@@ -34,6 +34,15 @@ spec:
resource: "uc1-load-generator-deployment.yaml"
properties:
loadGenMaxRecords: "15000"
+ slos:
+ - name: "lag trend"
+ sloType: "lag trend"
+ prometheusUrl: "http://prometheus-operated:9090"
+ offset: 0
+ properties:
+ threshold: 3000
+ externalSloUrl: "http://localhost:80/evaluate-slope"
+ warmup: 60 # in seconds
kafkaConfig:
bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092"
topics:
diff --git a/theodolite/src/test/resources/k8s-resource-files/test-execution-1.yaml b/theodolite/src/test/resources/k8s-resource-files/test-execution-1.yaml
index e250d8a9610be6c645ec2da4d6de5b9b59aaa929..077c4ab410e3fc78a9bb47c563c2b237b922c1ba 100644
--- a/theodolite/src/test/resources/k8s-resource-files/test-execution-1.yaml
+++ b/theodolite/src/test/resources/k8s-resource-files/test-execution-1.yaml
@@ -12,7 +12,7 @@ spec:
resourceType: "Instances"
resourceValues: [1, 2, 3, 4, 5]
slos:
- - sloType: "lag trend"
+ - name: "lag trend"
threshold: 2000
prometheusUrl: "http://prometheus-operated:9090"
externalSloUrl: "http://localhost:80/evaluate-slope"
diff --git a/theodolite/src/test/resources/k8s-resource-files/test-execution-update.yaml b/theodolite/src/test/resources/k8s-resource-files/test-execution-update.yaml
index 80476ba5c01a7c49b5058448117bc25e960eacee..504a73fe3b325368301897cacdc922e7f6e70430 100644
--- a/theodolite/src/test/resources/k8s-resource-files/test-execution-update.yaml
+++ b/theodolite/src/test/resources/k8s-resource-files/test-execution-update.yaml
@@ -12,13 +12,9 @@ spec:
resourceType: "Instances"
resourceValues: [1, 2, 3, 4, 5]
slos:
- - sloType: "lag trend"
- prometheusUrl: "http://prometheus-operated:9090"
- offset: 0
+ - name: "lag trend"
properties:
threshold: 2000
- externalSloUrl: "http://localhost:80/evaluate-slope"
- warmup: 60 # in seconds
execution:
strategy:
name: "RestrictionSearch"
diff --git a/theodolite/src/test/resources/k8s-resource-files/test-execution.yaml b/theodolite/src/test/resources/k8s-resource-files/test-execution.yaml
index 2df4760cd6deab72cae0303bb669852b3140baa6..2bd0bfc4b3602c52fc3cc5cce9729777c21f4ac4 100644
--- a/theodolite/src/test/resources/k8s-resource-files/test-execution.yaml
+++ b/theodolite/src/test/resources/k8s-resource-files/test-execution.yaml
@@ -12,13 +12,9 @@ spec:
resourceType: "Instances"
resourceValues: [1, 2, 3, 4, 5]
slos:
- - sloType: "lag trend"
- prometheusUrl: "http://prometheus-operated:9090"
- offset: 0
+ - name: "lag trend"
properties:
threshold: 2000
- externalSloUrl: "http://localhost:80/evaluate-slope"
- warmup: 60 # in seconds
execution:
strategy:
name: "RestrictionSearch"