diff --git a/theodolite/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt b/theodolite/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt
index a7fe55ec35442b42c48499fa9d697e1491ca06d9..281c68e318784ee8206473cd014f814b3f5152a9 100644
--- a/theodolite/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt
+++ b/theodolite/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt
@@ -13,7 +13,6 @@ import java.util.*
import java.util.regex.Pattern
private val logger = KotlinLogging.logger {}
-private val RECORD_LAG_QUERY = "sum by(group)(kafka_consumergroup_group_lag >= 0)"
/**
* Contains the analysis. Fetches a metric from Prometheus, documents it, and evaluates it.
@@ -51,7 +50,7 @@ class AnalysisExecutor(
fetcher.fetchMetric(
start = interval.first,
end = interval.second,
- query = RECORD_LAG_QUERY
+ query = SloConfigHandler.getQueryString(sloType = slo.sloType)
)
}
@@ -59,7 +58,7 @@ class AnalysisExecutor(
ioHandler.writeToCSVFile(
fileURL = "${fileURL}_${repetitionCounter++}",
data = data.getResultAsList(),
- columns = listOf("group", "timestamp", "value")
+ columns = listOf("labels", "timestamp", "value")
)
}
diff --git a/theodolite/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt b/theodolite/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt
index 448a2a05f8dbeb1aef153895360bfb40e7275224..d646286b70bc5880df1f603afdc2bda22bcc3259 100644
--- a/theodolite/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt
+++ b/theodolite/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt
@@ -36,13 +36,12 @@ class ExternalSloChecker(
*/
override fun evaluate(fetchedData: List<PrometheusResponse>): Boolean {
var counter = 0
- val data = Gson().toJson(
- mapOf(
- "total_lags" to fetchedData.map { it.data?.result },
- "threshold" to threshold,
- "warmup" to warmup
- )
- )
+ val data = SloJson.Builder()
+ .results(fetchedData.map { it.data?.result })
+ .addMetadata("threshold", threshold)
+ .addMetadata( "warmup", warmup)
+ .build()
+ .toJson()
while (counter < RETRIES) {
val result = post(externalSlopeURL, data = data, timeout = TIMEOUT)
diff --git a/theodolite/src/main/kotlin/theodolite/evaluation/SloCheckerFactory.kt b/theodolite/src/main/kotlin/theodolite/evaluation/SloCheckerFactory.kt
index 93e8e6180f5a99486e500af022869d896067d128..64f9110cd931feef41dc65f88d6623e82f4e03a2 100644
--- a/theodolite/src/main/kotlin/theodolite/evaluation/SloCheckerFactory.kt
+++ b/theodolite/src/main/kotlin/theodolite/evaluation/SloCheckerFactory.kt
@@ -43,22 +43,23 @@ class SloCheckerFactory {
properties: MutableMap<String, String>,
load: LoadDimension
): SloChecker {
- return when (sloType) {
- "lag trend" -> ExternalSloChecker(
+ return when (sloType.toLowerCase()) {
+ SloTypes.LAG_TREND.value, SloTypes.DROPPED_RECORDS.value -> ExternalSloChecker(
externalSlopeURL = properties["externalSloUrl"]
?: throw IllegalArgumentException("externalSloUrl expected"),
threshold = properties["threshold"]?.toInt() ?: throw IllegalArgumentException("threshold expected"),
warmup = properties["warmup"]?.toInt() ?: throw IllegalArgumentException("warmup expected")
)
- "lag trend ratio" -> {
- var thresholdRatio =
+
+ SloTypes.LAG_TREND_RATIO.value, SloTypes.DROPPED_RECORDS_RATIO.value -> {
+ val thresholdRatio =
properties["ratio"]?.toDouble()
?: throw IllegalArgumentException("ratio for threshold expected")
if (thresholdRatio < 0.0) {
throw IllegalArgumentException("Threshold ratio needs to be an Double greater or equal 0.0")
}
// cast to int, as rounding is not really necessary
- var threshold = (load.get() * thresholdRatio).toInt()
+ val threshold = (load.get() * thresholdRatio).toInt()
ExternalSloChecker(
externalSlopeURL = properties["externalSloUrl"]
diff --git a/theodolite/src/main/kotlin/theodolite/evaluation/SloConfigHandler.kt b/theodolite/src/main/kotlin/theodolite/evaluation/SloConfigHandler.kt
new file mode 100644
index 0000000000000000000000000000000000000000..93929218c822030ff065dafb19cce1fbaa69a179
--- /dev/null
+++ b/theodolite/src/main/kotlin/theodolite/evaluation/SloConfigHandler.kt
@@ -0,0 +1,20 @@
+package theodolite.evaluation
+
+import theodolite.util.InvalidPatcherConfigurationException
+import javax.enterprise.context.ApplicationScoped
+
+private const val CONSUMER_LAG_QUERY = "sum by(group)(kafka_consumergroup_group_lag >= 0)"
+private const val DROPPED_RECORDS_QUERY = "sum by(job) (kafka_streams_stream_task_metrics_dropped_records_total>=0)"
+
+@ApplicationScoped
+class SloConfigHandler() {
+ companion object {
+ fun getQueryString(sloType: String): String {
+ return when (sloType.toLowerCase()) {
+ SloTypes.LAG_TREND.value, SloTypes.LAG_TREND_RATIO.value -> CONSUMER_LAG_QUERY
+ SloTypes.DROPPED_RECORDS.value, SloTypes.DROPPED_RECORDS_RATIO.value -> DROPPED_RECORDS_QUERY
+ else -> throw InvalidPatcherConfigurationException("Could not find Prometheus query string for slo type $sloType")
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/theodolite/src/main/kotlin/theodolite/evaluation/SloJson.kt b/theodolite/src/main/kotlin/theodolite/evaluation/SloJson.kt
new file mode 100644
index 0000000000000000000000000000000000000000..fc9fe17b255dbb5ae68881538d8d2a50a191edb1
--- /dev/null
+++ b/theodolite/src/main/kotlin/theodolite/evaluation/SloJson.kt
@@ -0,0 +1,63 @@
+package theodolite.evaluation
+
+import com.google.gson.Gson
+import theodolite.util.PromResult
+
+class SloJson private constructor(
+ val results: List<List<PromResult>?>? = null,
+ var metadata: MutableMap<String, Any>? = null
+) {
+
+ data class Builder(
+ var results:List<List<PromResult>?>? = null,
+ var metadata: MutableMap<String, Any>? = null
+ ) {
+
+ /**
+ * Set the results
+ *
+ * @param results list of prometheus results
+ */
+ fun results(results: List<List<PromResult>?>) = apply { this.results = results }
+
+ /**
+ * Add metadata as key value pairs
+ *
+ * @param key key of the metadata to be added
+ * @param value value of the metadata to be added
+ */
+ fun addMetadata(key: String, value: String) = apply {
+ if (this.metadata.isNullOrEmpty()) {
+ this.metadata = mutableMapOf(key to value)
+ } else {
+ this.metadata!![key] = value
+ }
+ }
+
+ /**
+ * Add metadata as key value pairs
+ *
+ * @param key key of the metadata to be added
+ * @param value value of the metadata to be added
+ */
+ fun addMetadata(key: String, value: Int) = apply {
+ if (this.metadata.isNullOrEmpty()) {
+ this.metadata = mutableMapOf(key to value)
+ } else {
+ this.metadata!![key] = value
+ }
+ }
+
+ fun build() = SloJson(
+ results = results,
+ metadata = metadata
+ )
+ }
+
+ fun toJson(): String {
+ return Gson().toJson(mapOf(
+ "results" to this.results,
+ "metadata" to this.metadata
+ ))
+ }
+}
\ No newline at end of file
diff --git a/theodolite/src/main/kotlin/theodolite/evaluation/SloTypes.kt b/theodolite/src/main/kotlin/theodolite/evaluation/SloTypes.kt
new file mode 100644
index 0000000000000000000000000000000000000000..ac9de35861b0bd9c012bfb0b8cfcb2e1aa5aed68
--- /dev/null
+++ b/theodolite/src/main/kotlin/theodolite/evaluation/SloTypes.kt
@@ -0,0 +1,10 @@
+package theodolite.evaluation
+
+enum class SloTypes(val value: String) {
+ LAG_TREND("lag trend"),
+ LAG_TREND_RATIO("lag trend ratio"),
+ DROPPED_RECORDS("dropped records"),
+ DROPPED_RECORDS_RATIO("dropped records ratio")
+
+
+}
\ No newline at end of file
diff --git a/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt b/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt
index fcc0f3b728ac368ca061d7b0acdc0ffde53398b9..3238f447be06ce6486bb7f6ca1758700f36ba558 100644
--- a/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt
+++ b/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt
@@ -25,7 +25,7 @@ abstract class BenchmarkExecutor(
val results: Results,
val executionDuration: Duration,
val configurationOverrides: List<ConfigurationOverride?>,
- val slo: BenchmarkExecution.Slo,
+ val slos: List<BenchmarkExecution.Slo>,
val repetitions: Int,
val executionId: Int,
val loadGenerationDelay: Long,
diff --git a/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt b/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt
index f5f66de96e85dc937e2b4451142ec7327ec633e2..2e938be3a6e503a5e7e3f94c18a9454e173db5b0 100644
--- a/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt
+++ b/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt
@@ -18,7 +18,7 @@ class BenchmarkExecutorImpl(
results: Results,
executionDuration: Duration,
configurationOverrides: List<ConfigurationOverride?>,
- slo: BenchmarkExecution.Slo,
+ slos: List<BenchmarkExecution.Slo>,
repetitions: Int,
executionId: Int,
loadGenerationDelay: Long,
@@ -29,7 +29,7 @@ class BenchmarkExecutorImpl(
results,
executionDuration,
configurationOverrides,
- slo,
+ slos,
repetitions,
executionId,
loadGenerationDelay,
@@ -56,12 +56,16 @@ class BenchmarkExecutorImpl(
* Analyse the experiment, if [run] is true, otherwise the experiment was canceled by the user.
*/
if (this.run.get()) {
- result = AnalysisExecutor(slo = slo, executionId = executionId)
- .analyze(
- load = load,
- res = res,
- executionIntervals = executionIntervals
- )
+ val experimentResults = slos.map {
+ AnalysisExecutor(slo = it, executionId = executionId)
+ .analyze(
+ load = load,
+ res = res,
+ executionIntervals = executionIntervals
+ )
+ }
+
+ result = (false !in experimentResults)
this.results.setResult(Pair(load, res), result)
}
diff --git a/theodolite/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt b/theodolite/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt
index fc312888131a35f3004c37df228cebebce16530f..a5a4904f8ea8de152932333a1b8302f9539e260b 100644
--- a/theodolite/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt
+++ b/theodolite/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt
@@ -61,7 +61,7 @@ class TheodoliteExecutor(
results = results,
executionDuration = executionDuration,
configurationOverrides = config.configOverrides,
- slo = config.slos[0],
+ slos = config.slos,
repetitions = config.execution.repetitions,
executionId = config.executionId,
loadGenerationDelay = config.execution.loadGenerationDelay,
diff --git a/theodolite/src/main/kotlin/theodolite/util/PrometheusResponse.kt b/theodolite/src/main/kotlin/theodolite/util/PrometheusResponse.kt
index bf33fcf6104645727a13b92cf3a13d36e04a10c6..9b0b0dd4e0a5a48072ca576e874cb850c5f8df3b 100644
--- a/theodolite/src/main/kotlin/theodolite/util/PrometheusResponse.kt
+++ b/theodolite/src/main/kotlin/theodolite/util/PrometheusResponse.kt
@@ -23,7 +23,7 @@ data class PrometheusResponse(
* The format of the returned list is: `[[ group, timestamp, value ], [ group, timestamp, value ], ... ]`
*/
fun getResultAsList(): List<List<String>> {
- val group = data?.result?.get(0)?.metric?.group.toString()
+ val group = data?.result?.get(0)?.metric?.toString()!!
val values = data?.result?.get(0)?.values
val result = mutableListOf<List<String>>()
@@ -64,18 +64,9 @@ data class PromResult(
/**
* Label of the metric
*/
- var metric: PromMetric? = null,
+ var metric: Map<String, String>? = null,
/**
* Values of the metric (e.g. [ [ <unix_time>, "<sample_value>" ], ... ])
*/
var values: List<Any>? = null
-)
-
-/**
- * Corresponds to the metric field in the range-vector result format of a Prometheus range-query response.
- */
-@RegisterForReflection
-data class PromMetric(
- var group: String? = null
-)
-
+)
\ No newline at end of file
diff --git a/theodolite/src/test/kotlin/theodolite/CompositeStrategyTest.kt b/theodolite/src/test/kotlin/theodolite/CompositeStrategyTest.kt
index 49131352cfe517a382ddd7aa1be09d3fbe317466..580d9e747bde687a91ffb1bce2e7c9dfb6f166a2 100644
--- a/theodolite/src/test/kotlin/theodolite/CompositeStrategyTest.kt
+++ b/theodolite/src/test/kotlin/theodolite/CompositeStrategyTest.kt
@@ -31,7 +31,7 @@ class CompositeStrategyTest {
val results = Results()
val benchmark = TestBenchmark()
val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo()
- val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, sloChecker, 0, 0, 5)
+ val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, listOf(sloChecker), 0, 0, 5)
val linearSearch = LinearSearch(benchmarkExecutor)
val lowerBoundRestriction = LowerBoundRestriction(results)
val strategy =
@@ -65,7 +65,7 @@ class CompositeStrategyTest {
val benchmark = TestBenchmark()
val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo()
val benchmarkExecutorImpl =
- TestBenchmarkExecutorImpl(mockResults, benchmark, results, sloChecker, 0, 0, 0)
+ TestBenchmarkExecutorImpl(mockResults, benchmark, results, listOf(sloChecker), 0, 0, 0)
val binarySearch = BinarySearch(benchmarkExecutorImpl)
val lowerBoundRestriction = LowerBoundRestriction(results)
val strategy =
@@ -98,7 +98,7 @@ class CompositeStrategyTest {
val results = Results()
val benchmark = TestBenchmark()
val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo()
- val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, sloChecker, 0, 0, 0)
+ val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, listOf(sloChecker), 0, 0, 0)
val binarySearch = BinarySearch(benchmarkExecutor)
val lowerBoundRestriction = LowerBoundRestriction(results)
val strategy =
diff --git a/theodolite/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt b/theodolite/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt
index 948b60c66043dc78e0dc9800b4211d3f891779a3..2efddc48cb93a0870d1716c58a7018145c16e2ff 100644
--- a/theodolite/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt
+++ b/theodolite/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt
@@ -12,7 +12,7 @@ class TestBenchmarkExecutorImpl(
private val mockResults: Array<Array<Boolean>>,
benchmark: Benchmark,
results: Results,
- slo: BenchmarkExecution.Slo,
+ slo: List<BenchmarkExecution.Slo>,
executionId: Int,
loadGenerationDelay: Long,
afterTeardownDelay: Long
@@ -22,7 +22,7 @@ class TestBenchmarkExecutorImpl(
results,
executionDuration = Duration.ofSeconds(1),
configurationOverrides = emptyList(),
- slo = slo,
+ slos = slo,
repetitions = 1,
executionId = executionId,
loadGenerationDelay = loadGenerationDelay,