diff --git a/helm/jmx-custom-map.yaml b/helm/jmx-custom-map.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5abe283ce6ae506b2cd908083f2b34e714500e2a
--- /dev/null
+++ b/helm/jmx-custom-map.yaml
@@ -0,0 +1,11 @@
+apiVersion: v1
+data:
+ jmx-kafka-prometheus.yml: |
+ jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi
+ lowercaseOutputName: true
+ lowercaseOutputLabelNames: true
+ ssl: false
+kind: ConfigMap
+metadata:
+ name: theodolite-cp-kafka-jmx-configmap
+ namespace: default
diff --git a/helm/templates/theodolite/crd-execution.yaml b/helm/templates/theodolite/crd-execution.yaml
index 163835e9b37aca774983d4f019cc61d4bde06510..f6305ad3eb01c41a44ee3e92cff100eb67321f65 100644
--- a/helm/templates/theodolite/crd-execution.yaml
+++ b/helm/templates/theodolite/crd-execution.yaml
@@ -60,6 +60,9 @@ spec:
prometheusUrl:
description: Connection string for Promehteus.
type: string
+ query:
+ description: The prometheus query string
+ type: string
offset:
description: Hours by which the start and end timestamp will be shifted (for different timezones).
type: integer
diff --git a/slope-evaluator/app/main.py b/slope-evaluator/app/main.py
index 6f6788f0ca84b7710be5b509ca4f0641047e963d..621fa0cfc9c27e809fd92752de93f2795fa32c05 100644
--- a/slope-evaluator/app/main.py
+++ b/slope-evaluator/app/main.py
@@ -38,7 +38,7 @@ def calculate_slope_trend(results, warmup):
err_msg = 'Computing trend slope failed.'
logger.exception(err_msg)
logger.error('Mark this subexperiment as not successful and continue benchmark.')
- return False
+ return float('inf')
logger.info("Computed lag trend slope is '%s'", trend_slope)
return trend_slope
@@ -49,7 +49,7 @@ def check_service_level_objective(results, threshold):
@app.post("/evaluate-slope",response_model=bool)
async def evaluate_slope(request: Request):
data = json.loads(await request.body())
- results = [calculate_slope_trend(total_lag, data['warmup']) for total_lag in data['total_lags']]
- return check_service_level_objective(results=results, threshold=data["threshold"])
+ results = [calculate_slope_trend(total_lag, data['metadata']['warmup']) for total_lag in data['results']]
+ return check_service_level_objective(results=results, threshold=data['metadata']["threshold"])
logger.info("SLO evaluator is online")
\ No newline at end of file
diff --git a/slope-evaluator/resources/test-1-rep-success.json b/slope-evaluator/resources/test-1-rep-success.json
index 9e315c707be7b2a874c58fcb1093aa86f7676560..dfe11282720ebfcdd60582b7717da892bc85a923 100644
--- a/slope-evaluator/resources/test-1-rep-success.json
+++ b/slope-evaluator/resources/test-1-rep-success.json
@@ -1,5 +1,5 @@
{
- "total_lags": [
+ "results": [
[
{
"metric": {
@@ -134,6 +134,8 @@
}
]
],
- "threshold": 2000,
- "warmup": 0
+ "metadata": {
+ "threshold": 2000,
+ "warmup": 0
+ }
}
\ No newline at end of file
diff --git a/slope-evaluator/resources/test-3-rep-success.json b/slope-evaluator/resources/test-3-rep-success.json
index 485966cba40f01e4a646e626914510ba49b707bc..cf483f42f3783aecd1f428ac7bbbe2090c4cade0 100644
--- a/slope-evaluator/resources/test-3-rep-success.json
+++ b/slope-evaluator/resources/test-3-rep-success.json
@@ -1,5 +1,5 @@
{
- "total_lags": [
+ "results": [
[
{
"metric": {
@@ -284,6 +284,8 @@
}
]
],
- "threshold": 2000,
- "warmup": 0
+ "metadata": {
+ "threshold": 2000,
+ "warmup": 0
+ }
}
\ No newline at end of file
diff --git a/theodolite/crd/crd-execution.yaml b/theodolite/crd/crd-execution.yaml
index 47d0306f5150a8126f021c40bf3c4a4ce0e1abb1..0debaa46dcc8cfb4e5a22ce3f8bd2d9247012fcd 100644
--- a/theodolite/crd/crd-execution.yaml
+++ b/theodolite/crd/crd-execution.yaml
@@ -51,7 +51,7 @@ spec:
description: The type of the resource. It must match one of the resource types specified in the referenced benchmark.
type: string
resourceValues:
- descriptoin: List of resource values for the specified resource type.
+ description: List of resource values for the specified resource type.
type: array
items:
type: integer
@@ -68,6 +68,9 @@ spec:
prometheusUrl:
description: Connection string for Promehteus.
type: string
+ query:
+ description: The prometheus query string
+ type: string
offset:
description: Hours by which the start and end timestamp will be shifted (for different timezones).
type: integer
diff --git a/theodolite/examples/operator/example-execution.yaml b/theodolite/examples/operator/example-execution.yaml
index e2efb6e9a2bb6c08354b57a83506a601ac0ed96e..52bd499e5854d8940743c4215d0a4783352b3470 100644
--- a/theodolite/examples/operator/example-execution.yaml
+++ b/theodolite/examples/operator/example-execution.yaml
@@ -14,6 +14,7 @@ spec:
- sloType: "lag trend"
prometheusUrl: "http://prometheus-operated:9090"
offset: 0
+ query: "sum by(group)(kafka_consumergroup_group_lag >= 0)"
properties:
threshold: 2000
externalSloUrl: "http://localhost:80/evaluate-slope"
diff --git a/theodolite/examples/standalone/example-execution.yaml b/theodolite/examples/standalone/example-execution.yaml
index 6e649df957fe1d5dd962fdd5fe5152808e722de6..d25aa3866ae8b38208db62c85a4afe1339d0f241 100644
--- a/theodolite/examples/standalone/example-execution.yaml
+++ b/theodolite/examples/standalone/example-execution.yaml
@@ -10,6 +10,7 @@ slos:
- sloType: "lag trend"
prometheusUrl: "http://prometheus-operated:9090"
offset: 0
+ query: "sum by(group)(kafka_consumergroup_group_lag >= 0)"
properties:
threshold: 2000
externalSloUrl: "http://localhost:80/evaluate-slope"
diff --git a/theodolite/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt b/theodolite/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt
index f2dda487d390c5f771e4f47c0f9c7ebf2cf971e7..6aeeb2c542f109ad36a62bddc0cab94c0f389159 100644
--- a/theodolite/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt
+++ b/theodolite/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt
@@ -63,6 +63,7 @@ class BenchmarkExecution : KubernetesResource {
class Slo : KubernetesResource {
lateinit var sloType: String
lateinit var prometheusUrl: String
+ lateinit var query: String
var offset by Delegates.notNull<Int>()
lateinit var properties: MutableMap<String, String>
}
diff --git a/theodolite/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt b/theodolite/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt
index 777ea1c0ed43ac3af244dc0aaf770c69c11718cf..8e363a18e8f687c702771ae8797c44c122de4766 100644
--- a/theodolite/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt
+++ b/theodolite/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt
@@ -37,6 +37,7 @@ class AnalysisExecutor(
* @return true if the experiment succeeded.
*/
fun analyze(load: LoadDimension, res: Resource, executionIntervals: List<Pair<Instant, Instant>>): Boolean {
+
var result = false
var repetitionCounter = 1
@@ -50,7 +51,7 @@ class AnalysisExecutor(
fetcher.fetchMetric(
start = interval.first,
end = interval.second,
- query = RECORD_LAG_QUERY
+ query = slo.query
)
}
@@ -58,7 +59,7 @@ class AnalysisExecutor(
ioHandler.writeToCSVFile(
fileURL = "${fileURL}_${repetitionCounter++}",
data = data.getResultAsList(),
- columns = listOf("group", "timestamp", "value")
+ columns = listOf("labels", "timestamp", "value")
)
}
diff --git a/theodolite/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt b/theodolite/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt
index 448a2a05f8dbeb1aef153895360bfb40e7275224..d646286b70bc5880df1f603afdc2bda22bcc3259 100644
--- a/theodolite/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt
+++ b/theodolite/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt
@@ -36,13 +36,12 @@ class ExternalSloChecker(
*/
override fun evaluate(fetchedData: List<PrometheusResponse>): Boolean {
var counter = 0
- val data = Gson().toJson(
- mapOf(
- "total_lags" to fetchedData.map { it.data?.result },
- "threshold" to threshold,
- "warmup" to warmup
- )
- )
+ val data = SloJson.Builder()
+ .results(fetchedData.map { it.data?.result })
+ .addMetadata("threshold", threshold)
+ .addMetadata( "warmup", warmup)
+ .build()
+ .toJson()
while (counter < RETRIES) {
val result = post(externalSlopeURL, data = data, timeout = TIMEOUT)
diff --git a/theodolite/src/main/kotlin/theodolite/evaluation/SloJson.kt b/theodolite/src/main/kotlin/theodolite/evaluation/SloJson.kt
new file mode 100644
index 0000000000000000000000000000000000000000..fc9fe17b255dbb5ae68881538d8d2a50a191edb1
--- /dev/null
+++ b/theodolite/src/main/kotlin/theodolite/evaluation/SloJson.kt
@@ -0,0 +1,63 @@
+package theodolite.evaluation
+
+import com.google.gson.Gson
+import theodolite.util.PromResult
+
+class SloJson private constructor(
+ val results: List<List<PromResult>?>? = null,
+ var metadata: MutableMap<String, Any>? = null
+) {
+
+ data class Builder(
+ var results:List<List<PromResult>?>? = null,
+ var metadata: MutableMap<String, Any>? = null
+ ) {
+
+ /**
+ * Set the results
+ *
+ * @param results list of prometheus results
+ */
+ fun results(results: List<List<PromResult>?>) = apply { this.results = results }
+
+ /**
+ * Add metadata as key value pairs
+ *
+ * @param key key of the metadata to be added
+ * @param value value of the metadata to be added
+ */
+ fun addMetadata(key: String, value: String) = apply {
+ if (this.metadata.isNullOrEmpty()) {
+ this.metadata = mutableMapOf(key to value)
+ } else {
+ this.metadata!![key] = value
+ }
+ }
+
+ /**
+ * Add metadata as key value pairs
+ *
+ * @param key key of the metadata to be added
+ * @param value value of the metadata to be added
+ */
+ fun addMetadata(key: String, value: Int) = apply {
+ if (this.metadata.isNullOrEmpty()) {
+ this.metadata = mutableMapOf(key to value)
+ } else {
+ this.metadata!![key] = value
+ }
+ }
+
+ fun build() = SloJson(
+ results = results,
+ metadata = metadata
+ )
+ }
+
+ fun toJson(): String {
+ return Gson().toJson(mapOf(
+ "results" to this.results,
+ "metadata" to this.metadata
+ ))
+ }
+}
\ No newline at end of file
diff --git a/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt b/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt
index e7b511d8c83b5abccece1204aad2a4a9ecfdfd26..e699a6f8e4ddf2fe588c8acd1044a29acbac7f8a 100644
--- a/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt
+++ b/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt
@@ -25,7 +25,7 @@ abstract class BenchmarkExecutor(
val results: Results,
val executionDuration: Duration,
val configurationOverrides: List<ConfigurationOverride?>,
- val slo: BenchmarkExecution.Slo,
+ val slos: List<BenchmarkExecution.Slo>,
val repetitions: Int,
val executionId: Int,
val loadGenerationDelay: Long,
diff --git a/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt b/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt
index c54d1878d4957a0b8ec6a8fdfb18ec6342d7bfc1..e6f55d982f1ea226653ecf798189349c2104e772 100644
--- a/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt
+++ b/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt
@@ -20,7 +20,7 @@ class BenchmarkExecutorImpl(
results: Results,
executionDuration: Duration,
configurationOverrides: List<ConfigurationOverride?>,
- slo: BenchmarkExecution.Slo,
+ slos: List<BenchmarkExecution.Slo>,
repetitions: Int,
executionId: Int,
loadGenerationDelay: Long,
@@ -30,7 +30,7 @@ class BenchmarkExecutorImpl(
results,
executionDuration,
configurationOverrides,
- slo,
+ slos,
repetitions,
executionId,
loadGenerationDelay,
@@ -53,13 +53,19 @@ class BenchmarkExecutorImpl(
* Analyse the experiment, if [run] is true, otherwise the experiment was canceled by the user.
*/
if (this.run.get()) {
- result = AnalysisExecutor(slo = slo, executionId = executionId)
- .analyze(
- load = load,
- res = res,
- executionIntervals = executionIntervals
- )
+
+ val experimentResults = slos.map {
+ AnalysisExecutor(slo = it, executionId = executionId)
+ .analyze(
+ load = load,
+ res = res,
+ executionIntervals = executionIntervals
+ )
+ }
+
+ result = (false !in experimentResults)
this.results.setResult(Pair(load, res), result)
+
}
return result
}
diff --git a/theodolite/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt b/theodolite/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt
index bc9371763c30e4cef913a368b64e9989e7f2286b..0368435fa8e2f6ac3b664e1d761bcf910ca010a0 100644
--- a/theodolite/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt
+++ b/theodolite/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt
@@ -64,7 +64,7 @@ class TheodoliteExecutor(
results = results,
executionDuration = executionDuration,
configurationOverrides = config.configOverrides,
- slo = config.slos[0],
+ slos = config.slos,
repetitions = config.execution.repetitions,
executionId = config.executionId,
loadGenerationDelay = config.execution.loadGenerationDelay,
diff --git a/theodolite/src/main/kotlin/theodolite/util/PrometheusResponse.kt b/theodolite/src/main/kotlin/theodolite/util/PrometheusResponse.kt
index bf33fcf6104645727a13b92cf3a13d36e04a10c6..72fa6779cb16ebaeaeda63915671a0ecb800c97c 100644
--- a/theodolite/src/main/kotlin/theodolite/util/PrometheusResponse.kt
+++ b/theodolite/src/main/kotlin/theodolite/util/PrometheusResponse.kt
@@ -23,7 +23,7 @@ data class PrometheusResponse(
* The format of the returned list is: `[[ group, timestamp, value ], [ group, timestamp, value ], ... ]`
*/
fun getResultAsList(): List<List<String>> {
- val group = data?.result?.get(0)?.metric?.group.toString()
+ val group = data?.result?.get(0)?.metric?.toString()!!
val values = data?.result?.get(0)?.values
val result = mutableListOf<List<String>>()
@@ -64,18 +64,9 @@ data class PromResult(
/**
* Label of the metric
*/
- var metric: PromMetric? = null,
+ var metric: Map<String,String>? = null,
/**
* Values of the metric (e.g. [ [ <unix_time>, "<sample_value>" ], ... ])
*/
var values: List<Any>? = null
-)
-
-/**
- * Corresponds to the metric field in the range-vector result format of a Prometheus range-query response.
- */
-@RegisterForReflection
-data class PromMetric(
- var group: String? = null
-)
-
+)
\ No newline at end of file
diff --git a/theodolite/src/test/kotlin/theodolite/CompositeStrategyTest.kt b/theodolite/src/test/kotlin/theodolite/CompositeStrategyTest.kt
index 49131352cfe517a382ddd7aa1be09d3fbe317466..675ba9e3519ee9583a729933b7b6093ab26dd151 100644
--- a/theodolite/src/test/kotlin/theodolite/CompositeStrategyTest.kt
+++ b/theodolite/src/test/kotlin/theodolite/CompositeStrategyTest.kt
@@ -31,7 +31,7 @@ class CompositeStrategyTest {
val results = Results()
val benchmark = TestBenchmark()
val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo()
- val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, sloChecker, 0, 0, 5)
+ val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, listOf(sloChecker), 0, 0, 5)
val linearSearch = LinearSearch(benchmarkExecutor)
val lowerBoundRestriction = LowerBoundRestriction(results)
val strategy =
@@ -65,7 +65,7 @@ class CompositeStrategyTest {
val benchmark = TestBenchmark()
val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo()
val benchmarkExecutorImpl =
- TestBenchmarkExecutorImpl(mockResults, benchmark, results, sloChecker, 0, 0, 0)
+ TestBenchmarkExecutorImpl(mockResults, benchmark, results, listOf(sloChecker), 0, 0, 0)
val binarySearch = BinarySearch(benchmarkExecutorImpl)
val lowerBoundRestriction = LowerBoundRestriction(results)
val strategy =
@@ -98,7 +98,7 @@ class CompositeStrategyTest {
val results = Results()
val benchmark = TestBenchmark()
val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo()
- val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, sloChecker, 0, 0, 0)
+ val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, listOf(sloChecker), 0, 0, 0)
val binarySearch = BinarySearch(benchmarkExecutor)
val lowerBoundRestriction = LowerBoundRestriction(results)
val strategy =
@@ -114,4 +114,5 @@ class CompositeStrategyTest {
assertEquals(actual, expected)
}
+
}
diff --git a/theodolite/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt b/theodolite/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt
index cbd2d5926d61b0bfd4de6fab0c14422ddf88f190..ac7c7883f0c0929f8b7ea6df67c1a74b6b454f69 100644
--- a/theodolite/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt
+++ b/theodolite/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt
@@ -12,7 +12,7 @@ class TestBenchmarkExecutorImpl(
private val mockResults: Array<Array<Boolean>>,
benchmark: Benchmark,
results: Results,
- slo: BenchmarkExecution.Slo,
+ slo: List<BenchmarkExecution.Slo>,
executionId: Int,
loadGenerationDelay: Long,
afterTeardownDelay: Long
@@ -22,7 +22,7 @@ class TestBenchmarkExecutorImpl(
results,
executionDuration = Duration.ofSeconds(1),
configurationOverrides = emptyList(),
- slo = slo,
+ slos = slo,
repetitions = 1,
executionId = executionId,
loadGenerationDelay = loadGenerationDelay,
diff --git a/theodolite/src/test/resources/k8s-resource-files/test-execution-update.yaml b/theodolite/src/test/resources/k8s-resource-files/test-execution-update.yaml
index c075702da218397352f1dc1e5b283534fbb4d718..4689e468d379685463d2be71a0dde224f5766e9e 100644
--- a/theodolite/src/test/resources/k8s-resource-files/test-execution-update.yaml
+++ b/theodolite/src/test/resources/k8s-resource-files/test-execution-update.yaml
@@ -15,6 +15,7 @@ spec:
- sloType: "lag trend"
prometheusUrl: "http://prometheus-operated:9090"
offset: 0
+ query: "sum by(group)(kafka_consumergroup_group_lag >= 0)"
properties:
threshold: 2000
externalSloUrl: "http://localhost:80/evaluate-slope"
diff --git a/theodolite/src/test/resources/k8s-resource-files/test-execution.yaml b/theodolite/src/test/resources/k8s-resource-files/test-execution.yaml
index e12c851da5d8a79f57b1fa59b86239c219370c0f..816915ad81ebb9fe6382740111db302388c5e4e3 100644
--- a/theodolite/src/test/resources/k8s-resource-files/test-execution.yaml
+++ b/theodolite/src/test/resources/k8s-resource-files/test-execution.yaml
@@ -15,6 +15,7 @@ spec:
- sloType: "lag trend"
prometheusUrl: "http://prometheus-operated:9090"
offset: 0
+ query: "sum by(group)(kafka_consumergroup_group_lag >= 0)"
properties:
threshold: 2000
externalSloUrl: "http://localhost:80/evaluate-slope"