Made slo configurable + renaming of SLO to slo

slope-evaluator/api.py

@@ -14,7 +14,7 @@ logger = logging.getLogger("Api")
 
 logger.setLevel(logging.INFO)
 
-def execute(results, threshold):
+def execute(results, threshold, warmup):
     d = []
     for result in results:
         group = result['metric']['group']
@@ -26,7 +26,7 @@ def execute(results, threshold):
 
     logger.info(df)
     try:
-        trend_slope = trend_slope_computer.compute(df, 0)
+        trend_slope = trend_slope_computer.compute(df, warmup)
     except Exception as e:
         err_msg = 'Computing trend slope failed'
         logger.exception(err_msg)
@@ -41,4 +41,4 @@ def execute(results, threshold):
 async def evaluate_slope(request: Request):
     data = json.loads(await request.body())
     logger.info("Request received")
-    return execute(data['total_lag'], data['threshold'])
+    return execute(data['total_lag'], data['threshold'],data['warmup'])

theodolite-quarkus/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt

@@ -22,6 +22,10 @@ class BenchmarkExecution {
     class Slo {
         lateinit var sloType: String
         var threshold by Delegates.notNull<Int>()
+        lateinit var prometheusUrl: String
+        lateinit var externalSloUrl: String
+        var offset by Delegates.notNull<Int>()
+        var warmup by Delegates.notNull<Int>()
     }
 
     class LoadDefinition {

theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SLOChecker.kt → theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SloChecker.kt

@@ -2,6 +2,6 @@ package theodolite.evaluation
 
 import java.time.Instant
 
-interface SLOChecker {
+interface SloChecker {
     fun evaluate(start: Instant, end: Instant): Boolean
 }

theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SLOCheckerImpl.kt → theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SloCheckerImpl.kt

@@ -6,17 +6,25 @@ import java.net.ConnectException
 import java.time.Duration
 import java.time.Instant
 
-class SLOCheckerImpl(private val prometheusURL: String, private val threshold: Int, private val offset: Duration) :
-    SLOChecker {
+class SloCheckerImpl(
+    private val prometheusURL: String,
+    private val query: String,
+    private val externalSlopeURL: String,
+    private val threshold: Int,
+    private val offset: Duration,
+    private val warmup: Int
+) :
+    SloChecker {
 
     override fun evaluate(start: Instant, end: Instant): Boolean {
         var counter = 0
         val metricFetcher = MetricFetcher(prometheusURL = prometheusURL, offset = offset)
-        val fetchedData = metricFetcher.fetchMetric(start, end, "sum by(group)(kafka_consumergroup_group_lag >= 0)")
-        val data = Gson().toJson(mapOf("total_lag" to fetchedData.data?.result, "threshold" to threshold))
+        val fetchedData = metricFetcher.fetchMetric(start, end, query)
+        val data =
+            Gson().toJson(mapOf("total_lag" to fetchedData.data?.result, "threshold" to threshold, "warmup" to warmup))
 
         while (counter < 2) {
-            val result = post("http://127.0.0.1:8000/evaluate-slope", data = data, timeout = 60.0)
+            val result = post(externalSlopeURL, data = data, timeout = 60.0)
             if (result.statusCode != 200) {
                 counter++
             } else {

theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt

@@ -2,6 +2,7 @@ package theodolite.execution
 
 import mu.KotlinLogging
 import theodolite.benchmark.Benchmark
+import theodolite.benchmark.BenchmarkExecution
 import theodolite.util.ConfigurationOverride
 import theodolite.util.LoadDimension
 import theodolite.util.Resource
@@ -22,7 +23,8 @@ abstract class BenchmarkExecutor(
     val benchmark: Benchmark,
     val results: Results,
     val executionDuration: Duration,
-    configurationOverrides: List<ConfigurationOverride>
+    configurationOverrides: List<ConfigurationOverride>,
+    val slo: BenchmarkExecution.Slo
 ) {
 
     /**

theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt

@@ -2,7 +2,8 @@ package theodolite.execution
 
 import mu.KotlinLogging
 import theodolite.benchmark.Benchmark
-import theodolite.evaluation.SLOCheckerImpl
+import theodolite.benchmark.BenchmarkExecution
+import theodolite.evaluation.SloCheckerImpl
 import theodolite.util.ConfigurationOverride
 import theodolite.util.LoadDimension
 import theodolite.util.Resource
@@ -16,8 +17,9 @@ class BenchmarkExecutorImpl(
     benchmark: Benchmark,
     results: Results,
     executionDuration: Duration,
-    private val configurationOverrides: List<ConfigurationOverride>
-) : BenchmarkExecutor(benchmark, results, executionDuration, configurationOverrides) {
+    private val configurationOverrides: List<ConfigurationOverride>,
+    slo: BenchmarkExecution.Slo
+) : BenchmarkExecutor(benchmark, results, executionDuration, configurationOverrides, slo) {
     //TODO ADD SHUTDOWN HOOK HERE
     override fun runExperiment(load: LoadDimension, res: Resource): Boolean {
         val benchmarkDeployment = benchmark.buildDeployment(load, res, this.configurationOverrides)
@@ -28,7 +30,14 @@ class BenchmarkExecutorImpl(
 
         var result = false
         try {
-            result = SLOCheckerImpl("http://localhost:32656", 100, offset = Duration.ofSeconds(0))
+            result = SloCheckerImpl(
+                slo.prometheusUrl,
+                "sum by(group)(kafka_consumergroup_group_lag >= 0)",
+                slo.externalSloUrl,
+                slo.threshold,
+                Duration.ofHours(slo.offset.toLong()),
+                slo.warmup
+            )
                 .evaluate(
                     Instant.now().minus(executionDuration),
                     Instant.now()

theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt

@@ -20,7 +20,14 @@ class TheodoliteExecutor(
         val strategyFactory = StrategyFactory()
 
         val executionDuration = Duration.ofSeconds(config.execution.duration)
-        val executor = BenchmarkExecutorImpl(kubernetesBenchmark, results, executionDuration, config.configOverrides)
+        val executor =
+            BenchmarkExecutorImpl(
+                kubernetesBenchmark,
+                results,
+                executionDuration,
+                config.configOverrides,
+                config.slos[0]
+            )
 
         return Config(
             loads = config.load.loadValues.map { load -> LoadDimension(load, config.load.loadType) },

theodolite-quarkus/src/main/resources/yaml/BenchmarkExecution.yaml

@@ -11,6 +11,10 @@ resources:
 slos:
   - sloType: "slo type"
     threshold: 1000
+    prometheusUrl: "http://localhost:32656"
+    externalSloUrl: "http://127.0.0.1:8000/evaluate-slope"
+    offset: 0
+    warmup: 0
 execution:
   strategy: "LinearSearch"
   duration: 60