Merge branch '186-add-prefix-to-output-files' into 'theodolite-kotlin'

Add prefix to output files to link results and experiment

See merge request !111

theodolite-quarkus/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt

@@ -11,6 +11,7 @@ import kotlin.properties.Delegates
 @JsonDeserialize
 @RegisterForReflection
 class BenchmarkExecution : CustomResource(), Namespaced {
+    var executionId: Int = 0
     lateinit var name: String
     lateinit var benchmark: String
     lateinit var load: LoadDefinition

theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt

@@ -26,7 +26,6 @@ class KubernetesBenchmark : Benchmark, CustomResource(), Namespaced {
 
     private fun loadKubernetesResources(resources: List<String>): List<Pair<String, KubernetesResource>> {
         val parser = YamlParser()
-
         val loader = K8sResourceLoader(DefaultKubernetesClient().inNamespace(namespace))
         return resources
             .map { resource ->

theodolite-quarkus/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt

@@ -9,7 +9,10 @@ import java.time.Instant
 
 private val logger = KotlinLogging.logger {}
 
-class AnalysisExecutor(private val slo: BenchmarkExecution.Slo) {
+class AnalysisExecutor(
+    private val slo: BenchmarkExecution.Slo,
+    private val executionId: Int
+) {
 
     private val fetcher = MetricFetcher(
         prometheusURL = slo.prometheusUrl,
@@ -26,8 +29,7 @@ class AnalysisExecutor(private val slo: BenchmarkExecution.Slo) {
                 query = "sum by(group)(kafka_consumergroup_group_lag >= 0)"
             )
 
-            CsvExporter().toCsv(name = "${load.get()}_${res.get()}_${slo.sloType}", prom = prometheusData)
-
+            CsvExporter().toCsv(name = "$executionId-${load.get()}-${res.get()}-${slo.sloType}", prom = prometheusData)
             val sloChecker = SloCheckerFactory().create(
                 slotype = slo.sloType,
                 externalSlopeURL = slo.externalSloUrl,

theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt

@@ -25,7 +25,8 @@ abstract class BenchmarkExecutor(
     val results: Results,
     val executionDuration: Duration,
     configurationOverrides: List<ConfigurationOverride?>,
-    val slo: BenchmarkExecution.Slo
+    val slo: BenchmarkExecution.Slo,
+    val executionId: Int
 ) {
 
     var run: AtomicBoolean = AtomicBoolean(true)

theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt

@@ -19,8 +19,9 @@ class BenchmarkExecutorImpl(
     results: Results,
     executionDuration: Duration,
     private val configurationOverrides: List<ConfigurationOverride?>,
-    slo: BenchmarkExecution.Slo
-) : BenchmarkExecutor(benchmark, results, executionDuration, configurationOverrides, slo) {
+    slo: BenchmarkExecution.Slo,
+    executionId: Int
+) : BenchmarkExecutor(benchmark, results, executionDuration, configurationOverrides, slo, executionId) {
     override fun runExperiment(load: LoadDimension, res: Resource): Boolean {
         var result = false
         val benchmarkDeployment = benchmark.buildDeployment(load, res, this.configurationOverrides)
@@ -36,7 +37,7 @@ class BenchmarkExecutorImpl(
 
         if (this.run.get()) {
             result =
-                AnalysisExecutor(slo = slo).analyze(load = load, res = res, executionDuration = executionDuration)
+                AnalysisExecutor(slo = slo, executionId = executionId).analyze(load = load, res = res, executionDuration = executionDuration)
             this.results.setResult(Pair(load, res), result)
         }
         benchmarkDeployment.teardown()

theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt

 package theodolite.execution
 
+import com.google.gson.GsonBuilder
 import theodolite.benchmark.BenchmarkExecution
 import theodolite.benchmark.KubernetesBenchmark
 import theodolite.patcher.PatcherDefinitionFactory
@@ -9,6 +10,7 @@ import theodolite.util.Config
 import theodolite.util.LoadDimension
 import theodolite.util.Resource
 import theodolite.util.Results
+import java.io.PrintWriter
 import java.time.Duration
 
 class TheodoliteExecutor(
@@ -37,11 +39,12 @@ class TheodoliteExecutor(
 
         executor =
             BenchmarkExecutorImpl(
-                kubernetesBenchmark,
-                results,
-                executionDuration,
-                config.configOverrides,
-                config.slos[0]
+                benchmark = kubernetesBenchmark,
+                results = results,
+                executionDuration = executionDuration,
+                configurationOverrides = config.configOverrides,
+                slo = config.slos[0],
+                executionId = config.executionId
             )
 
         return Config(
@@ -72,6 +75,9 @@ class TheodoliteExecutor(
     }
 
     fun run() {
+        storeAsFile(this.config, "${this.config.executionId}-execution-configuration")
+        storeAsFile(kubernetesBenchmark, "${this.config.executionId}-benchmark-configuration")
+
         val config = buildConfig()
         // execute benchmarks for each load
         for (load in config.loads) {
@@ -79,5 +85,14 @@ class TheodoliteExecutor(
                 config.compositeStrategy.findSuitableResource(load, config.resources)
             }
         }
+        storeAsFile(config.compositeStrategy.benchmarkExecutor.results, "${this.config.executionId}-result")
+    }
+
+    private fun <T> storeAsFile(saveObject: T, filename: String) {
+        val gson = GsonBuilder().enableComplexMapKeySerialization().setPrettyPrinting().create()
+
+        PrintWriter(filename).use { pw ->
+            pw.println(gson.toJson(saveObject))
+        }
     }
 }

theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/TheodoliteController.kt

@@ -2,6 +2,7 @@ package theodolite.execution.operator
 
 import io.fabric8.kubernetes.client.NamespacedKubernetesClient
 import io.fabric8.kubernetes.client.dsl.base.CustomResourceDefinitionContext
+import khttp.patch
 import mu.KotlinLogging
 import theodolite.benchmark.BenchmarkExecution
 import theodolite.benchmark.KubernetesBenchmark
@@ -10,17 +11,20 @@ import java.lang.Thread.sleep
 import java.util.concurrent.ConcurrentHashMap
 import java.util.concurrent.ConcurrentLinkedDeque
 import java.util.concurrent.atomic.AtomicBoolean
+import java.util.concurrent.atomic.AtomicInteger
 
 private val logger = KotlinLogging.logger {}
 
 class TheodoliteController(
     val client: NamespacedKubernetesClient,
-    val executionContext: CustomResourceDefinitionContext
+    val executionContext: CustomResourceDefinitionContext,
+    val path: String
 ) {
     lateinit var executor: TheodoliteExecutor
     val executionsQueue: ConcurrentLinkedDeque<BenchmarkExecution> = ConcurrentLinkedDeque()
     val benchmarks: ConcurrentHashMap<String, KubernetesBenchmark> = ConcurrentHashMap()
     var isUpdated = AtomicBoolean(false)
+    var executionID = AtomicInteger(0)
 
     fun run() {
         while (true) {
@@ -59,7 +63,9 @@ class TheodoliteController(
 
     @Synchronized
     fun runExecution(execution: BenchmarkExecution, benchmark: KubernetesBenchmark) {
+        execution.executionId = executionID.getAndSet(executionID.get() + 1)
         isUpdated.set(false)
+        benchmark.path = path
         logger.info { "Start execution ${execution.name} with benchmark ${benchmark.name}." }
         executor = TheodoliteExecutor(config = execution, kubernetesBenchmark = benchmark)
         executor.run()

theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/TheodoliteOperator.kt

@@ -44,7 +44,8 @@ class TheodoliteOperator {
         val executionContext = contextFactory.create(API_VERSION, SCOPE, GROUP, EXECUTION_PLURAL)
         val benchmarkContext = contextFactory.create(API_VERSION, SCOPE, GROUP, BENCHMARK_PLURAL)
 
-        val controller = TheodoliteController(client = client, executionContext = executionContext)
+        val appResource = System.getenv("THEODOLITE_APP_RESOURCES") ?: "./config"
+        val controller = TheodoliteController(client = client, executionContext = executionContext, path = appResource)
 
         val informerFactory = client.informers()
         val informerExecution = informerFactory.sharedIndexInformerForCustomResource(

theodolite-quarkus/src/test/kotlin/theodolite/CompositeStrategyTest.kt

@@ -31,7 +31,7 @@ class CompositeStrategyTest {
         val results = Results()
         val benchmark = TestBenchmark()
         val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo()
-        val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, sloChecker)
+        val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, sloChecker, 0)
         val linearSearch = LinearSearch(benchmarkExecutor)
         val lowerBoundRestriction = LowerBoundRestriction(results)
         val strategy =
@@ -65,7 +65,7 @@ class CompositeStrategyTest {
         val benchmark = TestBenchmark()
         val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo()
         val benchmarkExecutorImpl =
-            TestBenchmarkExecutorImpl(mockResults, benchmark, results, sloChecker)
+            TestBenchmarkExecutorImpl(mockResults, benchmark, results, sloChecker, 0)
         val binarySearch = BinarySearch(benchmarkExecutorImpl)
         val lowerBoundRestriction = LowerBoundRestriction(results)
         val strategy =
@@ -98,7 +98,7 @@ class CompositeStrategyTest {
         val results = Results()
         val benchmark = TestBenchmark()
         val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo()
-        val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, sloChecker)
+        val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, sloChecker, 0)
         val binarySearch = BinarySearch(benchmarkExecutor)
         val lowerBoundRestriction = LowerBoundRestriction(results)
         val strategy =

theodolite-quarkus/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt

@@ -12,14 +12,16 @@ class TestBenchmarkExecutorImpl(
     private val mockResults: Array<Array<Boolean>>,
     benchmark: Benchmark,
     results: Results,
-    slo: BenchmarkExecution.Slo
+    slo: BenchmarkExecution.Slo,
+    executionId: Int
 ) :
     BenchmarkExecutor(
         benchmark,
         results,
         executionDuration = Duration.ofSeconds(1),
         configurationOverrides = emptyList(),
-        slo = slo
+        slo = slo,
+        executionId = executionId
     ) {
 
     override fun runExperiment(load: LoadDimension, res: Resource): Boolean {