Merge branch 'theodolite-kotlin' into flink-benchmark-definitions

execution/helm/.gitignore

+charts
\ No newline at end of file

execution/helm/templates/theodolite/random-scheduler/cluster-role-binding.yaml

+{{- if .Values.randomScheduler.rbac.create -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "theodolite.fullname" . }}-random-scheduler
+subjects:
+- kind: ServiceAccount
+  name: {{ include "theodolite.fullname" . }}-random-scheduler
+  namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  apiGroup: rbac.authorization.k8s.io
+  name: system:kube-scheduler
+{{- end }}

execution/helm/templates/theodolite/random-scheduler/deployment.yaml

+{{- if .Values.randomScheduler.enabled -}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "theodolite.fullname" . }}-random-scheduler
+  labels:
+    app: {{ include "theodolite.fullname" . }}
+    component: random-scheduler
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: {{ include "theodolite.fullname" . }}
+      component: random-scheduler
+  template:
+    metadata:
+      labels:
+        app: {{ include "theodolite.fullname" . }}
+        component: random-scheduler
+    spec:
+      serviceAccount: {{ include "theodolite.fullname" . }}-random-scheduler
+      containers:
+        - name: random-scheduler
+          image: ghcr.io/cau-se/theodolite-random-scheduler:theodolite-kotlin-latest
+          #imagePullPolicy: Always
+          env:
+            - name: TARGET_NAMESPACE
+              value: {{ .Release.Namespace }}
+{{- end }}

execution/helm/templates/theodolite/random-scheduler/service-account.yaml

+{{- if .Values.randomScheduler.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  namespace: kube-system
+  name: {{ include "theodolite.fullname" . }}-random-scheduler 
+  labels:
+    app: {{ include "theodolite.fullname" . }}
+    component: random-scheduler
+{{- end }}

execution/helm/values.yaml

@@ -92,8 +92,9 @@ cp-helm-charts:
       "replica.fetch.max.bytes": "134217728" # 128 MB
       #default.replication.factor: 1
       # "min.insync.replicas": 2
-      # "auto.create.topics.enable": false
-      "log.retention.ms": "10000" # 10s
+      "auto.create.topics.enable": false
+      #"log.retention.ms": "10000" # 10s
+      "log.retention.ms": "7200000" # 2h
       "metrics.sample.window.ms": "5000" #5s
       "advertised.listeners": |-
         EXTERNAL://${HOST_IP}:$((31090 + ${KAFKA_BROKER_ID}))
@@ -248,3 +249,10 @@ serviceAccount:
 
 rbac:
   create: true
+
+randomScheduler:
+  enabled: true
+  rbac:
+    create: true
+  serviceAccount:
+    create: true

slope-evaluator/README.md

@@ -8,7 +8,7 @@ For development:
 uvicorn main:app --reload
 ```
 
-Build the docker image:
+## Build the docker image:
 
 ```sh
 docker build . -t theodolite-evaluator

theodolite-quarkus/README.md

-# theodolite-quarkus project
+# Theodolite-quarkus project
 
 This project uses Quarkus, the Supersonic Subatomic Java Framework.
 
@@ -6,7 +6,8 @@ If you want to learn more about Quarkus, please visit its website: https://quark
 
 ## Running the application in dev mode
 
-You can run your application in dev mode that enables live coding using:
+You can run your application in dev mode using:
+
 ```shell script
 ./gradlew quarkusDev
 ```
@@ -14,13 +15,16 @@ You can run your application in dev mode that enables live coding using:
 ## Packaging and running the application
 
 The application can be packaged using:
+
 ```shell script
 ./gradlew build
 ```
-It produces the `theodolite-quarkus-1.0.0-SNAPSHOT-runner.jar` file in the `/build` directory.
-Be aware that it’s not an _über-jar_ as the dependencies are copied into the `build/lib` directory.
+
+It produces the `theodolite-quarkus-1.0.0-SNAPSHOT-runner.jar` file in the `/build` directory. Be aware that it’s not
+an _über-jar_ as the dependencies are copied into the `build/lib` directory.
 
 If you want to build an _über-jar_, execute the following command:
+
 ```shell script
 ./gradlew build -Dquarkus.package.type=uber-jar
 ```
@@ -30,11 +34,13 @@ The application is now runnable using `java -jar build/theodolite-quarkus-1.0.0-
 ## Creating a native executable
 
 You can create a native executable using:
+
 ```shell script
 ./gradlew build -Dquarkus.package.type=native
 ```
 
 Or, if you don't have GraalVM installed, you can run the native executable build in a container using:
+
 ```shell script
 ./gradlew build -Dquarkus.package.type=native -Dquarkus.native.container-build=true
 ```
@@ -44,20 +50,73 @@ You can then execute your native executable with:
 
 If you want to learn more about building native executables, please consult https://quarkus.io/guides/gradle-tooling.
 
-# RESTEasy JAX-RS
+## Build docker images
+
+For the jvm version use:
+
+```shell script
+./gradlew build
+docker build -f src/main/docker/Dockerfile.jvm -t theodolite-quarkus-jvm .
+```
+
+For the native image version use:
+
+```shell script
+./gradlew build -Dquarkus.package.type=native
+docker build -f src/main/docker/Dockerfile.native -t theodolite-quarkus-native .
+```
+
+## Execute docker images:
 
-<p>A Hello World RESTEasy resource</p>
+Remember to set the environment variables first.
 
-Guide: https://quarkus.io/guides/rest-json
+Jvm version:
 
-## Build and afterwards run the application in Docker container
+```shell script
+docker run -i --rm theodolite-quarkus-jvm
+```
+
+Native image version:
 
-```build_jvm.sh```   to build the jvm version
+```shell script
+docker run -i --rm theodolite-quarkus-native
+```
 
-```build_native.sh``` to build the native image graal version 
+## Environment variables
 
-## Install Detekt Code analysis Plugin
+**Production:** (Docker-Container)
 
+| Variables name               | Default value                      |Usage         |
+| -----------------------------|:----------------------------------:| ------------:|
+| `NAMESPACE`                  | `default`                          |Determines the namespace of the Theodolite will be executed in. Used in the KubernetesBenchmark|
+| `THEODOLITE_EXECUTION`       |  `./config/BenchmarkExecution.yaml`|The complete path to the benchmarkExecution file. Used in the TheodoliteYamlExecutor. |
+| `THEODOLITE_BENCHMARK_TYPE`  |  `./config/BenchmarkType.yaml`     |The complete path to the benchmarkType file. Used in the TheodoliteYamlExecutor.|
+| `THEODOLITE_APP_RESOURCES`   |  `./config`                        |The path under which the yamls for the resources for the subexperiments are found. Used in the KubernetesBenchmark|
+| `MODE`                       | `yaml-executor`                    |  Defines the mode of operation: either `yaml-executor` or `operator`|
+
+**Development:** (local via Intellij)
+
+When running Theodolite from within IntelliJ via
+[Run Configurations](https://www.jetbrains.com/help/idea/work-with-gradle-tasks.html#gradle_run_config), set the *
+Environment variables* field to:
+
+```
+NAMESPACE=default;THEODOLITE_BENCHMARK=./../../../../config/BenchmarkType.yaml;THEODOLITE_APP_RESOURCES=./../../../../config;THEODOLITE_EXECUTION=./../../../../config/BenchmarkExecution.yaml;MODE=operator
+```
+
+Alternative:
+
+``` sh
+export NAMESPACE=default
+export THEODOLITE_BENCHMARK=./../../../../config/BenchmarkType.yaml
+export THEODOLITE_APP_RESOURCES=./../../../../config
+export THEODOLITE_EXECUTION=./../../../../config/BenchmarkExecution.yaml
+export MODE=operator
+./gradlew quarkusDev
+
+```
+
+#### Install Detekt Code analysis Plugin
 
 Install https://plugins.jetbrains.com/plugin/10761-detekt
 
@@ -66,7 +125,6 @@ Install https://plugins.jetbrains.com/plugin/10761-detekt
 - Check Enable Detekt
 - Specify your detekt configuration and baseline file (optional)
 
-
 -> detekt issues will be annotated on-the-fly while coding
 
 **ingore Failures in build:** add

theodolite-quarkus/config/README.md

+## The Benchmark Object
+
+The *benchmark* object defines all static components of an execution of a benchmark with Theodolite.
+An exapmle for a benchmark object is given in [example-benchmark-yaml-resource](example-benchmark-yaml-resource.yaml).
+
+
+A **Benchmark** is a [*standard tool for the competitive evaluation and comparison of competing systems or components according to specific characteristics, such as performance, dependability, or security*](https://doi.org/10.1145/2668930.2688819). In Theodolite, we have [specification-based benchmarks](https://doi.org/10.1145/2668930.2688819), or at least something very close to that. That is, our benchmarks are architectural descriptions---in our case---[of typical use cases of stream processing in microservices](https://doi.org/10.1016/j.bdr.2021.100209) (e.g. our UC1). Hence, we don't really have a piece of software, which represents a benchmark. We only have implementations of benchmarks, e.g. an implementation of UC1 with Kafka Streams. For simplification, we call these *benchmark implementations* simply *benchmarks*.
+
+```yaml
+name: String
+appResource:
+  - String
+  ...
+loadGenResource:
+  - String
+  ...
+resourceTypes:
+  - typeName: String
+    patchers:
+      - type: String
+        resources: String
+        <Patcher Arguments> ...
+      ...
+loadTypes:
+  - typeName: String
+  patchers:
+    - type: String
+      resources: String
+      <Patcher Arguments> ...
+    ...
+kafkaConfig:
+  bootstrapServer: String
+  topics:
+    - name: String
+      numPartitions: UnsignedInt
+      replicationFactor: UnsignedInt
+    - name: String
+      removeOnly: bool
+    ...
+```
+
+The properties have the following definitions:
+
+* **name**: The name of the *benchmark*
+* **appResource**: A list of file names that reference Kubernetes resources that are deployed on the cluster for the system under test (SUT).
+* **loadGenResources**: A list of file names that reference Kubernetes resources that are deployed on the cluster for the load generator.
+* **resourceTypes**: A list of resource types that can be scaled for this *benchmark*. For each resource type the concrete values are defined in the *execution* object. Each resource type has the following structure:
+    * **typeName**: Name of the resource type.
+    * **patchers**: List of [patchers](#Patchers) used to scale this resource type. Each patcher has the following structure:
+        * **type**: Type of the [patcher](#Patchers). The concrete types can be looked up in the list of [patchers](#Patchers). 
+        * **resources**: Specifies the Kubernetes resource to be patched.
+        * **Patcher Arguments**: (Optional) Patcher specific additional arguments.
+* **loadTypes**: A list of load types that can be scaled for this *benchmark*. For each load type the concrete values are defined in the *execution* object. Each load type has the following structure:
+    * **typeName**: Name of the load type.
+    * **patchers**: List of patchers used to scale * **resourceTypes**: A list of resource types that can be scaled for this *benchmark*. For each resource type the concrete values are defined in the *execution* resource object.Each resource type has the following structure:
+    * **typeName**: Name of the resource type.
+    * **patchers**: List of patchers used to scale this resource type. Each patcher has the following structure:
+        * **type**: Type of the Patcher. The concrete types can be looked up in the list of patchers. 
+        * **resources**: Specifies the Kubernetes resource to be patched.
+        * **Patcher Arguments**: (Optional) Patcher specific additional arguments.
+* **kafkaConfig**: Contains the Kafka configuration.
+    * **bootstrapServers**: The bootstrap servers connection string.
+    * **topics**: List of topics to be created for each [experiment](#Experiment). Alternative theodolite offers the possibility to remove certain topics after each experiment.
+        * **name**: The name of the topic.
+        * **numPartitions**: The number of partitions of the topic.
+        * **replicationFactor**: The replication factor of the topic.
+        * **removeOnly**: determines if this topic should only be deleted after each experiement. For removeOnly topics the name can be a RegEx describing the topic.
+    
+
+## The Execution Object
+
+A benchmark can be executed for different SUTs, by different users and multiple times. We call such an execution of a benchmark simply an *execution*. The *execution* object defines all conrete values of an Execution.
+An exapmle for an execution object is given in [example-execution-yaml-resource](example-benchmark-yaml-resource.yaml).
+
+
+```yaml
+name: String
+benchmark: String
+load:
+  loadType: String
+  loadValues:
+    - UnsignedInt
+    ...
+resources:
+  resourceType: String
+  resourceValues:
+    - UnsignedInt
+    ...
+slos:
+  - sloType: String
+    threshold: UnsignedInt
+    prometheusUrl: String
+    externalSloUrl: String
+    offset: SignedInt
+    warmup: UnsignedInt
+  ...
+executions:
+  strategy: "LinearSearch" or "BinarySearch"
+  duration: UnsignedInt
+  repetition: UnsignedInt
+  restrictions:
+    - "LowerBound"
+    ...
+configurationOverrides:
+  - patcher:
+      type: String
+      resource: String
+      <Patcher Arguments> ...
+  ...
+```
+
+The properties have the following definitions:
+
+* **name**: The name of the *execution*
+* **benchmark**: The name of the *benchmark* this *execution* is referring to.
+* **load**: Specifies the load values that are benchmarked.
+  * **loadType**: The type of the load. It must match one of the load types specified in the referenced *benchmark*.
+  * **loadValues**: List of load values for the specified load type.
+* **resources**: Specifies the scaling resource that is benchmarked.
+  * **resourceType**: The type of the resource. It must match one of the resource types specified in the referenced *benchmark*.
+  * **resourceValues**: List of resource values for the specified resource type.
+* **slos**: List of the Service Level Objective (SLO) for this *execution*. Each SLO has the following fields:
+  * **sloType**: The type of the SLO. It must match 'lag trend'.
+  * **threshold**: The threshold the SUT should meet for a sucessful experiment.
+  * **prometheusUrl**: Connection string for promehteus.
+  * **externalSloUrl**: Connection string for a external slo analysis.
+  * **offset**: Hours by which the start and end timestamp will be shifted (for different timezones).
+  * **warmup**: Seconds of time that are ignored in the analysis.
+* **executions**: Defines the overall parameter for the execution.
+  * **strategy**: Defines the used strategy for the execution: either 'LinearSearch' or 'BinarySearch'
+  * **duration**: Defines the duration of each [experiment](#Experiment) in seconds.
+  * **repetition**: Unused.
+  * **restrictions**: List of restriction strategys used to delimit the search space.
+    **- LowerBound**: Currently only supported *restriction strategy*.
+* **configurationOverrides**: List of patchers that are used to override existing configurations.
+  * **patcher**: Patcher used to patch a resource. Each patcher has the following structure:
+        * **type**: Type of the Patcher. The concrete types can be looked up in the list of patchers. 
+        * **resources**: Specifies the Kubernetes resource to be patched.
+        * **Patcher Arguments**: (Optional) Patcher specific additional arguments.
+
+## Patchers
+
+* **ReplicaPatcher**: Allows to modify the number of Replicas for a kubernetes deployment.
+  * **type**: "ReplicaPatcher"
+  * **resource**: "uc1-kstreams-deployment.yaml"
+
+* **NumSensorsLoadGeneratorReplicaPatcher**: Allows to scale the nummer of load generators. Scales arcording to the following formular: (value + 15_000 - 1) / 15_000
+  * **type**: "NumSensorsLoadGeneratorReplicaPatcher"
+  * **resource**: "uc1-load-generator-deployment.yaml"
+
+* **NumNestedGroupsLoadGeneratorReplicaPatcher**: Allows to scale the nummer of load generators. Scales arcording to the following formular: (4^(value) + 15_000 -1) /15_000
+  * **type**: "NumNestedGroupsLoadGeneratorReplicaPatcher"
+  * **resource**: "uc1-load-generator-deployment.yaml"
+
+* **ReplicaPatcher**: Allows to modify the number of Replicas for a kubernetes deployment.
+  * **type**: "ReplicaPatcher"
+  * **resource**: "uc1-kstreams-deployment.yaml"
+
+* **EnvVarPatcher**: Allows to modify the value of an environment variable for a container in a kubernetes deployment. 
+  * **type**: "EnvVarPatcher"
+  * **resource**: "uc1-load-generator-deployment.yaml"
+  * **container**: "workload-generator"
+  * **variableName**: "NUM_SENSORS"
+
+* **NodeSelectorPatcher**: Changes the node selection field in kubernetes resources.
+  * **type**: "NodeSelectorPatcher"
+  * **resource**: "uc1-load-generator-deployment.yaml"
+  * **variableName**: "env"
+  * **value**: "prod"
+
+* **ResourceLimitPatcher**: Changes the resource limit for a kubernetes resource.
+  * **resource**: "uc1-kstreams-deployment.yaml"
+  * **container**: "uc-application"
+  * **variableName**: "cpu" or "memory"
+  * **value**:"1000m" or "2Gi"
+  
+* **SchedulerNamePatcher**: Changes the sheduler for kubernetes resources.
+  * **type**: "SchedulerNamePatcher"
+  * **resource**: "uc1-kstreams-deployment.yaml"
+  * **value**: "random-scheduler"
+
+* **ImagePatcher**: Changes the image of a kubernetes resource. Currently not fully implemented.
+  * **type**: "ImagePatcher"
+  * **resource**: "uc1-kstreams-deployment.yaml"
+  * **container**: "uc-application"
+  * **value**: "dockerhubrepo/imagename"
+
+
+
+## Experiment
+According to [our benchmarking method](https://doi.org/10.1016/j.bdr.2021.100209), the execution of a benchmark requires performing multiple **Experiments**. I think what is actually done within/during an experiment is another level of detail. (But just for the sake of completeness: In an experiment, the benchmark implementation is deployed, load is generated according to the benchmark specification, some SLOs are monitored continuously, etc.)
+
+
+

theodolite-quarkus/examples/operator/example-execution.yaml

@@ -22,6 +22,7 @@ execution:
   strategy: "LinearSearch"
   duration: 60
   repetitions: 1
+  delay: 30 # in seconds
   restrictions:
     - "LowerBound"
 configOverrides:

theodolite-quarkus/examples/standalone/example-benchmark.yaml

@@ -22,7 +22,7 @@ loadTypes:
       - type: "NumSensorsLoadGeneratorReplicaPatcher"
         resource: "uc1-load-generator-deployment.yaml"
 kafkaConfig:
-  bootstrapServer: "theodolite-cp-kafka:9092"
+  bootstrapServer: "localhost:31290"
   topics:
     - name: "input"
       numPartitions: 40

theodolite-quarkus/src/main/kotlin/theodolite/benchmark/Benchmark.kt

@@ -21,6 +21,8 @@ interface Benchmark {
     fun buildDeployment(
         load: LoadDimension,
         res: Resource,
-        configurationOverrides: List<ConfigurationOverride?>
+        configurationOverrides: List<ConfigurationOverride?>,
+        loadGenerationDelay: Long,
+        afterTeardownDelay: Long
     ): BenchmarkDeployment
 }

theodolite-quarkus/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt

@@ -47,6 +47,8 @@ class BenchmarkExecution : CustomResource(), Namespaced {
         var duration by Delegates.notNull<Long>()
         var repetitions by Delegates.notNull<Int>()
         lateinit var restrictions: List<String>
+        var loadGenerationDelay = 0L
+        var afterTeardownDelay = 5L
     }
 
     /**

theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt

@@ -22,7 +22,7 @@ private var DEFAULT_NAMESPACE = "default"
  * - [loadGenResource] resource that generates the load,
  * - [resourceTypes] types of scaling resources,
  * - [loadTypes] types of loads that can be scaled for the benchmark,
- * - [kafkaConfig] for the [TopicManager],
+ * - [kafkaConfig] for the [theodolite.k8s.TopicManager],
  * - [namespace] for the client,
  * - [path] under which the resource yamls can be found.
  *
@@ -63,38 +63,45 @@ class KubernetesBenchmark : Benchmark, CustomResource(), Namespaced {
      * First loads all required resources and then patches them to the concrete load and resources for the experiment.
      * Afterwards patches additional configurations(cluster depending) into the resources.
      * @param load concrete load that will be benchmarked in this experiment.
-     * @param res concrete resoruce that will be scaled for this experiment.
+     * @param res concrete resource that will be scaled for this experiment.
      * @param configurationOverrides
      * @return a [BenchmarkDeployment]
      */
     override fun buildDeployment(
         load: LoadDimension,
         res: Resource,
-        configurationOverrides: List<ConfigurationOverride?>
+        configurationOverrides: List<ConfigurationOverride?>,
+        loadGenerationDelay: Long,
+        afterTeardownDelay: Long
     ): BenchmarkDeployment {
         logger.info { "Using $namespace as namespace." }
         logger.info { "Using $path as resource path." }
 
-        val resources = loadKubernetesResources(this.appResource + this.loadGenResource)
+        val appResources = loadKubernetesResources(this.appResource)
+        val loadGenResources = loadKubernetesResources(this.loadGenResource)
+
         val patcherFactory = PatcherFactory()
 
         // patch the load dimension the resources
         load.getType().forEach { patcherDefinition ->
-            patcherFactory.createPatcher(patcherDefinition, resources).patch(load.get().toString())
+            patcherFactory.createPatcher(patcherDefinition, loadGenResources).patch(load.get().toString())
         }
         res.getType().forEach { patcherDefinition ->
-            patcherFactory.createPatcher(patcherDefinition, resources).patch(res.get().toString())
+            patcherFactory.createPatcher(patcherDefinition, appResources).patch(res.get().toString())
         }
 
         // Patch the given overrides
         configurationOverrides.forEach { override ->
             override?.let {
-                patcherFactory.createPatcher(it.patcher, resources).patch(override.value)
+                patcherFactory.createPatcher(it.patcher, appResources + loadGenResources).patch(override.value)
             }
         }
         return KubernetesBenchmarkDeployment(
             namespace = namespace,
-            resources = resources.map { r -> r.second },
+            appResources = appResources.map { it.second },
+            loadGenResources = loadGenResources.map { it.second },
+            loadGenerationDelay = loadGenerationDelay,
+            afterTeardownDelay = afterTeardownDelay,
             kafkaConfig = hashMapOf("bootstrap.servers" to kafkaConfig.bootstrapServer),
             topics = kafkaConfig.topics,
             client = DefaultKubernetesClient().inNamespace(namespace)

theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmarkDeployment.kt

@@ -8,6 +8,7 @@ import org.apache.kafka.clients.admin.NewTopic
 import theodolite.k8s.K8sManager
 import theodolite.k8s.TopicManager
 import theodolite.util.KafkaConfig
+import java.time.Duration
 
 private val logger = KotlinLogging.logger {}
 
@@ -22,7 +23,10 @@ private val logger = KotlinLogging.logger {}
 @RegisterForReflection
 class KubernetesBenchmarkDeployment(
     val namespace: String,
-    val resources: List<KubernetesResource>,
+    val appResources: List<KubernetesResource>,
+    val loadGenResources: List<KubernetesResource>,
+    private val loadGenerationDelay: Long,
+    private val afterTeardownDelay: Long,
     private val kafkaConfig: HashMap<String, Any>,
     private val topics: List<KafkaConfig.TopicWrapper>,
     private val client: NamespacedKubernetesClient
@@ -30,7 +34,6 @@ class KubernetesBenchmarkDeployment(
     private val kafkaController = TopicManager(this.kafkaConfig)
     private val kubernetesManager = K8sManager(client)
     private val LAG_EXPORTER_POD_LABEL = "app.kubernetes.io/name=kafka-lag-exporter"
-    private val SLEEP_AFTER_TEARDOWN = 5000L
 
     /**
      * Setup a [KubernetesBenchmark] using the [TopicManager] and the [K8sManager]:
@@ -41,7 +44,10 @@ class KubernetesBenchmarkDeployment(
         val kafkaTopics = this.topics.filter { !it.removeOnly }
             .map { NewTopic(it.name, it.numPartitions, it.replicationFactor) }
         kafkaController.createTopics(kafkaTopics)
-        resources.forEach { kubernetesManager.deploy(it) }
+        appResources.forEach { kubernetesManager.deploy(it) }
+        logger.info { "Wait ${this.loadGenerationDelay} seconds before starting the load generator." }
+        Thread.sleep(Duration.ofSeconds(this.loadGenerationDelay).toMillis())
+        loadGenResources.forEach { kubernetesManager.deploy(it) }
     }
 
     /**
@@ -51,12 +57,11 @@ class KubernetesBenchmarkDeployment(
      *  - Remove the [KubernetesResource]s.
      */
     override fun teardown() {
-        resources.forEach {
-            kubernetesManager.remove(it)
-        }
+        loadGenResources.forEach { kubernetesManager.remove(it) }
+        appResources.forEach { kubernetesManager.remove(it) }
         kafkaController.removeTopics(this.topics.map { topic -> topic.name })
         KafkaLagExporterRemover(client).remove(LAG_EXPORTER_POD_LABEL)
-        logger.info { "Teardown complete. Wait $SLEEP_AFTER_TEARDOWN ms to let everything come down." }
-        Thread.sleep(SLEEP_AFTER_TEARDOWN)
+        logger.info { "Teardown complete. Wait $afterTeardownDelay ms to let everything come down." }
+        Thread.sleep(Duration.ofSeconds(afterTeardownDelay).toMillis())
     }
 }

theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SloChecker.kt

@@ -4,7 +4,7 @@ import theodolite.util.PrometheusResponse
 import java.time.Instant
 
 /**
- * A SloChecker can be used to evaluate data from Promehteus.
+ * A SloChecker can be used to evaluate data from Prometheus.
  * @constructor Creates an empty SloChecker
  */
 interface SloChecker {

theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt

@@ -24,9 +24,11 @@ abstract class BenchmarkExecutor(
     val benchmark: Benchmark,
     val results: Results,
     val executionDuration: Duration,
-    configurationOverrides: List<ConfigurationOverride?>,
+    val configurationOverrides: List<ConfigurationOverride?>,
     val slo: BenchmarkExecution.Slo,
-    val executionId: Int
+    val executionId: Int,
+    val loadGenerationDelay: Long,
+    val afterTeardownDelay: Long
 ) {
 
     var run: AtomicBoolean = AtomicBoolean(true)

theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt

@@ -18,13 +18,15 @@ class BenchmarkExecutorImpl(
     benchmark: Benchmark,
     results: Results,
     executionDuration: Duration,
-    private val configurationOverrides: List<ConfigurationOverride?>,
+    configurationOverrides: List<ConfigurationOverride?>,
     slo: BenchmarkExecution.Slo,
-    executionId: Int
-) : BenchmarkExecutor(benchmark, results, executionDuration, configurationOverrides, slo, executionId) {
+    executionId: Int,
+    loadGenerationDelay: Long,
+    afterTeardownDelay: Long
+) : BenchmarkExecutor(benchmark, results, executionDuration, configurationOverrides, slo, executionId, loadGenerationDelay, afterTeardownDelay) {
     override fun runExperiment(load: LoadDimension, res: Resource): Boolean {
         var result = false
-        val benchmarkDeployment = benchmark.buildDeployment(load, res, this.configurationOverrides)
+        val benchmarkDeployment = benchmark.buildDeployment(load, res, configurationOverrides, loadGenerationDelay, this.afterTeardownDelay)
 
         try {
             benchmarkDeployment.setup()

theodolite-quarkus/src/main/kotlin/theodolite/execution/Shutdown.kt

@@ -30,7 +30,9 @@ class Shutdown(private val benchmarkExecution: BenchmarkExecution, private val b
             benchmark.buildDeployment(
                 load = LoadDimension(0, emptyList()),
                 res = Resource(0, emptyList()),
-                configurationOverrides = benchmarkExecution.configOverrides
+                configurationOverrides = benchmarkExecution.configOverrides,
+                loadGenerationDelay = 0L,
+                afterTeardownDelay = 5L
             )
             deployment.teardown()
         } catch (e: Exception) {

theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt

@@ -71,7 +71,9 @@ class TheodoliteExecutor(
                 executionDuration = executionDuration,
                 configurationOverrides = config.configOverrides,
                 slo = config.slos[0],
-                executionId = config.executionId
+                executionId = config.executionId,
+                loadGenerationDelay = config.execution.loadGenerationDelay,
+                afterTeardownDelay = config.execution.afterTeardownDelay
             )
 
         return Config(
@@ -128,7 +130,7 @@ class TheodoliteExecutor(
     fun run() {
         val resultsFolder = getResultFolderString()
         storeAsFile(this.config, "$resultsFolder${this.config.executionId}-execution-configuration")
-        storeAsFile(kubernetesBenchmark, "$resultsFolder/${this.config.executionId}-benchmark-configuration")
+        storeAsFile(kubernetesBenchmark, "$resultsFolder${this.config.executionId}-benchmark-configuration")
 
         val config = buildConfig()
         // execute benchmarks for each load