Skip to content
Snippets Groups Projects
Commit a7673c0d authored by Sören Henning's avatar Sören Henning
Browse files

Merge branch 'theodolite-kotlin' into flink-benchmark-definitions

parents 28798d93 42b91355
No related branches found
No related tags found
2 merge requests!159Re-implementation of Theodolite with Kotlin/Quarkus,!152Adjust Benchmark Definitions for Operator
Pipeline #3207 passed
Showing
with 389 additions and 49 deletions
charts
\ No newline at end of file
{{- if .Values.randomScheduler.rbac.create -}}
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: {{ include "theodolite.fullname" . }}-random-scheduler
subjects:
- kind: ServiceAccount
name: {{ include "theodolite.fullname" . }}-random-scheduler
namespace: kube-system
roleRef:
kind: ClusterRole
apiGroup: rbac.authorization.k8s.io
name: system:kube-scheduler
{{- end }}
{{- if .Values.randomScheduler.enabled -}}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "theodolite.fullname" . }}-random-scheduler
labels:
app: {{ include "theodolite.fullname" . }}
component: random-scheduler
namespace: kube-system
spec:
replicas: 1
selector:
matchLabels:
app: {{ include "theodolite.fullname" . }}
component: random-scheduler
template:
metadata:
labels:
app: {{ include "theodolite.fullname" . }}
component: random-scheduler
spec:
serviceAccount: {{ include "theodolite.fullname" . }}-random-scheduler
containers:
- name: random-scheduler
image: ghcr.io/cau-se/theodolite-random-scheduler:theodolite-kotlin-latest
#imagePullPolicy: Always
env:
- name: TARGET_NAMESPACE
value: {{ .Release.Namespace }}
{{- end }}
{{- if .Values.randomScheduler.serviceAccount.create -}}
apiVersion: v1
kind: ServiceAccount
metadata:
namespace: kube-system
name: {{ include "theodolite.fullname" . }}-random-scheduler
labels:
app: {{ include "theodolite.fullname" . }}
component: random-scheduler
{{- end }}
...@@ -92,8 +92,9 @@ cp-helm-charts: ...@@ -92,8 +92,9 @@ cp-helm-charts:
"replica.fetch.max.bytes": "134217728" # 128 MB "replica.fetch.max.bytes": "134217728" # 128 MB
#default.replication.factor: 1 #default.replication.factor: 1
# "min.insync.replicas": 2 # "min.insync.replicas": 2
# "auto.create.topics.enable": false "auto.create.topics.enable": false
"log.retention.ms": "10000" # 10s #"log.retention.ms": "10000" # 10s
"log.retention.ms": "7200000" # 2h
"metrics.sample.window.ms": "5000" #5s "metrics.sample.window.ms": "5000" #5s
"advertised.listeners": |- "advertised.listeners": |-
EXTERNAL://${HOST_IP}:$((31090 + ${KAFKA_BROKER_ID})) EXTERNAL://${HOST_IP}:$((31090 + ${KAFKA_BROKER_ID}))
...@@ -248,3 +249,10 @@ serviceAccount: ...@@ -248,3 +249,10 @@ serviceAccount:
rbac: rbac:
create: true create: true
randomScheduler:
enabled: true
rbac:
create: true
serviceAccount:
create: true
...@@ -8,7 +8,7 @@ For development: ...@@ -8,7 +8,7 @@ For development:
uvicorn main:app --reload uvicorn main:app --reload
``` ```
Build the docker image: ## Build the docker image:
```sh ```sh
docker build . -t theodolite-evaluator docker build . -t theodolite-evaluator
......
# theodolite-quarkus project # Theodolite-quarkus project
This project uses Quarkus, the Supersonic Subatomic Java Framework. This project uses Quarkus, the Supersonic Subatomic Java Framework.
...@@ -6,7 +6,8 @@ If you want to learn more about Quarkus, please visit its website: https://quark ...@@ -6,7 +6,8 @@ If you want to learn more about Quarkus, please visit its website: https://quark
## Running the application in dev mode ## Running the application in dev mode
You can run your application in dev mode that enables live coding using: You can run your application in dev mode using:
```shell script ```shell script
./gradlew quarkusDev ./gradlew quarkusDev
``` ```
...@@ -14,13 +15,16 @@ You can run your application in dev mode that enables live coding using: ...@@ -14,13 +15,16 @@ You can run your application in dev mode that enables live coding using:
## Packaging and running the application ## Packaging and running the application
The application can be packaged using: The application can be packaged using:
```shell script ```shell script
./gradlew build ./gradlew build
``` ```
It produces the `theodolite-quarkus-1.0.0-SNAPSHOT-runner.jar` file in the `/build` directory.
Be aware that it’s not an _über-jar_ as the dependencies are copied into the `build/lib` directory. It produces the `theodolite-quarkus-1.0.0-SNAPSHOT-runner.jar` file in the `/build` directory. Be aware that it’s not
an _über-jar_ as the dependencies are copied into the `build/lib` directory.
If you want to build an _über-jar_, execute the following command: If you want to build an _über-jar_, execute the following command:
```shell script ```shell script
./gradlew build -Dquarkus.package.type=uber-jar ./gradlew build -Dquarkus.package.type=uber-jar
``` ```
...@@ -30,11 +34,13 @@ The application is now runnable using `java -jar build/theodolite-quarkus-1.0.0- ...@@ -30,11 +34,13 @@ The application is now runnable using `java -jar build/theodolite-quarkus-1.0.0-
## Creating a native executable ## Creating a native executable
You can create a native executable using: You can create a native executable using:
```shell script ```shell script
./gradlew build -Dquarkus.package.type=native ./gradlew build -Dquarkus.package.type=native
``` ```
Or, if you don't have GraalVM installed, you can run the native executable build in a container using: Or, if you don't have GraalVM installed, you can run the native executable build in a container using:
```shell script ```shell script
./gradlew build -Dquarkus.package.type=native -Dquarkus.native.container-build=true ./gradlew build -Dquarkus.package.type=native -Dquarkus.native.container-build=true
``` ```
...@@ -44,20 +50,73 @@ You can then execute your native executable with: ...@@ -44,20 +50,73 @@ You can then execute your native executable with:
If you want to learn more about building native executables, please consult https://quarkus.io/guides/gradle-tooling. If you want to learn more about building native executables, please consult https://quarkus.io/guides/gradle-tooling.
# RESTEasy JAX-RS ## Build docker images
For the jvm version use:
```shell script
./gradlew build
docker build -f src/main/docker/Dockerfile.jvm -t theodolite-quarkus-jvm .
```
For the native image version use:
```shell script
./gradlew build -Dquarkus.package.type=native
docker build -f src/main/docker/Dockerfile.native -t theodolite-quarkus-native .
```
## Execute docker images:
<p>A Hello World RESTEasy resource</p> Remember to set the environment variables first.
Guide: https://quarkus.io/guides/rest-json Jvm version:
## Build and afterwards run the application in Docker container ```shell script
docker run -i --rm theodolite-quarkus-jvm
```
Native image version:
```build_jvm.sh``` to build the jvm version ```shell script
docker run -i --rm theodolite-quarkus-native
```
```build_native.sh``` to build the native image graal version ## Environment variables
## Install Detekt Code analysis Plugin **Production:** (Docker-Container)
| Variables name | Default value |Usage |
| -----------------------------|:----------------------------------:| ------------:|
| `NAMESPACE` | `default` |Determines the namespace of the Theodolite will be executed in. Used in the KubernetesBenchmark|
| `THEODOLITE_EXECUTION` | `./config/BenchmarkExecution.yaml`|The complete path to the benchmarkExecution file. Used in the TheodoliteYamlExecutor. |
| `THEODOLITE_BENCHMARK_TYPE` | `./config/BenchmarkType.yaml` |The complete path to the benchmarkType file. Used in the TheodoliteYamlExecutor.|
| `THEODOLITE_APP_RESOURCES` | `./config` |The path under which the yamls for the resources for the subexperiments are found. Used in the KubernetesBenchmark|
| `MODE` | `yaml-executor` | Defines the mode of operation: either `yaml-executor` or `operator`|
**Development:** (local via Intellij)
When running Theodolite from within IntelliJ via
[Run Configurations](https://www.jetbrains.com/help/idea/work-with-gradle-tasks.html#gradle_run_config), set the *
Environment variables* field to:
```
NAMESPACE=default;THEODOLITE_BENCHMARK=./../../../../config/BenchmarkType.yaml;THEODOLITE_APP_RESOURCES=./../../../../config;THEODOLITE_EXECUTION=./../../../../config/BenchmarkExecution.yaml;MODE=operator
```
Alternative:
``` sh
export NAMESPACE=default
export THEODOLITE_BENCHMARK=./../../../../config/BenchmarkType.yaml
export THEODOLITE_APP_RESOURCES=./../../../../config
export THEODOLITE_EXECUTION=./../../../../config/BenchmarkExecution.yaml
export MODE=operator
./gradlew quarkusDev
```
#### Install Detekt Code analysis Plugin
Install https://plugins.jetbrains.com/plugin/10761-detekt Install https://plugins.jetbrains.com/plugin/10761-detekt
...@@ -66,7 +125,6 @@ Install https://plugins.jetbrains.com/plugin/10761-detekt ...@@ -66,7 +125,6 @@ Install https://plugins.jetbrains.com/plugin/10761-detekt
- Check Enable Detekt - Check Enable Detekt
- Specify your detekt configuration and baseline file (optional) - Specify your detekt configuration and baseline file (optional)
-> detekt issues will be annotated on-the-fly while coding -> detekt issues will be annotated on-the-fly while coding
**ingore Failures in build:** add **ingore Failures in build:** add
......
## The Benchmark Object
The *benchmark* object defines all static components of an execution of a benchmark with Theodolite.
An exapmle for a benchmark object is given in [example-benchmark-yaml-resource](example-benchmark-yaml-resource.yaml).
A **Benchmark** is a [*standard tool for the competitive evaluation and comparison of competing systems or components according to specific characteristics, such as performance, dependability, or security*](https://doi.org/10.1145/2668930.2688819). In Theodolite, we have [specification-based benchmarks](https://doi.org/10.1145/2668930.2688819), or at least something very close to that. That is, our benchmarks are architectural descriptions---in our case---[of typical use cases of stream processing in microservices](https://doi.org/10.1016/j.bdr.2021.100209) (e.g. our UC1). Hence, we don't really have a piece of software, which represents a benchmark. We only have implementations of benchmarks, e.g. an implementation of UC1 with Kafka Streams. For simplification, we call these *benchmark implementations* simply *benchmarks*.
```yaml
name: String
appResource:
- String
...
loadGenResource:
- String
...
resourceTypes:
- typeName: String
patchers:
- type: String
resources: String
<Patcher Arguments> ...
...
loadTypes:
- typeName: String
patchers:
- type: String
resources: String
<Patcher Arguments> ...
...
kafkaConfig:
bootstrapServer: String
topics:
- name: String
numPartitions: UnsignedInt
replicationFactor: UnsignedInt
- name: String
removeOnly: bool
...
```
The properties have the following definitions:
* **name**: The name of the *benchmark*
* **appResource**: A list of file names that reference Kubernetes resources that are deployed on the cluster for the system under test (SUT).
* **loadGenResources**: A list of file names that reference Kubernetes resources that are deployed on the cluster for the load generator.
* **resourceTypes**: A list of resource types that can be scaled for this *benchmark*. For each resource type the concrete values are defined in the *execution* object. Each resource type has the following structure:
* **typeName**: Name of the resource type.
* **patchers**: List of [patchers](#Patchers) used to scale this resource type. Each patcher has the following structure:
* **type**: Type of the [patcher](#Patchers). The concrete types can be looked up in the list of [patchers](#Patchers).
* **resources**: Specifies the Kubernetes resource to be patched.
* **Patcher Arguments**: (Optional) Patcher specific additional arguments.
* **loadTypes**: A list of load types that can be scaled for this *benchmark*. For each load type the concrete values are defined in the *execution* object. Each load type has the following structure:
* **typeName**: Name of the load type.
* **patchers**: List of patchers used to scale * **resourceTypes**: A list of resource types that can be scaled for this *benchmark*. For each resource type the concrete values are defined in the *execution* resource object.Each resource type has the following structure:
* **typeName**: Name of the resource type.
* **patchers**: List of patchers used to scale this resource type. Each patcher has the following structure:
* **type**: Type of the Patcher. The concrete types can be looked up in the list of patchers.
* **resources**: Specifies the Kubernetes resource to be patched.
* **Patcher Arguments**: (Optional) Patcher specific additional arguments.
* **kafkaConfig**: Contains the Kafka configuration.
* **bootstrapServers**: The bootstrap servers connection string.
* **topics**: List of topics to be created for each [experiment](#Experiment). Alternative theodolite offers the possibility to remove certain topics after each experiment.
* **name**: The name of the topic.
* **numPartitions**: The number of partitions of the topic.
* **replicationFactor**: The replication factor of the topic.
* **removeOnly**: determines if this topic should only be deleted after each experiement. For removeOnly topics the name can be a RegEx describing the topic.
## The Execution Object
A benchmark can be executed for different SUTs, by different users and multiple times. We call such an execution of a benchmark simply an *execution*. The *execution* object defines all conrete values of an Execution.
An exapmle for an execution object is given in [example-execution-yaml-resource](example-benchmark-yaml-resource.yaml).
```yaml
name: String
benchmark: String
load:
loadType: String
loadValues:
- UnsignedInt
...
resources:
resourceType: String
resourceValues:
- UnsignedInt
...
slos:
- sloType: String
threshold: UnsignedInt
prometheusUrl: String
externalSloUrl: String
offset: SignedInt
warmup: UnsignedInt
...
executions:
strategy: "LinearSearch" or "BinarySearch"
duration: UnsignedInt
repetition: UnsignedInt
restrictions:
- "LowerBound"
...
configurationOverrides:
- patcher:
type: String
resource: String
<Patcher Arguments> ...
...
```
The properties have the following definitions:
* **name**: The name of the *execution*
* **benchmark**: The name of the *benchmark* this *execution* is referring to.
* **load**: Specifies the load values that are benchmarked.
* **loadType**: The type of the load. It must match one of the load types specified in the referenced *benchmark*.
* **loadValues**: List of load values for the specified load type.
* **resources**: Specifies the scaling resource that is benchmarked.
* **resourceType**: The type of the resource. It must match one of the resource types specified in the referenced *benchmark*.
* **resourceValues**: List of resource values for the specified resource type.
* **slos**: List of the Service Level Objective (SLO) for this *execution*. Each SLO has the following fields:
* **sloType**: The type of the SLO. It must match 'lag trend'.
* **threshold**: The threshold the SUT should meet for a sucessful experiment.
* **prometheusUrl**: Connection string for promehteus.
* **externalSloUrl**: Connection string for a external slo analysis.
* **offset**: Hours by which the start and end timestamp will be shifted (for different timezones).
* **warmup**: Seconds of time that are ignored in the analysis.
* **executions**: Defines the overall parameter for the execution.
* **strategy**: Defines the used strategy for the execution: either 'LinearSearch' or 'BinarySearch'
* **duration**: Defines the duration of each [experiment](#Experiment) in seconds.
* **repetition**: Unused.
* **restrictions**: List of restriction strategys used to delimit the search space.
**- LowerBound**: Currently only supported *restriction strategy*.
* **configurationOverrides**: List of patchers that are used to override existing configurations.
* **patcher**: Patcher used to patch a resource. Each patcher has the following structure:
* **type**: Type of the Patcher. The concrete types can be looked up in the list of patchers.
* **resources**: Specifies the Kubernetes resource to be patched.
* **Patcher Arguments**: (Optional) Patcher specific additional arguments.
## Patchers
* **ReplicaPatcher**: Allows to modify the number of Replicas for a kubernetes deployment.
* **type**: "ReplicaPatcher"
* **resource**: "uc1-kstreams-deployment.yaml"
* **NumSensorsLoadGeneratorReplicaPatcher**: Allows to scale the nummer of load generators. Scales arcording to the following formular: (value + 15_000 - 1) / 15_000
* **type**: "NumSensorsLoadGeneratorReplicaPatcher"
* **resource**: "uc1-load-generator-deployment.yaml"
* **NumNestedGroupsLoadGeneratorReplicaPatcher**: Allows to scale the nummer of load generators. Scales arcording to the following formular: (4^(value) + 15_000 -1) /15_000
* **type**: "NumNestedGroupsLoadGeneratorReplicaPatcher"
* **resource**: "uc1-load-generator-deployment.yaml"
* **ReplicaPatcher**: Allows to modify the number of Replicas for a kubernetes deployment.
* **type**: "ReplicaPatcher"
* **resource**: "uc1-kstreams-deployment.yaml"
* **EnvVarPatcher**: Allows to modify the value of an environment variable for a container in a kubernetes deployment.
* **type**: "EnvVarPatcher"
* **resource**: "uc1-load-generator-deployment.yaml"
* **container**: "workload-generator"
* **variableName**: "NUM_SENSORS"
* **NodeSelectorPatcher**: Changes the node selection field in kubernetes resources.
* **type**: "NodeSelectorPatcher"
* **resource**: "uc1-load-generator-deployment.yaml"
* **variableName**: "env"
* **value**: "prod"
* **ResourceLimitPatcher**: Changes the resource limit for a kubernetes resource.
* **resource**: "uc1-kstreams-deployment.yaml"
* **container**: "uc-application"
* **variableName**: "cpu" or "memory"
* **value**:"1000m" or "2Gi"
* **SchedulerNamePatcher**: Changes the sheduler for kubernetes resources.
* **type**: "SchedulerNamePatcher"
* **resource**: "uc1-kstreams-deployment.yaml"
* **value**: "random-scheduler"
* **ImagePatcher**: Changes the image of a kubernetes resource. Currently not fully implemented.
* **type**: "ImagePatcher"
* **resource**: "uc1-kstreams-deployment.yaml"
* **container**: "uc-application"
* **value**: "dockerhubrepo/imagename"
## Experiment
According to [our benchmarking method](https://doi.org/10.1016/j.bdr.2021.100209), the execution of a benchmark requires performing multiple **Experiments**. I think what is actually done within/during an experiment is another level of detail. (But just for the sake of completeness: In an experiment, the benchmark implementation is deployed, load is generated according to the benchmark specification, some SLOs are monitored continuously, etc.)
...@@ -22,6 +22,7 @@ execution: ...@@ -22,6 +22,7 @@ execution:
strategy: "LinearSearch" strategy: "LinearSearch"
duration: 60 duration: 60
repetitions: 1 repetitions: 1
delay: 30 # in seconds
restrictions: restrictions:
- "LowerBound" - "LowerBound"
configOverrides: configOverrides:
......
...@@ -22,7 +22,7 @@ loadTypes: ...@@ -22,7 +22,7 @@ loadTypes:
- type: "NumSensorsLoadGeneratorReplicaPatcher" - type: "NumSensorsLoadGeneratorReplicaPatcher"
resource: "uc1-load-generator-deployment.yaml" resource: "uc1-load-generator-deployment.yaml"
kafkaConfig: kafkaConfig:
bootstrapServer: "theodolite-cp-kafka:9092" bootstrapServer: "localhost:31290"
topics: topics:
- name: "input" - name: "input"
numPartitions: 40 numPartitions: 40
......
...@@ -21,6 +21,8 @@ interface Benchmark { ...@@ -21,6 +21,8 @@ interface Benchmark {
fun buildDeployment( fun buildDeployment(
load: LoadDimension, load: LoadDimension,
res: Resource, res: Resource,
configurationOverrides: List<ConfigurationOverride?> configurationOverrides: List<ConfigurationOverride?>,
loadGenerationDelay: Long,
afterTeardownDelay: Long
): BenchmarkDeployment ): BenchmarkDeployment
} }
...@@ -47,6 +47,8 @@ class BenchmarkExecution : CustomResource(), Namespaced { ...@@ -47,6 +47,8 @@ class BenchmarkExecution : CustomResource(), Namespaced {
var duration by Delegates.notNull<Long>() var duration by Delegates.notNull<Long>()
var repetitions by Delegates.notNull<Int>() var repetitions by Delegates.notNull<Int>()
lateinit var restrictions: List<String> lateinit var restrictions: List<String>
var loadGenerationDelay = 0L
var afterTeardownDelay = 5L
} }
/** /**
......
...@@ -22,7 +22,7 @@ private var DEFAULT_NAMESPACE = "default" ...@@ -22,7 +22,7 @@ private var DEFAULT_NAMESPACE = "default"
* - [loadGenResource] resource that generates the load, * - [loadGenResource] resource that generates the load,
* - [resourceTypes] types of scaling resources, * - [resourceTypes] types of scaling resources,
* - [loadTypes] types of loads that can be scaled for the benchmark, * - [loadTypes] types of loads that can be scaled for the benchmark,
* - [kafkaConfig] for the [TopicManager], * - [kafkaConfig] for the [theodolite.k8s.TopicManager],
* - [namespace] for the client, * - [namespace] for the client,
* - [path] under which the resource yamls can be found. * - [path] under which the resource yamls can be found.
* *
...@@ -63,38 +63,45 @@ class KubernetesBenchmark : Benchmark, CustomResource(), Namespaced { ...@@ -63,38 +63,45 @@ class KubernetesBenchmark : Benchmark, CustomResource(), Namespaced {
* First loads all required resources and then patches them to the concrete load and resources for the experiment. * First loads all required resources and then patches them to the concrete load and resources for the experiment.
* Afterwards patches additional configurations(cluster depending) into the resources. * Afterwards patches additional configurations(cluster depending) into the resources.
* @param load concrete load that will be benchmarked in this experiment. * @param load concrete load that will be benchmarked in this experiment.
* @param res concrete resoruce that will be scaled for this experiment. * @param res concrete resource that will be scaled for this experiment.
* @param configurationOverrides * @param configurationOverrides
* @return a [BenchmarkDeployment] * @return a [BenchmarkDeployment]
*/ */
override fun buildDeployment( override fun buildDeployment(
load: LoadDimension, load: LoadDimension,
res: Resource, res: Resource,
configurationOverrides: List<ConfigurationOverride?> configurationOverrides: List<ConfigurationOverride?>,
loadGenerationDelay: Long,
afterTeardownDelay: Long
): BenchmarkDeployment { ): BenchmarkDeployment {
logger.info { "Using $namespace as namespace." } logger.info { "Using $namespace as namespace." }
logger.info { "Using $path as resource path." } logger.info { "Using $path as resource path." }
val resources = loadKubernetesResources(this.appResource + this.loadGenResource) val appResources = loadKubernetesResources(this.appResource)
val loadGenResources = loadKubernetesResources(this.loadGenResource)
val patcherFactory = PatcherFactory() val patcherFactory = PatcherFactory()
// patch the load dimension the resources // patch the load dimension the resources
load.getType().forEach { patcherDefinition -> load.getType().forEach { patcherDefinition ->
patcherFactory.createPatcher(patcherDefinition, resources).patch(load.get().toString()) patcherFactory.createPatcher(patcherDefinition, loadGenResources).patch(load.get().toString())
} }
res.getType().forEach { patcherDefinition -> res.getType().forEach { patcherDefinition ->
patcherFactory.createPatcher(patcherDefinition, resources).patch(res.get().toString()) patcherFactory.createPatcher(patcherDefinition, appResources).patch(res.get().toString())
} }
// Patch the given overrides // Patch the given overrides
configurationOverrides.forEach { override -> configurationOverrides.forEach { override ->
override?.let { override?.let {
patcherFactory.createPatcher(it.patcher, resources).patch(override.value) patcherFactory.createPatcher(it.patcher, appResources + loadGenResources).patch(override.value)
} }
} }
return KubernetesBenchmarkDeployment( return KubernetesBenchmarkDeployment(
namespace = namespace, namespace = namespace,
resources = resources.map { r -> r.second }, appResources = appResources.map { it.second },
loadGenResources = loadGenResources.map { it.second },
loadGenerationDelay = loadGenerationDelay,
afterTeardownDelay = afterTeardownDelay,
kafkaConfig = hashMapOf("bootstrap.servers" to kafkaConfig.bootstrapServer), kafkaConfig = hashMapOf("bootstrap.servers" to kafkaConfig.bootstrapServer),
topics = kafkaConfig.topics, topics = kafkaConfig.topics,
client = DefaultKubernetesClient().inNamespace(namespace) client = DefaultKubernetesClient().inNamespace(namespace)
......
...@@ -8,6 +8,7 @@ import org.apache.kafka.clients.admin.NewTopic ...@@ -8,6 +8,7 @@ import org.apache.kafka.clients.admin.NewTopic
import theodolite.k8s.K8sManager import theodolite.k8s.K8sManager
import theodolite.k8s.TopicManager import theodolite.k8s.TopicManager
import theodolite.util.KafkaConfig import theodolite.util.KafkaConfig
import java.time.Duration
private val logger = KotlinLogging.logger {} private val logger = KotlinLogging.logger {}
...@@ -22,7 +23,10 @@ private val logger = KotlinLogging.logger {} ...@@ -22,7 +23,10 @@ private val logger = KotlinLogging.logger {}
@RegisterForReflection @RegisterForReflection
class KubernetesBenchmarkDeployment( class KubernetesBenchmarkDeployment(
val namespace: String, val namespace: String,
val resources: List<KubernetesResource>, val appResources: List<KubernetesResource>,
val loadGenResources: List<KubernetesResource>,
private val loadGenerationDelay: Long,
private val afterTeardownDelay: Long,
private val kafkaConfig: HashMap<String, Any>, private val kafkaConfig: HashMap<String, Any>,
private val topics: List<KafkaConfig.TopicWrapper>, private val topics: List<KafkaConfig.TopicWrapper>,
private val client: NamespacedKubernetesClient private val client: NamespacedKubernetesClient
...@@ -30,7 +34,6 @@ class KubernetesBenchmarkDeployment( ...@@ -30,7 +34,6 @@ class KubernetesBenchmarkDeployment(
private val kafkaController = TopicManager(this.kafkaConfig) private val kafkaController = TopicManager(this.kafkaConfig)
private val kubernetesManager = K8sManager(client) private val kubernetesManager = K8sManager(client)
private val LAG_EXPORTER_POD_LABEL = "app.kubernetes.io/name=kafka-lag-exporter" private val LAG_EXPORTER_POD_LABEL = "app.kubernetes.io/name=kafka-lag-exporter"
private val SLEEP_AFTER_TEARDOWN = 5000L
/** /**
* Setup a [KubernetesBenchmark] using the [TopicManager] and the [K8sManager]: * Setup a [KubernetesBenchmark] using the [TopicManager] and the [K8sManager]:
...@@ -41,7 +44,10 @@ class KubernetesBenchmarkDeployment( ...@@ -41,7 +44,10 @@ class KubernetesBenchmarkDeployment(
val kafkaTopics = this.topics.filter { !it.removeOnly } val kafkaTopics = this.topics.filter { !it.removeOnly }
.map { NewTopic(it.name, it.numPartitions, it.replicationFactor) } .map { NewTopic(it.name, it.numPartitions, it.replicationFactor) }
kafkaController.createTopics(kafkaTopics) kafkaController.createTopics(kafkaTopics)
resources.forEach { kubernetesManager.deploy(it) } appResources.forEach { kubernetesManager.deploy(it) }
logger.info { "Wait ${this.loadGenerationDelay} seconds before starting the load generator." }
Thread.sleep(Duration.ofSeconds(this.loadGenerationDelay).toMillis())
loadGenResources.forEach { kubernetesManager.deploy(it) }
} }
/** /**
...@@ -51,12 +57,11 @@ class KubernetesBenchmarkDeployment( ...@@ -51,12 +57,11 @@ class KubernetesBenchmarkDeployment(
* - Remove the [KubernetesResource]s. * - Remove the [KubernetesResource]s.
*/ */
override fun teardown() { override fun teardown() {
resources.forEach { loadGenResources.forEach { kubernetesManager.remove(it) }
kubernetesManager.remove(it) appResources.forEach { kubernetesManager.remove(it) }
}
kafkaController.removeTopics(this.topics.map { topic -> topic.name }) kafkaController.removeTopics(this.topics.map { topic -> topic.name })
KafkaLagExporterRemover(client).remove(LAG_EXPORTER_POD_LABEL) KafkaLagExporterRemover(client).remove(LAG_EXPORTER_POD_LABEL)
logger.info { "Teardown complete. Wait $SLEEP_AFTER_TEARDOWN ms to let everything come down." } logger.info { "Teardown complete. Wait $afterTeardownDelay ms to let everything come down." }
Thread.sleep(SLEEP_AFTER_TEARDOWN) Thread.sleep(Duration.ofSeconds(afterTeardownDelay).toMillis())
} }
} }
...@@ -4,7 +4,7 @@ import theodolite.util.PrometheusResponse ...@@ -4,7 +4,7 @@ import theodolite.util.PrometheusResponse
import java.time.Instant import java.time.Instant
/** /**
* A SloChecker can be used to evaluate data from Promehteus. * A SloChecker can be used to evaluate data from Prometheus.
* @constructor Creates an empty SloChecker * @constructor Creates an empty SloChecker
*/ */
interface SloChecker { interface SloChecker {
......
...@@ -24,9 +24,11 @@ abstract class BenchmarkExecutor( ...@@ -24,9 +24,11 @@ abstract class BenchmarkExecutor(
val benchmark: Benchmark, val benchmark: Benchmark,
val results: Results, val results: Results,
val executionDuration: Duration, val executionDuration: Duration,
configurationOverrides: List<ConfigurationOverride?>, val configurationOverrides: List<ConfigurationOverride?>,
val slo: BenchmarkExecution.Slo, val slo: BenchmarkExecution.Slo,
val executionId: Int val executionId: Int,
val loadGenerationDelay: Long,
val afterTeardownDelay: Long
) { ) {
var run: AtomicBoolean = AtomicBoolean(true) var run: AtomicBoolean = AtomicBoolean(true)
......
...@@ -18,13 +18,15 @@ class BenchmarkExecutorImpl( ...@@ -18,13 +18,15 @@ class BenchmarkExecutorImpl(
benchmark: Benchmark, benchmark: Benchmark,
results: Results, results: Results,
executionDuration: Duration, executionDuration: Duration,
private val configurationOverrides: List<ConfigurationOverride?>, configurationOverrides: List<ConfigurationOverride?>,
slo: BenchmarkExecution.Slo, slo: BenchmarkExecution.Slo,
executionId: Int executionId: Int,
) : BenchmarkExecutor(benchmark, results, executionDuration, configurationOverrides, slo, executionId) { loadGenerationDelay: Long,
afterTeardownDelay: Long
) : BenchmarkExecutor(benchmark, results, executionDuration, configurationOverrides, slo, executionId, loadGenerationDelay, afterTeardownDelay) {
override fun runExperiment(load: LoadDimension, res: Resource): Boolean { override fun runExperiment(load: LoadDimension, res: Resource): Boolean {
var result = false var result = false
val benchmarkDeployment = benchmark.buildDeployment(load, res, this.configurationOverrides) val benchmarkDeployment = benchmark.buildDeployment(load, res, configurationOverrides, loadGenerationDelay, this.afterTeardownDelay)
try { try {
benchmarkDeployment.setup() benchmarkDeployment.setup()
......
...@@ -30,7 +30,9 @@ class Shutdown(private val benchmarkExecution: BenchmarkExecution, private val b ...@@ -30,7 +30,9 @@ class Shutdown(private val benchmarkExecution: BenchmarkExecution, private val b
benchmark.buildDeployment( benchmark.buildDeployment(
load = LoadDimension(0, emptyList()), load = LoadDimension(0, emptyList()),
res = Resource(0, emptyList()), res = Resource(0, emptyList()),
configurationOverrides = benchmarkExecution.configOverrides configurationOverrides = benchmarkExecution.configOverrides,
loadGenerationDelay = 0L,
afterTeardownDelay = 5L
) )
deployment.teardown() deployment.teardown()
} catch (e: Exception) { } catch (e: Exception) {
......
...@@ -71,7 +71,9 @@ class TheodoliteExecutor( ...@@ -71,7 +71,9 @@ class TheodoliteExecutor(
executionDuration = executionDuration, executionDuration = executionDuration,
configurationOverrides = config.configOverrides, configurationOverrides = config.configOverrides,
slo = config.slos[0], slo = config.slos[0],
executionId = config.executionId executionId = config.executionId,
loadGenerationDelay = config.execution.loadGenerationDelay,
afterTeardownDelay = config.execution.afterTeardownDelay
) )
return Config( return Config(
...@@ -128,7 +130,7 @@ class TheodoliteExecutor( ...@@ -128,7 +130,7 @@ class TheodoliteExecutor(
fun run() { fun run() {
val resultsFolder = getResultFolderString() val resultsFolder = getResultFolderString()
storeAsFile(this.config, "$resultsFolder${this.config.executionId}-execution-configuration") storeAsFile(this.config, "$resultsFolder${this.config.executionId}-execution-configuration")
storeAsFile(kubernetesBenchmark, "$resultsFolder/${this.config.executionId}-benchmark-configuration") storeAsFile(kubernetesBenchmark, "$resultsFolder${this.config.executionId}-benchmark-configuration")
val config = buildConfig() val config = buildConfig()
// execute benchmarks for each load // execute benchmarks for each load
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment