Skip to content
Snippets Groups Projects
Commit 32e65a54 authored by Sören Henning's avatar Sören Henning
Browse files

Merge branch 'upgrade-flink' into upgrade-kstreams

parents 56561a8e d23cde42
No related branches found
No related tags found
2 merge requests!236Upgrade Kafka Streams Benchmarks to Kafka Streams 3.1,!230Upgrade Kafka Streams Benchmarks to Kafka Streams 3.1
Hide whitespace changes
Inline Side-by-side
Showing
with 410 additions and 148 deletions
.gitlab-ci.yml
+ 64
− 17
View file @ 32e65a54
...@@ -33,10 +33,16 @@ default: ...@@ -33,10 +33,16 @@ default:
script: script:
- mkdir -p /kaniko/.docker - mkdir -p /kaniko/.docker
- echo "{\"auths\":{\"${CR_HOST}\":{\"auth\":\"$(printf "%s:%s" "${CR_USER}" "${CR_PW}" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json - echo "{\"auths\":{\"${CR_HOST}\":{\"auth\":\"$(printf "%s:%s" "${CR_USER}" "${CR_PW}" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json
- DOCKER_TAG_NAME=$(echo $CI_COMMIT_REF_SLUG- | sed 's/^master-$//') - >
- "[ ! $CI_COMMIT_TAG ] && KANIKO_D=\"$KANIKO_D -d $CR_HOST/$CR_ORG/$IMAGE_NAME:${DOCKER_TAG_NAME}latest\"" if [ $IMAGE_TAG ]; then
- "[ ! $CI_COMMIT_TAG ] && KANIKO_D=\"$KANIKO_D -d $CR_HOST/$CR_ORG/$IMAGE_NAME:$DOCKER_TAG_NAME$CI_COMMIT_SHORT_SHA\"" KANIKO_D="$KANIKO_D -d $CR_HOST/$CR_ORG/$IMAGE_NAME:$IMAGE_TAG"
- "[ $CI_COMMIT_TAG ] && KANIKO_D=\"$KANIKO_D -d $CR_HOST/$CR_ORG/$IMAGE_NAME:$CI_COMMIT_TAG\"" elif [ $CI_COMMIT_TAG ]; then
KANIKO_D="$KANIKO_D -d $CR_HOST/$CR_ORG/$IMAGE_NAME:$CI_COMMIT_TAG"
else
DOCKER_TAG_NAME=$(echo $CI_COMMIT_REF_SLUG- | sed 's/^master-$//')
KANIKO_D="$KANIKO_D -d $CR_HOST/$CR_ORG/$IMAGE_NAME:${DOCKER_TAG_NAME}latest"
KANIKO_D="$KANIKO_D -d $CR_HOST/$CR_ORG/$IMAGE_NAME:$DOCKER_TAG_NAME$CI_COMMIT_SHORT_SHA"
fi
- "[ $DOCKERFILE ] && KANIKO_DOCKERFILE=\"--dockerfile $DOCKERFILE\"" - "[ $DOCKERFILE ] && KANIKO_DOCKERFILE=\"--dockerfile $DOCKERFILE\""
- /kaniko/executor --context `pwd`/$CONTEXT $KANIKO_DOCKERFILE $KANIKO_D - /kaniko/executor --context `pwd`/$CONTEXT $KANIKO_DOCKERFILE $KANIKO_D
...@@ -71,28 +77,39 @@ test-docs-links: ...@@ -71,28 +77,39 @@ test-docs-links:
extends: .docs extends: .docs
needs: needs:
- build-docs - build-docs
script: bundle exec htmlproofer --assume-extension --allow_hash_href --url-ignore "/favicon.ico" ./_site script: bundle exec htmlproofer --assume-extension --allow_hash_href ./_site
build-docs-crds:
stage: build
image:
name: ghcr.io/fybrik/crdoc:0.6.1
entrypoint: [""]
script: /crdoc --resources theodolite/crd/ --template docs/api-reference/crds.tmpl --output docs/api-reference/crds.ref.md
artifacts:
paths:
- docs/api-reference/crds.ref.md
expire_in: 1 week
rules:
- changes:
- docs/api-reference/crds.tmpl
- theodolite/crd/**/*
- when: manual
allow_failure: true
test-docs-crds-regression: test-docs-crds-regression:
stage: test stage: test
image: golang needs:
- build-docs-crds
image: alpine:3.15
before_script: before_script:
- cd docs - cd docs
- go install fybrik.io/crdoc@latest
script: script:
- crdoc --resources ../theodolite/crd/ --template api-reference/crds.tmpl --output api-reference/crds.ref.md
- cmp api-reference/crds.md api-reference/crds.ref.md - cmp api-reference/crds.md api-reference/crds.ref.md
artifacts: artifacts:
when: on_failure when: on_failure
paths: paths:
- docs/api-reference/crds.ref.md - docs/api-reference/crds.ref.md
expire_in: 1 week expire_in: 1 week
rules:
- changes:
- docs/api-reference/crds.tmpl
- theodolite/crd/**/*
- when: manual
allow_failure: true
# Theodolite Helm Chart # Theodolite Helm Chart
...@@ -104,6 +121,11 @@ lint-helm: ...@@ -104,6 +121,11 @@ lint-helm:
name: alpine/helm:3.5.2 name: alpine/helm:3.5.2
entrypoint: [""] entrypoint: [""]
script: helm lint helm/ script: helm lint helm/
rules:
- changes:
- helm/*
- when: manual
allow_failure: true
# Theodolite Benchmarks # Theodolite Benchmarks
...@@ -367,6 +389,11 @@ deploy-uc4-load-generator: ...@@ -367,6 +389,11 @@ deploy-uc4-load-generator:
before_script: before_script:
- export GRADLE_USER_HOME=`pwd`/.gradle - export GRADLE_USER_HOME=`pwd`/.gradle
- cd theodolite - cd theodolite
rules:
- changes:
- theodolite/**/*
- when: manual
allow_failure: true
build-theodolite-jvm: build-theodolite-jvm:
stage: build stage: build
...@@ -374,8 +401,10 @@ build-theodolite-jvm: ...@@ -374,8 +401,10 @@ build-theodolite-jvm:
script: ./gradlew --build-cache assemble script: ./gradlew --build-cache assemble
artifacts: artifacts:
paths: paths:
- "theodolite/build/lib/*" - "theodolite/build/quarkus-app/lib/"
- "theodolite/build/*-runner.jar" - "theodolite/build/quarkus-app/*.jar"
- "theodolite/build/quarkus-app/app/"
- "theodolite/build/quarkus-app/quarkus/"
expire_in: 6 hours expire_in: 6 hours
build-theodolite-native: build-theodolite-native:
...@@ -565,4 +594,22 @@ deploy-random-scheduler: ...@@ -565,4 +594,22 @@ deploy-random-scheduler:
- if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW" - if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW"
when: manual when: manual
allow_failure: true allow_failure: true
\ No newline at end of file deploy-buildimage-docker-compose-jq:
stage: deploy
extends:
- .kaniko-push
needs: []
variables:
DOCKER_VERSION: 20.10.12
IMAGE_NAME: theodolite-build-docker-compose-jq
IMAGE_TAG: $DOCKER_VERSION
before_script:
- cd buildimages/docker-compose-jq
rules:
- changes:
- buildimages/docker-compose-jq/Dockerfile
if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW"
- if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW && $CI_PIPELINE_SOURCE == 'web'"
when: manual
allow_failure: true
CITATION.cff
+ 1
− 1
View file @ 32e65a54
...@@ -8,7 +8,7 @@ authors: ...@@ -8,7 +8,7 @@ authors:
given-names: Wilhelm given-names: Wilhelm
orcid: "https://orcid.org/0000-0001-6625-4335" orcid: "https://orcid.org/0000-0001-6625-4335"
title: Theodolite title: Theodolite
version: "0.6.1" version: "0.6.3"
repository-code: "https://github.com/cau-se/theodolite" repository-code: "https://github.com/cau-se/theodolite"
license: "Apache-2.0" license: "Apache-2.0"
doi: "10.1016/j.bdr.2021.100209" doi: "10.1016/j.bdr.2021.100209"
......
buildimages/docker-compose-jq/Dockerfile 0 → 100644
+ 6
− 0
View file @ 32e65a54
FROM docker:${DOCKER_VERSION:-latest}
RUN apk update && \
apk add jq && \
apk add py-pip python3-dev libffi-dev openssl-dev gcc libc-dev rust cargo make && \
pip install docker-compose
codemeta.json
+ 2
− 2
View file @ 32e65a54
...@@ -5,10 +5,10 @@ ...@@ -5,10 +5,10 @@
"codeRepository": "https://github.com/cau-se/theodolite", "codeRepository": "https://github.com/cau-se/theodolite",
"dateCreated": "2020-03-13", "dateCreated": "2020-03-13",
"datePublished": "2020-07-27", "datePublished": "2020-07-27",
"dateModified": "2022-01-17", "dateModified": "2022-01-24",
"downloadUrl": "https://github.com/cau-se/theodolite/releases", "downloadUrl": "https://github.com/cau-se/theodolite/releases",
"name": "Theodolite", "name": "Theodolite",
"version": "0.6.1", "version": "0.6.3",
"description": "Theodolite is a framework for benchmarking the horizontal and vertical scalability of cloud-native applications.", "description": "Theodolite is a framework for benchmarking the horizontal and vertical scalability of cloud-native applications.",
"developmentStatus": "active", "developmentStatus": "active",
"relatedLink": [ "relatedLink": [
......
docs/README.md
+ 1
− 1
View file @ 32e65a54
...@@ -39,5 +39,5 @@ crdoc --resources ../theodolite/crd/ --template api-reference/crds.tmpl --outpu ...@@ -39,5 +39,5 @@ crdoc --resources ../theodolite/crd/ --template api-reference/crds.tmpl --outpu
With the following command, crdoc is executed in Docker: With the following command, crdoc is executed in Docker:
```sh ```sh
docker run --rm -v "`pwd`/../theodolite/crd/":/crd -u $UID -v "`pwd`/api-reference":/api-reference ghcr.io/fybrik/crdoc:0.6.0 --resources /crd/ --template /api-reference/crds.tmpl --output /api-reference/crds.md docker run --rm -v "`pwd`/../theodolite/crd/":/crd -v "`pwd`/api-reference":/api-reference ghcr.io/fybrik/crdoc:0.6.1 --resources /crd/ --template /api-reference/crds.tmpl --output /api-reference/crds.md
``` ```
docs/api-reference/crds.md
+ 94
− 94
View file @ 32e65a54
...@@ -94,13 +94,6 @@ Resource Types: ...@@ -94,13 +94,6 @@ Resource Types:
</tr> </tr>
</thead> </thead>
<tbody><tr> <tbody><tr>
<td><b><a href="#benchmarkspeckafkaconfig">kafkaConfig</a></b></td>
<td>object</td>
<td>
Contains the Kafka configuration.<br/>
</td>
<td>true</td>
</tr><tr>
<td><b><a href="#benchmarkspecloadgenerator">loadGenerator</a></b></td> <td><b><a href="#benchmarkspecloadgenerator">loadGenerator</a></b></td>
<td>object</td> <td>object</td>
<td> <td>
...@@ -138,103 +131,20 @@ Resource Types: ...@@ -138,103 +131,20 @@ Resource Types:
</td> </td>
<td>false</td> <td>false</td>
</tr><tr> </tr><tr>
<td><b>name</b></td> <td><b><a href="#benchmarkspeckafkaconfig">kafkaConfig</a></b></td>
<td>string</td> <td>object</td>
<td> <td>
This field exists only for technical reasons and should not be set by the user. The value of the field will be overwritten.<br/> Contains the Kafka configuration.<br/>
<br/>
<i>Default</i>: <br/>
</td> </td>
<td>false</td> <td>false</td>
</tr></tbody>
</table>
### benchmark.spec.kafkaConfig
<sup><sup>[↩ Parent](#benchmarkspec)</sup></sup>
Contains the Kafka configuration.
<table>
<thead>
<tr>
<th>Name</th>
<th>Type</th>
<th>Description</th>
<th>Required</th>
</tr>
</thead>
<tbody><tr>
<td><b>bootstrapServer</b></td>
<td>string</td>
<td>
The bootstrap servers connection string.<br/>
</td>
<td>true</td>
</tr><tr> </tr><tr>
<td><b><a href="#benchmarkspeckafkaconfigtopicsindex">topics</a></b></td>
<td>[]object</td>
<td>
List of topics to be created for each experiment. Alternative theodolite offers the possibility to remove certain topics after each experiment.<br/>
</td>
<td>true</td>
</tr></tbody>
</table>
### benchmark.spec.kafkaConfig.topics[index]
<sup><sup>[↩ Parent](#benchmarkspeckafkaconfig)</sup></sup>
<table>
<thead>
<tr>
<th>Name</th>
<th>Type</th>
<th>Description</th>
<th>Required</th>
</tr>
</thead>
<tbody><tr>
<td><b>name</b></td> <td><b>name</b></td>
<td>string</td> <td>string</td>
<td> <td>
The name of the topic.<br/> This field exists only for technical reasons and should not be set by the user. The value of the field will be overwritten.<br/>
<br/> <br/>
<i>Default</i>: <br/> <i>Default</i>: <br/>
</td> </td>
<td>true</td>
</tr><tr>
<td><b>numPartitions</b></td>
<td>integer</td>
<td>
The number of partitions of the topic.<br/>
<br/>
<i>Default</i>: 0<br/>
</td>
<td>false</td>
</tr><tr>
<td><b>removeOnly</b></td>
<td>boolean</td>
<td>
Determines if this topic should only be deleted after each experiement. For removeOnly topics the name can be a RegEx describing the topic.<br/>
<br/>
<i>Default</i>: false<br/>
</td>
<td>false</td>
</tr><tr>
<td><b>replicationFactor</b></td>
<td>integer</td>
<td>
The replication factor of the topic.<br/>
<br/>
<i>Default</i>: 0<br/>
</td>
<td>false</td> <td>false</td>
</tr></tbody> </tr></tbody>
</table> </table>
...@@ -1647,6 +1557,96 @@ The fileSystem resourceSet loads the Kubernetes manifests from the filesystem. ...@@ -1647,6 +1557,96 @@ The fileSystem resourceSet loads the Kubernetes manifests from the filesystem.
</table> </table>
### benchmark.spec.kafkaConfig
<sup><sup>[↩ Parent](#benchmarkspec)</sup></sup>
Contains the Kafka configuration.
<table>
<thead>
<tr>
<th>Name</th>
<th>Type</th>
<th>Description</th>
<th>Required</th>
</tr>
</thead>
<tbody><tr>
<td><b>bootstrapServer</b></td>
<td>string</td>
<td>
The bootstrap servers connection string.<br/>
</td>
<td>true</td>
</tr><tr>
<td><b><a href="#benchmarkspeckafkaconfigtopicsindex">topics</a></b></td>
<td>[]object</td>
<td>
List of topics to be created for each experiment. Alternative theodolite offers the possibility to remove certain topics after each experiment.<br/>
</td>
<td>true</td>
</tr></tbody>
</table>
### benchmark.spec.kafkaConfig.topics[index]
<sup><sup>[↩ Parent](#benchmarkspeckafkaconfig)</sup></sup>
<table>
<thead>
<tr>
<th>Name</th>
<th>Type</th>
<th>Description</th>
<th>Required</th>
</tr>
</thead>
<tbody><tr>
<td><b>name</b></td>
<td>string</td>
<td>
The name of the topic.<br/>
<br/>
<i>Default</i>: <br/>
</td>
<td>true</td>
</tr><tr>
<td><b>numPartitions</b></td>
<td>integer</td>
<td>
The number of partitions of the topic.<br/>
<br/>
<i>Default</i>: 0<br/>
</td>
<td>false</td>
</tr><tr>
<td><b>removeOnly</b></td>
<td>boolean</td>
<td>
Determines if this topic should only be deleted after each experiement. For removeOnly topics the name can be a RegEx describing the topic.<br/>
<br/>
<i>Default</i>: false<br/>
</td>
<td>false</td>
</tr><tr>
<td><b>replicationFactor</b></td>
<td>integer</td>
<td>
The replication factor of the topic.<br/>
<br/>
<i>Default</i>: 0<br/>
</td>
<td>false</td>
</tr></tbody>
</table>
### benchmark.status ### benchmark.status
<sup><sup>[↩ Parent](#benchmark)</sup></sup> <sup><sup>[↩ Parent](#benchmark)</sup></sup>
......
docs/creating-a-benchmark.md
+ 10
− 3
View file @ 32e65a54
...@@ -108,13 +108,20 @@ Suppose the resources needed by your benchmark are defined as YAML files, locate ...@@ -108,13 +108,20 @@ Suppose the resources needed by your benchmark are defined as YAML files, locate
Benchmarks need to specify at least one supported load and resource type for which scalability can be benchmarked. Benchmarks need to specify at least one supported load and resource type for which scalability can be benchmarked.
Load and resource types are described by a name (used for reference from an Execution) and a list of patchers. Load and resource types are described by a name (used for reference from an Execution) and a list of patchers.
Patchers can be seen as functions, which take a value as input and modify a Kubernetes resource in a patcher-specific way. Examples of patchers are the *ReplicaPatcher*, which modifies the replica specification of a deployment, or the *EnvVarPatcher*, which modifies an environment variable.
See the [patcher API reference](api-reference/patchers) for an overview of available patchers.
If a benchmark is [executed by an Execution](running-benchmarks), these patchers are used to configure SUT and load generator according to the [load and resource values](creating-an-execution) set in the Execution. If a benchmark is [executed by an Execution](running-benchmarks), these patchers are used to configure SUT and load generator according to the [load and resource values](creating-an-execution) set in the Execution.
## Kafka Configuration ## Kafka Configuration
Theodolite allows to automatically create and remove Kafka topics for each SLO experiment. Theodolite allows to automatically create and remove Kafka topics for each SLO experiment by setting a `kafkaConfig`.
Use the `removeOnly: True` property for topics which are created automatically by the SUT. It `bootstrapServer` needs to point your Kafka cluster and `topics` configures the list of Kafka topics to be created/removed.
For those topics, also wildcards are allowed in the topic name. For each topic, you configure its name, the number of partitions and the replication factor.
With the `removeOnly: True` property, you can also instruct Theodolite to only remove topics and not create them.
This is useful when benchmarking SUTs, which create topics on their own (e.g., Kafka Streams and Samza applications).
For those topics, also wildcards are allowed in the topic name and, of course, no partition count or replication factor must be provided.
<!-- Further information: API Reference --> <!-- Further information: API Reference -->
......
docs/favicon.ico 0 → 100644
+ 0
− 0
View file @ 32e65a54
docs/favicon.ico

33.7 KiB

docs/index.yaml
+ 71
− 1
View file @ 32e65a54
apiVersion: v1 apiVersion: v1
entries: entries:
theodolite: theodolite:
- apiVersion: v2
appVersion: 0.6.3
created: "2022-01-24T13:40:40.07330713+01:00"
dependencies:
- condition: grafana.enabled
name: grafana
repository: https://grafana.github.io/helm-charts
version: 6.17.5
- condition: kube-prometheus-stack.enabled
name: kube-prometheus-stack
repository: https://prometheus-community.github.io/helm-charts
version: 20.0.1
- condition: cp-helm-charts.enabled
name: cp-helm-charts
repository: https://soerenhenning.github.io/cp-helm-charts
version: 0.6.0
- condition: kafka-lag-exporter.enabled
name: kafka-lag-exporter
repository: https://lightbend.github.io/kafka-lag-exporter/repo/
version: 0.6.7
description: Theodolite is a framework for benchmarking the horizontal and vertical
scalability of cloud-native applications.
digest: ebf08e3bf084fcd96eb2ee0588d495258d1741c74019257e55ba40f574874525
home: https://www.theodolite.rocks
maintainers:
- email: soeren.henning@email.uni-kiel.de
name: Sören Henning
url: https://www.se.informatik.uni-kiel.de/en/team/soeren-henning-m-sc
name: theodolite
sources:
- https://github.com/cau-se/theodolite
type: application
urls:
- https://github.com/cau-se/theodolite/releases/download/v0.6.3/theodolite-0.6.3.tgz
version: 0.6.3
- apiVersion: v2
appVersion: 0.6.2
created: "2022-01-23T22:31:04.773793557+01:00"
dependencies:
- condition: grafana.enabled
name: grafana
repository: https://grafana.github.io/helm-charts
version: 6.17.5
- condition: kube-prometheus-stack.enabled
name: kube-prometheus-stack
repository: https://prometheus-community.github.io/helm-charts
version: 20.0.1
- condition: cp-helm-charts.enabled
name: cp-helm-charts
repository: https://soerenhenning.github.io/cp-helm-charts
version: 0.6.0
- condition: kafka-lag-exporter.enabled
name: kafka-lag-exporter
repository: https://lightbend.github.io/kafka-lag-exporter/repo/
version: 0.6.7
description: Theodolite is a framework for benchmarking the horizontal and vertical
scalability of cloud-native applications.
digest: f6514038741051230dc9be0a6bde3fbc6f92136ecb36c276343e98e550f2c6d0
home: https://www.theodolite.rocks
maintainers:
- email: soeren.henning@email.uni-kiel.de
name: Sören Henning
url: https://www.se.informatik.uni-kiel.de/en/team/soeren-henning-m-sc
name: theodolite
sources:
- https://github.com/cau-se/theodolite
type: application
urls:
- https://github.com/cau-se/theodolite/releases/download/v0.6.2/theodolite-0.6.2.tgz
version: 0.6.2
- apiVersion: v2 - apiVersion: v2
appVersion: 0.6.1 appVersion: 0.6.1
created: "2022-01-18T10:40:00.557347616+01:00" created: "2022-01-18T10:40:00.557347616+01:00"
...@@ -176,4 +246,4 @@ entries: ...@@ -176,4 +246,4 @@ entries:
urls: urls:
- https://github.com/cau-se/theodolite/releases/download/v0.4.0/theodolite-0.4.0.tgz - https://github.com/cau-se/theodolite/releases/download/v0.4.0/theodolite-0.4.0.tgz
version: 0.4.0 version: 0.4.0
generated: "2022-01-18T10:40:00.486387187+01:00" generated: "2022-01-24T13:40:40.036786105+01:00"
docs/running-benchmarks.md
+ 1
− 1
View file @ 32e65a54
...@@ -143,7 +143,7 @@ The easiest way to use them is at MyBinder: ...@@ -143,7 +143,7 @@ The easiest way to use them is at MyBinder:
Alternatively, you can also [run these notebook locally](https://github.com/cau-se/theodolite/tree/master/analysis), for example, with Docker or Visual Studio Code. Alternatively, you can also [run these notebook locally](https://github.com/cau-se/theodolite/tree/master/analysis), for example, with Docker or Visual Studio Code.
The notebooks allow to compute a scalability function using its *demand* metric and to visualize multiple such functions in plots: The notebooks allow to compute a scalability function using Theodolite's *demand* metric and to visualize multiple such functions in plots:
### Computing the *demand* metric with `demand-metric.ipynb` (optional) ### Computing the *demand* metric with `demand-metric.ipynb` (optional)
......
docs/theodolite-benchmarks/index.md
+ 2
− 2
View file @ 32e65a54
--- ---
title: Available Benchmarks title: Available Benchmarks
has_children: false has_children: true
nav_order: 7 nav_order: 7
--- ---
# Theodolite Benchmarks # Theodolite Benchmarks
Theodolite comes with 4 application benchmarks, which are based on typical use cases for stream processing within microservices. For each benchmark, a corresponding load generator is provided. Currently, Theodolite provides benchmark implementations for Apache Kafka Streams and Apache Flink. Theodolite comes with 4 application benchmarks, which are based on typical use cases for stream processing within microservices. For each benchmark, a corresponding [load generator](load-generator) is provided. Currently, Theodolite provides benchmark implementations for Apache Kafka Streams and Apache Flink.
Theodolite's benchmarks are based on typical use cases for stream processing within microservices. Specifically, all benchmarks represent some sort of microservice doing Industrial Internet of Things data analytics. Theodolite's benchmarks are based on typical use cases for stream processing within microservices. Specifically, all benchmarks represent some sort of microservice doing Industrial Internet of Things data analytics.
......
docs/theodolite-benchmarks/load-generator.md 0 → 100644
+ 87
− 0
View file @ 32e65a54
---
title: Load Generators
parent: Available Benchmarks
has_children: false
nav_order: 1
---
# Load Generator Framework
Theodolite's benchmarks come with a flexible load generator framework. It is used to create load on the [4 Theodolite benchmarks](#prebuilt-container-images), but can also be applied to create [custom load generators](#creating-a-custom-load-generator).
It is particularly designed for scalability: Just spin up multiple instances of the load generator and the instances automatically divide the load to be generated among themselves.
## Prebuilt container images
For each benchmark, we provide a [load generator as OCI container image](https://github.com/orgs/cau-se/packages?tab=packages&q=workload-generator). These load generators simulate smart power meters in an industrial facility, which generate measurement records at a fixed rate. Records are published to an Apache Kafka topic (default) or sent as POST requests to an HTTP endpoint.
You can simply run a load generator container, for example, for benchmark UC1 with:
```sh
docker run ghcr.io/cau-se/theodolite-uc1-workload-generator
```
### Message format
Messages generated by the load generators represent a single measurement of [active power](https://en.wikipedia.org/wiki/AC_power#Active,_reactive,_apparent,_and_complex_power_in_sinusoidal_steady-state). The corresponding message type is specified as [`ActivePowerRecords`](https://github.com/cau-se/titan-ccp-common/blob/master/src/main/avro/ActivePower.avdl)
defined with Avro. It consists of an identifier for simulated power sensor, a timestamp in epoch milliseconds and the actual measured (simulated) value in watts.
When sending generated records via Apache Kafka, these records are serialized with the [Confluent Schema Registry](https://docs.confluent.io/platform/current/schema-registry).
If the load generator is configured to send records as HTTP POST requests, records are serialized as JSON according to the following format:
```json
{
"identifier": "sensor-id",
"timestamp": 1645564942000,
"valueInW": 1234.56
}
```
### Configuration
The prebuilt container images can be configured with the following environment variables:
| Environment Variable | Description | Default |
|:----|:----|:----|
| `BOOTSTRAP_SERVER` | Address (`hostname:port`) of another load generator instance to form a cluster with. Can also be this instance. | `localhost:5701` |
| `KUBERNETES_DNS_NAME` | Kubernetes service name to discover other load generators to form a cluster with. Must be a fully qualified domain name (FQDN), e.g., something like `<service>.<namespace>.svc.cluster.local`. * Requires `BOOTSTRAP_SERVER` not to be set. | |
| `PORT` | Port used for for coordination among load generator instances. | 5701 |
| `PORT_AUTO_INCREMENT` | If set to true and the specified PORT is already used, use the next higher one. Useful if multiple instances should run on the same host, without configuring each instance individually. | true |
| `CLUSTER_NAME_PREFIX` | Only required if unrelated load generators form a cluster. | theodolite-load-generation |
| `TARGET` | The target system the load generator send messages to. Valid values are: `kafka`, `http`. | `kafka` |
| `KAFKA_BOOTSTRAP_SERVERS` | A list of host/port pairs to use for establishing the initial connection to the Kafka cluster. See [Kafka producer config: `bootstrap.servers`](https://kafka.apache.org/documentation/#producerconfigs_bootstrap.servers) for more information. Only used if Kafka is set as `TARGET`. | `localhost:9092` |
| `KAFKA_INPUT_TOPIC` | Name of the Kafka topic, which should receive the generated messages. Only used if Kafka is set as `TARGET`. | input |
| `SCHEMA_REGISTRY_URL` | URL of the [Confluent Schema Registry](https://docs.confluent.io/platform/current/schema-registry). | `http://localhost:8081` |
| `KAFKA_BATCH_SIZE` | Value for the Kafka producer configuration: [`batch.size`](https://kafka.apache.org/documentation/#producerconfigs_batch.size). Only used if Kafka is set as `TARGET`. | see Kafka producer config: [`batch.size`](https://kafka.apache.org/documentation/#producerconfigs_batch.size) |
| `KAFKA_LINGER_MS` | Value for the Kafka producer configuration: [`linger.ms`](https://kafka.apache.org/documentation/#producerconfigs_linger.ms). Only used if Kafka is set as `TARGET`. | see Kafka producer config: [`linger.ms`](https://kafka.apache.org/documentation/#producerconfigs_linger.ms) |
| `KAFKA_BUFFER_MEMORY` | Value for the Kafka producer configuration: [`buffer.memory`](https://kafka.apache.org/documentation/#producerconfigs_buffer.memory) Only used if Kafka is set as `TARGET`. | see Kafka producer config: [`buffer.memory`](https://kafka.apache.org/documentation/#producerconfigs_buffer.memory) |
| `HTTP_URL` | The URL the load generator should post messages to. Only used if HTTP is set as `TARGET`. | |
| `NUM_SENSORS` | The amount of simulated sensors. | 10 |
| `PERIOD_MS` | The time in milliseconds between generating two messages for the same sensor. With our Theodolite benchmarks, we apply an [open workload model](https://www.usenix.org/legacy/event/nsdi06/tech/full_papers/schroeder/schroeder.pdf) in which new messages are generated at a fixed rate, without considering the think time of the target server nor the time required for generating a message. | 1000 |
| `VALUE` | The constant `valueInW` of an `ActivePowerRecord`. | 10 |
| `THREADS` | Number of worker threads used to generate the load. | 4 |
Please note that there are some additional configuration options for benchmark [UC4's load generator](https://github.com/cau-se/theodolite/blob/master/theodolite-benchmarks/uc4-load-generator/src/main/java/theodolite/uc4/workloadgenerator/LoadGenerator.java).
## Creating a custom load generator
To create a custom load generator, you need to import the [load-generator-commons](https://github.com/cau-se/theodolite/tree/master/theodolite-benchmarks/load-generator-commons) project. You can then create an instance of the `LoadGenerator` object and call its `run` method:
```java
LoadGenerator loadGenerator = new LoadGenerator()
.setClusterConfig(clusterConfig)
.setLoadDefinition(new WorkloadDefinition(
new KeySpace(key_prefix, numSensors),
duration))
.setGeneratorConfig(new LoadGeneratorConfig(
recordGenerator,
recordSender))
.withThreads(threads);
loadGenerator.run();
```
Alternatively, you can also start with a load generator populated with a default configuration or created from environment variables and then adjust the `LoadGenerator` as desired:
```java
LoadGenerator loadGeneratorFromDefaults = LoadGenerator.fromDefaults()
LoadGenerator loadGeneratorFromEnv = LoadGenerator.fromEnvironment();
```
execution/theodolite.yaml
+ 4
− 5
View file @ 32e65a54
...@@ -21,17 +21,16 @@ spec: ...@@ -21,17 +21,16 @@ spec:
valueFrom: valueFrom:
fieldRef: fieldRef:
fieldPath: metadata.namespace fieldPath: metadata.namespace
# - name: MODE # - name: MODE
# value: yaml-executor # Default is `yaml-executor` # value: yaml-executor # Default is `yaml-executor`
- name: THEODOLITE_EXECUTION - name: THEODOLITE_EXECUTION
value: "execution/execution.yaml" # The name of this file must correspond to the filename of the execution, from which the config map is created. value: "/deployments/execution/execution.yaml" # The name of this file must correspond to the filename of the execution, from which the config map is created.
- name: THEODOLITE_BENCHMARK - name: THEODOLITE_BENCHMARK
value: "benchmark/benchmark.yaml" # The name of this file must correspond to the filename of the benchmark, from which the config map is created. value: "/deployments/benchmark/benchmark.yaml" # The name of this file must correspond to the filename of the benchmark, from which the config map is created.
- name: THEODOLITE_APP_RESOURCES - name: THEODOLITE_APP_RESOURCES
value: "benchmark-resources" value: "/deployments/benchmark-resources"
- name: RESULTS_FOLDER # Folder for saving results - name: RESULTS_FOLDER # Folder for saving results
value: results # Default is the pwd (/deployments) value: /deployments/results # Default is the pwd (/deployments)
# - name: CREATE_RESULTS_FOLDER # Specify whether the specified result folder should be created if it does not exist. # - name: CREATE_RESULTS_FOLDER # Specify whether the specified result folder should be created if it does not exist.
# value: "false" # Default is false. # value: "false" # Default is false.
volumeMounts: volumeMounts:
......
helm/templates/prometheus/prometheus.yaml
+ 1
− 4
View file @ 32e65a54
...@@ -5,10 +5,7 @@ metadata: ...@@ -5,10 +5,7 @@ metadata:
name: {{ template "theodolite.fullname" . }}-prometheus name: {{ template "theodolite.fullname" . }}-prometheus
spec: spec:
serviceAccountName: {{ template "theodolite.fullname" . }}-prometheus serviceAccountName: {{ template "theodolite.fullname" . }}-prometheus
serviceMonitorSelector: serviceMonitorSelector: {}
matchLabels:
#app: cp-kafka
appScope: titan-ccp
resources: resources:
requests: requests:
memory: 400Mi memory: 400Mi
......
helm/templates/theodolite/theodolite-operator.yaml
+ 30
− 2
View file @ 32e65a54
...@@ -27,11 +27,18 @@ spec: ...@@ -27,11 +27,18 @@ spec:
- name: MODE - name: MODE
value: operator value: operator
- name: RESULTS_FOLDER - name: RESULTS_FOLDER
value: "./results" value: "/deployments/results"
volumeMounts: volumeMounts:
- name: theodolite-results-volume - name: theodolite-results-volume
mountPath: "/deployments/results" mountPath: "/deployments/results"
{{- if .Values.operator.sloChecker.droppedRecordsKStreams.enabled }} resources:
requests:
memory: "512Mi"
cpu: "250m"
limits:
memory: "1024Mi"
cpu: "500m"
{{- if .Values.operator.sloChecker.generic.enabled }}
- name: slo-checker-generic - name: slo-checker-generic
image: "{{ .Values.operator.sloChecker.generic.image }}:{{ .Values.operator.sloChecker.generic.imageTag }}" image: "{{ .Values.operator.sloChecker.generic.image }}:{{ .Values.operator.sloChecker.generic.imageTag }}"
imagePullPolicy: "{{ .Values.operator.sloChecker.generic.imagePullPolicy }}" imagePullPolicy: "{{ .Values.operator.sloChecker.generic.imagePullPolicy }}"
...@@ -43,6 +50,13 @@ spec: ...@@ -43,6 +50,13 @@ spec:
value: "8082" value: "8082"
- name: LOG_LEVEL - name: LOG_LEVEL
value: INFO value: INFO
resources:
requests:
memory: "64Mi"
cpu: "50m"
limits:
memory: "128Mi"
cpu: "100m"
{{- end }} {{- end }}
{{- if .Values.operator.sloChecker.lagTrend.enabled }} {{- if .Values.operator.sloChecker.lagTrend.enabled }}
- name: lag-trend-slo-checker - name: lag-trend-slo-checker
...@@ -54,6 +68,13 @@ spec: ...@@ -54,6 +68,13 @@ spec:
env: env:
- name: LOG_LEVEL - name: LOG_LEVEL
value: INFO value: INFO
resources:
requests:
memory: "64Mi"
cpu: "50m"
limits:
memory: "128Mi"
cpu: "100m"
{{- end }} {{- end }}
{{- if .Values.operator.sloChecker.droppedRecordsKStreams.enabled }} {{- if .Values.operator.sloChecker.droppedRecordsKStreams.enabled }}
- name: slo-checker-dropped-records-kstreams - name: slo-checker-dropped-records-kstreams
...@@ -67,6 +88,13 @@ spec: ...@@ -67,6 +88,13 @@ spec:
value: "8081" value: "8081"
- name: LOG_LEVEL - name: LOG_LEVEL
value: INFO value: INFO
resources:
requests:
memory: "64Mi"
cpu: "50m"
limits:
memory: "128Mi"
cpu: "100m"
{{- end }} {{- end }}
{{- if .Values.operator.resultsVolume.accessSidecar.enabled }} {{- if .Values.operator.resultsVolume.accessSidecar.enabled }}
- name: results-access - name: results-access
......
slo-checker/dropped-records/Dockerfile
+ 13
− 4
View file @ 32e65a54
FROM tiangolo/uvicorn-gunicorn-fastapi:python3.7 FROM python:3.8
COPY requirements.txt requirements.txt WORKDIR /code
RUN pip install -r requirements.txt
COPY ./app /app COPY ./requirements.txt /code/requirements.txt
\ No newline at end of file RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
COPY ./app /code/app
WORKDIR /code/app
ENV HOST 0.0.0.0
ENV PORT 80
CMD ["sh", "-c", "uvicorn main:app --host $HOST --port $PORT"]
slo-checker/dropped-records/requirements.txt
+ 5
− 4
View file @ 32e65a54
fastapi==0.65.2
scikit-learn==0.20.3
pandas==1.0.3
uvicorn
requests requests
fastapi>=0.68.0,<0.69.0
uvicorn>=0.15.0,<0.16.0
#pydantic>=1.8.0,<2.0.0
#scikit-learn==0.22.2
pandas==1.0.3
slo-checker/generic/Dockerfile
+ 13
− 4
View file @ 32e65a54
FROM tiangolo/uvicorn-gunicorn-fastapi:python3.7 FROM python:3.8
COPY requirements.txt requirements.txt WORKDIR /code
RUN pip install -r requirements.txt
COPY ./app /app COPY ./requirements.txt /code/requirements.txt
\ No newline at end of file RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
COPY ./app /code/app
WORKDIR /code/app
ENV HOST 0.0.0.0
ENV PORT 80
CMD ["sh", "-c", "uvicorn main:app --host $HOST --port $PORT"]
slo-checker/generic/requirements.txt
+ 5
− 3
View file @ 32e65a54
fastapi==0.65.2
pandas==1.0.3
uvicorn
requests requests
fastapi>=0.68.0,<0.69.0
uvicorn>=0.15.0,<0.16.0
#pydantic>=1.8.0,<2.0.0
#scikit-learn==0.22.2
pandas==1.0.3
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment