Skip to content
Snippets Groups Projects
Commit e944be2e authored by Sören Henning's avatar Sören Henning
Browse files

Merge branch 'master' into pubsub-load-generator

parents bd9af6e8 47ec6dbc
No related branches found
No related tags found
1 merge request!225Add option to generate load via Google PubSub
Hide whitespace changes
Inline Side-by-side
Showing
with 406 additions and 146 deletions
.gitlab-ci.yml
+ 60
− 15
View file @ e944be2e
......@@ -33,10 +33,16 @@ default:
script:
- mkdir -p /kaniko/.docker
- echo "{\"auths\":{\"${CR_HOST}\":{\"auth\":\"$(printf "%s:%s" "${CR_USER}" "${CR_PW}" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json
- DOCKER_TAG_NAME=$(echo $CI_COMMIT_REF_SLUG- | sed 's/^master-$//')
- "[ ! $CI_COMMIT_TAG ] && KANIKO_D=\"$KANIKO_D -d $CR_HOST/$CR_ORG/$IMAGE_NAME:${DOCKER_TAG_NAME}latest\""
- "[ ! $CI_COMMIT_TAG ] && KANIKO_D=\"$KANIKO_D -d $CR_HOST/$CR_ORG/$IMAGE_NAME:$DOCKER_TAG_NAME$CI_COMMIT_SHORT_SHA\""
- "[ $CI_COMMIT_TAG ] && KANIKO_D=\"$KANIKO_D -d $CR_HOST/$CR_ORG/$IMAGE_NAME:$CI_COMMIT_TAG\""
- >
if [ $IMAGE_TAG ]; then
KANIKO_D="$KANIKO_D -d $CR_HOST/$CR_ORG/$IMAGE_NAME:$IMAGE_TAG"
elif [ $CI_COMMIT_TAG ]; then
KANIKO_D="$KANIKO_D -d $CR_HOST/$CR_ORG/$IMAGE_NAME:$CI_COMMIT_TAG"
else
DOCKER_TAG_NAME=$(echo $CI_COMMIT_REF_SLUG- | sed 's/^master-$//')
KANIKO_D="$KANIKO_D -d $CR_HOST/$CR_ORG/$IMAGE_NAME:${DOCKER_TAG_NAME}latest"
KANIKO_D="$KANIKO_D -d $CR_HOST/$CR_ORG/$IMAGE_NAME:$DOCKER_TAG_NAME$CI_COMMIT_SHORT_SHA"
fi
- "[ $DOCKERFILE ] && KANIKO_DOCKERFILE=\"--dockerfile $DOCKERFILE\""
- /kaniko/executor --context `pwd`/$CONTEXT $KANIKO_DOCKERFILE $KANIKO_D
......@@ -71,28 +77,39 @@ test-docs-links:
extends: .docs
needs:
- build-docs
script: bundle exec htmlproofer --assume-extension --allow_hash_href --url-ignore "/favicon.ico" ./_site
script: bundle exec htmlproofer --assume-extension --allow_hash_href ./_site
build-docs-crds:
stage: build
image:
name: ghcr.io/fybrik/crdoc:0.6.1
entrypoint: [""]
script: /crdoc --resources theodolite/crd/ --template docs/api-reference/crds.tmpl --output docs/api-reference/crds.ref.md
artifacts:
paths:
- docs/api-reference/crds.ref.md
expire_in: 1 week
rules:
- changes:
- docs/api-reference/crds.tmpl
- theodolite/crd/**/*
- when: manual
allow_failure: true
test-docs-crds-regression:
stage: test
image: golang
needs:
- build-docs-crds
image: alpine:3.15
before_script:
- cd docs
- go install fybrik.io/crdoc@latest
script:
- crdoc --resources ../theodolite/crd/ --template api-reference/crds.tmpl --output api-reference/crds.ref.md
- cmp api-reference/crds.md api-reference/crds.ref.md
artifacts:
when: on_failure
paths:
- docs/api-reference/crds.ref.md
expire_in: 1 week
rules:
- changes:
- docs/api-reference/crds.tmpl
- theodolite/crd/**/*
- when: manual
allow_failure: true
# Theodolite Helm Chart
......@@ -104,6 +121,11 @@ lint-helm:
name: alpine/helm:3.5.2
entrypoint: [""]
script: helm lint helm/
rules:
- changes:
- helm/*
- when: manual
allow_failure: true
# Theodolite Benchmarks
......@@ -367,6 +389,11 @@ deploy-uc4-load-generator:
before_script:
- export GRADLE_USER_HOME=`pwd`/.gradle
- cd theodolite
rules:
- changes:
- theodolite/**/*
- when: manual
allow_failure: true
build-theodolite-jvm:
stage: build
......@@ -567,4 +594,22 @@ deploy-random-scheduler:
- if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW"
when: manual
allow_failure: true
\ No newline at end of file
deploy-buildimage-docker-compose-jq:
stage: deploy
extends:
- .kaniko-push
needs: []
variables:
DOCKER_VERSION: 20.10.12
IMAGE_NAME: theodolite-build-docker-compose-jq
IMAGE_TAG: $DOCKER_VERSION
before_script:
- cd buildimages/docker-compose-jq
rules:
- changes:
- buildimages/docker-compose-jq/Dockerfile
if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW"
- if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW && $CI_PIPELINE_SOURCE == 'web'"
when: manual
allow_failure: true
CITATION.cff
+ 1
− 1
View file @ e944be2e
......@@ -8,7 +8,7 @@ authors:
given-names: Wilhelm
orcid: "https://orcid.org/0000-0001-6625-4335"
title: Theodolite
version: "0.6.1"
version: "0.6.3"
repository-code: "https://github.com/cau-se/theodolite"
license: "Apache-2.0"
doi: "10.1016/j.bdr.2021.100209"
......
buildimages/docker-compose-jq/Dockerfile 0 → 100644
+ 6
− 0
View file @ e944be2e
FROM docker:${DOCKER_VERSION:-latest}
RUN apk update && \
apk add jq && \
apk add py-pip python3-dev libffi-dev openssl-dev gcc libc-dev rust cargo make && \
pip install docker-compose
codemeta.json
+ 2
− 2
View file @ e944be2e
......@@ -5,10 +5,10 @@
"codeRepository": "https://github.com/cau-se/theodolite",
"dateCreated": "2020-03-13",
"datePublished": "2020-07-27",
"dateModified": "2022-01-17",
"dateModified": "2022-01-24",
"downloadUrl": "https://github.com/cau-se/theodolite/releases",
"name": "Theodolite",
"version": "0.6.1",
"version": "0.6.3",
"description": "Theodolite is a framework for benchmarking the horizontal and vertical scalability of cloud-native applications.",
"developmentStatus": "active",
"relatedLink": [
......
docs/README.md
+ 1
− 1
View file @ e944be2e
......@@ -39,5 +39,5 @@ crdoc --resources ../theodolite/crd/ --template api-reference/crds.tmpl --outpu
With the following command, crdoc is executed in Docker:
```sh
docker run --rm -v "`pwd`/../theodolite/crd/":/crd -u $UID -v "`pwd`/api-reference":/api-reference ghcr.io/fybrik/crdoc:0.6.0 --resources /crd/ --template /api-reference/crds.tmpl --output /api-reference/crds.md
docker run --rm -v "`pwd`/../theodolite/crd/":/crd -v "`pwd`/api-reference":/api-reference ghcr.io/fybrik/crdoc:0.6.1 --resources /crd/ --template /api-reference/crds.tmpl --output /api-reference/crds.md
```
docs/api-reference/crds.md
+ 94
− 94
View file @ e944be2e
......@@ -94,13 +94,6 @@ Resource Types:
</tr>
</thead>
<tbody><tr>
<td><b><a href="#benchmarkspeckafkaconfig">kafkaConfig</a></b></td>
<td>object</td>
<td>
Contains the Kafka configuration.<br/>
</td>
<td>true</td>
</tr><tr>
<td><b><a href="#benchmarkspecloadgenerator">loadGenerator</a></b></td>
<td>object</td>
<td>
......@@ -138,103 +131,20 @@ Resource Types:
</td>
<td>false</td>
</tr><tr>
<td><b>name</b></td>
<td>string</td>
<td><b><a href="#benchmarkspeckafkaconfig">kafkaConfig</a></b></td>
<td>object</td>
<td>
This field exists only for technical reasons and should not be set by the user. The value of the field will be overwritten.<br/>
<br/>
<i>Default</i>: <br/>
Contains the Kafka configuration.<br/>
</td>
<td>false</td>
</tr></tbody>
</table>
### benchmark.spec.kafkaConfig
<sup><sup>[↩ Parent](#benchmarkspec)</sup></sup>
Contains the Kafka configuration.
<table>
<thead>
<tr>
<th>Name</th>
<th>Type</th>
<th>Description</th>
<th>Required</th>
</tr>
</thead>
<tbody><tr>
<td><b>bootstrapServer</b></td>
<td>string</td>
<td>
The bootstrap servers connection string.<br/>
</td>
<td>true</td>
</tr><tr>
<td><b><a href="#benchmarkspeckafkaconfigtopicsindex">topics</a></b></td>
<td>[]object</td>
<td>
List of topics to be created for each experiment. Alternative theodolite offers the possibility to remove certain topics after each experiment.<br/>
</td>
<td>true</td>
</tr></tbody>
</table>
### benchmark.spec.kafkaConfig.topics[index]
<sup><sup>[↩ Parent](#benchmarkspeckafkaconfig)</sup></sup>
<table>
<thead>
<tr>
<th>Name</th>
<th>Type</th>
<th>Description</th>
<th>Required</th>
</tr>
</thead>
<tbody><tr>
<td><b>name</b></td>
<td>string</td>
<td>
The name of the topic.<br/>
This field exists only for technical reasons and should not be set by the user. The value of the field will be overwritten.<br/>
<br/>
<i>Default</i>: <br/>
</td>
<td>true</td>
</tr><tr>
<td><b>numPartitions</b></td>
<td>integer</td>
<td>
The number of partitions of the topic.<br/>
<br/>
<i>Default</i>: 0<br/>
</td>
<td>false</td>
</tr><tr>
<td><b>removeOnly</b></td>
<td>boolean</td>
<td>
Determines if this topic should only be deleted after each experiement. For removeOnly topics the name can be a RegEx describing the topic.<br/>
<br/>
<i>Default</i>: false<br/>
</td>
<td>false</td>
</tr><tr>
<td><b>replicationFactor</b></td>
<td>integer</td>
<td>
The replication factor of the topic.<br/>
<br/>
<i>Default</i>: 0<br/>
</td>
<td>false</td>
</tr></tbody>
</table>
......@@ -1647,6 +1557,96 @@ The fileSystem resourceSet loads the Kubernetes manifests from the filesystem.
</table>
### benchmark.spec.kafkaConfig
<sup><sup>[↩ Parent](#benchmarkspec)</sup></sup>
Contains the Kafka configuration.
<table>
<thead>
<tr>
<th>Name</th>
<th>Type</th>
<th>Description</th>
<th>Required</th>
</tr>
</thead>
<tbody><tr>
<td><b>bootstrapServer</b></td>
<td>string</td>
<td>
The bootstrap servers connection string.<br/>
</td>
<td>true</td>
</tr><tr>
<td><b><a href="#benchmarkspeckafkaconfigtopicsindex">topics</a></b></td>
<td>[]object</td>
<td>
List of topics to be created for each experiment. Alternative theodolite offers the possibility to remove certain topics after each experiment.<br/>
</td>
<td>true</td>
</tr></tbody>
</table>
### benchmark.spec.kafkaConfig.topics[index]
<sup><sup>[↩ Parent](#benchmarkspeckafkaconfig)</sup></sup>
<table>
<thead>
<tr>
<th>Name</th>
<th>Type</th>
<th>Description</th>
<th>Required</th>
</tr>
</thead>
<tbody><tr>
<td><b>name</b></td>
<td>string</td>
<td>
The name of the topic.<br/>
<br/>
<i>Default</i>: <br/>
</td>
<td>true</td>
</tr><tr>
<td><b>numPartitions</b></td>
<td>integer</td>
<td>
The number of partitions of the topic.<br/>
<br/>
<i>Default</i>: 0<br/>
</td>
<td>false</td>
</tr><tr>
<td><b>removeOnly</b></td>
<td>boolean</td>
<td>
Determines if this topic should only be deleted after each experiement. For removeOnly topics the name can be a RegEx describing the topic.<br/>
<br/>
<i>Default</i>: false<br/>
</td>
<td>false</td>
</tr><tr>
<td><b>replicationFactor</b></td>
<td>integer</td>
<td>
The replication factor of the topic.<br/>
<br/>
<i>Default</i>: 0<br/>
</td>
<td>false</td>
</tr></tbody>
</table>
### benchmark.status
<sup><sup>[↩ Parent](#benchmark)</sup></sup>
......
docs/creating-a-benchmark.md
+ 10
− 3
View file @ e944be2e
......@@ -108,13 +108,20 @@ Suppose the resources needed by your benchmark are defined as YAML files, locate
Benchmarks need to specify at least one supported load and resource type for which scalability can be benchmarked.
Load and resource types are described by a name (used for reference from an Execution) and a list of patchers.
Patchers can be seen as functions, which take a value as input and modify a Kubernetes resource in a patcher-specific way. Examples of patchers are the *ReplicaPatcher*, which modifies the replica specification of a deployment, or the *EnvVarPatcher*, which modifies an environment variable.
See the [patcher API reference](api-reference/patchers) for an overview of available patchers.
If a benchmark is [executed by an Execution](running-benchmarks), these patchers are used to configure SUT and load generator according to the [load and resource values](creating-an-execution) set in the Execution.
## Kafka Configuration
Theodolite allows to automatically create and remove Kafka topics for each SLO experiment.
Use the `removeOnly: True` property for topics which are created automatically by the SUT.
For those topics, also wildcards are allowed in the topic name.
Theodolite allows to automatically create and remove Kafka topics for each SLO experiment by setting a `kafkaConfig`.
It `bootstrapServer` needs to point your Kafka cluster and `topics` configures the list of Kafka topics to be created/removed.
For each topic, you configure its name, the number of partitions and the replication factor.
With the `removeOnly: True` property, you can also instruct Theodolite to only remove topics and not create them.
This is useful when benchmarking SUTs, which create topics on their own (e.g., Kafka Streams and Samza applications).
For those topics, also wildcards are allowed in the topic name and, of course, no partition count or replication factor must be provided.
<!-- Further information: API Reference -->
......
docs/favicon.ico 0 → 100644
+ 0
− 0
View file @ e944be2e
docs/favicon.ico

33.7 KiB

docs/index.yaml
+ 71
− 1
View file @ e944be2e
apiVersion: v1
entries:
theodolite:
- apiVersion: v2
appVersion: 0.6.3
created: "2022-01-24T13:40:40.07330713+01:00"
dependencies:
- condition: grafana.enabled
name: grafana
repository: https://grafana.github.io/helm-charts
version: 6.17.5
- condition: kube-prometheus-stack.enabled
name: kube-prometheus-stack
repository: https://prometheus-community.github.io/helm-charts
version: 20.0.1
- condition: cp-helm-charts.enabled
name: cp-helm-charts
repository: https://soerenhenning.github.io/cp-helm-charts
version: 0.6.0
- condition: kafka-lag-exporter.enabled
name: kafka-lag-exporter
repository: https://lightbend.github.io/kafka-lag-exporter/repo/
version: 0.6.7
description: Theodolite is a framework for benchmarking the horizontal and vertical
scalability of cloud-native applications.
digest: ebf08e3bf084fcd96eb2ee0588d495258d1741c74019257e55ba40f574874525
home: https://www.theodolite.rocks
maintainers:
- email: soeren.henning@email.uni-kiel.de
name: Sören Henning
url: https://www.se.informatik.uni-kiel.de/en/team/soeren-henning-m-sc
name: theodolite
sources:
- https://github.com/cau-se/theodolite
type: application
urls:
- https://github.com/cau-se/theodolite/releases/download/v0.6.3/theodolite-0.6.3.tgz
version: 0.6.3
- apiVersion: v2
appVersion: 0.6.2
created: "2022-01-23T22:31:04.773793557+01:00"
dependencies:
- condition: grafana.enabled
name: grafana
repository: https://grafana.github.io/helm-charts
version: 6.17.5
- condition: kube-prometheus-stack.enabled
name: kube-prometheus-stack
repository: https://prometheus-community.github.io/helm-charts
version: 20.0.1
- condition: cp-helm-charts.enabled
name: cp-helm-charts
repository: https://soerenhenning.github.io/cp-helm-charts
version: 0.6.0
- condition: kafka-lag-exporter.enabled
name: kafka-lag-exporter
repository: https://lightbend.github.io/kafka-lag-exporter/repo/
version: 0.6.7
description: Theodolite is a framework for benchmarking the horizontal and vertical
scalability of cloud-native applications.
digest: f6514038741051230dc9be0a6bde3fbc6f92136ecb36c276343e98e550f2c6d0
home: https://www.theodolite.rocks
maintainers:
- email: soeren.henning@email.uni-kiel.de
name: Sören Henning
url: https://www.se.informatik.uni-kiel.de/en/team/soeren-henning-m-sc
name: theodolite
sources:
- https://github.com/cau-se/theodolite
type: application
urls:
- https://github.com/cau-se/theodolite/releases/download/v0.6.2/theodolite-0.6.2.tgz
version: 0.6.2
- apiVersion: v2
appVersion: 0.6.1
created: "2022-01-18T10:40:00.557347616+01:00"
......@@ -176,4 +246,4 @@ entries:
urls:
- https://github.com/cau-se/theodolite/releases/download/v0.4.0/theodolite-0.4.0.tgz
version: 0.4.0
generated: "2022-01-18T10:40:00.486387187+01:00"
generated: "2022-01-24T13:40:40.036786105+01:00"
docs/running-benchmarks.md
+ 1
− 1
View file @ e944be2e
......@@ -143,7 +143,7 @@ The easiest way to use them is at MyBinder:
Alternatively, you can also [run these notebook locally](https://github.com/cau-se/theodolite/tree/master/analysis), for example, with Docker or Visual Studio Code.
The notebooks allow to compute a scalability function using its *demand* metric and to visualize multiple such functions in plots:
The notebooks allow to compute a scalability function using Theodolite's *demand* metric and to visualize multiple such functions in plots:
### Computing the *demand* metric with `demand-metric.ipynb` (optional)
......
docs/theodolite-benchmarks/index.md
+ 2
− 2
View file @ e944be2e
---
title: Available Benchmarks
has_children: false
has_children: true
nav_order: 7
---
# Theodolite Benchmarks
Theodolite comes with 4 application benchmarks, which are based on typical use cases for stream processing within microservices. For each benchmark, a corresponding load generator is provided. Currently, Theodolite provides benchmark implementations for Apache Kafka Streams and Apache Flink.
Theodolite comes with 4 application benchmarks, which are based on typical use cases for stream processing within microservices. For each benchmark, a corresponding [load generator](load-generator) is provided. Currently, Theodolite provides benchmark implementations for Apache Kafka Streams and Apache Flink.
Theodolite's benchmarks are based on typical use cases for stream processing within microservices. Specifically, all benchmarks represent some sort of microservice doing Industrial Internet of Things data analytics.
......
docs/theodolite-benchmarks/load-generator.md 0 → 100644
+ 87
− 0
View file @ e944be2e
---
title: Load Generators
parent: Available Benchmarks
has_children: false
nav_order: 1
---
# Load Generator Framework
Theodolite's benchmarks come with a flexible load generator framework. It is used to create load on the [4 Theodolite benchmarks](#prebuilt-container-images), but can also be applied to create [custom load generators](#creating-a-custom-load-generator).
It is particularly designed for scalability: Just spin up multiple instances of the load generator and the instances automatically divide the load to be generated among themselves.
## Prebuilt container images
For each benchmark, we provide a [load generator as OCI container image](https://github.com/orgs/cau-se/packages?tab=packages&q=workload-generator). These load generators simulate smart power meters in an industrial facility, which generate measurement records at a fixed rate. Records are published to an Apache Kafka topic (default) or sent as POST requests to an HTTP endpoint.
You can simply run a load generator container, for example, for benchmark UC1 with:
```sh
docker run ghcr.io/cau-se/theodolite-uc1-workload-generator
```
### Message format
Messages generated by the load generators represent a single measurement of [active power](https://en.wikipedia.org/wiki/AC_power#Active,_reactive,_apparent,_and_complex_power_in_sinusoidal_steady-state). The corresponding message type is specified as [`ActivePowerRecords`](https://github.com/cau-se/titan-ccp-common/blob/master/src/main/avro/ActivePower.avdl)
defined with Avro. It consists of an identifier for simulated power sensor, a timestamp in epoch milliseconds and the actual measured (simulated) value in watts.
When sending generated records via Apache Kafka, these records are serialized with the [Confluent Schema Registry](https://docs.confluent.io/platform/current/schema-registry).
If the load generator is configured to send records as HTTP POST requests, records are serialized as JSON according to the following format:
```json
{
"identifier": "sensor-id",
"timestamp": 1645564942000,
"valueInW": 1234.56
}
```
### Configuration
The prebuilt container images can be configured with the following environment variables:
| Environment Variable | Description | Default |
|:----|:----|:----|
| `BOOTSTRAP_SERVER` | Address (`hostname:port`) of another load generator instance to form a cluster with. Can also be this instance. | `localhost:5701` |
| `KUBERNETES_DNS_NAME` | Kubernetes service name to discover other load generators to form a cluster with. Must be a fully qualified domain name (FQDN), e.g., something like `<service>.<namespace>.svc.cluster.local`. * Requires `BOOTSTRAP_SERVER` not to be set. | |
| `PORT` | Port used for for coordination among load generator instances. | 5701 |
| `PORT_AUTO_INCREMENT` | If set to true and the specified PORT is already used, use the next higher one. Useful if multiple instances should run on the same host, without configuring each instance individually. | true |
| `CLUSTER_NAME_PREFIX` | Only required if unrelated load generators form a cluster. | theodolite-load-generation |
| `TARGET` | The target system the load generator send messages to. Valid values are: `kafka`, `http`. | `kafka` |
| `KAFKA_BOOTSTRAP_SERVERS` | A list of host/port pairs to use for establishing the initial connection to the Kafka cluster. See [Kafka producer config: `bootstrap.servers`](https://kafka.apache.org/documentation/#producerconfigs_bootstrap.servers) for more information. Only used if Kafka is set as `TARGET`. | `localhost:9092` |
| `KAFKA_INPUT_TOPIC` | Name of the Kafka topic, which should receive the generated messages. Only used if Kafka is set as `TARGET`. | input |
| `SCHEMA_REGISTRY_URL` | URL of the [Confluent Schema Registry](https://docs.confluent.io/platform/current/schema-registry). | `http://localhost:8081` |
| `KAFKA_BATCH_SIZE` | Value for the Kafka producer configuration: [`batch.size`](https://kafka.apache.org/documentation/#producerconfigs_batch.size). Only used if Kafka is set as `TARGET`. | see Kafka producer config: [`batch.size`](https://kafka.apache.org/documentation/#producerconfigs_batch.size) |
| `KAFKA_LINGER_MS` | Value for the Kafka producer configuration: [`linger.ms`](https://kafka.apache.org/documentation/#producerconfigs_linger.ms). Only used if Kafka is set as `TARGET`. | see Kafka producer config: [`linger.ms`](https://kafka.apache.org/documentation/#producerconfigs_linger.ms) |
| `KAFKA_BUFFER_MEMORY` | Value for the Kafka producer configuration: [`buffer.memory`](https://kafka.apache.org/documentation/#producerconfigs_buffer.memory) Only used if Kafka is set as `TARGET`. | see Kafka producer config: [`buffer.memory`](https://kafka.apache.org/documentation/#producerconfigs_buffer.memory) |
| `HTTP_URL` | The URL the load generator should post messages to. Only used if HTTP is set as `TARGET`. | |
| `NUM_SENSORS` | The amount of simulated sensors. | 10 |
| `PERIOD_MS` | The time in milliseconds between generating two messages for the same sensor. With our Theodolite benchmarks, we apply an [open workload model](https://www.usenix.org/legacy/event/nsdi06/tech/full_papers/schroeder/schroeder.pdf) in which new messages are generated at a fixed rate, without considering the think time of the target server nor the time required for generating a message. | 1000 |
| `VALUE` | The constant `valueInW` of an `ActivePowerRecord`. | 10 |
| `THREADS` | Number of worker threads used to generate the load. | 4 |
Please note that there are some additional configuration options for benchmark [UC4's load generator](https://github.com/cau-se/theodolite/blob/master/theodolite-benchmarks/uc4-load-generator/src/main/java/theodolite/uc4/workloadgenerator/LoadGenerator.java).
## Creating a custom load generator
To create a custom load generator, you need to import the [load-generator-commons](https://github.com/cau-se/theodolite/tree/master/theodolite-benchmarks/load-generator-commons) project. You can then create an instance of the `LoadGenerator` object and call its `run` method:
```java
LoadGenerator loadGenerator = new LoadGenerator()
.setClusterConfig(clusterConfig)
.setLoadDefinition(new WorkloadDefinition(
new KeySpace(key_prefix, numSensors),
duration))
.setGeneratorConfig(new LoadGeneratorConfig(
recordGenerator,
recordSender))
.withThreads(threads);
loadGenerator.run();
```
Alternatively, you can also start with a load generator populated with a default configuration or created from environment variables and then adjust the `LoadGenerator` as desired:
```java
LoadGenerator loadGeneratorFromDefaults = LoadGenerator.fromDefaults()
LoadGenerator loadGeneratorFromEnv = LoadGenerator.fromEnvironment();
```
execution/theodolite.yaml
+ 4
− 5
View file @ e944be2e
......@@ -21,17 +21,16 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.namespace
# - name: MODE
# value: yaml-executor # Default is `yaml-executor`
- name: THEODOLITE_EXECUTION
value: "execution/execution.yaml" # The name of this file must correspond to the filename of the execution, from which the config map is created.
value: "/deployments/execution/execution.yaml" # The name of this file must correspond to the filename of the execution, from which the config map is created.
- name: THEODOLITE_BENCHMARK
value: "benchmark/benchmark.yaml" # The name of this file must correspond to the filename of the benchmark, from which the config map is created.
value: "/deployments/benchmark/benchmark.yaml" # The name of this file must correspond to the filename of the benchmark, from which the config map is created.
- name: THEODOLITE_APP_RESOURCES
value: "benchmark-resources"
value: "/deployments/benchmark-resources"
- name: RESULTS_FOLDER # Folder for saving results
value: results # Default is the pwd (/deployments)
value: /deployments/results # Default is the pwd (/deployments)
# - name: CREATE_RESULTS_FOLDER # Specify whether the specified result folder should be created if it does not exist.
# value: "false" # Default is false.
volumeMounts:
......
helm/templates/prometheus/prometheus.yaml
+ 1
− 4
View file @ e944be2e
......@@ -5,10 +5,7 @@ metadata:
name: {{ template "theodolite.fullname" . }}-prometheus
spec:
serviceAccountName: {{ template "theodolite.fullname" . }}-prometheus
serviceMonitorSelector:
matchLabels:
#app: cp-kafka
appScope: titan-ccp
serviceMonitorSelector: {}
resources:
requests:
memory: 400Mi
......
helm/templates/theodolite/theodolite-operator.yaml
+ 30
− 2
View file @ e944be2e
......@@ -27,11 +27,18 @@ spec:
- name: MODE
value: operator
- name: RESULTS_FOLDER
value: "./results"
value: "/deployments/results"
volumeMounts:
- name: theodolite-results-volume
mountPath: "/deployments/results"
{{- if .Values.operator.sloChecker.droppedRecordsKStreams.enabled }}
resources:
requests:
memory: "512Mi"
cpu: "250m"
limits:
memory: "1024Mi"
cpu: "500m"
{{- if .Values.operator.sloChecker.generic.enabled }}
- name: slo-checker-generic
image: "{{ .Values.operator.sloChecker.generic.image }}:{{ .Values.operator.sloChecker.generic.imageTag }}"
imagePullPolicy: "{{ .Values.operator.sloChecker.generic.imagePullPolicy }}"
......@@ -43,6 +50,13 @@ spec:
value: "8082"
- name: LOG_LEVEL
value: INFO
resources:
requests:
memory: "64Mi"
cpu: "50m"
limits:
memory: "128Mi"
cpu: "100m"
{{- end }}
{{- if .Values.operator.sloChecker.lagTrend.enabled }}
- name: lag-trend-slo-checker
......@@ -54,6 +68,13 @@ spec:
env:
- name: LOG_LEVEL
value: INFO
resources:
requests:
memory: "64Mi"
cpu: "50m"
limits:
memory: "128Mi"
cpu: "100m"
{{- end }}
{{- if .Values.operator.sloChecker.droppedRecordsKStreams.enabled }}
- name: slo-checker-dropped-records-kstreams
......@@ -67,6 +88,13 @@ spec:
value: "8081"
- name: LOG_LEVEL
value: INFO
resources:
requests:
memory: "64Mi"
cpu: "50m"
limits:
memory: "128Mi"
cpu: "100m"
{{- end }}
{{- if .Values.operator.resultsVolume.accessSidecar.enabled }}
- name: results-access
......
slo-checker/dropped-records/Dockerfile
+ 13
− 4
View file @ e944be2e
FROM tiangolo/uvicorn-gunicorn-fastapi:python3.7
FROM python:3.8
COPY requirements.txt requirements.txt
RUN pip install -r requirements.txt
WORKDIR /code
COPY ./app /app
\ No newline at end of file
COPY ./requirements.txt /code/requirements.txt
RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
COPY ./app /code/app
WORKDIR /code/app
ENV HOST 0.0.0.0
ENV PORT 80
CMD ["sh", "-c", "uvicorn main:app --host $HOST --port $PORT"]
slo-checker/dropped-records/requirements.txt
+ 5
− 4
View file @ e944be2e
fastapi==0.65.2
scikit-learn==0.20.3
pandas==1.0.3
uvicorn
requests
fastapi>=0.68.0,<0.69.0
uvicorn>=0.15.0,<0.16.0
#pydantic>=1.8.0,<2.0.0
#scikit-learn==0.22.2
pandas==1.0.3
slo-checker/generic/Dockerfile
+ 13
− 4
View file @ e944be2e
FROM tiangolo/uvicorn-gunicorn-fastapi:python3.7
FROM python:3.8
COPY requirements.txt requirements.txt
RUN pip install -r requirements.txt
WORKDIR /code
COPY ./app /app
\ No newline at end of file
COPY ./requirements.txt /code/requirements.txt
RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
COPY ./app /code/app
WORKDIR /code/app
ENV HOST 0.0.0.0
ENV PORT 80
CMD ["sh", "-c", "uvicorn main:app --host $HOST --port $PORT"]
slo-checker/generic/requirements.txt
+ 5
− 3
View file @ e944be2e
fastapi==0.65.2
pandas==1.0.3
uvicorn
requests
fastapi>=0.68.0,<0.69.0
uvicorn>=0.15.0,<0.16.0
#pydantic>=1.8.0,<2.0.0
#scikit-learn==0.22.2
pandas==1.0.3
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment