Skip to content
Snippets Groups Projects
Commit cbba407e authored by Lorenz Boguhn's avatar Lorenz Boguhn
Browse files

Merge branch 'master' into feature/374-improve-hazelcastjet-structure

parents 909f29a8 7af98977
No related branches found
No related tags found
1 merge request!275Refactor hazelcast jet benchmarks:
Pipeline #9270 passed
Stage: build
Stage: test
Stage: check
Stage: smoketest
Expand all Show whitespace changes
Inline Side-by-side
Showing
with 1207 additions and 199 deletions
.gitlab-ci.yml
+ 26
− 26
View file @ cbba407e
......@@ -33,18 +33,18 @@ default:
entrypoint: [""]
script:
- mkdir -p /kaniko/.docker
- echo "{\"auths\":{\"${CR_HOST}\":{\"auth\":\"$(printf "%s:%s" "${CR_USER}" "${CR_PW}" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json
- echo "{\"auths\":{\"${CR_PUBLIC_HOST}\":{\"auth\":\"$(printf "%s:%s" "${CR_PUBLIC_USER}" "${CR_PUBLIC_PW}" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json
- >
if [ $IMAGE_TAG ]; then
KANIKO_D="$KANIKO_D -d $CR_HOST/$CR_ORG/$IMAGE_NAME:$IMAGE_TAG"
KANIKO_D="$KANIKO_D -d $CR_PUBLIC_HOST/$CR_PUBLIC_ORG/$IMAGE_NAME:$IMAGE_TAG"
export PUBLISHED_IMAGE_TAG=$IMAGE_TAG
elif [ $CI_COMMIT_TAG ]; then
KANIKO_D="$KANIKO_D -d $CR_HOST/$CR_ORG/$IMAGE_NAME:$CI_COMMIT_TAG"
KANIKO_D="$KANIKO_D -d $CR_PUBLIC_HOST/$CR_PUBLIC_ORG/$IMAGE_NAME:$CI_COMMIT_TAG"
export PUBLISHED_IMAGE_TAG=$CI_COMMIT_TAG
else
DOCKER_TAG_NAME=$(echo $CI_COMMIT_REF_SLUG- | sed 's/^master-$//')
KANIKO_D="$KANIKO_D -d $CR_HOST/$CR_ORG/$IMAGE_NAME:${DOCKER_TAG_NAME}latest"
KANIKO_D="$KANIKO_D -d $CR_HOST/$CR_ORG/$IMAGE_NAME:$DOCKER_TAG_NAME$CI_COMMIT_SHORT_SHA"
DOCKER_TAG_NAME=$(echo $CI_COMMIT_REF_SLUG- | sed 's/^main-$//')
KANIKO_D="$KANIKO_D -d $CR_PUBLIC_HOST/$CR_PUBLIC_ORG/$IMAGE_NAME:${DOCKER_TAG_NAME}latest"
KANIKO_D="$KANIKO_D -d $CR_PUBLIC_HOST/$CR_PUBLIC_ORG/$IMAGE_NAME:$DOCKER_TAG_NAME$CI_COMMIT_SHORT_SHA"
export PUBLISHED_IMAGE_TAG=$DOCKER_TAG_NAME$CI_COMMIT_SHORT_SHA
fi
- "[ $DOCKERFILE ] && KANIKO_DOCKERFILE=\"--dockerfile $DOCKERFILE\""
......@@ -86,7 +86,7 @@ test-docs-links:
extends: .docs
needs:
- build-docs
script: bundle exec htmlproofer --assume-extension --allow_hash_href ./_site
script: bundle exec htmlproofer --assume-extension --allow_hash_href --http-status-ignore 999 ./_site
build-docs-crds:
stage: build
......@@ -161,10 +161,10 @@ test-helm:
- cd helm
- helm dependencies update .
- helm install theodolite . -f preconfigs/minimal.yaml --wait
- helm test theodolite
- kubectl get nodes -o wide
- kubectl get pods --all-namespaces -o wide
- kubectl get services --all-namespaces -o wide
- helm test theodolite
after_script:
- k3d cluster delete $CLUSTERNAME
......@@ -269,8 +269,8 @@ spotbugs-benchmarks:
- theodolite-benchmarks/*
- theodolite-benchmarks/$JAVA_PROJECT_NAME/**/*
- theodolite-benchmarks/{commons,$JAVA_PROJECT_DEPS}/**/*
if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW && $IMAGE_NAME && $JAVA_PROJECT_NAME && $JAVA_PROJECT_DEPS"
- if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW && $IMAGE_NAME && $JAVA_PROJECT_NAME"
if: "$CR_PUBLIC_HOST && $CR_PUBLIC_ORG && $CR_PUBLIC_USER && $CR_PUBLIC_PW && $IMAGE_NAME && $JAVA_PROJECT_NAME && $JAVA_PROJECT_DEPS"
- if: "$CR_PUBLIC_HOST && $CR_PUBLIC_ORG && $CR_PUBLIC_USER && $CR_PUBLIC_PW && $IMAGE_NAME && $JAVA_PROJECT_NAME"
when: manual
allow_failure: true
......@@ -469,8 +469,8 @@ deploy-http-bridge:
- changes:
- theodolite-benchmarks/*
- theodolite-benchmarks/{$JAVA_PROJECT_DEPS}/**/*
if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW && $DOCKER_COMPOSE_DIR && $JAVA_PROJECT_DEPS"
- if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW && $DOCKER_COMPOSE_DIR && $JAVA_PROJECT_DEPS"
if: "$CR_PUBLIC_HOST && $CR_PUBLIC_ORG && $CR_PUBLIC_USER && $CR_PUBLIC_PW && $DOCKER_COMPOSE_DIR && $JAVA_PROJECT_DEPS"
- if: "$CR_PUBLIC_HOST && $CR_PUBLIC_ORG && $CR_PUBLIC_USER && $CR_PUBLIC_PW && $DOCKER_COMPOSE_DIR && $JAVA_PROJECT_DEPS"
when: manual
allow_failure: true
......@@ -736,8 +736,8 @@ deploy-theodolite:
rules:
- changes:
- theodolite/**/*
if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW"
- if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW"
if: "$CR_PUBLIC_HOST && $CR_PUBLIC_ORG && $CR_PUBLIC_USER && $CR_PUBLIC_PW"
- if: "$CR_PUBLIC_HOST && $CR_PUBLIC_ORG && $CR_PUBLIC_USER && $CR_PUBLIC_PW"
when: manual
allow_failure: true
......@@ -805,8 +805,8 @@ deploy-slo-checker-lag-trend:
rules:
- changes:
- slo-checker/record-lag/**/*
if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW"
- if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW"
if: "$CR_PUBLIC_HOST && $CR_PUBLIC_ORG && $CR_PUBLIC_USER && $CR_PUBLIC_PW"
- if: "$CR_PUBLIC_HOST && $CR_PUBLIC_ORG && $CR_PUBLIC_USER && $CR_PUBLIC_PW"
when: manual
allow_failure: true
......@@ -823,8 +823,8 @@ deploy-slo-checker-dropped-records-kstreams:
rules:
- changes:
- slo-checker/dropped-records/**/*
if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW"
- if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW"
if: "$CR_PUBLIC_HOST && $CR_PUBLIC_ORG && $CR_PUBLIC_USER && $CR_PUBLIC_PW"
- if: "$CR_PUBLIC_HOST && $CR_PUBLIC_ORG && $CR_PUBLIC_USER && $CR_PUBLIC_PW"
when: manual
allow_failure: true
......@@ -841,8 +841,8 @@ deploy-slo-checker-generic:
rules:
- changes:
- slo-checker/generic/**/*
if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW"
- if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW"
if: "$CR_PUBLIC_HOST && $CR_PUBLIC_ORG && $CR_PUBLIC_USER && $CR_PUBLIC_PW"
- if: "$CR_PUBLIC_HOST && $CR_PUBLIC_ORG && $CR_PUBLIC_USER && $CR_PUBLIC_PW"
when: manual
allow_failure: true
......@@ -861,8 +861,8 @@ deploy-random-scheduler:
rules:
- changes:
- execution/infrastructure/random-scheduler/**/*
if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW"
- if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW"
if: "$CR_PUBLIC_HOST && $CR_PUBLIC_ORG && $CR_PUBLIC_USER && $CR_PUBLIC_PW"
- if: "$CR_PUBLIC_HOST && $CR_PUBLIC_ORG && $CR_PUBLIC_USER && $CR_PUBLIC_PW"
when: manual
allow_failure: true
......@@ -880,8 +880,8 @@ deploy-buildimage-docker-compose-jq:
rules:
- changes:
- buildimages/docker-compose-jq/Dockerfile
if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW"
- if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW && $CI_PIPELINE_SOURCE == 'web'"
if: "$CR_PUBLIC_HOST && $CR_PUBLIC_ORG && $CR_PUBLIC_USER && $CR_PUBLIC_PW"
- if: "$CR_PUBLIC_HOST && $CR_PUBLIC_ORG && $CR_PUBLIC_USER && $CR_PUBLIC_PW && $CI_PIPELINE_SOURCE == 'web'"
when: manual
allow_failure: true
......@@ -901,7 +901,7 @@ deploy-buildimage-k3d-helm:
rules:
- changes:
- buildimages/k3d-helm/Dockerfile
if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW"
- if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW && $CI_PIPELINE_SOURCE == 'web'"
if: "$CR_PUBLIC_HOST && $CR_PUBLIC_ORG && $CR_PUBLIC_USER && $CR_PUBLIC_PW"
- if: "$CR_PUBLIC_HOST && $CR_PUBLIC_ORG && $CR_PUBLIC_USER && $CR_PUBLIC_PW && $CI_PIPELINE_SOURCE == 'web'"
when: manual
allow_failure: true
CITATION.cff
+ 1
− 1
View file @ cbba407e
......@@ -8,7 +8,7 @@ authors:
given-names: Wilhelm
orcid: "https://orcid.org/0000-0001-6625-4335"
title: Theodolite
version: "0.7.0"
version: "0.8.0"
repository-code: "https://github.com/cau-se/theodolite"
license: "Apache-2.0"
doi: "10.1016/j.bdr.2021.100209"
......
......
codemeta.json
+ 2
− 2
View file @ cbba407e
......@@ -5,10 +5,10 @@
"codeRepository": "https://github.com/cau-se/theodolite",
"dateCreated": "2020-03-13",
"datePublished": "2020-07-27",
"dateModified": "2022-05-11",
"dateModified": "2022-07-18",
"downloadUrl": "https://github.com/cau-se/theodolite/releases",
"name": "Theodolite",
"version": "0.7.0",
"version": "0.8.0",
"description": "Theodolite is a framework for benchmarking the horizontal and vertical scalability of cloud-native applications.",
"developmentStatus": "active",
"relatedLink": [
......
......
docs/.gitignore
+ 1
− 0
View file @ cbba407e
_site
.bundle
.sass-cache
.jekyll-cache
.jekyll-metadata
......
......
docs/Gemfile.lock
+ 1
− 1
View file @ cbba407e
......@@ -273,7 +273,7 @@ GEM
thread_safe (0.3.6)
typhoeus (1.4.0)
ethon (>= 0.9.0)
tzinfo (1.2.9)
tzinfo (1.2.10)
thread_safe (~> 0.1)
unf (0.1.4)
unf_ext
......
......
docs/_includes/head_custom.html 0 → 100644
+ 5
− 0
View file @ cbba407e
{% if page.usemathjax %}
<script type="text/javascript" async
src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-MML-AM_CHTML">
</script>
{% endif %}
\ No newline at end of file
docs/_sass/custom/custom.scss
+ 15
− 1
View file @ cbba407e
.main-content a {
white-space: normal;
}
.theodolite-logo {
height: 18em;
}
.MJXc-display {
overflow-x: auto;
}
table.fixed-colums {
th, td {
width: 1%;
}
}
docs/api-reference/crds.md
+ 98
− 96
View file @ cbba407e
......@@ -10,9 +10,9 @@ nav_order: 1
Packages:
- [theodolite.com/v1](#theodolitecomv1)
- [theodolite.rocks/v1beta1](#theodoliterocksv1beta1)
# theodolite.com/v1
# theodolite.rocks/v1beta1
Resource Types:
......@@ -24,7 +24,7 @@ Resource Types:
## benchmark
<sup><sup>[↩ Parent](#theodolitecomv1 )</sup></sup>
<sup><sup>[↩ Parent](#theodoliterocksv1beta1 )</sup></sup>
......@@ -45,7 +45,7 @@ Resource Types:
<tbody><tr>
<td><b>apiVersion</b></td>
<td>string</td>
<td>theodolite.com/v1</td>
<td>theodolite.rocks/v1beta1</td>
<td>true</td>
</tr>
<tr>
......@@ -2209,7 +2209,7 @@ Contains the Kafka configuration.
</table>
## execution
<sup><sup>[↩ Parent](#theodolitecomv1 )</sup></sup>
<sup><sup>[↩ Parent](#theodoliterocksv1beta1 )</sup></sup>
......@@ -2230,7 +2230,7 @@ Contains the Kafka configuration.
<tbody><tr>
<td><b>apiVersion</b></td>
<td>string</td>
<td>theodolite.com/v1</td>
<td>theodolite.rocks/v1beta1</td>
<td>true</td>
</tr>
<tr>
......@@ -2285,13 +2285,6 @@ Contains the Kafka configuration.
The name of the benchmark this execution is referring to.<br/>
</td>
<td>true</td>
</tr><tr>
<td><b><a href="#executionspecconfigoverridesindex">configOverrides</a></b></td>
<td>[]object</td>
<td>
List of patchers that are used to override existing configurations.<br/>
</td>
<td>true</td>
</tr><tr>
<td><b><a href="#executionspecexecution">execution</a></b></td>
<td>object</td>
......@@ -2300,7 +2293,7 @@ Contains the Kafka configuration.
</td>
<td>true</td>
</tr><tr>
<td><b><a href="#executionspecloads">loads</a></b></td>
<td><b><a href="#executionspecload">load</a></b></td>
<td>object</td>
<td>
Specifies the load values that are benchmarked.<br/>
......@@ -2314,100 +2307,28 @@ Contains the Kafka configuration.
</td>
<td>true</td>
</tr><tr>
<td><b>name</b></td>
<td>string</td>
<td>
This field exists only for technical reasons and should not be set by the user. The value of the field will be overwritten.<br/>
<br/>
<i>Default</i>: <br/>
</td>
<td>false</td>
</tr><tr>
<td><b><a href="#executionspecslosindex">slos</a></b></td>
<td><b><a href="#executionspecconfigoverridesindex">configOverrides</a></b></td>
<td>[]object</td>
<td>
List of SLOs with their properties, which differ from the benchmark definition.<br/>
</td>
<td>false</td>
</tr></tbody>
</table>
### execution.spec.configOverrides[index]
<sup><sup>[↩ Parent](#executionspec)</sup></sup>
<table>
<thead>
<tr>
<th>Name</th>
<th>Type</th>
<th>Description</th>
<th>Required</th>
</tr>
</thead>
<tbody><tr>
<td><b><a href="#executionspecconfigoverridesindexpatcher">patcher</a></b></td>
<td>object</td>
<td>
Patcher used to patch a resource<br/>
</td>
<td>false</td>
</tr><tr>
<td><b>value</b></td>
<td>string</td>
<td>
List of patchers that are used to override existing configurations.<br/>
<br/>
<i>Default</i>: []<br/>
</td>
<td>false</td>
</tr></tbody>
</table>
### execution.spec.configOverrides[index].patcher
<sup><sup>[↩ Parent](#executionspecconfigoverridesindex)</sup></sup>
Patcher used to patch a resource
<table>
<thead>
<tr>
<th>Name</th>
<th>Type</th>
<th>Description</th>
<th>Required</th>
</tr>
</thead>
<tbody><tr>
<td><b>resource</b></td>
<td>string</td>
<td>
Specifies the Kubernetes resource to be patched.<br/>
<br/>
<i>Default</i>: <br/>
</td>
<td>true</td>
</tr><tr>
<td><b>type</b></td>
<td><b>name</b></td>
<td>string</td>
<td>
Type of the Patcher.<br/>
This field exists only for technical reasons and should not be set by the user. The value of the field will be overwritten.<br/>
<br/>
<i>Default</i>: <br/>
</td>
<td>true</td>
<td>false</td>
</tr><tr>
<td><b>properties</b></td>
<td>map[string]string</td>
<td><b><a href="#executionspecslosindex">slos</a></b></td>
<td>[]object</td>
<td>
(Optional) Patcher specific additional arguments.<br/>
<br/>
<i>Default</i>: map[]<br/>
List of SLOs with their properties, which differ from the benchmark definition.<br/>
</td>
<td>false</td>
</tr></tbody>
......@@ -2517,7 +2438,7 @@ Defines the used strategy for the execution, either 'LinearSearch', 'BinarySearc
</table>
### execution.spec.loads
### execution.spec.load
<sup><sup>[↩ Parent](#executionspec)</sup></sup>
......@@ -2585,6 +2506,87 @@ Specifies the scaling resource that is benchmarked.
</table>
### execution.spec.configOverrides[index]
<sup><sup>[↩ Parent](#executionspec)</sup></sup>
<table>
<thead>
<tr>
<th>Name</th>
<th>Type</th>
<th>Description</th>
<th>Required</th>
</tr>
</thead>
<tbody><tr>
<td><b><a href="#executionspecconfigoverridesindexpatcher">patcher</a></b></td>
<td>object</td>
<td>
Patcher used to patch a resource<br/>
</td>
<td>false</td>
</tr><tr>
<td><b>value</b></td>
<td>string</td>
<td>
<br/>
</td>
<td>false</td>
</tr></tbody>
</table>
### execution.spec.configOverrides[index].patcher
<sup><sup>[↩ Parent](#executionspecconfigoverridesindex)</sup></sup>
Patcher used to patch a resource
<table>
<thead>
<tr>
<th>Name</th>
<th>Type</th>
<th>Description</th>
<th>Required</th>
</tr>
</thead>
<tbody><tr>
<td><b>resource</b></td>
<td>string</td>
<td>
Specifies the Kubernetes resource to be patched.<br/>
<br/>
<i>Default</i>: <br/>
</td>
<td>true</td>
</tr><tr>
<td><b>type</b></td>
<td>string</td>
<td>
Type of the Patcher.<br/>
<br/>
<i>Default</i>: <br/>
</td>
<td>true</td>
</tr><tr>
<td><b>properties</b></td>
<td>map[string]string</td>
<td>
(Optional) Patcher specific additional arguments.<br/>
<br/>
<i>Default</i>: map[]<br/>
</td>
<td>false</td>
</tr></tbody>
</table>
### execution.spec.slos[index]
<sup><sup>[↩ Parent](#executionspec)</sup></sup>
......
......
docs/api-reference/index.md
+ 1
− 1
View file @ cbba407e
---
title: API Reference
has_children: true
nav_order: 9
nav_order: 11
---
\ No newline at end of file
docs/assets/images/capacity.svg 0 → 100644
+ 420
− 0
View file @ cbba407e
This diff is collapsed.
docs/assets/images/demand.svg 0 → 100644
+ 391
− 0
View file @ cbba407e
This diff is collapsed.
docs/benchmarks-and-executions.md → docs/concepts/benchmarks-and-executions.md
+ 7
− 2
View file @ cbba407e
---
title: Fundamental Concepts
title: Data and Role Model
has_children: false
nav_order: 2
parent: Fundamental Concepts
nav_order: 3
---
# Benchmarks and Executions
......@@ -41,3 +42,7 @@ In contrast to benchmarks, execution have a life-cycle. They can be planned,
executed, or aborted. Each execution of a benchmark is represented by an
individual entity. This supports repeatability as executions can be archived
and shared.
## Further Reading
S. Henning and W. Hasselbring. “[A Configurable Method for Benchmarking Scalability of Cloud-Native Applications](https://doi.org/10.1007/s10664-022-10162-1)”. In: *Empirical Software Engineering* 27. 2022. DOI: [10.1007/s10664-022-10162-1](https://doi.org/10.1007/s10664-022-10162-1).
docs/concepts/index.md 0 → 100644
+ 19
− 0
View file @ cbba407e
---
title: Fundamental Concepts
has_children: true
nav_order: 2
---
# Fundamental Concepts
> A theodolite is a precision optical instrument for measuring angles between designated visible points in the horizontal and vertical planes. -- <cite>[Wikipedia](https://en.wikipedia.org/wiki/Theodolite)</cite>
Theodolite is a framework for benchmarking the horizontal and vertical scalability of cloud-native applications.
Theodolite adopts established definitions of scalability in cloud computing for its benchmarking method. It quantifies
scalability by running isolated experiments for different load intensities and provisioned resource amounts, which assess whether specified SLOs are fulfilled. [Two metrics are available](metrics): The demand metric describes how the amount of minimal required resources evolve with increasing load intensities, while the capacity metric describes how the maximal processable load evolves with increasing resources. Hence, both metrics are functions. <!--Example?-->
The terms load, resources and SLOs are consciously kept abstract as Theodolite leaves it to the benchmark designer to define what type of load, resources, and SLOs should be evaluated. For example, horizontal scalability can be benchmarked by varying the amount of Kubernetes Pods, while vertical scalability can be benchmarked by varying CPU and memory constraints of Pods.
To balance statistical grounding and time-efficient benchmark execution, Theodolite comes with different [heuristic for
evaluating the search space](search-strategies) of load and resource combinations. Other configuration options include the number of repetitions, the experiment and warm-up duration, as well as the amount of different load and resource values to be evaluated.
\ No newline at end of file
docs/concepts/metrics.md 0 → 100644
+ 67
− 0
View file @ cbba407e
---
title: Scalability Metrics
has_children: false
parent: Fundamental Concepts
nav_order: 1
usemathjax: true
---
# Theodolite's Scalability Metrics
Theodolite's scalability metrics are based on the following definition:
> Scalability is the ability of [a] system to sustain increasing workloads by making use of additional resources. -- <cite>[Herbst et al. (2013)](https://www.usenix.org/conference/icac13/technical-sessions/presentation/herbst)</cite>
Based on this definition, scalability can be characterized by the following three attributes:
* **Load intensity** is the input variable to which a system is subjected. Scalability is evaluated within a range of load intensities.
* **Service levels objectives (SLOs)** are measurable quality criteria that have to be fulfilled for every load intensity.
* **Provisioned resources** can be increased to meet the SLOs if load intensities increase.
## Scalability Metrics
Theodolite uses these attributes to define two scalability metrics:
| Resource Demand Metric | Load Capacity Metric |
|:----|:----|
| The resource demand metric quantifies scalability by describing how the amount of minimal required resources (i.e., all SLOs are fulfilled) evolves with increasing load intensities. | The load capacity metric quantifies scalability by describing how the maximal processable load (i.e., all SLOs are fulfilled) evolves with increasing resources. |
| ![Example for resource demand metric](../../assets/images/demand.svg){: .d-block .mx-auto } *Example: Scalability of two stream processing engines measured with the demand metric.*{: .d-block .text-center } | ![Example for load capacity metric](../../assets/images/capacity.svg){: .d-block .mx-auto } *Example: Scalability of two stream processing engines measured with the capacity metric.*{: .d-block .text-center } |
{: .fixed-colums }
<!--
## Resource Demand Metric
![](../../assets/images/capacity.svg){: .d-block .mx-auto }
## Load Capacity Metric
![](../../assets/images/demand.svg){: .d-block .mx-auto }
-->
## Formal Definition
For a more formal definition of both metrics, we define the load type as the set of possible load intensities for that
type, denoted as $$L$$.
Similarly, we define the resource type as the set of possible resources, denoted as $$R$$.
We also require that there exists an ordering on both sets $$L$$ and $$R$$.
We define the set of all SLOs as $$S$$ and denote an SLO $$s \in S$$ as Boolean-valued function
$$\text{slo}_s: L \times R \to \{\text{false},\text{true}\}$$ with $$\text{slo}_s(l,r) = \text{true}$$ if a system deployed with $$r$$ resource amounts does not violate SLO $$s$$ when processing load intensity $$l$$.
We can denote our **resource demand** metric as $$\text{demand: } L \to R$$, defined as:
$$
\forall l \in L: \text{demand}(l) = \min\{r \in R \mid \forall s \in S: \text{slo}_s(l,r) = \text{true}\}
$$
And similarly denote our **resource capacity** metric as $$\text{capacity: } R \to L$$, defined as:
$$
\forall r \in R: \text{capacity}(r) = \max\{l \in L \mid \forall s \in S: \text{slo}_s(l,r) = \text{true}\}
$$
## Further Reading
S. Henning and W. Hasselbring. “[A Configurable Method for Benchmarking Scalability of Cloud-Native Applications](https://doi.org/10.1007/s10664-022-10162-1)”. In: *Empirical Software Engineering* 27. 2022. DOI: [10.1007/s10664-022-10162-1](https://doi.org/10.1007/s10664-022-10162-1).
docs/concepts/search-strategies.md 0 → 100644
+ 40
− 0
View file @ cbba407e
---
title: Search Strategies
has_children: false
parent: Fundamental Concepts
nav_order: 2
---
# Theodolite's Search Strategies
Theodolite measures [its metrics](metrics) by performing isolated experiments for different load intensities and provisioned resource amounts.
However, to determine a system's resource demand or load capacity or a good approximation of those, it is often not necessary to evaluate each possible combination. Instead, Theodolite provides search strategies, which decide at benchmark runtime which combinations of load and resources to evaluate.
The following search strategies are available:
### Full Search
The full search strategy performs SLO experiments for each combination of resource configuration and load intensity. Its advantage is that it allows for extensive evaluation after the benchmark has been executed. This also includes that based on the same SLO experiments, both the demand and the capacity metric can be evaluated. However, this comes at the cost of significantly longer execution times.
### Linear Search
The linear search strategy reduces the overall execution time by not running SLO experiments whose results are not required by the metric.
For the resource demand metric this means, as soon as a sufficient resource configuration for a certain load intensity is found, no further resource configurations are tested for that load.
### Binary Search
The binary search strategy adopts the well known binary search algorithm for sorted arrays.
For the resource demand metric this means, the strategy starts by performing the SLO experiments for the middle resource configuration.
Depending on whether this experiment was successful or not, it then continues searching in the lower or upper half, respectively.
The binary search is particularly advantageous if the search space is very large.
However it is based on the assumption that with additional resources for the same load, performance does not substantially decrease.
### Lower Bound Restriction Search
The lower bound restriction is a search strategy that uses the results of already performed SLO experiments to narrow the search space.
For the resource demand, it starts searching (with another strategy) beginning from the minimal required resources of all lower load intensities.
The lower bound restriction is based on the assumption that with increasing load intensity, the resource demand never decreases.
## Further Reading
S. Henning and W. Hasselbring. “[A Configurable Method for Benchmarking Scalability of Cloud-Native Applications](https://doi.org/10.1007/s10664-022-10162-1)”. In: *Empirical Software Engineering* 27. 2022. DOI: [10.1007/s10664-022-10162-1](https://doi.org/10.1007/s10664-022-10162-1).
docs/creating-a-benchmark.md
+ 80
− 27
View file @ cbba407e
......@@ -6,12 +6,12 @@ nav_order: 5
# Creating a Benchmark
Please note that to simply run a benchmark, it is not required to define one. Theodolite comes with a [set of benchmarks](theodolite-benchmarks), which are ready to be executed. See the [fundamental concepts](benchmarks-and-executions) page to learn more about our distinction between benchmarks and executions.
Please note that to simply run a benchmark, it is not required to define one. Theodolite comes with a [set of benchmarks](theodolite-benchmarks), which are ready to be executed. See the [fundamental concepts](concepts) page to learn more about our distinction between benchmarks and executions.
A typical benchmark looks like this:
```yaml
apiVersion: theodolite.com/v1
apiVersion: theodolite.rocks/v1beta1
kind: benchmark
metadata:
name: example-benchmark
......@@ -29,6 +29,11 @@ spec:
files:
- uc1-load-generator-service.yaml
- uc1-load-generator-deployment.yaml
resourceTypes:
- typeName: "Instances"
patchers:
- type: "ReplicaPatcher"
resource: "uc1-kstreams-deployment.yaml"
loadTypes:
- typeName: "NumSensors"
patchers:
......@@ -41,6 +46,15 @@ spec:
resource: "uc1-load-generator-deployment.yaml"
properties:
loadGenMaxRecords: "150000"
slos:
- name: "lag trend"
sloType: "lag trend"
prometheusUrl: "http://prometheus-operated:9090"
offset: 0
properties:
threshold: 3000
externalSloUrl: "http://localhost:80/evaluate-slope"
warmup: 60 # in seconds
kafkaConfig:
bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092"
topics:
......@@ -60,8 +74,6 @@ Infrastructure resources live over the entire duration of a benchmark run. They
### Resources
#### ConfigMap
The recommended way to link Kubernetes resources files from a Benchmark is by bundling them in one or multiple ConfigMaps and refer to that ConfigMap from `sut.resources`, `loadGenerator.resources` or `infrastructure.resources`.
To create a ConfigMap from all the Kubernetes resources in a directory run:
......@@ -79,21 +91,13 @@ configMap:
- example-service.yaml
```
#### Filesystem
Alternatively, resources can also be read from the filesystem, Theodolite has access to. This usually requires that the Benchmark resources are available in a volume, which is mounted into the Theodolite container.
```yaml
filesystem:
path: example/path/to/files
files:
- example-deployment.yaml
- example-service.yaml
```
### Actions
Sometimes it is not sufficient to just define resources that are created and deleted when running a benchmark. Instead, it might be necessary to define certain actions that will be executed before running or after stopping the benchmark.
Theodolite supports *actions*, which can run before (`beforeActions`) or after `afterActions` all `sut`, `loadGenerator` or `infrastructure` resources are deployed.
Theodolite provides two types of actions:
#### Exec Actions
Theodolite allows to execute commands on running pods. This is similar to `kubectl exec` or Kubernetes' [container lifecycle handlers](https://kubernetes.io/docs/tasks/configure-pod-container/attach-handler-lifecycle-event/). Theodolite actions can run before (`beforeActions`) or after `afterActions` all `sut`, `loadGenerator` or `infrastructure` resources are deployed.
For example, the following actions will create a file in a pod with label `app: logger` before the SUT is started and delete if after the SUT is stopped:
......@@ -102,26 +106,48 @@ For example, the following actions will create a file in a pod with label `app:
sut:
resources: # ...
beforeActions:
- selector:
- exec:
selector:
pod:
matchLabels:
app: logger
exec:
container: logger # optional
command: ["touch", "file-used-by-logger.txt"]
timeoutSeconds: 90
afterActions:
- selector:
- exec:
selector:
pod:
matchLabels:
app: logger
exec:
container: logger # optional
command: [ "rm", "file-used-by-logger.txt" ]
timeoutSeconds: 90
```
Theodolite checks if all referenced pods are available for the specified actions. That means these pods must either be defined in `infrastructure` or already deployed in the cluster. If not all referenced pods are available, the benchmark will not be set as `Ready`. Consequently, an action cannot be executed on a pod that is defined as an SUT or load generator resource.
*Note: Actions should be used sparingly. While it is possible to define entire benchmarks imperatively as actions, it is considered better practice to define as much as possible using declarative, native Kubernetes resource files.*
*Note: Exec actions should be used sparingly. While it is possible to define entire benchmarks imperatively as actions, it is considered better practice to define as much as possible using declarative, native Kubernetes resource files.*
#### Delete Actions
Sometimes it is required to delete Kubernetes resources before or after running a benchmark.
This is typically the case for resources that are automatically created while running a benchmark.
For example, Kafka Streams creates internal Kafka topics. When using the [Strimzi](https://strimzi.io/) Kafka operator, we can delete these topics by deleting the corresponding Kafka topic resource.
As shown in the following example, delete actions select the resources to be deleted by specifying their *apiVersion*, *kind* and a [regular expression](https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html) for their name.
```yaml
sut:
resources: # ...
beforeActions:
- delete:
selector:
apiVersion: kafka.strimzi.io/v1beta2
kind: KafkaTopic
nameRegex: ^some-internal-topic-.*
```
<!--
A Benchmark refers to other Kubernetes resources (e.g., Deployments, Services, ConfigMaps), which describe the system under test, the load generator and infrastructure components such as a middleware used in the benchmark. To manage those resources, Theodolite needs to have access to them. This is done by bundling resources in ConfigMaps.
......@@ -140,6 +166,33 @@ See the [patcher API reference](api-reference/patchers) for an overview of avail
If a benchmark is [executed by an Execution](running-benchmarks), these patchers are used to configure SUT and load generator according to the [load and resource values](creating-an-execution) set in the Execution.
## Service Level Objectives SLOs
SLOs provide a way to quantify whether a certain load intensity can be handled by a certain amount of provisioned resources.
In Theodolite, SLOs are evaluated by requesting monitoring data from Prometheus and analyzing it in a benchmark-specific way.
An Execution must at least define one SLO to be checked.
A good choice to get started is defining an SLO of type `generic`:
```yaml
- sloType: "generic"
prometheusUrl: "http://prometheus-operated:9090"
offset: 0
properties:
externalSloUrl: "http://localhost:8082"
promQLQuery: "sum by(job) (kafka_streams_stream_task_metrics_dropped_records_total>=0)"
warmup: 60 # in seconds
queryAggregation: max
repetitionAggregation: median
operator: lte
threshold: 1000
```
All you have to do is to define a [PromQL query](https://prometheus.io/docs/prometheus/latest/querying/basics/) describing which metrics should be requested (`promQLQuery`) and how the resulting time series should be evaluated. With `queryAggregation` you specify how the resulting time series is aggregated to a single value and `repetitionAggregation` describes how the results of multiple repetitions are aggregated. Possible values are
`mean`, `median`, `mode`, `sum`, `count`, `max`, `min`, `std`, `var`, `skew`, `kurt` as well as percentiles such as `p99` or `p99.9`. The result of aggregation all repetitions is checked against `threshold`. This check is performed using an `operator`, which describes that the result must be "less than" (`lt`), "less than equal" (`lte`), "greater than" (`gt`) or "greater than equal" (`gte`) to the threshold.
In case you need to evaluate monitoring data in a more flexible fashion, you can also change the value of `externalSloUrl` to your custom SLO checker. Have a look at the source code of the [generic SLO checker](https://github.com/cau-se/theodolite/tree/main/slo-checker/generic) to get started.
## Kafka Configuration
Theodolite allows to automatically create and remove Kafka topics for each SLO experiment by setting a `kafkaConfig`.
......
......
docs/creating-an-execution.md
+ 16
− 38
View file @ cbba407e
......@@ -11,12 +11,12 @@ Theodolite Executions look similar to the following example.
<!-- TODO align with upstream -->
```yaml
apiVersion: theodolite.com/v1
apiVersion: theodolite.rocks/v1beta1
kind: execution
metadata:
name: theodolite-example-execution
spec:
benchmark: "uc1-kstreams"
benchmark: "example-benchmark"
load:
loadType: "NumSensors"
loadValues: [25000, 50000]
......@@ -24,20 +24,19 @@ spec:
resourceType: "Instances"
resourceValues: [1, 2]
slos:
- sloType: "lag trend"
prometheusUrl: "http://prometheus-operated:9090"
offset: 0
- name: "lag trend"
properties:
threshold: 2000
externalSloUrl: "http://localhost:80/evaluate-slope"
warmup: 60 # in seconds
execution:
strategy: "LinearSearch"
metric: "demand"
strategy:
name: "RestrictionSearch"
restrictions:
- "LowerBound"
searchStrategy: "LinearSearch"
duration: 300 # in seconds
repetitions: 1
loadGenerationDelay: 30 # in seconds
restrictions:
- "LowerBound"
configOverrides:
- patcher:
type: "SchedulerNamePatcher"
......@@ -53,41 +52,20 @@ Similar to [Kubernetes Jobs](https://kubernetes.io/docs/concepts/workloads/contr
An Execution always refers to a Benchmark. For the Execution to run, the Benchmark must be registered with Kubernetes and it must be in state *Ready*. If this is not the case, the Execution will remain in state *Pending*.
As a Benchmark may define multiple supported load and resource types, an Execution has to pick exactly one of each by its name. Additionally, it defines the set of load values and resource values the benchmark should be executed with. Both these values are represented as integers, which are interpreted in a [Benchmark-specific way](creating-a-benchmark) to configure the SUT and load generator.
## Definition of SLOs
SLOs provide a way to quantify whether a certain load intensity can be handled by a certain amount of provisioned resources.
In Theodolite, SLO are evaluated by requesting monitoring data from Prometheus and analyzing it in a benchmark-specific way.
An Execution must at least define one SLO to be checked.
A good choice to get started is defining an SLO of type `generic`:
```yaml
- sloType: "generic"
prometheusUrl: "http://prometheus-operated:9090"
offset: 0
properties:
externalSloUrl: "http://localhost:8082"
promQLQuery: "sum by(job) (kafka_streams_stream_task_metrics_dropped_records_total>=0)"
warmup: 60 # in seconds
queryAggregation: max
repetitionAggregation: median
operator: lte
threshold: 1000
```
All you have to do is to define a [PromQL query](https://prometheus.io/docs/prometheus/latest/querying/basics/) describing which metrics should be requested (`promQLQuery`) and how the resulting time series should be evaluated. With `queryAggregation` you specify how the resulting time series is aggregated to a single value and `repetitionAggregation` describes how the results of multiple repetitions are aggregated. Possible values are
`mean`, `median`, `mode`, `sum`, `count`, `max`, `min`, `std`, `var`, `skew`, `kurt` as well as percentiles such as `p99` or `p99.9`. The result of aggregation all repetitions is checked against `threshold`. This check is performed using an `operator`, which describes that the result must be "less than" (`lt`), "less than equal" (`lte`), "greater than" (`gt`) or "greater than equal" (`gte`) to the threshold.
## Selecting Load Type, Resource Type and SLOs
In case you need to evaluate monitoring data in a more flexible fashion, you can also change the value of `externalSloUrl` to your custom SLO checker. Have a look at the source code of the [generic SLO checker](https://github.com/cau-se/theodolite/tree/master/slo-checker/generic) to get started.
As a Benchmark may define multiple supported load and resource types, an Execution has to pick exactly one of each by its name. Additionally, it defines the set of load values and resource values the benchmark should be executed with.
Both these values are represented as integers, which are interpreted in a [Benchmark-specific way](creating-a-benchmark#load-and-resource-types) to configure the SUT and load generator.
Similarly, an Execution must select a subset of the [SLOs defined in the Benchmark](creating-a-benchmark#service-level-objectives-slos). Additionally, these SLOs can be configured by their `properties`.
<!-- TODO: What happpens if slos are not set? -->
## Experimental Setup
According to Theodolite's measurement method, isolated SLO experiments are performed for different combinations of load intensity and resource amounts.
The experimental setup can be configured by:
* A search strategy (`strategy`), which determines which load and resource combinations should be tested. Supported values are `FullSearch`, `LinearSearch` and `BinarySearch`. Additionally, a `restrictions` can be set to `LowerBound`.
* A [scalability metric](concepts/metrics) (`metric`). Supported values are `demand` and `capacity`, with `demand` being the default.
* A [search strategy](concepts/search-strategies) (`strategy`), which determines which load and resource combinations should be tested. Supported values are `FullSearch`, `LinearSearch` and `BinarySearch`. Additionally, a `restrictions` can be set to `LowerBound`.
* The `duration` per SLO experiment in seconds.
* The number of repetitions (`repetitions`) for each SLO experiment.
* A `loadGenerationDelay`, specifying the time in seconds before the load generation starts.
......
......
docs/development/index.md
+ 1
− 1
View file @ cbba407e
---
title: Development
has_children: true
nav_order: 8
nav_order: 10
---
\ No newline at end of file
docs/development/release-process.md
+ 3
− 3
View file @ cbba407e
......@@ -13,7 +13,7 @@ We assume that we are creating the release `v0.3.1`. Please make sure to adjust
the following steps according to the release, you are actually performing.
1. Create a new branch `v0.3` if it does not already exist. This branch will never
again be merged into master.
again be merged into main.
2. Checkout the `v0.3` branch.
......@@ -43,7 +43,7 @@ again be merged into master.
8. Create *releases* on GitLab and GitHub. Upload the generated Helm package to these releases via the UIs of GitLab and GitHub.
9. Switch to the `master` branch.
9. Switch to the `main` branch.
10. Re-run `./update-index.sh v0.3.1` to include the latest release in the *upstream* Helm repository. You can now delete the packaged Helm chart.
......@@ -57,4 +57,4 @@ again be merged into master.
4. Update the `CITATION.cff` file according to Step 3.
12. Commit these changes to the `master` branch.
12. Commit these changes to the `main` branch.
docs/drafts/filesystem.md 0 → 100644
+ 13
− 0
View file @ cbba407e
## Creating a Benchmark with Filesystem Resources
#### Filesystem
Alternatively, resources can also be read from the filesystem, Theodolite has access to. This usually requires that the Benchmark resources are available in a volume, which is mounted into the Theodolite container.
```yaml
filesystem:
path: example/path/to/files
files:
- example-deployment.yaml
- example-service.yaml
```
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please to comment