diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7559bf2824b2d2db24ec4915667a4ba4a0fc073e..df49c1d6b0f2e22bd12156d7f956b64f2f8c0418 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -341,7 +341,18 @@ test-slo-checker-lag-trend:
tags:
- exec-docker
script:
- - cd slope-evaluator
+ - cd slo-checker/record-lag
+ - pip install -r requirements.txt
+ - cd app
+ - python -m unittest
+
+test-slo-checker-dropped-records-kstreams:
+ stage: test
+ image: python:3.7-slim
+ tags:
+ - exec-docker
+ script:
+ - cd slo-checker/dropped-records
- pip install -r requirements.txt
- cd app
- python -m unittest
@@ -354,7 +365,7 @@ deploy-slo-checker-lag-trend:
- test-slo-checker-lag-trend
script:
- DOCKER_TAG_NAME=$(echo $CI_COMMIT_REF_SLUG- | sed 's/^master-$//')
- - docker build --pull -t theodolite-slo-checker-lag-trend slope-evaluator
+ - docker build --pull -t theodolite-slo-checker-lag-trend slo-checker/record-lag
- "[ ! $CI_COMMIT_TAG ] && docker tag theodolite-slo-checker-lag-trend $CR_HOST/$CR_ORG/theodolite-slo-checker-lag-trend:${DOCKER_TAG_NAME}latest"
- "[ $CI_COMMIT_TAG ] && docker tag theodolite-slo-checker-lag-trend $CR_HOST/$CR_ORG/theodolite-slo-checker-lag-trend:$CI_COMMIT_TAG"
- echo $CR_PW | docker login $CR_HOST -u $CR_USER --password-stdin
@@ -364,7 +375,32 @@ deploy-slo-checker-lag-trend:
- if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW && $CI_COMMIT_TAG"
when: always
- changes:
- - slope-evaluator/**/*
+ - slo-checker/record-lag/**/*
+ if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW"
+ when: always
+ - if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW"
+ when: manual
+ allow_failure: true
+
+deploy-slo-checker-dropped-records-kstreams:
+ stage: deploy
+ extends:
+ - .dind
+ needs:
+ - test-slo-checker-dropped-records-kstreams
+ script:
+ - DOCKER_TAG_NAME=$(echo $CI_COMMIT_REF_SLUG- | sed 's/^master-$//')
+ - docker build --pull -t theodolite-slo-checker-dropped-records-kstreams slo-checker/dropped-records
+ - "[ ! $CI_COMMIT_TAG ] && docker tag theodolite-slo-checker-dropped-records-kstreams $CR_HOST/$CR_ORG/theodolite-slo-checker-dropped-records-kstreams:${DOCKER_TAG_NAME}latest"
+ - "[ $CI_COMMIT_TAG ] && docker tag theodolite-slo-checker-dropped-records-kstreams $CR_HOST/$CR_ORG/theodolite-slo-checker-dropped-records-kstreams:$CI_COMMIT_TAG"
+ - echo $CR_PW | docker login $CR_HOST -u $CR_USER --password-stdin
+ - docker push $CR_HOST/$CR_ORG/theodolite-slo-checker-dropped-records-kstreams
+ - docker logout
+ rules:
+ - if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW && $CI_COMMIT_TAG"
+ when: always
+ - changes:
+ - slo-checker/dropped-records/**/*
if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW"
when: always
- if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW"
diff --git a/CITATION.cff b/CITATION.cff
index b6afbc8d75810b530e3a15538301c03323e7c6e2..ca94e1c5039d3aeac3a4535767d5217de4960a6f 100644
--- a/CITATION.cff
+++ b/CITATION.cff
@@ -1,16 +1,29 @@
cff-version: "1.1.0"
message: "If you use Theodolite, please cite it using these metadata."
authors:
- -
- family-names: Henning
+ - family-names: Henning
given-names: "Sören"
orcid: "https://orcid.org/0000-0001-6912-2549"
- -
- family-names: Hasselbring
+ - family-names: Hasselbring
given-names: Wilhelm
orcid: "https://orcid.org/0000-0001-6625-4335"
title: Theodolite
-version: "0.4.0"
+version: "0.5.1"
repository-code: "https://github.com/cau-se/theodolite"
license: "Apache-2.0"
doi: "10.1016/j.bdr.2021.100209"
+preferred-citation:
+ type: article
+ authors:
+ - family-names: Henning
+ given-names: "Sören"
+ orcid: "https://orcid.org/0000-0001-6912-2549"
+ - family-names: Hasselbring
+ given-names: Wilhelm
+ orcid: "https://orcid.org/0000-0001-6625-4335"
+ doi: "10.1016/j.bdr.2021.100209"
+ journal: "Big Data Research"
+ month: 7
+ title: "Theodolite: Scalability Benchmarking of Distributed Stream Processing Engines in Microservice Architectures"
+ volume: 25
+ year: 2021
diff --git a/README.md b/README.md
index f2673f4b9ed0c46987963f8b455e19def802db79..804a193df21f3883ecf9a727af5a743b77a9cceb 100644
--- a/README.md
+++ b/README.md
@@ -4,20 +4,17 @@
Theodolite is a framework for benchmarking the horizontal and vertical scalability of stream processing engines. It consists of three modules:
-## Theodolite Benchmarks
-
-Theodolite contains 4 application benchmarks, which are based on typical use cases for stream processing within microservices. For each benchmark, a corresponding workload generator is provided. Currently, this repository provides benchmark implementations for Apache Kafka Streams and Apache Flink. The benchmark sources can be found in [Thedolite benchmarks](benchmarks).
-
-
-## Theodolite Execution Framework
-
-Theodolite aims to benchmark scalability of stream processing engines for real use cases. Microservices that apply stream processing techniques are usually deployed in elastic cloud environments. Hence, Theodolite's cloud-native benchmarking framework deploys its components in a cloud environment, orchestrated by Kubernetes. More information on how to execute scalability benchmarks can be found in [Thedolite execution framework](execution).
+## Theodolite Benchmarking Tool
+Theodolite aims to benchmark scalability of stream processing engines for real use cases. Microservices that apply stream processing techniques are usually deployed in elastic cloud environments. Hence, Theodolite's cloud-native benchmarking framework deploys its components in a cloud environment, orchestrated by Kubernetes. It is recommended to install Theodolite with the package manager Helm. The Theodolite Helm chart along with instructions how to install it can be found in the [`helm`](helm) directory.
## Theodolite Analysis Tools
-Theodolite's benchmarking method creates a *scalability graph* allowing to draw conclusions about the scalability of a stream processing engine or its deployment. A scalability graph shows how resource demand evolves with an increasing workload. Theodolite provides Jupyter notebooks for creating such scalability graphs based on benchmarking results from the execution framework. More information can be found in [Theodolite analysis tool](analysis).
+Theodolite's benchmarking method maps load intensities to the resource amounts that are required for processing them. A plot showing how resource demand evolves with an increasing load allows to draw conclusions about the scalability of a stream processing engine or its deployment. Theodolite provides Jupyter notebooks for creating such plots based on benchmarking results from the execution framework. More information can be found in [Theodolite analysis tool](analysis).
+
+## Theodolite Benchmarks
+Theodolite comes with 4 application benchmarks, which are based on typical use cases for stream processing within microservices. For each benchmark, a corresponding load generator is provided. Currently, this repository provides benchmark implementations for Apache Kafka Streams and Apache Flink. The benchmark sources can be found in [Thedolite benchmarks](theodolite-benchmarks).
## How to Cite
diff --git a/analysis/demand-metric-plot.ipynb b/analysis/demand-metric-plot.ipynb
index 90ef227dbf6a4566760329b615d5f59b4cc2bc25..71e08f0590f819a63b1bdd6bf13b57ac665f65bc 100644
--- a/analysis/demand-metric-plot.ipynb
+++ b/analysis/demand-metric-plot.ipynb
@@ -1,22 +1,22 @@
{
"cells": [
{
+ "cell_type": "markdown",
+ "metadata": {},
"source": [
"# Theodolite Analysis - Plotting the Demand Metric\n",
"\n",
"This notebook creates a plot, showing scalability as a function that maps load intensities to the resources required for processing them. It is able to combine multiple such plots in one figure, for example, to compare multiple systems or configurations.\n",
"\n",
"The notebook takes a CSV file for each plot mapping load intensities to minimum required resources, computed by the `demand-metric-plot.ipynb` notebook."
- ],
- "cell_type": "markdown",
- "metadata": {}
+ ]
},
{
+ "cell_type": "markdown",
+ "metadata": {},
"source": [
"First, we need to import some libraries, which are required for creating the plots."
- ],
- "cell_type": "markdown",
- "metadata": {}
+ ]
},
{
"cell_type": "code",
@@ -33,11 +33,11 @@
]
},
{
+ "cell_type": "markdown",
+ "metadata": {},
"source": [
"We need to specify the directory, where the demand CSV files can be found, and a dictionary that maps a system description (e.g. its name) to the corresponding CSV file (prefix). To use Unicode narrow non-breaking spaces in the description format it as `u\"1000\\u202FmCPU\"`."
- ],
- "cell_type": "markdown",
- "metadata": {}
+ ]
},
{
"cell_type": "code",
@@ -53,11 +53,11 @@
]
},
{
+ "cell_type": "markdown",
+ "metadata": {},
"source": [
"Now, we combie all systems described in `experiments`."
- ],
- "cell_type": "markdown",
- "metadata": {}
+ ]
},
{
"cell_type": "code",
@@ -71,11 +71,11 @@
]
},
{
+ "cell_type": "markdown",
+ "metadata": {},
"source": [
"We might want to display the mappings before we plot it."
- ],
- "cell_type": "markdown",
- "metadata": {}
+ ]
},
{
"cell_type": "code",
@@ -87,11 +87,11 @@
]
},
{
+ "cell_type": "markdown",
+ "metadata": {},
"source": [
"The following code creates a MatPlotLib figure showing the scalability plots for all specified systems. You might want to adjust its styling etc. according to your preferences. Make sure to also set a filename."
- ],
- "cell_type": "markdown",
- "metadata": {}
+ ]
},
{
"cell_type": "code",
@@ -149,27 +149,33 @@
}
],
"metadata": {
+ "file_extension": ".py",
+ "interpreter": {
+ "hash": "e9e076445e1891a25f59b525adcc71b09846b3f9cf034ce4147fc161b19af121"
+ },
+ "kernelspec": {
+ "display_name": "Python 3.8.10 64-bit ('.venv': venv)",
+ "name": "python3"
+ },
"language_info": {
- "name": "python",
"codemirror_mode": {
"name": "ipython",
"version": 3
},
- "version": "3.8.5-final"
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.10"
},
- "orig_nbformat": 2,
- "file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"npconvert_exporter": "python",
+ "orig_nbformat": 2,
"pygments_lexer": "ipython3",
- "version": 3,
- "kernelspec": {
- "name": "python37064bitvenvvenv6c432ee1239d4f3cb23f871068b0267d",
- "display_name": "Python 3.7.0 64-bit ('.venv': venv)",
- "language": "python"
- }
+ "version": 3
},
"nbformat": 4,
"nbformat_minor": 2
-}
\ No newline at end of file
+}
diff --git a/analysis/demand-metric.ipynb b/analysis/demand-metric.ipynb
index bcea129b7cb07465fa99f32b6f8b2b6115e8a0aa..fbf3ee02960a1e06457eef5dda96cb6d0a1a75ac 100644
--- a/analysis/demand-metric.ipynb
+++ b/analysis/demand-metric.ipynb
@@ -1,6 +1,8 @@
{
"cells": [
{
+ "cell_type": "markdown",
+ "metadata": {},
"source": [
"# Theodolite Analysis - Demand Metric\n",
"\n",
@@ -9,11 +11,11 @@
"Theodolite's *demand* metric is a function, mapping load intensities to the minimum required resources (e.g., instances) that are required to process this load. With this notebook, the *demand* metric function is approximated by a map of tested load intensities to their minimum required resources.\n",
"\n",
"The final output when running this notebook will be a CSV file, providig this mapping. It can be used to create nice plots of a system's scalability using the `demand-metric-plot.ipynb` notebook."
- ],
- "cell_type": "markdown",
- "metadata": {}
+ ]
},
{
+ "cell_type": "markdown",
+ "metadata": {},
"source": [
"In the following cell, we need to specifiy:\n",
"\n",
@@ -22,9 +24,7 @@
"* `max_lag_trend_slope`: The maximum tolerable increase in queued messages per second.\n",
"* `measurement_dir`: The directory where the measurement data files are to be found.\n",
"* `results_dir`: The directory where the computed demand CSV files are to be stored."
- ],
- "cell_type": "markdown",
- "metadata": {}
+ ]
},
{
"cell_type": "code",
@@ -40,11 +40,11 @@
]
},
{
+ "cell_type": "markdown",
+ "metadata": {},
"source": [
"With the following call, we compute our demand mapping."
- ],
- "cell_type": "markdown",
- "metadata": {}
+ ]
},
{
"cell_type": "code",
@@ -58,11 +58,11 @@
]
},
{
+ "cell_type": "markdown",
+ "metadata": {},
"source": [
"We might already want to plot a simple visualization here:"
- ],
- "cell_type": "markdown",
- "metadata": {}
+ ]
},
{
"cell_type": "code",
@@ -74,11 +74,11 @@
]
},
{
+ "cell_type": "markdown",
+ "metadata": {},
"source": [
"Finally we store the results in a CSV file."
- ],
- "cell_type": "markdown",
- "metadata": {}
+ ]
},
{
"cell_type": "code",
@@ -93,27 +93,33 @@
}
],
"metadata": {
+ "file_extension": ".py",
+ "interpreter": {
+ "hash": "e9e076445e1891a25f59b525adcc71b09846b3f9cf034ce4147fc161b19af121"
+ },
+ "kernelspec": {
+ "display_name": "Python 3.8.10 64-bit ('.venv': venv)",
+ "name": "python3"
+ },
"language_info": {
- "name": "python",
"codemirror_mode": {
"name": "ipython",
"version": 3
},
- "version": "3.8.5-final"
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.10"
},
- "orig_nbformat": 2,
- "file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"npconvert_exporter": "python",
+ "orig_nbformat": 2,
"pygments_lexer": "ipython3",
- "version": 3,
- "kernelspec": {
- "name": "python37064bitvenvvenv6c432ee1239d4f3cb23f871068b0267d",
- "display_name": "Python 3.7.0 64-bit ('.venv': venv)",
- "language": "python"
- }
+ "version": 3
},
"nbformat": 4,
"nbformat_minor": 2
-}
\ No newline at end of file
+}
diff --git a/analysis/src/demand.py b/analysis/src/demand.py
index dfb20c05af8e9a134eedd2cdb584c961a82369f5..2178ab7c5dc5f7e4c04ebb58d4c14c9bf8b1aeff 100644
--- a/analysis/src/demand.py
+++ b/analysis/src/demand.py
@@ -1,59 +1,51 @@
import os
from datetime import datetime, timedelta, timezone
import pandas as pd
+from pandas.core.frame import DataFrame
from sklearn.linear_model import LinearRegression
def demand(exp_id, directory, threshold, warmup_sec):
raw_runs = []
- # Compute SL, i.e., lag trend, for each tested configuration
- filenames = [filename for filename in os.listdir(directory) if filename.startswith(f"exp{exp_id}") and filename.endswith("totallag.csv")]
+ # Compute SLI, i.e., lag trend, for each tested configuration
+ filenames = [filename for filename in os.listdir(directory) if filename.startswith(f"exp{exp_id}") and "lag-trend" in filename and filename.endswith(".csv")]
for filename in filenames:
- #print(filename)
run_params = filename[:-4].split("_")
- dim_value = run_params[2]
- instances = run_params[3]
+ dim_value = run_params[1]
+ instances = run_params[2]
df = pd.read_csv(os.path.join(directory, filename))
- #input = df.loc[df['topic'] == "input"]
input = df
- #print(input)
+
input['sec_start'] = input.loc[0:, 'timestamp'] - input.iloc[0]['timestamp']
- #print(input)
- #print(input.iloc[0, 'timestamp'])
+
regress = input.loc[input['sec_start'] >= warmup_sec] # Warm-Up
- #regress = input
- #input.plot(kind='line',x='timestamp',y='value',color='red')
- #plt.show()
+ X = regress.iloc[:, 1].values.reshape(-1, 1) # values converts it into a numpy array
+ Y = regress.iloc[:, 2].values.reshape(-1, 1) # -1 means that calculate the dimension of rows, but have 1 column
- X = regress.iloc[:, 2].values.reshape(-1, 1) # values converts it into a numpy array
- Y = regress.iloc[:, 3].values.reshape(-1, 1) # -1 means that calculate the dimension of rows, but have 1 column
linear_regressor = LinearRegression() # create object for the class
linear_regressor.fit(X, Y) # perform linear regression
Y_pred = linear_regressor.predict(X) # make predictions
trend_slope = linear_regressor.coef_[0][0]
- #print(linear_regressor.coef_)
row = {'load': int(dim_value), 'resources': int(instances), 'trend_slope': trend_slope}
- #print(row)
raw_runs.append(row)
runs = pd.DataFrame(raw_runs)
- # Set suitable = True if SLOs are met, i.e., lag trend is below threshold
- runs["suitable"] = runs.apply(lambda row: row['trend_slope'] < threshold, axis=1)
-
- # Sort results table (unsure if required)
- runs.columns = runs.columns.str.strip()
- runs.sort_values(by=["load", "resources"])
+ # Group by the load and resources to handle repetitions, and take from the reptitions the median
+ # for even reptitions, the mean of the two middle values is used
+ medians = runs.groupby(by=['load', 'resources'], as_index=False).median()
- # Filter only suitable configurations
- filtered = runs[runs.apply(lambda x: x['suitable'], axis=1)]
-
- # Compute demand per load intensity
- grouped = filtered.groupby(['load'])['resources'].min()
- demand_per_load = grouped.to_frame().reset_index()
+ # Set suitable = True if SLOs are met, i.e., lag trend slope is below threshold
+ medians["suitable"] = medians.apply(lambda row: row['trend_slope'] < threshold, axis=1)
+ suitable = medians[medians.apply(lambda x: x['suitable'], axis=1)]
+
+ # Compute minimal demand per load intensity
+ demand_per_load = suitable.groupby(by=['load'], as_index=False)['resources'].min()
+
return demand_per_load
+
diff --git a/codemeta.json b/codemeta.json
index 5696996592f63bf8ece23239d8204e0f25b9cce1..a158e30eb7f1ab433779678aba3a1cc3b7e33c80 100644
--- a/codemeta.json
+++ b/codemeta.json
@@ -5,10 +5,10 @@
"codeRepository": "https://github.com/cau-se/theodolite",
"dateCreated": "2020-03-13",
"datePublished": "2020-07-27",
- "dateModified": "2021-03-18",
+ "dateModified": "2021-11-12",
"downloadUrl": "https://github.com/cau-se/theodolite/releases",
"name": "Theodolite",
- "version": "0.4.0",
+ "version": "0.5.1",
"description": "Theodolite is a framework for benchmarking the horizontal and vertical scalability of stream processing engines.",
"developmentStatus": "active",
"referencePublication": "https://doi.org/10.1016/j.bdr.2021.100209",
diff --git a/docs/README.md b/docs/README.md
index a1e43024a89c05a56aec2410b6c130dde0645ba0..142ced47fdd2056982e397327f3cd91ddadd5401 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -11,16 +11,20 @@ permalink: /
Theodolite is a framework for benchmarking the horizontal and vertical scalability of stream processing engines. It consists of three modules:
-## Theodolite Benchmarks
+## Theodolite Benchmarking Tool
-Theodolite contains 4 application benchmarks, which are based on typical use cases for stream processing within microservices. For each benchmark, a corresponding workload generator is provided. Currently, this repository provides benchmark implementations for Kafka Streams.
+Theodolite aims to benchmark scalability of stream processing engines for real use cases. Microservices that apply stream processing techniques are usually deployed in elastic cloud environments. Hence, Theodolite's cloud-native benchmarking framework deploys its components in a cloud environment, orchestrated by Kubernetes. It is recommended to install Theodolite with the package manager Helm. The Theodolite Helm chart along with instructions how to install it can be found in the [`helm`](helm) directory.
+## Theodolite Analysis Tools
-## Theodolite Execution Framework
+Theodolite's benchmarking method maps load intensities to the resource amounts that are required for processing them. A plot showing how resource demand evolves with an increasing load allows to draw conclusions about the scalability of a stream processing engine or its deployment. Theodolite provides Jupyter notebooks for creating such plots based on benchmarking results from the execution framework. More information can be found in [Theodolite analysis tool](analysis).
-Theodolite aims to benchmark scalability of stream processing engines for real use cases. Microservices that apply stream processing techniques are usually deployed in elastic cloud environments. Hence, Theodolite's cloud-native benchmarking framework deploys as components in a cloud environment, orchestrated by Kubernetes. More information on how to execute scalability benchmarks can be found in [Thedolite execution framework](execution).
+## Theodolite Benchmarks
+Theodolite comes with 4 application benchmarks, which are based on typical use cases for stream processing within microservices. For each benchmark, a corresponding load generator is provided. Currently, this repository provides benchmark implementations for Apache Kafka Streams and Apache Flink. The benchmark sources can be found in [Thedolite benchmarks](theodolite-benchmarks).
-## Theodolite Analysis Tools
+## How to Cite
+
+If you use Theodolite, please cite
-Theodolite's benchmarking method create a *scalability graph* allowing to draw conclusions about the scalability of a stream processing engine or its deployment. A scalability graph shows how resource demand evolves with an increasing workload. Theodolite provides Jupyter notebooks for creating such scalability graphs based on benchmarking results from the execution framework. More information can be found in [Theodolite analysis tool](analysis).
+> Sören Henning and Wilhelm Hasselbring. (2021). Theodolite: Scalability Benchmarking of Distributed Stream Processing Engines in Microservice Architectures. Big Data Research, Volume 25. DOI: [10.1016/j.bdr.2021.100209](https://doi.org/10.1016/j.bdr.2021.100209). arXiv:[2009.00304](https://arxiv.org/abs/2009.00304).
diff --git a/docs/index.yaml b/docs/index.yaml
index 087124d158794e1b48dfc880e26da2c91d78808f..54580ea45f1c678443dae96c7139f53fdac37f60 100644
--- a/docs/index.yaml
+++ b/docs/index.yaml
@@ -1,6 +1,76 @@
apiVersion: v1
entries:
theodolite:
+ - apiVersion: v2
+ appVersion: 0.5.1
+ created: "2021-11-12T16:15:01.629937292+01:00"
+ dependencies:
+ - condition: grafana.enabled
+ name: grafana
+ repository: https://grafana.github.io/helm-charts
+ version: 6.17.5
+ - condition: kube-prometheus-stack.enabled
+ name: kube-prometheus-stack
+ repository: https://prometheus-community.github.io/helm-charts
+ version: 12.0.0
+ - condition: cp-helm-charts.enabled
+ name: cp-helm-charts
+ repository: https://soerenhenning.github.io/cp-helm-charts
+ version: 0.6.0
+ - condition: kafka-lag-exporter.enabled
+ name: kafka-lag-exporter
+ repository: https://lightbend.github.io/kafka-lag-exporter/repo/
+ version: 0.6.6
+ description: Theodolite is a framework for benchmarking the scalability stream
+ processing engines.
+ digest: a67374c4cb2b0e8b2d711468364c6b4a486a910bd1c667dbf3c5614e36e0680c
+ home: https://cau-se.github.io/theodolite
+ maintainers:
+ - email: soeren.henning@email.uni-kiel.de
+ name: Sören Henning
+ url: https://www.se.informatik.uni-kiel.de/en/team/soeren-henning-m-sc
+ name: theodolite
+ sources:
+ - https://github.com/cau-se/theodolite
+ type: application
+ urls:
+ - https://github.com/cau-se/theodolite/releases/download/v0.5.1/theodolite-0.5.1.tgz
+ version: 0.5.1
+ - apiVersion: v2
+ appVersion: 0.5.0
+ created: "2021-11-04T17:45:14.153231798+01:00"
+ dependencies:
+ - condition: grafana.enabled
+ name: grafana
+ repository: https://grafana.github.io/helm-charts
+ version: 6.0.0
+ - condition: kube-prometheus-stack.enabled
+ name: kube-prometheus-stack
+ repository: https://prometheus-community.github.io/helm-charts
+ version: 12.0.0
+ - condition: cp-helm-charts.enabled
+ name: cp-helm-charts
+ repository: https://soerenhenning.github.io/cp-helm-charts
+ version: 0.6.0
+ - condition: kafka-lag-exporter.enabled
+ name: kafka-lag-exporter
+ repository: https://lightbend.github.io/kafka-lag-exporter/repo/
+ version: 0.6.6
+ description: Theodolite is a framework for benchmarking the scalability stream
+ processing engines.
+ digest: 8a4f218e44341eb8fb09ddc58c6aaa0a14aded685f3423088c21fe0ffc112281
+ home: https://cau-se.github.io/theodolite
+ maintainers:
+ - email: soeren.henning@email.uni-kiel.de
+ name: Sören Henning
+ url: https://www.se.informatik.uni-kiel.de/en/team/soeren-henning-m-sc
+ name: theodolite
+ sources:
+ - https://github.com/cau-se/theodolite
+ type: application
+ urls:
+ - https://github.com/cau-se/theodolite/releases/download/v0.5.0/theodolite-0.5.0.tgz
+ version: 0.5.0
- apiVersion: v2
appVersion: 0.4.0
created: "2021-03-18T15:50:50.930902088+01:00"
@@ -36,4 +106,4 @@ entries:
urls:
- https://github.com/cau-se/theodolite/releases/download/v0.4.0/theodolite-0.4.0.tgz
version: 0.4.0
-generated: "2021-03-18T15:50:50.897801281+01:00"
+generated: "2021-11-12T16:15:01.591258889+01:00"
diff --git a/docs/patchers.md b/docs/patchers.md
new file mode 100644
index 0000000000000000000000000000000000000000..572f107fb38ba295cd013abeff5dd53c2702527b
--- /dev/null
+++ b/docs/patchers.md
@@ -0,0 +1,50 @@
+## Patchers
+
+* **ReplicaPatcher**: Allows to modify the number of Replicas for a kubernetes deployment.
+ * **type**: "ReplicaPatcher"
+ * **resource**: "uc1-kstreams-deployment.yaml"
+
+* **NumSensorsLoadGeneratorReplicaPatcher**: Allows to scale the nummer of load generators. Scales arcording to the following formular: (value + 15_000 - 1) / 15_000
+ * **type**: "NumSensorsLoadGeneratorReplicaPatcher"
+ * **resource**: "uc1-load-generator-deployment.yaml"
+
+* **NumNestedGroupsLoadGeneratorReplicaPatcher**: Allows to scale the nummer of load generators. Scales arcording to the following formular: (4^(value) + 15_000 -1) /15_000
+ * **type**: "NumNestedGroupsLoadGeneratorReplicaPatcher"
+ * **resource**: "uc1-load-generator-deployment.yaml"
+
+* **ReplicaPatcher**: Allows to modify the number of Replicas for a kubernetes deployment.
+ * **type**: "ReplicaPatcher"
+ * **resource**: "uc1-kstreams-deployment.yaml"
+
+* **EnvVarPatcher**: Allows to modify the value of an environment variable for a container in a kubernetes deployment.
+ * **type**: "EnvVarPatcher"
+ * **resource**: "uc1-load-generator-deployment.yaml"
+ * **properties**:
+ * container: "workload-generator"
+ * variableName: "NUM_SENSORS"
+
+* **NodeSelectorPatcher**: Changes the node selection field in kubernetes resources.
+ * **type**: "NodeSelectorPatcher"
+ * **resource**: "uc1-load-generator-deployment.yaml"
+ * **properties**:
+ * variableName: "env"
+ * **value**: "prod"
+
+* **ResourceLimitPatcher**: Changes the resource limit for a kubernetes resource.
+ * **resource**: "uc1-kstreams-deployment.yaml"
+ * **properties**:
+ * container: "uc-application"
+ * variableName: "cpu" or "memory"
+ * **value**:"1000m" or "2Gi"
+
+* **SchedulerNamePatcher**: Changes the sheduler for kubernetes resources.
+ * **type**: "SchedulerNamePatcher"
+ * **resource**: "uc1-kstreams-deployment.yaml"
+ * **value**: "random-scheduler"
+
+* **ImagePatcher**: Changes the image of a kubernetes resource. Currently not fully implemented.
+ * **type**: "ImagePatcher"
+ * **resource**: "uc1-kstreams-deployment.yaml"
+ * **properties**:
+ * container: "uc-application"
+ * **value**: "dockerhubrepo/imagename"
diff --git a/docs/release-process.md b/docs/release-process.md
index 28456f44291a2938974e53a6be5254882215d237..a54b2c8505eabfeedc41177164fde0b80442c82c 100644
--- a/docs/release-process.md
+++ b/docs/release-process.md
@@ -18,8 +18,11 @@ again be merged into master.
3. Update all references to artifacts which are versioned. This includes:
- 1. Update all references to Theodolite Docker images to tag `v0.3.1`. These are the Kubernetes resource definitions in
-`execution`, the references to *latest* in `run_uc.py`, the Docker Compose files in `theodolite-benchmarks/docker-test` and the example `theodolite.yaml` job.
+ 1. Update all references to Theodolite Docker images to tag `v0.3.1`. These are:
+ 1. the default `helm/values.yaml` file,
+ 2. the example `execution/theodolite.yaml` job,
+ 3. the Kubernetes benchmark resources in `theodolite-benchmarks/definitions/**/resources` and
+ 2. the Docker Compose files in `theodolite-benchmarks/docker-test`.
2. Update both, the `version` and the `appVersion` fields, in the Helm `Charts.yaml` file to `0.3.1`.
diff --git a/execution/.dockerignore b/execution/.dockerignore
deleted file mode 100644
index 68e5f21c503a80d7db64722d700351a303ddb9dd..0000000000000000000000000000000000000000
--- a/execution/.dockerignore
+++ /dev/null
@@ -1,9 +0,0 @@
-*
-!requirements.txt
-!uc-workload-generator
-!uc-application
-!strategies
-!lib
-!theodolite.py
-!run_uc.py
-!lag_analysis.py
diff --git a/execution/.gitignore b/execution/.gitignore
deleted file mode 100644
index bac9a5d1eeb12d9e40d38376904e8fb69c0e5231..0000000000000000000000000000000000000000
--- a/execution/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-exp_counter.txt
-results
diff --git a/execution/theodolite.yaml b/execution/theodolite.yaml
index 749c94b12736b2d8546fa3cb46559ec1b4ea0f27..ae18a68ee61c71e20008a71537357cdf9521216a 100644
--- a/execution/theodolite.yaml
+++ b/execution/theodolite.yaml
@@ -25,11 +25,11 @@ spec:
# - name: MODE
# value: yaml-executor # Default is `yaml-executor`
- name: THEODOLITE_EXECUTION
- value: /etc/execution/example-execution-yaml-resource.yaml # The name of this file must correspond to the filename of the execution, from which the config map is created.
+ value: "execution/execution.yaml" # The name of this file must correspond to the filename of the execution, from which the config map is created.
- name: THEODOLITE_BENCHMARK
- value: /etc/benchmark/example-benchmark-yaml-resource.yaml # The name of this file must correspond to the filename of the benchmark, from which the config map is created.
+ value: "benchmark/benchmark.yaml" # The name of this file must correspond to the filename of the benchmark, from which the config map is created.
- name: THEODOLITE_APP_RESOURCES
- value: /etc/app-resources
+ value: "benchmark-resources"
- name: RESULTS_FOLDER # Folder for saving results
value: results # Default is the pwd (/deployments)
# - name: CREATE_RESULTS_FOLDER # Specify whether the specified result folder should be created if it does not exist.
@@ -37,11 +37,11 @@ spec:
volumeMounts:
- mountPath: "/deployments/results" # the mounted path must corresponds to the value of `RESULT_FOLDER`.
name: theodolite-pv-storage
- - mountPath: "/etc/app-resources" # must be corresponds to the value of `THEODOLITE_APP_RESOURCES`.
- name: app-resources
- - mountPath: "/etc/benchmark" # must be corresponds to the value of `THEODOLITE_BENCHMARK`.
+ - mountPath: "/deployments/benchmark-resources" # must correspond to the value of `THEODOLITE_APP_RESOURCES`.
+ name: benchmark-resources
+ - mountPath: "/deployments/benchmark" # must correspond to the value of `THEODOLITE_BENCHMARK`.
name: benchmark
- - mountPath: "/etc/execution" # must be corresponds to the value of `THEODOLITE_EXECUTION`.
+ - mountPath: "/deployments/execution" # must correspond to the value of `THEODOLITE_EXECUTION`.
name: execution
restartPolicy: Never
# Uncomment if RBAC is enabled and configured
@@ -52,9 +52,9 @@ spec:
- name: theodolite-pv-storage
persistentVolumeClaim:
claimName: theodolite-pv-claim
- - name: app-resources
+ - name: benchmark-resources
configMap:
- name: app-resources-configmap
+ name: benchmark-resources-configmap
- name: benchmark
configMap:
name: benchmark-configmap
diff --git a/helm/Chart.yaml b/helm/Chart.yaml
index 30cfa8249391750071a679b95d2ce2b01be9c20b..856644feb587fd1783162d55a7e6ee8c596c0ac0 100644
--- a/helm/Chart.yaml
+++ b/helm/Chart.yaml
@@ -13,11 +13,11 @@ type: application
dependencies:
- name: grafana
- version: 6.0.0
+ version: 6.17.5
repository: https://grafana.github.io/helm-charts
condition: grafana.enabled
- name: kube-prometheus-stack
- version: 12.0.0
+ version: 20.0.1
repository: https://prometheus-community.github.io/helm-charts
condition: kube-prometheus-stack.enabled
- name: cp-helm-charts
@@ -25,10 +25,10 @@ dependencies:
repository: https://soerenhenning.github.io/cp-helm-charts
condition: cp-helm-charts.enabled
- name: kafka-lag-exporter
- version: 0.6.6
+ version: 0.6.7
repository: https://lightbend.github.io/kafka-lag-exporter/repo/
condition: kafka-lag-exporter.enabled
-version: 0.5.0-SNAPSHOT
+version: 0.6.0-SNAPSHOT
-appVersion: 0.5.0-SNAPSHOT
+appVersion: 0.6.0-SNAPSHOT
diff --git a/helm/README.md b/helm/README.md
index c545804aaec8eb8ed91054f1f7ee97dd293816a4..1a3428b5e601de0c6c33f9dab236321e95592c6c 100644
--- a/helm/README.md
+++ b/helm/README.md
@@ -2,55 +2,47 @@
## Installation
-Install the chart via:
+The Theodolite Helm chart with all its dependencies can be installed via:
```sh
helm dependencies update .
helm install theodolite .
```
-This chart installs requirements to execute benchmarks with Theodolite.
+## Customize Installation
-Dependencies and subcharts:
+As usual, the installation with Helm can be configured by passing a values YAML file:
-- Prometheus Operator
-- Prometheus
-- Grafana (incl. dashboard and data source configuration)
-- Kafka
-- Zookeeper
-- A Kafka client pod
-
-## Test
-
-Test the installation:
-
-```sh
-helm test theodolite
+```
+helm install theodolite . -f <your-config.yaml>
```
-Our test files are located [here](templates/../../theodolite-chart/templates/tests). Many subcharts have their own tests, these are also executed and are placed in the respective /templates folders.
-
-Please note: If a test fails, Helm will stop testing.
+We provide a minimal configuration, especially suited for development environments, with the `preconfigs/minimal.yaml`
+file.
-It is possible that the tests are not running successfully at the moment. This is because the Helm tests of the subchart cp-confluent receive a timeout exception. There is an [issue](https://github.com/confluentinc/cp-helm-charts/issues/318) for this problem on GitHub.
+Per default, Helm installs the Theodolite CRDs used for the operator. If Theodolite will not be used as operator or if
+the CRDs are already installed, you can skip their installation by adding the flag `--skip-crds`.
-## Configuration
+## Test Installation
-In development environments Kubernetes resources are often low. To reduce resource consumption, we provide an `one-broker-value.yaml` file. This file can be used with:
+Test the installation with:
```sh
-helm install theodolite . -f preconfigs/one-broker-values.yaml
+helm test theodolite
```
+Our test files are located [here](templates/tests). Many subcharts have their own tests, which are also executed.
+Please note: If a test fails, Helm will stop testing.
+
## Uninstall this Chart
-To uninstall/delete the `theodolite` deployment:
+The Theodolite Helm can easily be removed with:
```sh
-helm delete theodolite
+helm uninstall theodolite
```
-This command does not remove the CRDs which are created by this chart. Remove them manually with:
+Helm does not remove any CRDs created by this chart. You can remove them manually with:
```sh
# CRDs from Theodolite
@@ -69,9 +61,20 @@ kubectl delete crd thanosrulers.monitoring.coreos.com
## Development
-**Hints**:
+### Dependencies
+
+The following 3rd party charts are used by Theodolite:
+
+- Kube Prometheus Stack (to install the Prometheus Operator, which is used to create a Prometheus instances)
+- Grafana (including a dashboard and a data source configuration)
+- Confluent Platform (for Kafka and Zookeeper)
+- Kafka Lag Exporter (used to collect monitoring data of the Kafka lag)
+
+### Hints
+
+#### Grafana
-- Grafana configuration: Grafana ConfigMaps contains expressions like {{ topic }}. Helm uses the same syntax for template function. More information [here](https://github.com/helm/helm/issues/2798)
+Grafana ConfigMaps contain expressions like `{{ topic }}`. Helm uses the same syntax for template function. More information [here](https://github.com/helm/helm/issues/2798)
- Escape braces: {{ "{{" topic }}
- Let Helm render the template as raw string: {{ `{{ <config>}}` }}
\ No newline at end of file
diff --git a/helm/crds/benchmark.yaml b/helm/crds/benchmark.yaml
new file mode 120000
index 0000000000000000000000000000000000000000..fb100de7a1407462bfb6488a54b7f70014a58474
--- /dev/null
+++ b/helm/crds/benchmark.yaml
@@ -0,0 +1 @@
+./../../theodolite/crd/crd-benchmark.yaml
\ No newline at end of file
diff --git a/helm/crds/execution.yaml b/helm/crds/execution.yaml
new file mode 120000
index 0000000000000000000000000000000000000000..62d268c23c391cd7bbfbaffeaee8af1697dc446a
--- /dev/null
+++ b/helm/crds/execution.yaml
@@ -0,0 +1 @@
+./../../theodolite/crd/crd-execution.yaml
\ No newline at end of file
diff --git a/helm/preconfigs/minimal.yaml b/helm/preconfigs/minimal.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b0828c2f424e8456933dc626a66a199cd60aa5da
--- /dev/null
+++ b/helm/preconfigs/minimal.yaml
@@ -0,0 +1,12 @@
+cp-helm-charts:
+ cp-zookeeper:
+ servers: 1
+
+ cp-kafka:
+ brokers: 1
+ configurationOverrides:
+ offsets.topic.replication.factor: "1"
+
+operator:
+ resultsVolume:
+ enabled: false
diff --git a/helm/preconfigs/oci.yaml b/helm/preconfigs/oci.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..35fe5dcf423eed77cb2d3d4298088738125fa9fe
--- /dev/null
+++ b/helm/preconfigs/oci.yaml
@@ -0,0 +1,6 @@
+operator:
+ resultsVolume:
+ persistent:
+ enabled: true
+ storageClassName: "oci-bv"
+ size: 50Gi # minimal size in OCI
diff --git a/helm/preconfigs/one-broker-values.yaml b/helm/preconfigs/one-broker-values.yaml
deleted file mode 100644
index c53c1f1eb8bc7a17f192d70a6f10f8cacc09c98f..0000000000000000000000000000000000000000
--- a/helm/preconfigs/one-broker-values.yaml
+++ /dev/null
@@ -1,15 +0,0 @@
-cp-helm-charts:
- ## ------------------------------------------------------
- ## Zookeeper
- ## ------------------------------------------------------
- cp-zookeeper:
- servers: 1 # default: 3
-
- ## ------------------------------------------------------
- ## Kafka
- ## ------------------------------------------------------
- cp-kafka:
- brokers: 1 # default: 10
-
- configurationOverrides:
- offsets.topic.replication.factor: "1"
\ No newline at end of file
diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl
index f59f74d369b64ec89a44cbf2048fda9e844df92b..569cf454a950f3f621c23472f0346c8bbd52229d 100644
--- a/helm/templates/_helpers.tpl
+++ b/helm/templates/_helpers.tpl
@@ -60,3 +60,10 @@ Create the name of the service account to use
{{- default "default" .Values.serviceAccount.name }}
{{- end }}
{{- end }}
+
+{{/*
+Create the name of the results volume to use
+*/}}
+{{- define "theodolite.resultsClaimName" -}}
+{{- default (printf "%s-results" (include "theodolite.fullname" .)) .Values.operator.resultsVolume.persistent.existingClaim }}
+{{- end }}
diff --git a/helm/templates/grafana/dashboard-config-map.yaml b/helm/templates/grafana/dashboard-config-map.yaml
index 41365e5efefaddc92a9f2f25f867a9d895e4ca3d..0df01b20efa0fb1100fe4b7289b00b3058eb032f 100644
--- a/helm/templates/grafana/dashboard-config-map.yaml
+++ b/helm/templates/grafana/dashboard-config-map.yaml
@@ -24,7 +24,7 @@ data:
"editable": true,
"gnetId": null,
"graphTooltip": 0,
- "id": 2,
+ "id": 1,
"iteration": 1589140028684,
"links": [],
"panels": [
@@ -1004,4 +1004,4 @@ data:
"uid": "dad0CNlZz",
"version": 25
}`}}
-{{- end }}
\ No newline at end of file
+{{- end }}
diff --git a/helm/templates/grafana/osp-dashboard-config-map.yaml b/helm/templates/grafana/osp-dashboard-config-map.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..22685adf8e93d95614bdbb71ef6bc993a009a150
--- /dev/null
+++ b/helm/templates/grafana/osp-dashboard-config-map.yaml
@@ -0,0 +1,1029 @@
+{{- if .Values.grafana.enabled -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: {{ template "theodolite.fullname" . }}-grafana-scalability-osp
+ labels:
+ grafana_dashboard: "2"
+data:
+ osp-dashboard.json: |-
+ {{`{
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": "-- Grafana --",
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 0,
+ "id": 2,
+ "iteration": 1631777972723,
+ "links": [],
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 0
+ },
+ "hiddenSeries": false,
+ "id": 2,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(cp_kafka_server_brokertopicmetrics_messagesinpersec_topic_ndwflow)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "ndwflow",
+ "refId": "A"
+ },
+ {
+ "expr": "sum(cp_kafka_server_brokertopicmetrics_messagesinpersec_topic_ndwspeed)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "ndwspeed",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(cp_kafka_server_brokertopicmetrics_messagesinpersec_topic_ndwflow+cp_kafka_server_brokertopicmetrics_messagesinpersec_topic_ndwspeed)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "ndwflow+ndwspeed",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Messages In Per Second",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 0
+ },
+ "hiddenSeries": false,
+ "id": 3,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(cp_kafka_server_brokertopicmetrics_messagesinpersec_topic_metrics)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "Metrics",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Messages Out Per Second",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 7
+ },
+ "hiddenSeries": false,
+ "id": 9,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum by(group, topic) (kafka_consumergroup_group_lag >= 0)",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{topic}}",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Record Lag",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 7
+ },
+ "hiddenSeries": false,
+ "id": 5,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "count(count (kafka_consumer_consumer_fetch_manager_metrics_records_lag) by(pod))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "instances",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Number of Instances",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "decimals": 0,
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 15
+ },
+ "hiddenSeries": false,
+ "id": 10,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum by(group,topic) (kafka_consumergroup_group_offset >= 0)",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{topic}}",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Records Consumed",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 15
+ },
+ "hiddenSeries": false,
+ "id": 12,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "count by(group,topic) (kafka_consumergroup_group_offset >= 0)",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{topic}}",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Number of Partitions",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 23
+ },
+ "hiddenSeries": false,
+ "id": 11,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum by(group,topic) (kafka_partition_latest_offset)",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{topic}}",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Records Produced (Kafka Lag Exporter)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 23
+ },
+ "hiddenSeries": false,
+ "id": 8,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "count by(job, topic) (kafka_consumer_consumer_fetch_manager_metrics_records_lag)",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{topic}}",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Number of Partitions (Kafka Streams Export)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "decimals": null,
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 31
+ },
+ "hiddenSeries": false,
+ "id": 4,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum by(job, topic) (kafka_consumer_consumer_fetch_manager_metrics_records_lag)",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{topic}}",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Record Lag (Kafka Streams Export)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 31
+ },
+ "hiddenSeries": false,
+ "id": 13,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum by(group) (kafka_consumergroup_group_lag >= 0)",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "total lag",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Total Record Lag (Kafka Lag Exporter)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ }
+ ],
+ "refresh": "10s",
+ "schemaVersion": 22,
+ "style": "dark",
+ "tags": [],
+ "templating": {
+ "list": [
+ {
+ "allValue": null,
+ "current": {
+ "tags": [],
+ "text": "titan-ccp-aggregation",
+ "value": "titan-ccp-aggregation"
+ },
+ "datasource": "Prometheus",
+ "definition": "label_values(kafka_consumer_consumer_fetch_manager_metrics_records_lag, job)",
+ "hide": 0,
+ "includeAll": false,
+ "index": -1,
+ "label": "Job",
+ "multi": false,
+ "name": "Job",
+ "options": [
+ {
+ "selected": true,
+ "text": "titan-ccp-aggregation",
+ "value": "titan-ccp-aggregation"
+ }
+ ],
+ "query": "label_values(kafka_consumer_consumer_fetch_manager_metrics_records_lag, job)",
+ "refresh": 0,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tags": [],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ]
+ },
+ "timezone": "",
+ "title": "OSPBench",
+ "uid": "01O646v7z",
+ "variables": {
+ "list": []
+ },
+ "version": 1
+ }`}}
+{{- end }}
diff --git a/helm/templates/theodolite/crd-benchmark.yaml b/helm/templates/theodolite/crd-benchmark.yaml
deleted file mode 100644
index 084480e1f9e2ef827fb145cd823bbd2f68a20bac..0000000000000000000000000000000000000000
--- a/helm/templates/theodolite/crd-benchmark.yaml
+++ /dev/null
@@ -1,119 +0,0 @@
-{{- if .Values.operator.benchmarkCRD.create -}}
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
- name: benchmarks.theodolite.com
-spec:
- group: theodolite.com
- names:
- kind: benchmark
- plural: benchmarks
- shortNames:
- - bench
- versions:
- - name: v1
- served: true
- storage: true
- schema:
- openAPIV3Schema:
- type: object
- required: ["spec"]
- properties:
- spec:
- type: object
- required: []
- properties:
- name:
- type: string
- appResource:
- type: array
- minItems: 1
- items:
- type: string
- loadGenResource:
- type: array
- minItems: 1
- items:
- type: string
- resourceTypes:
- type: array
- minItems: 1
- items:
- type: object
- properties:
- typeName:
- type: string
- patchers:
- type: array
- minItems: 1
- items:
- type: object
- properties:
- type:
- type: string
- default: ""
- resource:
- type: string
- default: ""
- properties:
- type: object
- additionalProperties: true
- x-kubernetes-map-type: "granular"
- default: {}
- loadTypes:
- type: array
- minItems: 1
- items:
- type: object
- properties:
- typeName:
- type: string
- patchers:
- type: array
- minItems: 1
- items:
- type: object
- properties:
- type:
- type: string
- default: ""
- resource:
- type: string
- default: ""
- properties:
- type: object
- additionalProperties: true
- x-kubernetes-map-type: "granular"
- default: {}
- kafkaConfig:
- type: object
- properties:
- bootstrapServer:
- type: string
- topics:
- type: array
- minItems: 1
- items:
- type: object
- required: []
- properties:
- name:
- type: string
- default: ""
- numPartitions:
- type: integer
- default: 0
- replicationFactor:
- type: integer
- default: 0
- removeOnly:
- type: boolean
- default: false
- additionalPrinterColumns:
- - name: Age
- type: date
- jsonPath: .metadata.creationTimestamp
- subresources:
- status: {}
- scope: Namespaced
-{{- end }}
diff --git a/helm/templates/theodolite/crd-execution.yaml b/helm/templates/theodolite/crd-execution.yaml
deleted file mode 100644
index d9212e0e0576bb54ffcaf51a227e47f42894a742..0000000000000000000000000000000000000000
--- a/helm/templates/theodolite/crd-execution.yaml
+++ /dev/null
@@ -1,130 +0,0 @@
-{{- if .Values.operator.executionCRD.create -}}
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
- name: executions.theodolite.com
-spec:
- group: theodolite.com
- names:
- kind: execution
- plural: executions
- shortNames:
- - exec
- versions:
- - name: v1
- served: true
- storage: true
- schema:
- openAPIV3Schema:
- type: object
- required: ["spec"]
- properties:
- spec:
- type: object
- required: ["benchmark", "load", "resources", "slos", "execution", "configOverrides"]
- properties:
- name:
- type: string
- default: ""
- benchmark:
- type: string
- load: # definition of the load dimension
- type: object
- required: ["loadType", "loadValues"]
- properties:
- loadType:
- type: string
- loadValues:
- type: array
- items:
- type: integer
- resources: # definition of the resource dimension
- type: object
- required: ["resourceType", "resourceValues"]
- properties:
- resourceType:
- type: string
- resourceValues:
- type: array
- items:
- type: integer
- slos: # def of service level objectives
- type: array
- items:
- type: object
- required: ["sloType", "threshold", "prometheusUrl", "externalSloUrl", "offset", "warmup"]
- properties:
- sloType:
- type: string
- threshold:
- type: integer
- prometheusUrl:
- type: string
- externalSloUrl:
- type: string
- offset:
- type: integer
- warmup:
- type: integer
- execution: # def execution config
- type: object
- required: ["strategy", "duration", "repetitions", "restrictions"]
- properties:
- strategy:
- type: string
- duration:
- type: integer
- repetitions:
- type: integer
- loadGenerationDelay:
- type: integer
- restrictions:
- type: array
- items:
- type: string
- configOverrides:
- type: array
- items:
- type: object
- properties:
- patcher:
- type: object
- properties:
- type:
- type: string
- default: ""
- resource:
- type: string
- default: ""
- properties:
- type: object
- additionalProperties: true
- x-kubernetes-map-type: "granular"
- default: {}
- value:
- type: string
- status:
- type: object
- properties:
- executionState:
- description: ""
- type: string
- executionDuration:
- description: "Duration of the execution in seconds"
- type: string
- additionalPrinterColumns:
- - name: STATUS
- type: string
- description: State of the execution
- jsonPath: .status.executionState
- - name: Duration
- type: string
- description: Duration of the execution
- jsonPath: .status.executionDuration
- - name: Age
- type: date
- jsonPath: .metadata.creationTimestamp
- subresources:
- status: {}
- scope: Namespaced
-{{- end }}
diff --git a/helm/templates/theodolite/random-scheduler/deployment.yaml b/helm/templates/theodolite/random-scheduler/deployment.yaml
index a1ea535d52d3dce971806dd638a90e9acb81c5d0..55b6e4ad5f8fafccc9623e69ef1df1fccf81ed39 100644
--- a/helm/templates/theodolite/random-scheduler/deployment.yaml
+++ b/helm/templates/theodolite/random-scheduler/deployment.yaml
@@ -22,8 +22,8 @@ spec:
serviceAccount: {{ include "theodolite.fullname" . }}-random-scheduler
containers:
- name: random-scheduler
- image: ghcr.io/cau-se/theodolite-random-scheduler:latest
- #imagePullPolicy: Always
+ image: "{{ .Values.randomScheduler.image }}:{{ .Values.randomScheduler.imageTag }}"
+ imagePullPolicy: "{{ .Values.randomScheduler.imagePullPolicy }}"
env:
- name: TARGET_NAMESPACE
value: {{ .Release.Namespace }}
diff --git a/helm/templates/theodolite/results-volume/pvc.yaml b/helm/templates/theodolite/results-volume/pvc.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..26ac56e42889ccaebbc669791ad4d318b8318fec
--- /dev/null
+++ b/helm/templates/theodolite/results-volume/pvc.yaml
@@ -0,0 +1,18 @@
+{{- if and .Values.operator.resultsVolume.persistent.enabled (not .Values.operator.resultsVolume.persistent.existingClaim) -}}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ name: {{ include "theodolite.resultsClaimName" . }}
+spec:
+ {{- if .Values.operator.resultsVolume.persistent.storageClassName }}
+ storageClassName: {{ .Values.operator.resultsVolume.persistent.storageClassName }}
+ {{- end }}
+ accessModes:
+ - ReadWriteOnce
+ {{- range .Values.operator.resultsVolume.persistent.accessModes }}
+ - {{ . | quote }}
+ {{- end }}
+ resources:
+ requests:
+ storage: {{ .Values.operator.resultsVolume.persistent.size | quote }}
+{{- end }}
\ No newline at end of file
diff --git a/helm/templates/theodolite/role-binding.yaml b/helm/templates/theodolite/role-binding.yaml
index 93d8c34e7bc544c3b0c231e986bc58c792cce38e..3b327bb246f9716be0939416db55fc1b2cc5dd70 100644
--- a/helm/templates/theodolite/role-binding.yaml
+++ b/helm/templates/theodolite/role-binding.yaml
@@ -1,5 +1,5 @@
{{- if .Values.rbac.create -}}
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: {{ include "theodolite.fullname" . }}
diff --git a/helm/templates/theodolite/theodolite-operator.yaml b/helm/templates/theodolite/theodolite-operator.yaml
index 3487b9a4fabb16897b302d8be65f065a647ffb16..3d1d186306a5d1103a921b4c39e1425621548db8 100644
--- a/helm/templates/theodolite/theodolite-operator.yaml
+++ b/helm/templates/theodolite/theodolite-operator.yaml
@@ -31,10 +31,8 @@ spec:
- name: RESULTS_FOLDER
value: "./results"
volumeMounts:
- {{- if .Values.operator.resultsVolume.enabled }}
- - name: theodolite-pv-storage
+ - name: theodolite-results-volume
mountPath: "/deployments/results"
- {{- end }}
- name: benchmark-resources-uc1-kstreams
mountPath: /deployments/benchmark-resources/uc1-kstreams
- name: benchmark-resources-uc2-kstreams
@@ -64,7 +62,20 @@ spec:
- name: LOG_LEVEL
value: INFO
{{- end }}
- {{- if and .Values.operator.resultsVolume.enabled .Values.operator.resultsVolume.accessSidecar.enabled }}
+ {{- if .Values.operator.sloChecker.droppedRecordsKStreams.enabled }}
+ - name: slo-checker-dropped-records-kstreams
+ image: "{{ .Values.operator.sloChecker.droppedRecordsKStreams.image }}:{{ .Values.operator.sloChecker.droppedRecordsKStreams.imageTag }}"
+ imagePullPolicy: "{{ .Values.operator.sloChecker.droppedRecordsKStreams.imagePullPolicy }}"
+ ports:
+ - containerPort: 8081
+ name: analysis
+ env:
+ - name: PORT
+ value: "8081"
+ - name: LOG_LEVEL
+ value: INFO
+ {{- end }}
+ {{- if .Values.operator.resultsVolume.accessSidecar.enabled }}
- name: results-access
image: busybox:stable
image: "{{ .Values.operator.resultsVolume.accessSidecar.image }}:{{ .Values.operator.resultsVolume.accessSidecar.imageTag }}"
@@ -75,14 +86,16 @@ spec:
- exec tail -f /dev/null
volumeMounts:
- mountPath: /results
- name: theodolite-pv-storage
+ name: theodolite-results-volume
{{- end }}
volumes:
- {{- if .Values.operator.resultsVolume.enabled }}
- - name: theodolite-pv-storage
+ - name: theodolite-results-volume
+ {{- if .Values.operator.resultsVolume.persistent.enabled }}
persistentVolumeClaim:
- claimName: {{ .Values.operator.resultsVolume.persistentVolumeClaim.name | quote }}
- {{- end }}
+ claimName: {{ include "theodolite.resultsClaimName" . | quote }}
+ {{- else }}
+ emptyDir: {}
+ {{- end }}
- name: benchmark-resources-uc1-kstreams
configMap:
name: benchmark-resources-uc1-kstreams
diff --git a/helm/update-index.sh b/helm/update-index.sh
index 286724dd87718387df58ed993af417bf0fd4d8ec..66c55bb8b79e18e3d06d156cb1859f2a53078999 100755
--- a/helm/update-index.sh
+++ b/helm/update-index.sh
@@ -3,7 +3,7 @@
RELEASE_NAME=$1 # Supposed to be equal to tag, e.g., v0.3.0
RELEASE_PATH="https://github.com/cau-se/theodolite/releases/download"
-REPO_INDEX="../../docs/index.yaml"
+REPO_INDEX="../docs/index.yaml"
helm repo index . --url $RELEASE_PATH/$RELEASE_NAME --merge $REPO_INDEX && \
mv index.yaml $REPO_INDEX
\ No newline at end of file
diff --git a/helm/values.yaml b/helm/values.yaml
index 917ec5cdbda7541b56b366aff2412ccf7d856f3b..c45e62aed8126a8b9f20a6dd0ca6083cd162bc2b 100644
--- a/helm/values.yaml
+++ b/helm/values.yaml
@@ -250,22 +250,26 @@ operator:
imageTag: latest
imagePullPolicy: Always
- executionCRD:
- create: true
- benchmarkCRD:
- create: true
-
sloChecker:
lagTrend:
enabled: true
image: ghcr.io/cau-se/theodolite-slo-checker-lag-trend
imageTag: latest
imagePullPolicy: Always
+ droppedRecordsKStreams:
+ enabled: true
+ image: ghcr.io/cau-se/theodolite-slo-checker-dropped-records-kstreams
+ imageTag: latest
+ imagePullPolicy: Always
resultsVolume:
- enabled: true
- persistentVolumeClaim:
- name: theodolite-pv-claim
+ persistent:
+ enabled: false
+ # existingClaim:
+ # storageClassName:
+ accessModes:
+ - ReadWriteOnce
+ size: 1Gi
accessSidecar:
enabled: true
image: busybox
@@ -281,6 +285,9 @@ rbac:
randomScheduler:
enabled: true
+ image: ghcr.io/cau-se/theodolite-random-scheduler
+ imageTag: latest
+ imagePullPolicy: Always
rbac:
create: true
serviceAccount:
diff --git a/slope-evaluator/Dockerfile b/slo-checker/dropped-records/Dockerfile
similarity index 100%
rename from slope-evaluator/Dockerfile
rename to slo-checker/dropped-records/Dockerfile
diff --git a/slo-checker/dropped-records/README.md b/slo-checker/dropped-records/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..3a1ea982a399201143ad50f173c934ff58abbf4a
--- /dev/null
+++ b/slo-checker/dropped-records/README.md
@@ -0,0 +1,80 @@
+# Kafka Streams Dropped Record SLO Evaluator
+
+## Execution
+
+For development:
+
+```sh
+uvicorn main:app --reload --port 81 # run this command inside the app/ folder
+```
+
+## Build the docker image:
+
+```sh
+docker build . -t theodolite-evaluator
+```
+
+Run the Docker image:
+
+```sh
+docker run -p 80:81 theodolite-evaluator
+```
+
+## Configuration
+
+You can set the `HOST` and the `PORT` (and a lot of more parameters) via environment variables. Default is `0.0.0.0:80`.
+For more information see the [Gunicorn/FastAPI Docker docs](https://github.com/tiangolo/uvicorn-gunicorn-fastapi-docker#advanced-usage).
+
+## API Documentation
+
+The running webserver provides a REST API with the following route:
+
+* /dropped-records
+ * Method: POST
+ * Body:
+ * results
+ * metric-metadata
+ * values
+ * metadata
+ * threshold
+ * warmup
+
+The body of the request must be a JSON string that satisfies the following conditions:
+
+* **dropped records**: This property is based on the [Range Vector type](https://www.prometheus.io/docs/prometheus/latest/querying/api/#range-vectors) from Prometheus and must have the following JSON *structure*:
+
+ ```json
+ {
+ "results": [
+ [
+ {
+ "metric": {
+ "<label-name>": "<label-value>"
+ },
+ "values": [
+ [
+ <unix_timestamp>, // 1.634624989695E9
+ "<sample_value>" // integer
+ ]
+ ]
+ }
+ ]
+ ],
+ "metadata": {
+ "threshold": 2000000,
+ "warmup": 60
+ }
+ }
+ ```
+
+### description
+
+* results:
+ * metric-metadata:
+ * Labels of this metric. The `dropped-records` slo checker does not use labels in the calculation of the service level objective.
+ * results
+ * The `<unix_timestamp>` provided as the first element of each element in the "values" array must be the timestamp of the measurement value in seconds (with optional decimal precision)
+ * The `<sample_value>` must be the measurement value as string.
+* metadata: For the calculation of the service level objective require metadata.
+ * **threshold**: Must be an unsigned integer that specifies the threshold for the SLO evaluation. The SLO is considered fulfilled, if the result value is below the threshold. If the result value is equal or above the threshold, the SLO is considered not fulfilled.
+ * **warmup**: Specifieds the warmup time in seconds that are ignored for evaluating the SLO.
diff --git a/slo-checker/dropped-records/app/main.py b/slo-checker/dropped-records/app/main.py
new file mode 100644
index 0000000000000000000000000000000000000000..c1577f9c11ed5a2798ee0b4505ae0739301ab2a8
--- /dev/null
+++ b/slo-checker/dropped-records/app/main.py
@@ -0,0 +1,32 @@
+from fastapi import FastAPI,Request
+import logging
+import os
+import json
+import sys
+
+app = FastAPI()
+
+logging.basicConfig(stream=sys.stdout,
+ format="%(asctime)s %(levelname)s %(name)s: %(message)s")
+logger = logging.getLogger("API")
+
+
+if os.getenv('LOG_LEVEL') == 'INFO':
+ logger.setLevel(logging.INFO)
+elif os.getenv('LOG_LEVEL') == 'WARNING':
+ logger.setLevel(logging.WARNING)
+elif os.getenv('LOG_LEVEL') == 'DEBUG':
+ logger.setLevel(logging.DEBUG)
+
+
+def check_service_level_objective(results, threshold):
+ return max(results) < threshold
+
+@app.post("/dropped-records",response_model=bool)
+async def evaluate_slope(request: Request):
+ data = json.loads(await request.body())
+ warmup = int(data['results'][0][0]['values'][0][0]) + int(data['metadata']['warmup'])
+ results = [int(val[1]) if(int(val[0]>=warmup)) else 0 for result in data['results'] for r in result for val in r['values'] ]
+ return check_service_level_objective(results=results, threshold=data['metadata']["threshold"])
+
+logger.info("SLO evaluator is online")
\ No newline at end of file
diff --git a/slo-checker/dropped-records/app/test.py b/slo-checker/dropped-records/app/test.py
new file mode 100644
index 0000000000000000000000000000000000000000..3c657c914002066357d58d88d7f8e4afe920db45
--- /dev/null
+++ b/slo-checker/dropped-records/app/test.py
@@ -0,0 +1,23 @@
+import unittest
+from main import app, check_service_level_objective
+import numpy as np
+import json
+from fastapi.testclient import TestClient
+
+class TestSloEvaluation(unittest.TestCase):
+ client = TestClient(app)
+
+ def test_1_rep(self):
+ with open('../resources/test-1-rep-success.json') as json_file:
+ data = json.load(json_file)
+ response = self.client.post("/dropped-records", json=data)
+ self.assertEquals(response.json(), True)
+
+ def test_check_service_level_objective(self):
+ list = [ x for x in range(-100, 100) ]
+
+ self.assertEquals(check_service_level_objective(list, 90), False)
+ self.assertEquals(check_service_level_objective(list, 110), True)
+
+if __name__ == '__main__':
+ unittest.main()
\ No newline at end of file
diff --git a/slope-evaluator/requirements.txt b/slo-checker/dropped-records/requirements.txt
similarity index 100%
rename from slope-evaluator/requirements.txt
rename to slo-checker/dropped-records/requirements.txt
diff --git a/slo-checker/dropped-records/resources/test-1-rep-success.json b/slo-checker/dropped-records/resources/test-1-rep-success.json
new file mode 100644
index 0000000000000000000000000000000000000000..0964c30fed60e34c1ac4cf6b6b89f81d95a2f0eb
--- /dev/null
+++ b/slo-checker/dropped-records/resources/test-1-rep-success.json
@@ -0,0 +1,273 @@
+{
+ "results": [
+ [
+ {
+ "metric": {
+ "job": "titan-ccp-aggregation"
+ },
+ "values": [
+ [
+ 1.634624674695E9,
+ "0"
+ ],
+ [
+ 1.634624679695E9,
+ "0"
+ ],
+ [
+ 1.634624684695E9,
+ "0"
+ ],
+ [
+ 1.634624689695E9,
+ "0"
+ ],
+ [
+ 1.634624694695E9,
+ "0"
+ ],
+ [
+ 1.634624699695E9,
+ "0"
+ ],
+ [
+ 1.634624704695E9,
+ "0"
+ ],
+ [
+ 1.634624709695E9,
+ "0"
+ ],
+ [
+ 1.634624714695E9,
+ "0"
+ ],
+ [
+ 1.634624719695E9,
+ "0"
+ ],
+ [
+ 1.634624724695E9,
+ "0"
+ ],
+ [
+ 1.634624729695E9,
+ "0"
+ ],
+ [
+ 1.634624734695E9,
+ "0"
+ ],
+ [
+ 1.634624739695E9,
+ "0"
+ ],
+ [
+ 1.634624744695E9,
+ "1"
+ ],
+ [
+ 1.634624749695E9,
+ "3"
+ ],
+ [
+ 1.634624754695E9,
+ "4"
+ ],
+ [
+ 1.634624759695E9,
+ "4"
+ ],
+ [
+ 1.634624764695E9,
+ "4"
+ ],
+ [
+ 1.634624769695E9,
+ "4"
+ ],
+ [
+ 1.634624774695E9,
+ "4"
+ ],
+ [
+ 1.634624779695E9,
+ "4"
+ ],
+ [
+ 1.634624784695E9,
+ "4"
+ ],
+ [
+ 1.634624789695E9,
+ "4"
+ ],
+ [
+ 1.634624794695E9,
+ "4"
+ ],
+ [
+ 1.634624799695E9,
+ "4"
+ ],
+ [
+ 1.634624804695E9,
+ "176"
+ ],
+ [
+ 1.634624809695E9,
+ "176"
+ ],
+ [
+ 1.634624814695E9,
+ "176"
+ ],
+ [
+ 1.634624819695E9,
+ "176"
+ ],
+ [
+ 1.634624824695E9,
+ "176"
+ ],
+ [
+ 1.634624829695E9,
+ "159524"
+ ],
+ [
+ 1.634624834695E9,
+ "209870"
+ ],
+ [
+ 1.634624839695E9,
+ "278597"
+ ],
+ [
+ 1.634624844695E9,
+ "460761"
+ ],
+ [
+ 1.634624849695E9,
+ "460761"
+ ],
+ [
+ 1.634624854695E9,
+ "460761"
+ ],
+ [
+ 1.634624859695E9,
+ "460761"
+ ],
+ [
+ 1.634624864695E9,
+ "460761"
+ ],
+ [
+ 1.634624869695E9,
+ "606893"
+ ],
+ [
+ 1.634624874695E9,
+ "653534"
+ ],
+ [
+ 1.634624879695E9,
+ "755796"
+ ],
+ [
+ 1.634624884695E9,
+ "919317"
+ ],
+ [
+ 1.634624889695E9,
+ "919317"
+ ],
+ [
+ 1.634624894695E9,
+ "955926"
+ ],
+ [
+ 1.634624899695E9,
+ "955926"
+ ],
+ [
+ 1.634624904695E9,
+ "955926"
+ ],
+ [
+ 1.634624909695E9,
+ "955926"
+ ],
+ [
+ 1.634624914695E9,
+ "955926"
+ ],
+ [
+ 1.634624919695E9,
+ "1036530"
+ ],
+ [
+ 1.634624924695E9,
+ "1078477"
+ ],
+ [
+ 1.634624929695E9,
+ "1194775"
+ ],
+ [
+ 1.634624934695E9,
+ "1347755"
+ ],
+ [
+ 1.634624939695E9,
+ "1352151"
+ ],
+ [
+ 1.634624944695E9,
+ "1360428"
+ ],
+ [
+ 1.634624949695E9,
+ "1360428"
+ ],
+ [
+ 1.634624954695E9,
+ "1360428"
+ ],
+ [
+ 1.634624959695E9,
+ "1360428"
+ ],
+ [
+ 1.634624964695E9,
+ "1360428"
+ ],
+ [
+ 1.634624969695E9,
+ "1525685"
+ ],
+ [
+ 1.634624974695E9,
+ "1689296"
+ ],
+ [
+ 1.634624979695E9,
+ "1771358"
+ ],
+ [
+ 1.634624984695E9,
+ "1854284"
+ ],
+ [
+ 1.634624989695E9,
+ "1854284"
+ ]
+ ]
+ }
+ ]
+ ],
+ "metadata": {
+ "threshold": 2000000,
+ "warmup": 60
+ }
+}
\ No newline at end of file
diff --git a/slo-checker/record-lag/Dockerfile b/slo-checker/record-lag/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..032b8153a6989ca04631ba553289dacb3620a38d
--- /dev/null
+++ b/slo-checker/record-lag/Dockerfile
@@ -0,0 +1,6 @@
+FROM tiangolo/uvicorn-gunicorn-fastapi:python3.7
+
+COPY requirements.txt requirements.txt
+RUN pip install -r requirements.txt
+
+COPY ./app /app
\ No newline at end of file
diff --git a/slo-checker/record-lag/README.md b/slo-checker/record-lag/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..b4882eeaf54aadfb8cbf33a957e6052a7b74123b
--- /dev/null
+++ b/slo-checker/record-lag/README.md
@@ -0,0 +1,80 @@
+# Lag Trend SLO Evaluator
+
+## Execution
+
+For development:
+
+```sh
+uvicorn main:app --reload # run this command inside the app/ folder
+```
+
+## Build the docker image:
+
+```sh
+docker build . -t theodolite-evaluator
+```
+
+Run the Docker image:
+
+```sh
+docker run -p 80:80 theodolite-evaluator
+```
+
+## Configuration
+
+You can set the `HOST` and the `PORT` (and a lot of more parameters) via environment variables. Default is `0.0.0.0:80`.
+For more information see the [Gunicorn/FastAPI Docker docs](https://github.com/tiangolo/uvicorn-gunicorn-fastapi-docker#advanced-usage).
+
+# API Documentation
+
+The running webserver provides a REST API with the following route:
+
+* /dropped-records
+ * Method: POST
+ * Body:
+ * results
+ * metric-metadata
+ * values
+ * metadata
+ * threshold
+ * warmup
+
+The body of the request must be a JSON string that satisfies the following conditions:
+
+* **total_lag**: This property is based on the [Range Vector type](https://www.prometheus.io/docs/prometheus/latest/querying/api/#range-vectors) from Prometheus and must have the following JSON *structure*:
+
+ ```json
+ {
+ "results": [
+ [
+ {
+ "metric": {
+ "<label-name>": "<label-value>"
+ },
+ "values": [
+ [
+ <unix_timestamp>, // 1.634624989695E9
+ "<sample_value>" // integer
+ ]
+ ]
+ }
+ ]
+ ],
+ "metadata": {
+ "threshold": 2000000,
+ "warmup": 60
+ }
+ }
+ ```
+
+### description
+
+* results:
+ * metric-metadata:
+ * Labels of this metric. The `dropped-records` slo checker does not use labels in the calculation of the service level objective.
+ * results
+ * The `<unix_timestamp>` provided as the first element of each element in the "values" array must be the timestamp of the measurement value in seconds (with optional decimal precision)
+ * The `<sample_value>` must be the measurement value as string.
+* metadata: For the calculation of the service level objective require metadata.
+ * **threshold**: Must be an unsigned integer that specifies the threshold for the SLO evaluation. The SLO is considered fulfilled, if the result value is below the threshold. If the result value is equal or above the threshold, the SLO is considered not fulfilled.
+ * **warmup**: Specifieds the warmup time in seconds that are ignored for evaluating the SLO.
diff --git a/slope-evaluator/app/main.py b/slo-checker/record-lag/app/main.py
similarity index 90%
rename from slope-evaluator/app/main.py
rename to slo-checker/record-lag/app/main.py
index 6f6788f0ca84b7710be5b509ca4f0641047e963d..621fa0cfc9c27e809fd92752de93f2795fa32c05 100644
--- a/slope-evaluator/app/main.py
+++ b/slo-checker/record-lag/app/main.py
@@ -38,7 +38,7 @@ def calculate_slope_trend(results, warmup):
err_msg = 'Computing trend slope failed.'
logger.exception(err_msg)
logger.error('Mark this subexperiment as not successful and continue benchmark.')
- return False
+ return float('inf')
logger.info("Computed lag trend slope is '%s'", trend_slope)
return trend_slope
@@ -49,7 +49,7 @@ def check_service_level_objective(results, threshold):
@app.post("/evaluate-slope",response_model=bool)
async def evaluate_slope(request: Request):
data = json.loads(await request.body())
- results = [calculate_slope_trend(total_lag, data['warmup']) for total_lag in data['total_lags']]
- return check_service_level_objective(results=results, threshold=data["threshold"])
+ results = [calculate_slope_trend(total_lag, data['metadata']['warmup']) for total_lag in data['results']]
+ return check_service_level_objective(results=results, threshold=data['metadata']["threshold"])
logger.info("SLO evaluator is online")
\ No newline at end of file
diff --git a/slope-evaluator/app/test.py b/slo-checker/record-lag/app/test.py
similarity index 99%
rename from slope-evaluator/app/test.py
rename to slo-checker/record-lag/app/test.py
index 9b165ea479bb9a552edaba7692df4fd4ef3f4ab4..c8d81f86b16255dcdce5337d8f00e922b98b4f82 100644
--- a/slope-evaluator/app/test.py
+++ b/slo-checker/record-lag/app/test.py
@@ -17,7 +17,7 @@ class TestSloEvaluation(unittest.TestCase):
data = json.load(json_file)
response = self.client.post("/evaluate-slope", json=data)
self.assertEquals(response.json(), True)
-
+
def test_check_service_level_objective(self):
list = [1,2,3,4]
self.assertEquals(check_service_level_objective(list, 2), False)
diff --git a/slope-evaluator/app/trend_slope_computer.py b/slo-checker/record-lag/app/trend_slope_computer.py
similarity index 100%
rename from slope-evaluator/app/trend_slope_computer.py
rename to slo-checker/record-lag/app/trend_slope_computer.py
diff --git a/slo-checker/record-lag/requirements.txt b/slo-checker/record-lag/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..8b6c3863226c2bd5e8bcd7982b2674dee593f192
--- /dev/null
+++ b/slo-checker/record-lag/requirements.txt
@@ -0,0 +1,5 @@
+fastapi==0.65.2
+scikit-learn==0.20.3
+pandas==1.0.3
+uvicorn
+requests
diff --git a/slope-evaluator/resources/test-1-rep-success.json b/slo-checker/record-lag/resources/test-1-rep-success.json
similarity index 97%
rename from slope-evaluator/resources/test-1-rep-success.json
rename to slo-checker/record-lag/resources/test-1-rep-success.json
index 9e315c707be7b2a874c58fcb1093aa86f7676560..dfe11282720ebfcdd60582b7717da892bc85a923 100644
--- a/slope-evaluator/resources/test-1-rep-success.json
+++ b/slo-checker/record-lag/resources/test-1-rep-success.json
@@ -1,5 +1,5 @@
{
- "total_lags": [
+ "results": [
[
{
"metric": {
@@ -134,6 +134,8 @@
}
]
],
- "threshold": 2000,
- "warmup": 0
+ "metadata": {
+ "threshold": 2000,
+ "warmup": 0
+ }
}
\ No newline at end of file
diff --git a/slope-evaluator/resources/test-3-rep-success.json b/slo-checker/record-lag/resources/test-3-rep-success.json
similarity index 98%
rename from slope-evaluator/resources/test-3-rep-success.json
rename to slo-checker/record-lag/resources/test-3-rep-success.json
index 485966cba40f01e4a646e626914510ba49b707bc..cf483f42f3783aecd1f428ac7bbbe2090c4cade0 100644
--- a/slope-evaluator/resources/test-3-rep-success.json
+++ b/slo-checker/record-lag/resources/test-3-rep-success.json
@@ -1,5 +1,5 @@
{
- "total_lags": [
+ "results": [
[
{
"metric": {
@@ -284,6 +284,8 @@
}
]
],
- "threshold": 2000,
- "warmup": 0
+ "metadata": {
+ "threshold": 2000,
+ "warmup": 0
+ }
}
\ No newline at end of file
diff --git a/slope-evaluator/README.md b/slope-evaluator/README.md
deleted file mode 100644
index cd9e6820ed46452ce44d57d0c7e5cd5ae05e5a3b..0000000000000000000000000000000000000000
--- a/slope-evaluator/README.md
+++ /dev/null
@@ -1,61 +0,0 @@
-# Lag Trend SLO Evaluator
-
-## Execution
-
-For development:
-
-```sh
-uvicorn main:app --reload # run this command inside the app/ folder
-```
-
-## Build the docker image:
-
-```sh
-docker build . -t theodolite-evaluator
-```
-
-Run the Docker image:
-
-```sh
-docker run -p 80:80 theodolite-evaluator
-```
-
-## Configuration
-
-You can set the `HOST` and the `PORT` (and a lot of more parameters) via environment variables. Default is `0.0.0.0:80`.
-For more information see the [Gunicorn/FastAPI Docker docs](https://github.com/tiangolo/uvicorn-gunicorn-fastapi-docker#advanced-usage).
-
-## API Documentation
-
-The running webserver provides a REST API with the following route:
-
-* /evaluate-slope
- * Method: POST
- * Body:
- * total_lags
- * threshold
- * warmup
-
-The body of the request must be a JSON string that satisfies the following conditions:
-
-* **total_lag**: This property is based on the [Range Vector type](https://www.prometheus.io/docs/prometheus/latest/querying/api/#range-vectors) from Prometheus and must have the following JSON structure:
- ```
- {
- [
- "metric": {
- "group": "<label_value>"
- },
- "values": [
- [
- <unix_timestamp>,
- "<sample_value>"
- ]
- ]
- ]
- }
- ```
- * The `<label_value>` provided in "metric.group" must be equal to the id of the Kafka consumer group.
- * The `<unix_timestamp>` provided as the first element of each element in the "values" array must be the timestamp of the measurement value in seconds (with optional decimal precision)
- * The `<sample_value>` must be the measurement value as string.
-* **threshold**: Must be an unsigned integer that specifies the threshold for the SLO evaluation. The SLO is considered fulfilled, if the result value is below the threshold. If the result value is equal or above the threshold, the SLO is considered not fulfilled.
-* **warmup**: Specifieds the warmup time in seconds that are ignored for evaluating the SLO.
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc1-flink/resources/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc1-flink/resources/jobmanager-deployment.yaml
index 512d4fe3c786e1b2c44e6ec57fccadf41a2e2eeb..1f328b1cd553c8036e570d28b97795fb2b00ec81 100644
--- a/theodolite-benchmarks/definitions/uc1-flink/resources/jobmanager-deployment.yaml
+++ b/theodolite-benchmarks/definitions/uc1-flink/resources/jobmanager-deployment.yaml
@@ -14,6 +14,7 @@ spec:
app: flink
component: jobmanager
spec:
+ terminationGracePeriodSeconds: 0
containers:
- name: jobmanager
image: ghcr.io/cau-se/theodolite-uc1-flink:latest
@@ -90,4 +91,4 @@ spec:
emptyDir: {}
# - name: job-artifacts-volume
# hostPath:
-# path: /host/path/to/job/artifacts
\ No newline at end of file
+# path: /host/path/to/job/artifacts
diff --git a/theodolite-benchmarks/definitions/uc1-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc1-flink/resources/taskmanager-deployment.yaml
index 7d46554692696b194736df6023eed5686040497d..c2266a4aeb21302262279f147e6512d5264e1dc1 100644
--- a/theodolite-benchmarks/definitions/uc1-flink/resources/taskmanager-deployment.yaml
+++ b/theodolite-benchmarks/definitions/uc1-flink/resources/taskmanager-deployment.yaml
@@ -14,6 +14,7 @@ spec:
app: flink
component: taskmanager
spec:
+ terminationGracePeriodSeconds: 0
containers:
- name: taskmanager
image: ghcr.io/cau-se/theodolite-uc1-flink:latest
diff --git a/theodolite-benchmarks/definitions/uc1-kstreams/uc1-benchmark-standalone.yaml b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-benchmark-standalone.yaml
index 12cbd8ea310423d28e35de8185288b27257c15ec..871210c213b70070e2aae8f4986058763be74b7e 100644
--- a/theodolite-benchmarks/definitions/uc1-kstreams/uc1-benchmark-standalone.yaml
+++ b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-benchmark-standalone.yaml
@@ -22,7 +22,7 @@ loadTypes:
- type: NumSensorsLoadGeneratorReplicaPatcher
resource: "uc1-load-generator-deployment.yaml"
properties:
- loadGenMaxRecords: "15000"
+ loadGenMaxRecords: "150000"
kafkaConfig:
bootstrapServer: "theodolite-cp-kafka:9092"
topics:
diff --git a/theodolite-benchmarks/definitions/uc2-flink/resources/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc2-flink/resources/jobmanager-deployment.yaml
index cece4286d49a3f6ff139ca7f1e01c647acd5d9f3..87ea174f71c592bbffab4e5fc9ce6e3963596b9c 100644
--- a/theodolite-benchmarks/definitions/uc2-flink/resources/jobmanager-deployment.yaml
+++ b/theodolite-benchmarks/definitions/uc2-flink/resources/jobmanager-deployment.yaml
@@ -14,6 +14,7 @@ spec:
app: flink
component: jobmanager
spec:
+ terminationGracePeriodSeconds: 0
containers:
- name: jobmanager
image: ghcr.io/cau-se/theodolite-uc2-flink:latest
@@ -90,4 +91,4 @@ spec:
emptyDir: {}
# - name: job-artifacts-volume
# hostPath:
-# path: /host/path/to/job/artifacts
\ No newline at end of file
+# path: /host/path/to/job/artifacts
diff --git a/theodolite-benchmarks/definitions/uc2-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc2-flink/resources/taskmanager-deployment.yaml
index c0a10f65aae92e4ac1fd8fb92bae97794c142232..c37df972a334a4a0e27f0420030f99f1dff15b53 100644
--- a/theodolite-benchmarks/definitions/uc2-flink/resources/taskmanager-deployment.yaml
+++ b/theodolite-benchmarks/definitions/uc2-flink/resources/taskmanager-deployment.yaml
@@ -14,6 +14,7 @@ spec:
app: flink
component: taskmanager
spec:
+ terminationGracePeriodSeconds: 0
containers:
- name: taskmanager
image: ghcr.io/cau-se/theodolite-uc2-flink:latest
diff --git a/theodolite-benchmarks/definitions/uc2-kstreams/uc2-benchmark-standalone.yaml b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-benchmark-standalone.yaml
index e38f83f5b05d05febb59c2f775a29b2d545acf0e..48269b38a086f074ead80964df3bd4633742743e 100644
--- a/theodolite-benchmarks/definitions/uc2-kstreams/uc2-benchmark-standalone.yaml
+++ b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-benchmark-standalone.yaml
@@ -23,7 +23,7 @@ loadTypes:
- type: NumSensorsLoadGeneratorReplicaPatcher
resource: "uc2-load-generator-deployment.yaml"
properties:
- loadGenMaxRecords: "15000"
+ loadGenMaxRecords: "150000"
kafkaConfig:
bootstrapServer: "theodolite-cp-kafka:9092"
topics:
diff --git a/theodolite-benchmarks/definitions/uc3-flink/resources/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc3-flink/resources/jobmanager-deployment.yaml
index 33bf1f1121a9764785db7a504799314a7ed40cf3..d01123b13fe2d63637ee4000051091a99bad0546 100644
--- a/theodolite-benchmarks/definitions/uc3-flink/resources/jobmanager-deployment.yaml
+++ b/theodolite-benchmarks/definitions/uc3-flink/resources/jobmanager-deployment.yaml
@@ -14,6 +14,7 @@ spec:
app: flink
component: jobmanager
spec:
+ terminationGracePeriodSeconds: 0
containers:
- name: jobmanager
image: ghcr.io/cau-se/theodolite-uc3-flink:latest
@@ -90,4 +91,4 @@ spec:
emptyDir: {}
# - name: job-artifacts-volume
# hostPath:
-# path: /host/path/to/job/artifacts
\ No newline at end of file
+# path: /host/path/to/job/artifacts
diff --git a/theodolite-benchmarks/definitions/uc3-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc3-flink/resources/taskmanager-deployment.yaml
index 8f70b7308429f79cfd8f8bda7a7a96e2bc8d8689..495f97817e43d692c30fe898c4ef3118cae682d7 100644
--- a/theodolite-benchmarks/definitions/uc3-flink/resources/taskmanager-deployment.yaml
+++ b/theodolite-benchmarks/definitions/uc3-flink/resources/taskmanager-deployment.yaml
@@ -14,6 +14,7 @@ spec:
app: flink
component: taskmanager
spec:
+ terminationGracePeriodSeconds: 0
containers:
- name: taskmanager
image: ghcr.io/cau-se/theodolite-uc3-flink:latest
diff --git a/theodolite-benchmarks/definitions/uc3-kstreams/uc3-benchmark-standalone.yaml b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-benchmark-standalone.yaml
index e00c1672c4a5a02128c2618b525573a4cddd6c72..0c2311388f1dfa87e8a182eb8399020bc83ae4ce 100644
--- a/theodolite-benchmarks/definitions/uc3-kstreams/uc3-benchmark-standalone.yaml
+++ b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-benchmark-standalone.yaml
@@ -23,7 +23,7 @@ loadTypes:
- type: NumSensorsLoadGeneratorReplicaPatcher
resource: "uc3-load-generator-deployment.yaml"
properties:
- loadGenMaxRecords: "15000"
+ loadGenMaxRecords: "150000"
kafkaConfig:
bootstrapServer: "theodolite-cp-kafka:9092"
topics:
diff --git a/theodolite-benchmarks/definitions/uc4-flink/resources/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc4-flink/resources/jobmanager-deployment.yaml
index b6533a2c4355e227a16aeface2080253bce19958..032499ea498f8155fd80e42ec4cbdd850498b217 100644
--- a/theodolite-benchmarks/definitions/uc4-flink/resources/jobmanager-deployment.yaml
+++ b/theodolite-benchmarks/definitions/uc4-flink/resources/jobmanager-deployment.yaml
@@ -14,6 +14,7 @@ spec:
app: flink
component: jobmanager
spec:
+ terminationGracePeriodSeconds: 0
containers:
- name: jobmanager
image: ghcr.io/cau-se/theodolite-uc4-flink:latest
@@ -90,4 +91,4 @@ spec:
emptyDir: {}
# - name: job-artifacts-volume
# hostPath:
-# path: /host/path/to/job/artifacts
\ No newline at end of file
+# path: /host/path/to/job/artifacts
diff --git a/theodolite-benchmarks/definitions/uc4-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc4-flink/resources/taskmanager-deployment.yaml
index 7363b013b21ad29b481e449113ccf31538505634..7af13f20b6b2edf3c8878adf4f381dc1c1add115 100644
--- a/theodolite-benchmarks/definitions/uc4-flink/resources/taskmanager-deployment.yaml
+++ b/theodolite-benchmarks/definitions/uc4-flink/resources/taskmanager-deployment.yaml
@@ -14,6 +14,7 @@ spec:
app: flink
component: taskmanager
spec:
+ terminationGracePeriodSeconds: 0
containers:
- name: taskmanager
image: ghcr.io/cau-se/theodolite-uc4-flink:latest
diff --git a/theodolite-benchmarks/definitions/uc4-kstreams/uc4-benchmark-standalone.yaml b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-benchmark-standalone.yaml
index 96e72c9b6d726267044464cce6deb32f60442e96..ec7a67db4ea24547bc23d5c57e7b907ba489859c 100644
--- a/theodolite-benchmarks/definitions/uc4-kstreams/uc4-benchmark-standalone.yaml
+++ b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-benchmark-standalone.yaml
@@ -23,7 +23,7 @@ loadTypes:
- type: "NumNestedGroupsLoadGeneratorReplicaPatcher"
resource: "uc4-load-generator-deployment.yaml"
properties:
- loadGenMaxRecords: "15000"
+ loadGenMaxRecords: "150000"
numSensors: "4.0"
kafkaConfig:
bootstrapServer: "theodolite-cp-kafka:9092"
diff --git a/theodolite-benchmarks/kstreams-commons/build.gradle b/theodolite-benchmarks/kstreams-commons/build.gradle
index e177aa5c4770c1a77fc21084a7766741fdb9bdec..c5a880acd4377056cc0b0f06b33a2d74c9f87c4e 100644
--- a/theodolite-benchmarks/kstreams-commons/build.gradle
+++ b/theodolite-benchmarks/kstreams-commons/build.gradle
@@ -7,6 +7,9 @@ repositories {
maven {
url "https://oss.sonatype.org/content/repositories/snapshots/"
}
+ maven {
+ url 'https://packages.confluent.io/maven/'
+ }
}
dependencies {
diff --git a/theodolite/.dockerignore b/theodolite/.dockerignore
index d95caadc42523460fa9d78cf17629c8ee231acc9..680e535674de90720f521c92a5ad518100f906b8 100644
--- a/theodolite/.dockerignore
+++ b/theodolite/.dockerignore
@@ -3,4 +3,3 @@
!build/*-runner.jar
!build/lib/*
!build/quarkus-app/*
-!config/*
\ No newline at end of file
diff --git a/theodolite/README.md b/theodolite/README.md
index 26efe96b4756316971378be810f69f6138613958..e0d7b9b46473a877d982e912cb85e2455e09be79 100644
--- a/theodolite/README.md
+++ b/theodolite/README.md
@@ -2,13 +2,13 @@
This project uses Quarkus, the Supersonic Subatomic Java Framework.
-If you want to learn more about Quarkus, please visit its website: https://quarkus.io/ .
+If you want to learn more about Quarkus, please visit its website: <https://quarkus.io/> .
## Running the application in dev mode
You can run your application in dev mode using:
-```shell script
+```sh
./gradlew quarkusDev
```
@@ -16,7 +16,7 @@ You can run your application in dev mode using:
The application can be packaged using:
-```shell script
+```sh
./gradlew build
```
@@ -25,7 +25,7 @@ an _über-jar_ as the dependencies are copied into the `build/lib` directory.
If you want to build an _über-jar_, execute the following command:
-```shell script
+```sh
./gradlew build -Dquarkus.package.type=uber-jar
```
@@ -33,92 +33,108 @@ The application is now runnable using `java -jar build/theodolite-1.0.0-SNAPSHOT
## Creating a native executable
+It is recommended to use the native GraalVM images to create executable jars from Theodolite. For more information please visit the [Native Image guide](https://www.graalvm.org/reference-manual/native-image/).
+
You can create a native executable using:
-```shell script
+```sh
./gradlew build -Dquarkus.package.type=native
```
Or, if you don't have GraalVM installed, you can run the native executable build in a container using:
-```shell script
+```sh
./gradlew build -Dquarkus.package.type=native -Dquarkus.native.container-build=true
```
You can then execute your native executable with:
```./build/theodolite-1.0.0-SNAPSHOT-runner```
-If you want to learn more about building native executables, please consult https://quarkus.io/guides/gradle-tooling.
+If you want to learn more about building native executables, please consult <https://quarkus.io/guides/gradle-tooling>.
## Build docker images
For the jvm version use:
-```shell script
+```sh
./gradlew build
docker build -f src/main/docker/Dockerfile.jvm -t theodolite-jvm .
```
For the native image version use:
-```shell script
+```sh
./gradlew build -Dquarkus.package.type=native
docker build -f src/main/docker/Dockerfile.native -t theodolite-native .
```
-## Execute docker images:
+## Execute docker images
Remember to set the environment variables first.
Jvm version:
-```shell script
+```sh
docker run -i --rm theodolite-jvm
```
Native image version:
-```shell script
+```sh
docker run -i --rm theodolite-native
```
## Environment variables
-**Production:** (Docker-Container)
+**Execution in Docker**:
| Variables name | Default value |Usage |
| -----------------------------|:----------------------------------:| ------------:|
| `NAMESPACE` | `default` |Determines the namespace of the Theodolite will be executed in. Used in the KubernetesBenchmark|
-| `THEODOLITE_EXECUTION` | `./config/BenchmarkExecution.yaml`|The complete path to the benchmarkExecution file. Used in the TheodoliteYamlExecutor. |
-| `THEODOLITE_BENCHMARK_TYPE` | `./config/BenchmarkType.yaml` |The complete path to the benchmarkType file. Used in the TheodoliteYamlExecutor.|
-| `THEODOLITE_APP_RESOURCES` | `./config` |The path under which the yamls for the resources for the subexperiments are found. Used in the KubernetesBenchmark|
-| `MODE` | `yaml-executor` | Defines the mode of operation: either `yaml-executor` or `operator`|
+| `THEODOLITE_EXECUTION` | `execution/execution.yaml` |The complete path to the benchmarkExecution file. Used in the TheodoliteYamlExecutor. |
+| `THEODOLITE_BENCHMARK_TYPE` | `benchmark/benchmark.yaml` |The complete path to the benchmarkType file. Used in the TheodoliteYamlExecutor.|
+| `THEODOLITE_APP_RESOURCES` | `./benchmark-resources` |The path under which the yamls for the resources for the subexperiments are found. Used in the KubernetesBenchmark|
+| `MODE` | `standalone` |Defines the mode of operation: either `standalone` or `operator`
-**Development:** (local via Intellij)
+**Execution in IntelliJ**:
When running Theodolite from within IntelliJ via
-[Run Configurations](https://www.jetbrains.com/help/idea/work-with-gradle-tasks.html#gradle_run_config), set the *
-Environment variables* field to:
+[Run Configurations](https://www.jetbrains.com/help/idea/work-with-gradle-tasks.html#gradle_run_config), set the *Environment variables* field to:
+
+Set the following environment variables to run the example in the `standalone` mode within the IDE:
+
+```sh
+THEODOLITE_BENCHMARK=./../../../../examples/standalone/example-benchmark.yaml;THEODOLITE_EXECUTION=./../../../../examples/standalone/example-execution.yaml;THEODOLITE_APP_RESOURCES=./../../../../examples/resources;
+```
+
+Alternative:
+``` sh
+export THEODOLITE_BENCHMARK=./../../../../examples/standalone/example-benchmark.yaml
+export THEODOLITE_APP_RESOURCES=./../../../../examples/resources;
+export THEODOLITE_EXECUTION=./../../../../examples/standalone/example-execution.yaml
+./gradlew quarkusDev
```
-NAMESPACE=default;THEODOLITE_BENCHMARK=./../../../../config/BenchmarkType.yaml;THEODOLITE_APP_RESOURCES=./../../../../config;THEODOLITE_EXECUTION=./../../../../config/BenchmarkExecution.yaml;MODE=operator
+
+Set the following environment variables to run the example in the `operator` mode within the IDE:
+
+```sh
+THEODOLITE_APP_RESOURCES=./../../../../examples/resources;MODE=operator
```
Alternative:
``` sh
-export NAMESPACE=default
-export THEODOLITE_BENCHMARK=./../../../../config/BenchmarkType.yaml
-export THEODOLITE_APP_RESOURCES=./../../../../config
-export THEODOLITE_EXECUTION=./../../../../config/BenchmarkExecution.yaml
+export THEODOLITE_APP_RESOURCES=./../../../../examples/resources;
export MODE=operator
./gradlew quarkusDev
-
```
-#### Install Detekt Code analysis Plugin
+Additionally, the benchmark and execution resources must be installed.
+
+### Install Detekt Code analysis Plugin
-Install https://plugins.jetbrains.com/plugin/10761-detekt
+Install <https://plugins.jetbrains.com/plugin/10761-detekt>
- Install the plugin
- Navigate to Settings/Preferences -> Tools -> Detekt
@@ -127,7 +143,7 @@ Install https://plugins.jetbrains.com/plugin/10761-detekt
-> detekt issues will be annotated on-the-fly while coding
-**ingore Failures in build:** add
+**ingore Failures in build**: add
```ignoreFailures = true```
diff --git a/theodolite/config/README.md b/theodolite/config/README.md
deleted file mode 100644
index 23337d77375ebba8f624e7a11f714502fe3d5e67..0000000000000000000000000000000000000000
--- a/theodolite/config/README.md
+++ /dev/null
@@ -1,201 +0,0 @@
-## The Benchmark Object
-
-The *benchmark* object defines all static components of an execution of a benchmark with Theodolite.
-An exapmle for a benchmark object is given in [example-benchmark-yaml-resource](example-benchmark-yaml-resource.yaml).
-
-
-A **Benchmark** is a [*standard tool for the competitive evaluation and comparison of competing systems or components according to specific characteristics, such as performance, dependability, or security*](https://doi.org/10.1145/2668930.2688819). In Theodolite, we have [specification-based benchmarks](https://doi.org/10.1145/2668930.2688819), or at least something very close to that. That is, our benchmarks are architectural descriptions---in our case---[of typical use cases of stream processing in microservices](https://doi.org/10.1016/j.bdr.2021.100209) (e.g. our UC1). Hence, we don't really have a piece of software, which represents a benchmark. We only have implementations of benchmarks, e.g. an implementation of UC1 with Kafka Streams. For simplification, we call these *benchmark implementations* simply *benchmarks*.
-
-```yaml
-name: String
-appResource:
- - String
- ...
-loadGenResource:
- - String
- ...
-resourceTypes:
- - typeName: String
- patchers:
- - type: String
- resources: String
- properties:
- <Patcher Arguments> ...
- ...
-loadTypes:
- - typeName: String
- patchers:
- - type: String
- resources: String
- properties:
- <Patcher Arguments> ...
- ...
-kafkaConfig:
- bootstrapServer: String
- topics:
- - name: String
- numPartitions: UnsignedInt
- replicationFactor: UnsignedInt
- - name: String
- removeOnly: bool
- ...
-```
-
-The properties have the following definitions:
-
-* **name**: The name of the *benchmark*
-* **appResource**: A list of file names that reference Kubernetes resources that are deployed on the cluster for the system under test (SUT).
-* **loadGenResources**: A list of file names that reference Kubernetes resources that are deployed on the cluster for the load generator.
-* **resourceTypes**: A list of resource types that can be scaled for this *benchmark*. For each resource type the concrete values are defined in the *execution* object. Each resource type has the following structure:
- * **typeName**: Name of the resource type.
- * **patchers**: List of [patchers](#Patchers) used to scale this resource type. Each patcher has the following structure:
- * **type**: Type of the [patcher](#Patchers). The concrete types can be looked up in the list of [patchers](#Patchers).
- * **resources**: Specifies the Kubernetes resource to be patched.
- * **properties**: *Patcher Arguments*: (Optional) Patcher specific additional arguments.
-* **loadTypes**: A list of load types that can be scaled for this *benchmark*. For each load type the concrete values are defined in the *execution* object. Each load type has the following structure:
- * **typeName**: Name of the load type.
- * **patchers**: List of patchers used to scale * **resourceTypes**: A list of resource types that can be scaled for this *benchmark*. For each resource type the concrete values are defined in the *execution* resource object.Each resource type has the following structure:
- * **typeName**: Name of the resource type.
- * **patchers**: List of patchers used to scale this resource type. Each patcher has the following structure:
- * **type**: Type of the Patcher. The concrete types can be looked up in the list of patchers.
- * **resources**: Specifies the Kubernetes resource to be patched.
- * **properties**: *Patcher Arguments*: (Optional) Patcher specific additional arguments as Map<String, String>.
-* **kafkaConfig**: Contains the Kafka configuration.
- * **bootstrapServers**: The bootstrap servers connection string.
- * **topics**: List of topics to be created for each [experiment](#Experiment). Alternative theodolite offers the possibility to remove certain topics after each experiment.
- * **name**: The name of the topic.
- * **numPartitions**: The number of partitions of the topic.
- * **replicationFactor**: The replication factor of the topic.
- * **removeOnly**: determines if this topic should only be deleted after each experiement. For removeOnly topics the name can be a RegEx describing the topic.
-
-
-## The Execution Object
-
-A benchmark can be executed for different SUTs, by different users and multiple times. We call such an execution of a benchmark simply an *execution*. The *execution* object defines all conrete values of an Execution.
-An exapmle for an execution object is given in [example-execution-yaml-resource](example-benchmark-yaml-resource.yaml).
-
-
-```yaml
-name: String
-benchmark: String
-load:
- loadType: String
- loadValues:
- - UnsignedInt
- ...
-resources:
- resourceType: String
- resourceValues:
- - UnsignedInt
- ...
-slos:
- - sloType: String
- threshold: UnsignedInt
- prometheusUrl: String
- externalSloUrl: String
- offset: SignedInt
- warmup: UnsignedInt
- ...
-executions:
- strategy: "LinearSearch" or "BinarySearch"
- duration: UnsignedInt
- repetition: UnsignedInt
- restrictions:
- - "LowerBound"
- ...
-configurationOverrides:
- - patcher:
- type: String
- resource: String
- properties:
- <Patcher Arguments> ...
- ...
-```
-
-The properties have the following definitions:
-
-* **name**: The name of the *execution*
-* **benchmark**: The name of the *benchmark* this *execution* is referring to.
-* **load**: Specifies the load values that are benchmarked.
- * **loadType**: The type of the load. It must match one of the load types specified in the referenced *benchmark*.
- * **loadValues**: List of load values for the specified load type.
-* **resources**: Specifies the scaling resource that is benchmarked.
- * **resourceType**: The type of the resource. It must match one of the resource types specified in the referenced *benchmark*.
- * **resourceValues**: List of resource values for the specified resource type.
-* **slos**: List of the Service Level Objective (SLO) for this *execution*. Each SLO has the following fields:
- * **sloType**: The type of the SLO. It must match 'lag trend'.
- * **threshold**: The threshold the SUT should meet for a sucessful experiment.
- * **prometheusUrl**: Connection string for promehteus.
- * **externalSloUrl**: Connection string for a external slo analysis.
- * **offset**: Hours by which the start and end timestamp will be shifted (for different timezones).
- * **warmup**: Seconds of time that are ignored in the analysis.
-* **executions**: Defines the overall parameter for the execution.
- * **strategy**: Defines the used strategy for the execution: either 'LinearSearch' or 'BinarySearch'
- * **duration**: Defines the duration of each [experiment](#Experiment) in seconds.
- * **repetition**: Unused.
- * **restrictions**: List of restriction strategys used to delimit the search space.
- **- LowerBound**: Currently only supported *restriction strategy*.
-* **configurationOverrides**: List of patchers that are used to override existing configurations.
- * **patcher**: Patcher used to patch a resource. Each patcher has the following structure:
- * **type**: Type of the Patcher. The concrete types can be looked up in the list of patchers.
- * **resources**: Specifies the Kubernetes resource to be patched.
- * **properties**: *Patcher Arguments*: (Optional) Patcher specific additional arguments.
-
-## Patchers
-
-* **ReplicaPatcher**: Allows to modify the number of Replicas for a kubernetes deployment.
- * **type**: "ReplicaPatcher"
- * **resource**: "uc1-kstreams-deployment.yaml"
-
-* **NumSensorsLoadGeneratorReplicaPatcher**: Allows to scale the nummer of load generators. Scales arcording to the following formular: (value + 15_000 - 1) / 15_000
- * **type**: "NumSensorsLoadGeneratorReplicaPatcher"
- * **resource**: "uc1-load-generator-deployment.yaml"
-
-* **NumNestedGroupsLoadGeneratorReplicaPatcher**: Allows to scale the nummer of load generators. Scales arcording to the following formular: (4^(value) + 15_000 -1) /15_000
- * **type**: "NumNestedGroupsLoadGeneratorReplicaPatcher"
- * **resource**: "uc1-load-generator-deployment.yaml"
-
-* **ReplicaPatcher**: Allows to modify the number of Replicas for a kubernetes deployment.
- * **type**: "ReplicaPatcher"
- * **resource**: "uc1-kstreams-deployment.yaml"
-
-* **EnvVarPatcher**: Allows to modify the value of an environment variable for a container in a kubernetes deployment.
- * **type**: "EnvVarPatcher"
- * **resource**: "uc1-load-generator-deployment.yaml"
- * **properties**:
- * container: "workload-generator"
- * variableName: "NUM_SENSORS"
-
-* **NodeSelectorPatcher**: Changes the node selection field in kubernetes resources.
- * **type**: "NodeSelectorPatcher"
- * **resource**: "uc1-load-generator-deployment.yaml"
- * **properties**:
- * variableName: "env"
- * **value**: "prod"
-
-* **ResourceLimitPatcher**: Changes the resource limit for a kubernetes resource.
- * **resource**: "uc1-kstreams-deployment.yaml"
- * **properties**:
- * container: "uc-application"
- * variableName: "cpu" or "memory"
- * **value**:"1000m" or "2Gi"
-
-* **SchedulerNamePatcher**: Changes the sheduler for kubernetes resources.
- * **type**: "SchedulerNamePatcher"
- * **resource**: "uc1-kstreams-deployment.yaml"
- * **value**: "random-scheduler"
-
-* **ImagePatcher**: Changes the image of a kubernetes resource. Currently not fully implemented.
- * **type**: "ImagePatcher"
- * **resource**: "uc1-kstreams-deployment.yaml"
- * **properties**:
- * container: "uc-application"
- * **value**: "dockerhubrepo/imagename"
-
-
-
-## Experiment
-According to [our benchmarking method](https://doi.org/10.1016/j.bdr.2021.100209), the execution of a benchmark requires performing multiple **Experiments**. I think what is actually done within/during an experiment is another level of detail. (But just for the sake of completeness: In an experiment, the benchmark implementation is deployed, load is generated according to the benchmark specification, some SLOs are monitored continuously, etc.)
-
-
-
diff --git a/theodolite/config/aggregation-service.yaml b/theodolite/config/aggregation-service.yaml
deleted file mode 100644
index 85432d04f225c30469f3232153ef6bd72bd02bdf..0000000000000000000000000000000000000000
--- a/theodolite/config/aggregation-service.yaml
+++ /dev/null
@@ -1,17 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
- name: titan-ccp-aggregation
- labels:
- app: titan-ccp-aggregation
-spec:
- #type: NodePort
- selector:
- app: titan-ccp-aggregation
- ports:
- - name: http
- port: 80
- targetPort: 80
- protocol: TCP
- - name: metrics
- port: 5556
diff --git a/theodolite/config/jmx-configmap.yaml b/theodolite/config/jmx-configmap.yaml
deleted file mode 100644
index 78496a86b1242a89b9e844ead3e700fd0b9a9667..0000000000000000000000000000000000000000
--- a/theodolite/config/jmx-configmap.yaml
+++ /dev/null
@@ -1,10 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: aggregation-jmx-configmap
-data:
- jmx-kafka-prometheus.yml: |+
- jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi
- lowercaseOutputName: true
- lowercaseOutputLabelNames: true
- ssl: false
diff --git a/theodolite/config/service-monitor.yaml b/theodolite/config/service-monitor.yaml
deleted file mode 100644
index 4e7e758cacb5086305efa26292ddef2afc958096..0000000000000000000000000000000000000000
--- a/theodolite/config/service-monitor.yaml
+++ /dev/null
@@ -1,14 +0,0 @@
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
- labels:
- app: titan-ccp-aggregation
- appScope: titan-ccp
- name: titan-ccp-aggregation
-spec:
- selector:
- matchLabels:
- app: titan-ccp-aggregation
- endpoints:
- - port: metrics
- interval: 10s
diff --git a/theodolite/config/uc1-service-monitor.yaml b/theodolite/config/uc1-service-monitor.yaml
deleted file mode 100644
index 4e7e758cacb5086305efa26292ddef2afc958096..0000000000000000000000000000000000000000
--- a/theodolite/config/uc1-service-monitor.yaml
+++ /dev/null
@@ -1,14 +0,0 @@
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
- labels:
- app: titan-ccp-aggregation
- appScope: titan-ccp
- name: titan-ccp-aggregation
-spec:
- selector:
- matchLabels:
- app: titan-ccp-aggregation
- endpoints:
- - port: metrics
- interval: 10s
diff --git a/theodolite/crd/crd-benchmark.yaml b/theodolite/crd/crd-benchmark.yaml
index 9de29fc031e26b9e4e16517492740b0805fd4af9..7a861276a651af520ccec853b65cb57009a45cc5 100644
--- a/theodolite/crd/crd-benchmark.yaml
+++ b/theodolite/crd/crd-benchmark.yaml
@@ -118,7 +118,7 @@ spec:
minItems: 1
items:
type: object
- required: ["name", "numPartitions", "replicationFactor"]
+ required: ["name"]
properties:
name:
description: The name of the topic.
diff --git a/theodolite/crd/crd-execution.yaml b/theodolite/crd/crd-execution.yaml
index f0588ecf98d40aadc10c1804f32d4bce72d22b03..47d0306f5150a8126f021c40bf3c4a4ce0e1abb1 100644
--- a/theodolite/crd/crd-execution.yaml
+++ b/theodolite/crd/crd-execution.yaml
@@ -60,26 +60,23 @@ spec:
type: array
items:
type: object
- required: ["sloType", "threshold", "prometheusUrl", "externalSloUrl", "offset", "warmup"]
+ required: ["sloType", "prometheusUrl", "offset"]
properties:
sloType:
description: The type of the SLO. It must match 'lag trend'.
type: string
- threshold:
- description: The threshold the SUT should meet for a sucessful experiment.
- type: integer
prometheusUrl:
description: Connection string for Promehteus.
type: string
- externalSloUrl:
- description: Connection string for a external slo analysis.
- type: string
offset:
description: Hours by which the start and end timestamp will be shifted (for different timezones).
type: integer
- warmup:
- description: Seconds of time that are ignored in the analysis.
- type: integer
+ properties:
+ description: (Optional) SLO specific additional arguments.
+ type: object
+ additionalProperties: true
+ x-kubernetes-map-type: "granular"
+ default: {}
execution: # def execution config
description: Defines the overall parameter for the execution.
type: object
@@ -152,4 +149,4 @@ spec:
jsonPath: .metadata.creationTimestamp
subresources:
status: {}
- scope: Namespaced
\ No newline at end of file
+ scope: Namespaced
diff --git a/theodolite/examples/operator/example-benchmark.yaml b/theodolite/examples/operator/example-benchmark.yaml
index 91d9f8f1f7dfed31d9edcb59947af4e832ca2843..2bd75f3299f3504a6d156c185162acad7e19b8cf 100644
--- a/theodolite/examples/operator/example-benchmark.yaml
+++ b/theodolite/examples/operator/example-benchmark.yaml
@@ -5,9 +5,6 @@ metadata:
spec:
appResource:
- "uc1-kstreams-deployment.yaml"
- - "aggregation-service.yaml"
- - "jmx-configmap.yaml"
- - "uc1-service-monitor.yaml"
loadGenResource:
- "uc1-load-generator-deployment.yaml"
- "uc1-load-generator-service.yaml"
diff --git a/theodolite/examples/operator/example-execution.yaml b/theodolite/examples/operator/example-execution.yaml
index 5386fd7c8665e01302067da81c5dd4caf87fc602..e2efb6e9a2bb6c08354b57a83506a601ac0ed96e 100644
--- a/theodolite/examples/operator/example-execution.yaml
+++ b/theodolite/examples/operator/example-execution.yaml
@@ -12,11 +12,12 @@ spec:
resourceValues: [1, 2, 3, 4, 5]
slos:
- sloType: "lag trend"
- threshold: 2000
prometheusUrl: "http://prometheus-operated:9090"
- externalSloUrl: "http://localhost:80/evaluate-slope"
offset: 0
- warmup: 60 # in seconds
+ properties:
+ threshold: 2000
+ externalSloUrl: "http://localhost:80/evaluate-slope"
+ warmup: 60 # in seconds
execution:
strategy: "LinearSearch"
duration: 300 # in seconds
@@ -24,7 +25,7 @@ spec:
loadGenerationDelay: 30 # in seconds
restrictions:
- "LowerBound"
- configOverrides:
+ configOverrides: []
# - patcher:
# type: "NodeSelectorPatcher"
# resource: "uc1-load-generator-deployment.yaml"
diff --git a/theodolite/config/uc1-kstreams-deployment.yaml b/theodolite/examples/resources/uc1-kstreams-deployment.yaml
similarity index 57%
rename from theodolite/config/uc1-kstreams-deployment.yaml
rename to theodolite/examples/resources/uc1-kstreams-deployment.yaml
index 171c3446db2719ee91bd8954233015316851fcf9..fdd1ff867ac83beb10856baec53569c88169232e 100644
--- a/theodolite/config/uc1-kstreams-deployment.yaml
+++ b/theodolite/examples/resources/uc1-kstreams-deployment.yaml
@@ -31,25 +31,4 @@ spec:
resources:
limits:
memory: 4Gi
- cpu: 1000m
- - name: prometheus-jmx-exporter
- image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143"
- command:
- - java
- - -XX:+UnlockExperimentalVMOptions
- - -XX:+UseCGroupMemoryLimitForHeap
- - -XX:MaxRAMFraction=1
- - -XshowSettings:vm
- - -jar
- - jmx_prometheus_httpserver.jar
- - "5556"
- - /etc/jmx-aggregation/jmx-kafka-prometheus.yml
- ports:
- - containerPort: 5556
- volumeMounts:
- - name: jmx-config
- mountPath: /etc/jmx-aggregation
- volumes:
- - name: jmx-config
- configMap:
- name: aggregation-jmx-configmap
\ No newline at end of file
+ cpu: 1000m
\ No newline at end of file
diff --git a/theodolite/config/uc1-load-generator-deployment.yaml b/theodolite/examples/resources/uc1-load-generator-deployment.yaml
similarity index 88%
rename from theodolite/config/uc1-load-generator-deployment.yaml
rename to theodolite/examples/resources/uc1-load-generator-deployment.yaml
index 374dd60113e133ef0a793149e3786efb38973287..9f9ccc6ae39407bb1f027e1e23cb152944b869e0 100644
--- a/theodolite/config/uc1-load-generator-deployment.yaml
+++ b/theodolite/examples/resources/uc1-load-generator-deployment.yaml
@@ -20,10 +20,6 @@ spec:
- containerPort: 5701
name: coordination
env:
- - name: NUM_SENSORS
- value: "25000"
- - name: NUM_NESTED_GROUPS
- value: "5"
- name: KUBERNETES_NAMESPACE
valueFrom:
fieldRef:
diff --git a/theodolite/config/uc1-load-generator-service.yaml b/theodolite/examples/resources/uc1-load-generator-service.yaml
similarity index 100%
rename from theodolite/config/uc1-load-generator-service.yaml
rename to theodolite/examples/resources/uc1-load-generator-service.yaml
diff --git a/theodolite/examples/standalone/example-benchmark.yaml b/theodolite/examples/standalone/example-benchmark.yaml
index 83edce93834ca9b8eef5606c1e5884ce40bdd7d8..e3b37a348c592e065937abf6c8ca99556804867b 100644
--- a/theodolite/examples/standalone/example-benchmark.yaml
+++ b/theodolite/examples/standalone/example-benchmark.yaml
@@ -1,9 +1,6 @@
name: "uc1-kstreams"
appResource:
- "uc1-kstreams-deployment.yaml"
- - "aggregation-service.yaml"
- - "jmx-configmap.yaml"
- - "uc1-service-monitor.yaml"
loadGenResource:
- "uc1-load-generator-deployment.yaml"
- "uc1-load-generator-service.yaml"
@@ -25,7 +22,7 @@ loadTypes:
properties:
loadGenMaxRecords: "15000"
kafkaConfig:
- bootstrapServer: "localhost:31290"
+ bootstrapServer: "theodolite-cp-kafka:9092"
topics:
- name: "input"
numPartitions: 40
diff --git a/theodolite/examples/standalone/example-execution.yaml b/theodolite/examples/standalone/example-execution.yaml
index 24b2b7f32e803553a4a13b76869ccf4cf3f6e5a5..6e649df957fe1d5dd962fdd5fe5152808e722de6 100644
--- a/theodolite/examples/standalone/example-execution.yaml
+++ b/theodolite/examples/standalone/example-execution.yaml
@@ -8,11 +8,12 @@ resources:
resourceValues: [1, 2, 3, 4, 5]
slos:
- sloType: "lag trend"
- threshold: 2000
prometheusUrl: "http://prometheus-operated:9090"
- externalSloUrl: "http://localhost:80/evaluate-slope"
offset: 0
- warmup: 60 # in seconds
+ properties:
+ threshold: 2000
+ externalSloUrl: "http://localhost:80/evaluate-slope"
+ warmup: 60 # in seconds
execution:
strategy: "LinearSearch"
duration: 300 # in seconds
@@ -20,4 +21,4 @@ execution:
loadGenerationDelay: 30 # in seconds, optional field, default is 0 seconds
restrictions:
- "LowerBound"
-configOverrides: []
\ No newline at end of file
+configOverrides: []
diff --git a/theodolite/src/main/docker/Dockerfile.jvm b/theodolite/src/main/docker/Dockerfile.jvm
index 4800a03181194772f854a85a9b0ba0eed17365ec..4d51240e0225bb571cc4a625e40c9ec76fd8f10d 100644
--- a/theodolite/src/main/docker/Dockerfile.jvm
+++ b/theodolite/src/main/docker/Dockerfile.jvm
@@ -44,7 +44,6 @@ RUN microdnf install curl ca-certificates ${JAVA_PACKAGE} \
ENV JAVA_OPTIONS="-Dquarkus.http.host=0.0.0.0 -Djava.util.logging.manager=org.jboss.logmanager.LogManager"
COPY build/lib/* /deployments/lib/
COPY build/*-runner.jar /deployments/app.jar
-COPY config/ /deployments/config/
EXPOSE 8080
USER 1001
diff --git a/theodolite/src/main/docker/Dockerfile.native b/theodolite/src/main/docker/Dockerfile.native
index d03e77564f783b76d202986ebd7c1e336f013779..95ef4fb51d7dc1ac520fb4c5a9af1b2d0a32fd09 100644
--- a/theodolite/src/main/docker/Dockerfile.native
+++ b/theodolite/src/main/docker/Dockerfile.native
@@ -20,7 +20,6 @@ RUN chown 1001 /deployments \
&& chmod "g+rwX" /deployments \
&& chown 1001:root /deployments
COPY --chown=1001:root build/*-runner /deployments/application
-COPY config/ /deployments/config/
EXPOSE 8080
USER 1001
diff --git a/theodolite/src/main/kotlin/theodolite/benchmark/Benchmark.kt b/theodolite/src/main/kotlin/theodolite/benchmark/Benchmark.kt
index d57a28e8bbcf4dc101e4814ecaa0d52fe28c08a9..05d021b1bcfb77fa8ffeb0522510d49e39ef501c 100644
--- a/theodolite/src/main/kotlin/theodolite/benchmark/Benchmark.kt
+++ b/theodolite/src/main/kotlin/theodolite/benchmark/Benchmark.kt
@@ -1,8 +1,5 @@
package theodolite.benchmark
-import io.fabric8.kubernetes.api.model.KubernetesResource
-import io.fabric8.kubernetes.api.model.Namespaced
-import io.fabric8.kubernetes.client.CustomResource
import io.quarkus.runtime.annotations.RegisterForReflection
import theodolite.util.ConfigurationOverride
import theodolite.util.LoadDimension
diff --git a/theodolite/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt b/theodolite/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt
index 62ab75898d16ff2732ab6aa5c254ec8f87fb7266..f2dda487d390c5f771e4f47c0f9c7ebf2cf971e7 100644
--- a/theodolite/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt
+++ b/theodolite/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt
@@ -18,7 +18,7 @@ import kotlin.properties.Delegates
* - An [execution] that encapsulates: the strategy, the duration, and the restrictions
* for the execution of the benchmark.
* - [configOverrides] additional configurations.
- * This class is used for parsing(in [theodolite.execution.TheodoliteYamlExecutor]) and
+ * This class is used for parsing(in [theodolite.execution.TheodoliteStandalone]) and
* for the deserializing in the [theodolite.execution.operator.TheodoliteOperator].
* @constructor construct an empty BenchmarkExecution.
*/
@@ -62,11 +62,9 @@ class BenchmarkExecution : KubernetesResource {
@RegisterForReflection
class Slo : KubernetesResource {
lateinit var sloType: String
- var threshold by Delegates.notNull<Int>()
lateinit var prometheusUrl: String
- lateinit var externalSloUrl: String
var offset by Delegates.notNull<Int>()
- var warmup by Delegates.notNull<Int>()
+ lateinit var properties: MutableMap<String, String>
}
/**
diff --git a/theodolite/src/main/kotlin/theodolite/benchmark/KafkaLagExporterRemover.kt b/theodolite/src/main/kotlin/theodolite/benchmark/KafkaLagExporterRemover.kt
deleted file mode 100644
index e8179b42d40e40e7ed45a8f5c48fe26f235be334..0000000000000000000000000000000000000000
--- a/theodolite/src/main/kotlin/theodolite/benchmark/KafkaLagExporterRemover.kt
+++ /dev/null
@@ -1,22 +0,0 @@
-package theodolite.benchmark
-
-import io.fabric8.kubernetes.client.NamespacedKubernetesClient
-import mu.KotlinLogging
-
-private val logger = KotlinLogging.logger {}
-
-/**
- * Used to reset the KafkaLagExporter by deleting the pod.
- * @param client NamespacedKubernetesClient used for the deletion.
- */
-class KafkaLagExporterRemover(private val client: NamespacedKubernetesClient) {
-
- /**
- * Deletes all pods with the selected label.
- * @param [label] of the pod that should be deleted.
- */
- fun remove(label: String) {
- this.client.pods().withLabel(label).delete()
- logger.info { "Pod with label: $label deleted" }
- }
-}
diff --git a/theodolite/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt b/theodolite/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt
index aa9c36ad912437e3b104dccf6ff1f4dea5905946..b9a2fc7f18b92664b4d93b11755280a9e18b170d 100644
--- a/theodolite/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt
+++ b/theodolite/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt
@@ -2,8 +2,6 @@ package theodolite.benchmark
import com.fasterxml.jackson.databind.annotation.JsonDeserialize
import io.fabric8.kubernetes.api.model.KubernetesResource
-import io.fabric8.kubernetes.api.model.Namespaced
-import io.fabric8.kubernetes.client.CustomResource
import io.fabric8.kubernetes.client.DefaultKubernetesClient
import io.quarkus.runtime.annotations.RegisterForReflection
import mu.KotlinLogging
@@ -14,6 +12,7 @@ import theodolite.util.*
private val logger = KotlinLogging.logger {}
private var DEFAULT_NAMESPACE = "default"
+private var DEFAULT_THEODOLITE_APP_RESOURCES = "./benchmark-resources"
/**
* Represents a benchmark in Kubernetes. An example for this is the BenchmarkType.yaml
@@ -27,13 +26,13 @@ private var DEFAULT_NAMESPACE = "default"
* - [namespace] for the client,
* - [path] under which the resource yamls can be found.
*
- * This class is used for the parsing(in the [theodolite.execution.TheodoliteYamlExecutor]) and
+ * This class is used for the parsing(in the [theodolite.execution.TheodoliteStandalone]) and
* for the deserializing in the [theodolite.execution.operator.TheodoliteOperator].
* @constructor construct an empty Benchmark.
*/
@JsonDeserialize
@RegisterForReflection
-class KubernetesBenchmark: KubernetesResource, Benchmark{
+class KubernetesBenchmark : KubernetesResource, Benchmark {
lateinit var name: String
lateinit var appResource: List<String>
lateinit var loadGenResource: List<String>
@@ -41,7 +40,6 @@ class KubernetesBenchmark: KubernetesResource, Benchmark{
lateinit var loadTypes: List<TypeName>
lateinit var kafkaConfig: KafkaConfig
var namespace = System.getenv("NAMESPACE") ?: DEFAULT_NAMESPACE
- var path = System.getenv("THEODOLITE_APP_RESOURCES") ?: "./config"
/**
@@ -50,8 +48,11 @@ class KubernetesBenchmark: KubernetesResource, Benchmark{
* the [K8sResourceLoader]
*/
private fun loadKubernetesResources(resources: List<String>): List<Pair<String, KubernetesResource>> {
+ val path = System.getenv("THEODOLITE_APP_RESOURCES") ?: DEFAULT_THEODOLITE_APP_RESOURCES
+ logger.info { "Using $path as resource path." }
+
val parser = YamlParser()
- val loader = K8sResourceLoader(DefaultKubernetesClient().inNamespace(namespace))
+ val loader = K8sResourceLoader(DefaultKubernetesClient())
return resources
.map { resource ->
val resourcePath = "$path/$resource"
@@ -78,7 +79,6 @@ class KubernetesBenchmark: KubernetesResource, Benchmark{
afterTeardownDelay: Long
): BenchmarkDeployment {
logger.info { "Using $namespace as namespace." }
- logger.info { "Using $path as resource path." }
val appResources = loadKubernetesResources(this.appResource)
val loadGenResources = loadKubernetesResources(this.loadGenResource)
@@ -100,7 +100,6 @@ class KubernetesBenchmark: KubernetesResource, Benchmark{
}
}
return KubernetesBenchmarkDeployment(
- namespace = namespace,
appResources = appResources.map { it.second },
loadGenResources = loadGenResources.map { it.second },
loadGenerationDelay = loadGenerationDelay,
diff --git a/theodolite/src/main/kotlin/theodolite/benchmark/KubernetesBenchmarkDeployment.kt b/theodolite/src/main/kotlin/theodolite/benchmark/KubernetesBenchmarkDeployment.kt
index 6cf239676ddb24752f4754a85fc62657f9eb6603..423ac92c654ff55057796d9642c2cb408bc62fe5 100644
--- a/theodolite/src/main/kotlin/theodolite/benchmark/KubernetesBenchmarkDeployment.kt
+++ b/theodolite/src/main/kotlin/theodolite/benchmark/KubernetesBenchmarkDeployment.kt
@@ -6,6 +6,7 @@ import io.quarkus.runtime.annotations.RegisterForReflection
import mu.KotlinLogging
import org.apache.kafka.clients.admin.NewTopic
import theodolite.k8s.K8sManager
+import theodolite.k8s.ResourceByLabelHandler
import theodolite.k8s.TopicManager
import theodolite.util.KafkaConfig
import java.time.Duration
@@ -22,7 +23,6 @@ private val logger = KotlinLogging.logger {}
*/
@RegisterForReflection
class KubernetesBenchmarkDeployment(
- val namespace: String,
val appResources: List<KubernetesResource>,
val loadGenResources: List<KubernetesResource>,
private val loadGenerationDelay: Long,
@@ -33,7 +33,8 @@ class KubernetesBenchmarkDeployment(
) : BenchmarkDeployment {
private val kafkaController = TopicManager(this.kafkaConfig)
private val kubernetesManager = K8sManager(client)
- private val LAG_EXPORTER_POD_LABEL = "app.kubernetes.io/name=kafka-lag-exporter"
+ private val LAG_EXPORTER_POD_LABEL_NAME = "app.kubernetes.io/name"
+ private val LAG_EXPORTER_POD_LABEL_VALUE = "kafka-lag-exporter"
/**
* Setup a [KubernetesBenchmark] using the [TopicManager] and the [K8sManager]:
@@ -60,7 +61,10 @@ class KubernetesBenchmarkDeployment(
loadGenResources.forEach { kubernetesManager.remove(it) }
appResources.forEach { kubernetesManager.remove(it) }
kafkaController.removeTopics(this.topics.map { topic -> topic.name })
- KafkaLagExporterRemover(client).remove(LAG_EXPORTER_POD_LABEL)
+ ResourceByLabelHandler(client).removePods(
+ labelName = LAG_EXPORTER_POD_LABEL_NAME,
+ labelValue = LAG_EXPORTER_POD_LABEL_VALUE
+ )
logger.info { "Teardown complete. Wait $afterTeardownDelay ms to let everything come down." }
Thread.sleep(Duration.ofSeconds(afterTeardownDelay).toMillis())
}
diff --git a/theodolite/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt b/theodolite/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt
index ef4d371173c7099eb091f90cddbe26d31e6522be..777ea1c0ed43ac3af244dc0aaf770c69c11718cf 100644
--- a/theodolite/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt
+++ b/theodolite/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt
@@ -12,6 +12,7 @@ import java.util.*
import java.util.regex.Pattern
private val logger = KotlinLogging.logger {}
+private val RECORD_LAG_QUERY = "sum by(group)(kafka_consumergroup_group_lag >= 0)"
/**
* Contains the analysis. Fetches a metric from Prometheus, documents it, and evaluates it.
@@ -32,7 +33,7 @@ class AnalysisExecutor(
* First fetches data from prometheus, then documents them and afterwards evaluate it via a [slo].
* @param load of the experiment.
* @param res of the experiment.
- * @param executionDuration of the experiment.
+ * @param executionIntervals list of start and end points of experiments
* @return true if the experiment succeeded.
*/
fun analyze(load: LoadDimension, res: Resource, executionIntervals: List<Pair<Instant, Instant>>): Boolean {
@@ -45,28 +46,32 @@ class AnalysisExecutor(
val fileURL = "${resultsFolder}exp${executionId}_${load.get()}_${res.get()}_${slo.sloType.toSlug()}"
val prometheusData = executionIntervals
- .map { interval -> fetcher.fetchMetric(
+ .map { interval ->
+ fetcher.fetchMetric(
start = interval.first,
end = interval.second,
- query = "sum by(group)(kafka_consumergroup_group_lag >= 0)") }
+ query = RECORD_LAG_QUERY
+ )
+ }
- prometheusData.forEach{ data ->
+ prometheusData.forEach { data ->
ioHandler.writeToCSVFile(
fileURL = "${fileURL}_${repetitionCounter++}",
data = data.getResultAsList(),
- columns = listOf("group", "timestamp", "value"))
+ columns = listOf("group", "timestamp", "value")
+ )
}
val sloChecker = SloCheckerFactory().create(
sloType = slo.sloType,
- externalSlopeURL = slo.externalSloUrl,
- threshold = slo.threshold,
- warmup = slo.warmup
+ properties = slo.properties,
+ load = load
)
result = sloChecker.evaluate(prometheusData)
} catch (e: Exception) {
+ // TODO(throw exception in order to make it possible to mark an experiment as unsuccessfully)
logger.error { "Evaluation failed for resource '${res.get()}' and load '${load.get()}'. Error: $e" }
}
return result
@@ -75,7 +80,7 @@ class AnalysisExecutor(
private val NONLATIN: Pattern = Pattern.compile("[^\\w-]")
private val WHITESPACE: Pattern = Pattern.compile("[\\s]")
- fun String.toSlug(): String {
+ private fun String.toSlug(): String {
val noWhitespace: String = WHITESPACE.matcher(this).replaceAll("-")
val normalized: String = Normalizer.normalize(noWhitespace, Normalizer.Form.NFD)
val slug: String = NONLATIN.matcher(normalized).replaceAll("")
diff --git a/theodolite/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt b/theodolite/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt
index f7ebee8faf740583dbe6a37381a599e9bde19280..448a2a05f8dbeb1aef153895360bfb40e7275224 100644
--- a/theodolite/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt
+++ b/theodolite/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt
@@ -5,7 +5,6 @@ import khttp.post
import mu.KotlinLogging
import theodolite.util.PrometheusResponse
import java.net.ConnectException
-import java.time.Instant
/**
* [SloChecker] that uses an external source for the concrete evaluation.
@@ -37,10 +36,13 @@ class ExternalSloChecker(
*/
override fun evaluate(fetchedData: List<PrometheusResponse>): Boolean {
var counter = 0
- val data = Gson().toJson(mapOf(
- "total_lags" to fetchedData.map { it.data?.result},
- "threshold" to threshold,
- "warmup" to warmup))
+ val data = Gson().toJson(
+ mapOf(
+ "total_lags" to fetchedData.map { it.data?.result },
+ "threshold" to threshold,
+ "warmup" to warmup
+ )
+ )
while (counter < RETRIES) {
val result = post(externalSlopeURL, data = data, timeout = TIMEOUT)
diff --git a/theodolite/src/main/kotlin/theodolite/evaluation/SloChecker.kt b/theodolite/src/main/kotlin/theodolite/evaluation/SloChecker.kt
index 9ee5fe7ef34ce5b6214882ce2c1d19677f1d7130..af70fa5dca3f0556d38791ed96c2af30b9a44a68 100644
--- a/theodolite/src/main/kotlin/theodolite/evaluation/SloChecker.kt
+++ b/theodolite/src/main/kotlin/theodolite/evaluation/SloChecker.kt
@@ -8,13 +8,10 @@ import theodolite.util.PrometheusResponse
*/
interface SloChecker {
/**
- * Evaluates [fetchedData] and returns if the experiment was successful.
- * Returns if the evaluated experiment was successful.
+ * Evaluates [fetchedData] and returns if the experiments were successful.
*
- * @param start of the experiment
- * @param end of the experiment
* @param fetchedData from Prometheus that will be evaluated.
- * @return true if experiment was successful. Otherwise false.
+ * @return true if experiments were successful. Otherwise false.
*/
fun evaluate(fetchedData: List<PrometheusResponse>): Boolean
}
diff --git a/theodolite/src/main/kotlin/theodolite/evaluation/SloCheckerFactory.kt b/theodolite/src/main/kotlin/theodolite/evaluation/SloCheckerFactory.kt
index 20c421acdfcd76f5d2ebc2ab2c30142bcca3841a..93e8e6180f5a99486e500af022869d896067d128 100644
--- a/theodolite/src/main/kotlin/theodolite/evaluation/SloCheckerFactory.kt
+++ b/theodolite/src/main/kotlin/theodolite/evaluation/SloCheckerFactory.kt
@@ -1,5 +1,7 @@
package theodolite.evaluation
+import theodolite.util.LoadDimension
+
/**
* Factory used to potentially create different [SloChecker]s.
* Supports: lag type.
@@ -8,28 +10,63 @@ class SloCheckerFactory {
/**
* Creates different [SloChecker]s.
- * Supports: lag type.
+ *
+ * Supports: `lag trend` and `lag trend percent` as arguments for `sloType`
+ *
+ * ### `lag trend`
+ * Creates an [ExternalSloChecker] with defined parameters.
+ *
+ * The properties map needs the following fields:
+ * - `externalSlopeURL`: Url to the concrete SLO checker service.
+ * - `threshold`: fixed value used for the slope.
+ * - `warmup`: time from the beginning to skip in the analysis.
+ *
+ *
+ * ### `lag trend ratio`
+ * Creates an [ExternalSloChecker] with defined parameters.
+ * The required threshold is computed using a ratio and the load of the experiment.
+ *
+ * The properties map needs the following fields:
+ * - `externalSlopeURL`: Url to the concrete SLO checker service.
+ * - `ratio`: of the executed load that is accepted for the slope.
+ * - `warmup`: time from the beginning to skip in the analysis.
*
* @param sloType Type of the [SloChecker].
- * @param externalSlopeURL Url to the concrete [SloChecker].
- * @param threshold for the [SloChecker].
- * @param warmup for the [SloChecker].
+ * @param properties map of properties to use for the SLO checker creation.
+ * @param load that is executed in the experiment.
*
* @return A [SloChecker]
* @throws IllegalArgumentException If [sloType] not supported.
*/
fun create(
sloType: String,
- externalSlopeURL: String,
- threshold: Int,
- warmup: Int
+ properties: MutableMap<String, String>,
+ load: LoadDimension
): SloChecker {
return when (sloType) {
"lag trend" -> ExternalSloChecker(
- externalSlopeURL = externalSlopeURL,
- threshold = threshold,
- warmup = warmup
+ externalSlopeURL = properties["externalSloUrl"]
+ ?: throw IllegalArgumentException("externalSloUrl expected"),
+ threshold = properties["threshold"]?.toInt() ?: throw IllegalArgumentException("threshold expected"),
+ warmup = properties["warmup"]?.toInt() ?: throw IllegalArgumentException("warmup expected")
)
+ "lag trend ratio" -> {
+ var thresholdRatio =
+ properties["ratio"]?.toDouble()
+ ?: throw IllegalArgumentException("ratio for threshold expected")
+ if (thresholdRatio < 0.0) {
+ throw IllegalArgumentException("Threshold ratio needs to be an Double greater or equal 0.0")
+ }
+ // cast to int, as rounding is not really necessary
+ var threshold = (load.get() * thresholdRatio).toInt()
+
+ ExternalSloChecker(
+ externalSlopeURL = properties["externalSloUrl"]
+ ?: throw IllegalArgumentException("externalSloUrl expected"),
+ threshold = threshold,
+ warmup = properties["warmup"]?.toInt() ?: throw IllegalArgumentException("warmup expected")
+ )
+ }
else -> throw IllegalArgumentException("Slotype $sloType not found.")
}
}
diff --git a/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt b/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt
index 3afc85f0a8cb67011763498a662b447ce2c07f0f..c54d1878d4957a0b8ec6a8fdfb18ec6342d7bfc1 100644
--- a/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt
+++ b/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt
@@ -5,7 +5,10 @@ import mu.KotlinLogging
import theodolite.benchmark.Benchmark
import theodolite.benchmark.BenchmarkExecution
import theodolite.evaluation.AnalysisExecutor
-import theodolite.util.*
+import theodolite.util.ConfigurationOverride
+import theodolite.util.LoadDimension
+import theodolite.util.Resource
+import theodolite.util.Results
import java.time.Duration
import java.time.Instant
@@ -22,7 +25,17 @@ class BenchmarkExecutorImpl(
executionId: Int,
loadGenerationDelay: Long,
afterTeardownDelay: Long
-) : BenchmarkExecutor(benchmark, results, executionDuration, configurationOverrides, slo, repetitions, executionId, loadGenerationDelay, afterTeardownDelay) {
+) : BenchmarkExecutor(
+ benchmark,
+ results,
+ executionDuration,
+ configurationOverrides,
+ slo,
+ repetitions,
+ executionId,
+ loadGenerationDelay,
+ afterTeardownDelay
+) {
override fun runExperiment(load: LoadDimension, res: Resource): Boolean {
var result = false
val executionIntervals: MutableList<Pair<Instant, Instant>> = ArrayList()
@@ -30,7 +43,7 @@ class BenchmarkExecutorImpl(
for (i in 1.rangeTo(repetitions)) {
logger.info { "Run repetition $i/$repetitions" }
if (this.run.get()) {
- executionIntervals.add(runSingleExperiment(load,res))
+ executionIntervals.add(runSingleExperiment(load, res))
} else {
break
}
@@ -40,19 +53,27 @@ class BenchmarkExecutorImpl(
* Analyse the experiment, if [run] is true, otherwise the experiment was canceled by the user.
*/
if (this.run.get()) {
- result =AnalysisExecutor(slo = slo, executionId = executionId)
- .analyze(
- load = load,
- res = res,
- executionIntervals = executionIntervals)
+ result = AnalysisExecutor(slo = slo, executionId = executionId)
+ .analyze(
+ load = load,
+ res = res,
+ executionIntervals = executionIntervals
+ )
this.results.setResult(Pair(load, res), result)
}
return result
}
private fun runSingleExperiment(load: LoadDimension, res: Resource): Pair<Instant, Instant> {
- val benchmarkDeployment = benchmark.buildDeployment(load, res, this.configurationOverrides, this.loadGenerationDelay, this.afterTeardownDelay)
+ val benchmarkDeployment = benchmark.buildDeployment(
+ load,
+ res,
+ this.configurationOverrides,
+ this.loadGenerationDelay,
+ this.afterTeardownDelay
+ )
val from = Instant.now()
+ // TODO(restructure try catch in order to throw exceptions if there are significant problems by running a experiment)
try {
benchmarkDeployment.setup()
this.waitAndLog()
@@ -68,6 +89,6 @@ class BenchmarkExecutorImpl(
logger.warn { "Error while tearing down the benchmark deployment." }
logger.debug { "Teardown failed, caused by: $e" }
}
- return Pair(from,to)
+ return Pair(from, to)
}
}
diff --git a/theodolite/src/main/kotlin/theodolite/execution/Main.kt b/theodolite/src/main/kotlin/theodolite/execution/Main.kt
index bf883529967a8b24229fe8256ba0e4edd11b342c..7d5fca859422a194e81468d9766a9e7ba29fb998 100644
--- a/theodolite/src/main/kotlin/theodolite/execution/Main.kt
+++ b/theodolite/src/main/kotlin/theodolite/execution/Main.kt
@@ -17,8 +17,8 @@ object Main {
logger.info { "Start Theodolite with mode $mode" }
when (mode) {
- "standalone" -> TheodoliteYamlExecutor().start()
- "yaml-executor" -> TheodoliteYamlExecutor().start() // TODO remove (#209)
+ "standalone" -> TheodoliteStandalone().start()
+ "yaml-executor" -> TheodoliteStandalone().start() // TODO remove (#209)
"operator" -> TheodoliteOperator().start()
else -> {
logger.error { "MODE $mode not found" }
diff --git a/theodolite/src/main/kotlin/theodolite/execution/Shutdown.kt b/theodolite/src/main/kotlin/theodolite/execution/Shutdown.kt
index 0ff8379a0af4b11154214dde021d7c60609631d1..e795ada3e3bcb2dba19f1e088f426f38a824f4a7 100644
--- a/theodolite/src/main/kotlin/theodolite/execution/Shutdown.kt
+++ b/theodolite/src/main/kotlin/theodolite/execution/Shutdown.kt
@@ -5,7 +5,6 @@ import theodolite.benchmark.BenchmarkExecution
import theodolite.benchmark.KubernetesBenchmark
import theodolite.util.LoadDimension
import theodolite.util.Resource
-import java.lang.Exception
private val logger = KotlinLogging.logger {}
@@ -25,22 +24,26 @@ class Shutdown(private val benchmarkExecution: BenchmarkExecution, private val b
override fun run() {
// Build Configuration to teardown
try {
- logger.info { "Received shutdown signal -> Shutting down" }
- val deployment =
- benchmark.buildDeployment(
- load = LoadDimension(0, emptyList()),
- res = Resource(0, emptyList()),
- configurationOverrides = benchmarkExecution.configOverrides,
- loadGenerationDelay = 0L,
- afterTeardownDelay = 5L
- )
+ logger.info { "Received shutdown signal -> Shutting down" }
+ val deployment =
+ benchmark.buildDeployment(
+ load = LoadDimension(0, emptyList()),
+ res = Resource(0, emptyList()),
+ configurationOverrides = benchmarkExecution.configOverrides,
+ loadGenerationDelay = 0L,
+ afterTeardownDelay = 5L
+ )
deployment.teardown()
} catch (e: Exception) {
- logger.warn { "Could not delete all specified resources from Kubernetes. " +
- "This could be the case, if not all resources are deployed and running." }
+ // TODO(throw exception in order to make it possible to mark an experiment as unsuccessfully)
+ logger.warn {
+ "Could not delete all specified resources from Kubernetes. " +
+ "This could be the case, if not all resources are deployed and running."
+ }
}
- logger.info { "Teardown everything deployed" }
- logger.info { "Teardown completed" }
+ logger.info {
+ "Finished teardown of all benchmark resources."
+ }
}
}
diff --git a/theodolite/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt b/theodolite/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt
index c73aaae08489c25a40163d4edb1607247fae010a..addf30acde31ee8e3e53c20a5e2b57a03587d08e 100644
--- a/theodolite/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt
+++ b/theodolite/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt
@@ -70,14 +70,18 @@ class TheodoliteExecutor(
if (config.load.loadValues != config.load.loadValues.sorted()) {
config.load.loadValues = config.load.loadValues.sorted()
- logger.info { "Load values are not sorted correctly, Theodolite sorts them in ascending order." +
- "New order is: ${config.load.loadValues}" }
+ logger.info {
+ "Load values are not sorted correctly, Theodolite sorts them in ascending order." +
+ "New order is: ${config.load.loadValues}"
+ }
}
if (config.resources.resourceValues != config.resources.resourceValues.sorted()) {
config.resources.resourceValues = config.resources.resourceValues.sorted()
- logger.info { "Load values are not sorted correctly, Theodolite sorts them in ascending order." +
- "New order is: ${config.resources.resourceValues}" }
+ logger.info {
+ "Load values are not sorted correctly, Theodolite sorts them in ascending order." +
+ "New order is: ${config.resources.resourceValues}"
+ }
}
return Config(
@@ -103,10 +107,6 @@ class TheodoliteExecutor(
return this.config
}
- fun getBenchmark(): KubernetesBenchmark {
- return this.kubernetesBenchmark
- }
-
/**
* Run all experiments which are specified in the corresponding
* execution and benchmark objects.
@@ -114,9 +114,12 @@ class TheodoliteExecutor(
fun run() {
val ioHandler = IOHandler()
val resultsFolder = ioHandler.getResultFolderURL()
- this.config.executionId = getAndIncrementExecutionID(resultsFolder+"expID.txt")
- ioHandler.writeToJSONFile(this.config, "$resultsFolder${this.config.executionId}-execution-configuration")
- ioHandler.writeToJSONFile(kubernetesBenchmark, "$resultsFolder${this.config.executionId}-benchmark-configuration")
+ this.config.executionId = getAndIncrementExecutionID(resultsFolder + "expID.txt")
+ ioHandler.writeToJSONFile(this.config, "${resultsFolder}exp${this.config.executionId}-execution-configuration")
+ ioHandler.writeToJSONFile(
+ kubernetesBenchmark,
+ "${resultsFolder}exp${this.config.executionId}-benchmark-configuration"
+ )
val config = buildConfig()
// execute benchmarks for each load
@@ -125,17 +128,20 @@ class TheodoliteExecutor(
config.compositeStrategy.findSuitableResource(load, config.resources)
}
}
- ioHandler.writeToJSONFile(config.compositeStrategy.benchmarkExecutor.results, "$resultsFolder${this.config.executionId}-result")
+ ioHandler.writeToJSONFile(
+ config.compositeStrategy.benchmarkExecutor.results,
+ "${resultsFolder}exp${this.config.executionId}-result"
+ )
}
- private fun getAndIncrementExecutionID(fileURL: String): Int {
- val ioHandler = IOHandler()
- var executionID = 0
- if (File(fileURL).exists()) {
- executionID = ioHandler.readFileAsString(fileURL).toInt() + 1
- }
- ioHandler.writeStringToTextFile(fileURL, (executionID).toString())
- return executionID
+ private fun getAndIncrementExecutionID(fileURL: String): Int {
+ val ioHandler = IOHandler()
+ var executionID = 0
+ if (File(fileURL).exists()) {
+ executionID = ioHandler.readFileAsString(fileURL).toInt() + 1
+ }
+ ioHandler.writeStringToTextFile(fileURL, (executionID).toString())
+ return executionID
}
}
diff --git a/theodolite/src/main/kotlin/theodolite/execution/TheodoliteYamlExecutor.kt b/theodolite/src/main/kotlin/theodolite/execution/TheodoliteStandalone.kt
similarity index 93%
rename from theodolite/src/main/kotlin/theodolite/execution/TheodoliteYamlExecutor.kt
rename to theodolite/src/main/kotlin/theodolite/execution/TheodoliteStandalone.kt
index b9977029703c8012ada7fb3d7766bfa321a836c3..76fd7f707a3e190ff6c61052ae4b5aaf50459418 100644
--- a/theodolite/src/main/kotlin/theodolite/execution/TheodoliteYamlExecutor.kt
+++ b/theodolite/src/main/kotlin/theodolite/execution/TheodoliteStandalone.kt
@@ -25,14 +25,14 @@ private val logger = KotlinLogging.logger {}
*
* @constructor Create empty Theodolite yaml executor
*/
-class TheodoliteYamlExecutor {
+class TheodoliteStandalone {
private val parser = YamlParser()
fun start() {
logger.info { "Theodolite started" }
- val executionPath = System.getenv("THEODOLITE_EXECUTION") ?: "./config/example-execution-yaml-resource.yaml"
- val benchmarkPath = System.getenv("THEODOLITE_BENCHMARK") ?: "./config/example-benchmark-yaml-resource.yaml"
+ val executionPath = System.getenv("THEODOLITE_EXECUTION") ?: "execution/execution.yaml"
+ val benchmarkPath = System.getenv("THEODOLITE_BENCHMARK") ?: "benchmark/benchmark.yaml"
logger.info { "Using $executionPath for BenchmarkExecution" }
logger.info { "Using $benchmarkPath for BenchmarkType" }
diff --git a/theodolite/src/main/kotlin/theodolite/execution/operator/AbstractStateHandler.kt b/theodolite/src/main/kotlin/theodolite/execution/operator/AbstractStateHandler.kt
index a7a40cd569f8034f3b8e062dad3031d5643a12e3..9d7436526f18081c7130870956d8a5eea5fc8997 100644
--- a/theodolite/src/main/kotlin/theodolite/execution/operator/AbstractStateHandler.kt
+++ b/theodolite/src/main/kotlin/theodolite/execution/operator/AbstractStateHandler.kt
@@ -4,47 +4,50 @@ import io.fabric8.kubernetes.api.model.HasMetadata
import io.fabric8.kubernetes.api.model.KubernetesResourceList
import io.fabric8.kubernetes.api.model.Namespaced
import io.fabric8.kubernetes.client.CustomResource
-import io.fabric8.kubernetes.client.KubernetesClient
+import io.fabric8.kubernetes.client.NamespacedKubernetesClient
import io.fabric8.kubernetes.client.dsl.MixedOperation
import io.fabric8.kubernetes.client.dsl.Resource
import java.lang.Thread.sleep
-abstract class AbstractStateHandler<T,L,D>(
- private val client: KubernetesClient,
+abstract class AbstractStateHandler<T, L, D>(
+ private val client: NamespacedKubernetesClient,
private val crd: Class<T>,
private val crdList: Class<L>
- ): StateHandler<T> where T : CustomResource<*, *>?, T: HasMetadata, T: Namespaced, L: KubernetesResourceList<T> {
+) : StateHandler<T> where T : CustomResource<*, *>?, T : HasMetadata, T : Namespaced, L : KubernetesResourceList<T> {
- private val crdClient: MixedOperation<T, L,Resource<T>> =
+ private val crdClient: MixedOperation<T, L, Resource<T>> =
this.client.customResources(this.crd, this.crdList)
@Synchronized
override fun setState(resourceName: String, f: (T) -> T?) {
this.crdClient
- .inNamespace(this.client.namespace)
.list().items
- .filter { item -> item.metadata.name == resourceName }
+ .filter { it.metadata.name == resourceName }
.map { customResource -> f(customResource) }
.forEach { this.crdClient.updateStatus(it) }
- }
+ }
@Synchronized
override fun getState(resourceName: String, f: (T) -> String?): String? {
return this.crdClient
- .inNamespace(this.client.namespace)
.list().items
- .filter { item -> item.metadata.name == resourceName }
+ .filter { it.metadata.name == resourceName }
.map { customResource -> f(customResource) }
.firstOrNull()
}
@Synchronized
- override fun blockUntilStateIsSet(resourceName: String, desiredStatusString: String, f: (T) -> String?, maxTries: Int): Boolean {
+ override fun blockUntilStateIsSet(
+ resourceName: String,
+ desiredStatusString: String,
+ f: (T) -> String?,
+ maxTries: Int
+ ): Boolean {
for (i in 0.rangeTo(maxTries)) {
val currentStatus = getState(resourceName, f)
- if(currentStatus == desiredStatusString) {
- return true
- }
+ if (currentStatus == desiredStatusString) {
+ return true
+ }
sleep(50)
}
return false
diff --git a/theodolite/src/main/kotlin/theodolite/execution/operator/ClusterSetup.kt b/theodolite/src/main/kotlin/theodolite/execution/operator/ClusterSetup.kt
index 8fc951d09598187bcaf4cb7e4a39d322be722792..c3a2b7b25ed71e797c45d8b497bad6cad15e21e8 100644
--- a/theodolite/src/main/kotlin/theodolite/execution/operator/ClusterSetup.kt
+++ b/theodolite/src/main/kotlin/theodolite/execution/operator/ClusterSetup.kt
@@ -4,9 +4,9 @@ import io.fabric8.kubernetes.client.NamespacedKubernetesClient
import io.fabric8.kubernetes.client.dsl.MixedOperation
import io.fabric8.kubernetes.client.dsl.Resource
import mu.KotlinLogging
-import org.json.JSONObject
import theodolite.execution.Shutdown
import theodolite.k8s.K8sContextFactory
+import theodolite.k8s.ResourceByLabelHandler
import theodolite.model.crd.*
private val logger = KotlinLogging.logger {}
@@ -16,7 +16,7 @@ class ClusterSetup(
private val benchmarkCRDClient: MixedOperation<BenchmarkCRD, KubernetesBenchmarkList, Resource<BenchmarkCRD>>,
private val client: NamespacedKubernetesClient
- ) {
+) {
private val serviceMonitorContext = K8sContextFactory().create(
api = "v1",
scope = "Namespaced",
@@ -24,18 +24,24 @@ class ClusterSetup(
plural = "servicemonitors"
)
- fun clearClusterState(){
+ fun clearClusterState() {
stopRunningExecution()
clearByLabel()
}
+ /**
+ * This function searches for executions in the cluster that have the status running and tries to stop the execution.
+ * For this the corresponding benchmark is searched and terminated.
+ *
+ * Throws [IllegalStateException] if no suitable benchmark can be found.
+ *
+ */
private fun stopRunningExecution() {
executionCRDClient
- .inNamespace(client.namespace)
.list()
.items
.asSequence()
- .filter { it.status.executionState == States.RUNNING.value }
+ .filter { it.status.executionState == States.RUNNING.value }
.forEach { execution ->
val benchmark = benchmarkCRDClient
.inNamespace(client.namespace)
@@ -50,27 +56,35 @@ class ClusterSetup(
} else {
logger.error {
"Execution with state ${States.RUNNING.value} was found, but no corresponding benchmark. " +
- "Could not initialize cluster." }
+ "Could not initialize cluster."
+ }
+ throw IllegalStateException("Cluster state is invalid, required Benchmark for running execution not found.")
}
-
-
}
- }
-
- private fun clearByLabel() {
- this.client.services().withLabel("app.kubernetes.io/created-by=theodolite").delete()
- this.client.apps().deployments().withLabel("app.kubernetes.io/created-by=theodolite").delete()
- this.client.apps().statefulSets().withLabel("app.kubernetes.io/created-by=theodolite").delete()
- this.client.configMaps().withLabel("app.kubernetes.io/created-by=theodolite").delete()
+ }
- val serviceMonitors = JSONObject(
- this.client.customResource(serviceMonitorContext)
- .list(client.namespace, mapOf(Pair("app.kubernetes.io/created-by", "theodolite")))
+ private fun clearByLabel() {
+ val resourceRemover = ResourceByLabelHandler(client = client)
+ resourceRemover.removeServices(
+ labelName = "app.kubernetes.io/created-by",
+ labelValue = "theodolite"
+ )
+ resourceRemover.removeDeployments(
+ labelName = "app.kubernetes.io/created-by",
+ labelValue = "theodolite"
+ )
+ resourceRemover.removeStatefulSets(
+ labelName = "app.kubernetes.io/created-by",
+ labelValue = "theodolite"
+ )
+ resourceRemover.removeConfigMaps(
+ labelName = "app.kubernetes.io/created-by",
+ labelValue = "theodolite"
+ )
+ resourceRemover.removeCR(
+ labelName = "app.kubernetes.io/created-by",
+ labelValue = "theodolite",
+ context = serviceMonitorContext
)
- .getJSONArray("items")
-
- (0 until serviceMonitors.length())
- .map { serviceMonitors.getJSONObject(it).getJSONObject("metadata").getString("name") }
- .forEach { this.client.customResource(serviceMonitorContext).delete(client.namespace, it) }
}
}
\ No newline at end of file
diff --git a/theodolite/src/main/kotlin/theodolite/execution/operator/ExecutionEventHandler.kt b/theodolite/src/main/kotlin/theodolite/execution/operator/ExecutionEventHandler.kt
index 653366010725a8db816c92aece7bb572b659426b..1209195ee09cebe382f010f38e955dea1c860cd1 100644
--- a/theodolite/src/main/kotlin/theodolite/execution/operator/ExecutionEventHandler.kt
+++ b/theodolite/src/main/kotlin/theodolite/execution/operator/ExecutionEventHandler.kt
@@ -36,10 +36,10 @@ class ExecutionHandler(
States.NO_STATE -> this.stateHandler.setExecutionState(execution.spec.name, States.PENDING)
States.RUNNING -> {
this.stateHandler.setExecutionState(execution.spec.name, States.RESTART)
- if(this.controller.isExecutionRunning(execution.spec.name)){
- this.controller.stop(restart=true)
- }
+ if (this.controller.isExecutionRunning(execution.spec.name)) {
+ this.controller.stop(restart = true)
}
+ }
}
}
@@ -55,20 +55,21 @@ class ExecutionHandler(
override fun onUpdate(oldExecution: ExecutionCRD, newExecution: ExecutionCRD) {
newExecution.spec.name = newExecution.metadata.name
oldExecution.spec.name = oldExecution.metadata.name
- if(gson.toJson(oldExecution.spec) != gson.toJson(newExecution.spec)) {
+ if (gson.toJson(oldExecution.spec) != gson.toJson(newExecution.spec)) {
logger.info { "Receive update event for execution ${oldExecution.metadata.name}" }
- when(this.stateHandler.getExecutionState(newExecution.metadata.name)) {
+ when (this.stateHandler.getExecutionState(newExecution.metadata.name)) {
States.RUNNING -> {
- this.stateHandler.setExecutionState(newExecution.spec.name, States.RESTART)
- if (this.controller.isExecutionRunning(newExecution.spec.name)){
- this.controller.stop(restart=true)
- }
- }
- States.RESTART -> {} // should this set to pending?
- else -> this.stateHandler.setExecutionState(newExecution.spec.name, States.PENDING)
+ this.stateHandler.setExecutionState(newExecution.spec.name, States.RESTART)
+ if (this.controller.isExecutionRunning(newExecution.spec.name)) {
+ this.controller.stop(restart = true)
+ }
}
+ States.RESTART -> {
+ } // should this set to pending?
+ else -> this.stateHandler.setExecutionState(newExecution.spec.name, States.PENDING)
}
}
+ }
/**
* Delete an execution from the queue of the TheodoliteController.
@@ -78,8 +79,9 @@ class ExecutionHandler(
@Synchronized
override fun onDelete(execution: ExecutionCRD, b: Boolean) {
logger.info { "Delete execution ${execution.metadata.name}" }
- if(execution.status.executionState == States.RUNNING.value
- && this.controller.isExecutionRunning(execution.spec.name)) {
+ if (execution.status.executionState == States.RUNNING.value
+ && this.controller.isExecutionRunning(execution.metadata.name)
+ ) {
this.controller.stop()
}
}
diff --git a/theodolite/src/main/kotlin/theodolite/execution/operator/ExecutionStateHandler.kt b/theodolite/src/main/kotlin/theodolite/execution/operator/ExecutionStateHandler.kt
index df5e77695d1beb562408f1b5830f6f4353543c75..bcc86c8f2a9b233fa9a1972a866936e14688ecf8 100644
--- a/theodolite/src/main/kotlin/theodolite/execution/operator/ExecutionStateHandler.kt
+++ b/theodolite/src/main/kotlin/theodolite/execution/operator/ExecutionStateHandler.kt
@@ -1,6 +1,6 @@
package theodolite.execution.operator
-import io.fabric8.kubernetes.client.KubernetesClient
+import io.fabric8.kubernetes.client.NamespacedKubernetesClient
import theodolite.model.crd.BenchmarkExecutionList
import theodolite.model.crd.ExecutionCRD
import theodolite.model.crd.ExecutionStatus
@@ -10,8 +10,8 @@ import java.time.Duration
import java.time.Instant
import java.util.concurrent.atomic.AtomicBoolean
-class ExecutionStateHandler(val client: KubernetesClient):
- AbstractStateHandler<ExecutionCRD, BenchmarkExecutionList, ExecutionStatus >(
+class ExecutionStateHandler(val client: NamespacedKubernetesClient) :
+ AbstractStateHandler<ExecutionCRD, BenchmarkExecutionList, ExecutionStatus>(
client = client,
crd = ExecutionCRD::class.java,
crdList = BenchmarkExecutionList::class.java
@@ -24,13 +24,13 @@ class ExecutionStateHandler(val client: KubernetesClient):
private fun getDurationLambda() = { cr: ExecutionCRD -> cr.status.executionDuration }
fun setExecutionState(resourceName: String, status: States): Boolean {
- setState(resourceName) {cr -> cr.status.executionState = status.value; cr}
+ setState(resourceName) { cr -> cr.status.executionState = status.value; cr }
return blockUntilStateIsSet(resourceName, status.value, getExecutionLambda())
}
- fun getExecutionState(resourceName: String) : States {
+ fun getExecutionState(resourceName: String): States {
val status = this.getState(resourceName, getExecutionLambda())
- return if(status.isNullOrBlank()){
+ return if (status.isNullOrBlank()) {
States.NO_STATE
} else {
States.values().first { it.value == status }
diff --git a/theodolite/src/main/kotlin/theodolite/execution/operator/LeaderElector.kt b/theodolite/src/main/kotlin/theodolite/execution/operator/LeaderElector.kt
index 9d093e4851e5c43d29a3fea3057ccf01be612e63..1ce94c2fdd1ce13d50a21e01b9d4692c87d0da6f 100644
--- a/theodolite/src/main/kotlin/theodolite/execution/operator/LeaderElector.kt
+++ b/theodolite/src/main/kotlin/theodolite/execution/operator/LeaderElector.kt
@@ -13,14 +13,15 @@ import kotlin.reflect.KFunction0
private val logger = KotlinLogging.logger {}
class LeaderElector(
- val client: NamespacedKubernetesClient,
+ val client: NamespacedKubernetesClient,
val name: String
- ) {
+) {
+ // TODO(what is the name of the lock? .withName() or LeaseLock(..,name..) ?)
fun getLeadership(leader: KFunction0<Unit>) {
val lockIdentity: String = UUID.randomUUID().toString()
- DefaultKubernetesClient().use { kc ->
- kc.leaderElector()
+ DefaultKubernetesClient().use { kc ->
+ kc.leaderElector()
.withConfig(
LeaderElectionConfigBuilder()
.withName("Theodolite")
@@ -29,10 +30,10 @@ class LeaderElector(
.withRenewDeadline(Duration.ofSeconds(10L))
.withRetryPeriod(Duration.ofSeconds(2L))
.withLeaderCallbacks(LeaderCallbacks(
- { Thread{leader()}.start() },
+ { Thread { leader() }.start() },
{ logger.info { "STOPPED LEADERSHIP" } }
) { newLeader: String? ->
- logger.info { "New leader elected $newLeader" }
+ logger.info { "New leader elected $newLeader" }
})
.build()
)
diff --git a/theodolite/src/main/kotlin/theodolite/execution/operator/StateHandler.kt b/theodolite/src/main/kotlin/theodolite/execution/operator/StateHandler.kt
index cefcf2ec97986375205205fd95ddcd2ff7eacf5a..e2cfaa354443cdc940abf92ef2c7474d028daecf 100644
--- a/theodolite/src/main/kotlin/theodolite/execution/operator/StateHandler.kt
+++ b/theodolite/src/main/kotlin/theodolite/execution/operator/StateHandler.kt
@@ -9,6 +9,7 @@ interface StateHandler<T> {
resourceName: String,
desiredStatusString: String,
f: (T) -> String?,
- maxTries: Int = MAX_TRIES): Boolean
+ maxTries: Int = MAX_TRIES
+ ): Boolean
}
\ No newline at end of file
diff --git a/theodolite/src/main/kotlin/theodolite/execution/operator/TheodoliteController.kt b/theodolite/src/main/kotlin/theodolite/execution/operator/TheodoliteController.kt
index 6bcff9aeed2e756f115f27fbf25cc2aa35230d2d..eb51a445ee05f1811a8780ff64c570f0bbdff4d0 100644
--- a/theodolite/src/main/kotlin/theodolite/execution/operator/TheodoliteController.kt
+++ b/theodolite/src/main/kotlin/theodolite/execution/operator/TheodoliteController.kt
@@ -7,11 +7,15 @@ import theodolite.benchmark.BenchmarkExecution
import theodolite.benchmark.KubernetesBenchmark
import theodolite.execution.TheodoliteExecutor
import theodolite.model.crd.*
-import theodolite.util.ConfigurationOverride
-import theodolite.util.PatcherDefinition
+import theodolite.patcher.ConfigOverrideModifier
+import theodolite.util.ExecutionStateComparator
import java.lang.Thread.sleep
private val logger = KotlinLogging.logger {}
+const val DEPLOYED_FOR_EXECUTION_LABEL_NAME = "deployed-for-execution"
+const val DEPLOYED_FOR_BENCHMARK_LABEL_NAME = "deployed-for-benchmark"
+const val CREATED_BY_LABEL_NAME = "app.kubernetes.io/created-by"
+const val CREATED_BY_LABEL_VALUE = "theodolite"
/**
* The controller implementation for Theodolite.
@@ -22,12 +26,12 @@ private val logger = KotlinLogging.logger {}
*/
class TheodoliteController(
- val path: String,
private val executionCRDClient: MixedOperation<ExecutionCRD, BenchmarkExecutionList, Resource<ExecutionCRD>>,
private val benchmarkCRDClient: MixedOperation<BenchmarkCRD, KubernetesBenchmarkList, Resource<BenchmarkCRD>>,
private val executionStateHandler: ExecutionStateHandler
) {
lateinit var executor: TheodoliteExecutor
+
/**
*
* Runs the TheodoliteController forever.
@@ -61,18 +65,22 @@ class TheodoliteController(
* @see BenchmarkExecution
*/
private fun runExecution(execution: BenchmarkExecution, benchmark: KubernetesBenchmark) {
- setAdditionalLabels(execution.name,
- "deployed-for-execution",
- benchmark.appResource + benchmark.loadGenResource,
- execution)
- setAdditionalLabels(benchmark.name,
- "deployed-for-benchmark",
- benchmark.appResource + benchmark.loadGenResource,
- execution)
- setAdditionalLabels("theodolite",
- "app.kubernetes.io/created-by",
- benchmark.appResource + benchmark.loadGenResource,
- execution)
+ val modifier = ConfigOverrideModifier(
+ execution = execution,
+ resources = benchmark.appResource + benchmark.loadGenResource
+ )
+ modifier.setAdditionalLabels(
+ labelValue = execution.name,
+ labelName = DEPLOYED_FOR_EXECUTION_LABEL_NAME
+ )
+ modifier.setAdditionalLabels(
+ labelValue = benchmark.name,
+ labelName = DEPLOYED_FOR_BENCHMARK_LABEL_NAME
+ )
+ modifier.setAdditionalLabels(
+ labelValue = CREATED_BY_LABEL_VALUE,
+ labelName = CREATED_BY_LABEL_NAME
+ )
executionStateHandler.setExecutionState(execution.name, States.RUNNING)
executionStateHandler.startDurationStateTimer(execution.name)
@@ -100,9 +108,6 @@ class TheodoliteController(
if (!::executor.isInitialized) return
if (restart) {
executionStateHandler.setExecutionState(this.executor.getExecution().name, States.RESTART)
- } else {
- executionStateHandler.setExecutionState(this.executor.getExecution().name, States.INTERRUPTED)
- logger.warn { "Execution ${executor.getExecution().name} unexpected interrupted" }
}
this.executor.executor.run.set(false)
}
@@ -114,9 +119,10 @@ class TheodoliteController(
return this.benchmarkCRDClient
.list()
.items
- .map { it.spec.name = it.metadata.name; it }
- .map { it.spec.path = path; it } // TODO check if we can remove the path field from the KubernetesBenchmark
- .map { it.spec }
+ .map {
+ it.spec.name = it.metadata.name
+ it.spec
+ }
}
/**
@@ -131,6 +137,7 @@ class TheodoliteController(
* @return the next execution or null
*/
private fun getNextExecution(): BenchmarkExecution? {
+ val comparator = ExecutionStateComparator(States.RESTART)
val availableBenchmarkNames = getBenchmarks()
.map { it.name }
@@ -144,46 +151,13 @@ class TheodoliteController(
it.status.executionState == States.RESTART.value
}
.filter { availableBenchmarkNames.contains(it.spec.benchmark) }
- .sortedWith(stateComparator().thenBy { it.metadata.creationTimestamp })
+ .sortedWith(comparator.thenBy { it.metadata.creationTimestamp })
.map { it.spec }
.firstOrNull()
}
- /**
- * Simple comparator which can be used to order a list of [ExecutionCRD] such that executions with
- * status [States.RESTART] are before all other executions.
- */
- private fun stateComparator() = Comparator<ExecutionCRD> { a, b ->
- when {
- (a == null && b == null) -> 0
- (a.status.executionState == States.RESTART.value) -> -1
- else -> 1
- }
- }
-
fun isExecutionRunning(executionName: String): Boolean {
if (!::executor.isInitialized) return false
return this.executor.getExecution().name == executionName
}
-
- private fun setAdditionalLabels(
- labelValue: String,
- labelName: String,
- resources: List<String>,
- execution: BenchmarkExecution
- ) {
- val additionalConfigOverrides = mutableListOf<ConfigurationOverride>()
- resources.forEach {
- run {
- val configurationOverride = ConfigurationOverride()
- configurationOverride.patcher = PatcherDefinition()
- configurationOverride.patcher.type = "LabelPatcher"
- configurationOverride.patcher.properties = mutableMapOf("variableName" to labelName)
- configurationOverride.patcher.resource = it
- configurationOverride.value = labelValue
- additionalConfigOverrides.add(configurationOverride)
- }
- }
- execution.configOverrides.addAll(additionalConfigOverrides)
- }
}
\ No newline at end of file
diff --git a/theodolite/src/main/kotlin/theodolite/execution/operator/TheodoliteOperator.kt b/theodolite/src/main/kotlin/theodolite/execution/operator/TheodoliteOperator.kt
index 5318abc174d9204c19466729698a63f2ad29a54c..2aaba77c03884d94c4d5745db270e84324482878 100644
--- a/theodolite/src/main/kotlin/theodolite/execution/operator/TheodoliteOperator.kt
+++ b/theodolite/src/main/kotlin/theodolite/execution/operator/TheodoliteOperator.kt
@@ -28,7 +28,6 @@ private val logger = KotlinLogging.logger {}
*/
class TheodoliteOperator {
private val namespace = System.getenv("NAMESPACE") ?: DEFAULT_NAMESPACE
- private val appResource = System.getenv("THEODOLITE_APP_RESOURCES") ?: "./config"
private val client: NamespacedKubernetesClient = DefaultKubernetesClient().inNamespace(namespace)
private lateinit var controller: TheodoliteController
@@ -38,7 +37,7 @@ class TheodoliteOperator {
fun start() {
LeaderElector(
client = client,
- name = "theodolite-operator"
+ name = "theodolite-operator" // TODO(make leaslock name configurable via env var)
)
.getLeadership(::startOperator)
}
@@ -76,7 +75,10 @@ class TheodoliteOperator {
}
}
- fun getExecutionEventHandler(controller: TheodoliteController, client: NamespacedKubernetesClient): SharedInformerFactory {
+ fun getExecutionEventHandler(
+ controller: TheodoliteController,
+ client: NamespacedKubernetesClient
+ ): SharedInformerFactory {
val factory = client.informers()
.inNamespace(client.namespace)
@@ -105,7 +107,6 @@ class TheodoliteOperator {
): TheodoliteController {
if (!::controller.isInitialized) {
this.controller = TheodoliteController(
- path = this.appResource,
benchmarkCRDClient = getBenchmarkClient(client),
executionCRDClient = getExecutionClient(client),
executionStateHandler = executionStateHandler
diff --git a/theodolite/src/main/kotlin/theodolite/k8s/CustomResourceWrapper.kt b/theodolite/src/main/kotlin/theodolite/k8s/CustomResourceWrapper.kt
index 31a95be04e3290e0797dca5c588394ea36279b0c..ab355677ec53216072fb58a170610aa5f12dba28 100644
--- a/theodolite/src/main/kotlin/theodolite/k8s/CustomResourceWrapper.kt
+++ b/theodolite/src/main/kotlin/theodolite/k8s/CustomResourceWrapper.kt
@@ -7,7 +7,10 @@ import mu.KotlinLogging
private val logger = KotlinLogging.logger {}
-class CustomResourceWrapper(val crAsMap: Map<String, String>, private val context: CustomResourceDefinitionContext) : KubernetesResource {
+class CustomResourceWrapper(
+ private val crAsMap: Map<String, String>,
+ private val context: CustomResourceDefinitionContext
+) : KubernetesResource {
/**
* Deploy a service monitor
*
@@ -41,9 +44,4 @@ class CustomResourceWrapper(val crAsMap: Map<String, String>, private val contex
val metadataAsMap = this.crAsMap["metadata"]!! as Map<String, String>
return metadataAsMap["name"]!!
}
-
- fun getLabels(): Map<String, String>{
- val metadataAsMap = this.crAsMap["metadata"]!! as Map<String, String>
- return metadataAsMap["labels"]!! as Map<String, String>
- }
}
diff --git a/theodolite/src/main/kotlin/theodolite/k8s/K8sManager.kt b/theodolite/src/main/kotlin/theodolite/k8s/K8sManager.kt
index 77350868500ffa974ab2b9fadfb8cfd915c8aaf2..389d5eefad556df502c218862e2f253ef8ad2100 100644
--- a/theodolite/src/main/kotlin/theodolite/k8s/K8sManager.kt
+++ b/theodolite/src/main/kotlin/theodolite/k8s/K8sManager.kt
@@ -43,9 +43,11 @@ class K8sManager(private val client: NamespacedKubernetesClient) {
fun remove(resource: KubernetesResource) {
when (resource) {
is Deployment -> {
- val label = resource.spec.selector.matchLabels["app"]!!
this.client.apps().deployments().delete(resource)
- blockUntilPodsDeleted(label)
+ ResourceByLabelHandler(client = client)
+ .blockUntilPodsDeleted(
+ matchLabels = resource.spec.selector.matchLabels
+ )
logger.info { "Deployment '${resource.metadata.name}' deleted." }
}
is Service ->
@@ -53,21 +55,15 @@ class K8sManager(private val client: NamespacedKubernetesClient) {
is ConfigMap ->
this.client.configMaps().delete(resource)
is StatefulSet -> {
- val label = resource.spec.selector.matchLabels["app"]!!
this.client.apps().statefulSets().delete(resource)
- blockUntilPodsDeleted(label)
+ ResourceByLabelHandler(client = client)
+ .blockUntilPodsDeleted(
+ matchLabels = resource.spec.selector.matchLabels
+ )
logger.info { "StatefulSet '$resource.metadata.name' deleted." }
}
is CustomResourceWrapper -> resource.delete(client)
else -> throw IllegalArgumentException("Unknown Kubernetes resource.")
}
}
-
- private fun blockUntilPodsDeleted(podLabel: String) {
- while (!this.client.pods().withLabel(podLabel).list().items.isNullOrEmpty()) {
- logger.info { "Wait for pods with label '$podLabel' to be deleted." }
- Thread.sleep(1000)
- }
- }
-
}
diff --git a/theodolite/src/main/kotlin/theodolite/k8s/K8sResourceLoader.kt b/theodolite/src/main/kotlin/theodolite/k8s/K8sResourceLoader.kt
index ab4bef3eaa0d93032ab9edacb510ba1b750e2dd6..faae5ade28deb579df6a463007cbdfbc9cc7706e 100644
--- a/theodolite/src/main/kotlin/theodolite/k8s/K8sResourceLoader.kt
+++ b/theodolite/src/main/kotlin/theodolite/k8s/K8sResourceLoader.kt
@@ -24,7 +24,7 @@ class K8sResourceLoader(private val client: NamespacedKubernetesClient) {
* @return Service from fabric8
*/
private fun loadService(path: String): Service {
- return loadGenericResource(path) { x: String -> client.services().load(x).get() }
+ return loadGenericResource(path) { client.services().load(it).get() }
}
@@ -34,17 +34,20 @@ class K8sResourceLoader(private val client: NamespacedKubernetesClient) {
* @param context specific crd context for this custom resource
* @return CustomResourceWrapper from fabric8
*/
- private fun loadCustomResourceWrapper(path: String, context: CustomResourceDefinitionContext): CustomResourceWrapper {
- return loadGenericResource(path) {
- CustomResourceWrapper(
- YamlParser().parse(
- path,
- HashMap<String, String>()::class.java
- )!!,
- context
- )
- }
- }
+ private fun loadCustomResourceWrapper(
+ path: String,
+ context: CustomResourceDefinitionContext
+ ): CustomResourceWrapper {
+ return loadGenericResource(path) {
+ CustomResourceWrapper(
+ YamlParser().parse(
+ path,
+ HashMap<String, String>()::class.java
+ )!!,
+ context
+ )
+ }
+ }
private fun loadServiceMonitor(path: String): CustomResourceWrapper {
val context = K8sContextFactory().create(
@@ -83,7 +86,7 @@ class K8sResourceLoader(private val client: NamespacedKubernetesClient) {
* @return Deployment from fabric8
*/
private fun loadDeployment(path: String): Deployment {
- return loadGenericResource(path) { x: String -> client.apps().deployments().load(x).get() }
+ return loadGenericResource(path) { client.apps().deployments().load(it).get() }
}
/**
@@ -92,7 +95,7 @@ class K8sResourceLoader(private val client: NamespacedKubernetesClient) {
* @return ConfigMap from fabric8
*/
private fun loadConfigmap(path: String): ConfigMap {
- return loadGenericResource(path) { x: String -> client.configMaps().load(x).get() }
+ return loadGenericResource(path) { client.configMaps().load(it).get() }
}
/**
@@ -101,7 +104,7 @@ class K8sResourceLoader(private val client: NamespacedKubernetesClient) {
* @return StatefulSet from fabric8
*/
private fun loadStatefulSet(path: String): KubernetesResource {
- return loadGenericResource(path) { x: String -> client.apps().statefulSets().load(x).get() }
+ return loadGenericResource(path) { client.apps().statefulSets().load(it).get() }
}
@@ -117,7 +120,7 @@ class K8sResourceLoader(private val client: NamespacedKubernetesClient) {
try {
resource = f(path)
} catch (e: Exception) {
- logger.warn { "You potentially misspelled the path: $path" }
+ logger.warn { "You potentially misspelled the path: $path" }
logger.warn { e }
}
@@ -146,7 +149,7 @@ class K8sResourceLoader(private val client: NamespacedKubernetesClient) {
"Benchmark" -> loadBenchmark(path)
else -> {
logger.error { "Error during loading of unspecified resource Kind" }
- throw java.lang.IllegalArgumentException("error while loading resource with kind: $kind")
+ throw IllegalArgumentException("error while loading resource with kind: $kind")
}
}
}
diff --git a/theodolite/src/main/kotlin/theodolite/k8s/ResourceByLabelHandler.kt b/theodolite/src/main/kotlin/theodolite/k8s/ResourceByLabelHandler.kt
new file mode 100644
index 0000000000000000000000000000000000000000..28a72c8947bffe7b57203cacf2460d7080fa7b51
--- /dev/null
+++ b/theodolite/src/main/kotlin/theodolite/k8s/ResourceByLabelHandler.kt
@@ -0,0 +1,115 @@
+package theodolite.k8s
+
+import io.fabric8.kubernetes.client.NamespacedKubernetesClient
+import io.fabric8.kubernetes.client.dsl.base.CustomResourceDefinitionContext
+import mu.KotlinLogging
+import org.json.JSONObject
+
+private val logger = KotlinLogging.logger {}
+
+/**
+ * The ResourceByLabelHandler provides basic functions to manage Kubernetes resources through their labels.
+ * @param client NamespacedKubernetesClient used for the deletion.
+ */
+class ResourceByLabelHandler(private val client: NamespacedKubernetesClient) {
+
+ /**
+ * Deletes all pods with the selected label.
+ * @param [labelName] the label name
+ * @param [labelValue] the value of this label
+ */
+ fun removePods(labelName: String, labelValue: String) {
+ this.client
+ .pods()
+ .withLabel("$labelName=$labelValue").delete()
+ logger.info { "Pod with label: $labelName=$labelValue deleted" }
+ }
+
+ /**
+ * Deletes all services with the selected label.
+ * @param [labelName] the label name
+ * @param [labelValue] the value of this label
+ */
+ fun removeServices(labelName: String, labelValue: String) {
+ this.client
+ .services()
+ .withLabel("$labelName=$labelValue")
+ .delete()
+ }
+
+ /**
+ * Deletes all deployments with the selected label.
+ * @param [labelName] the label name
+ * @param [labelValue] the value of this label
+ */
+ fun removeDeployments(labelName: String, labelValue: String) {
+ this.client
+ .apps()
+ .deployments()
+ .withLabel("$labelName=$labelValue")
+ .delete()
+
+ }
+
+ /**
+ * Deletes all stateful sets with the selected label.
+ * @param [labelName] the label name
+ * @param [labelValue] the value of this label
+ */
+ fun removeStatefulSets(labelName: String, labelValue: String) {
+ this.client
+ .apps()
+ .statefulSets()
+ .withLabel("$labelName=$labelValue")
+ .delete()
+ }
+
+ /**
+ * Deletes all configmaps with the selected label.
+ * @param [labelName] the label name
+ * @param [labelValue] the value of this label
+ */
+ fun removeConfigMaps(labelName: String, labelValue: String) {
+ this.client
+ .configMaps()
+ .withLabel("$labelName=$labelValue")
+ .delete()
+ }
+
+ /**
+ * Deletes all custom resources sets with the selected label.
+ * @param [labelName] the label name
+ * @param [labelValue] the value of this label
+ */
+ fun removeCR(labelName: String, labelValue: String, context: CustomResourceDefinitionContext) {
+ val customResources = JSONObject(
+ this.client.customResource(context)
+ .list(client.namespace, mapOf(Pair(labelName, labelValue)))
+ )
+ .getJSONArray("items")
+
+ (0 until customResources.length())
+ .map { customResources.getJSONObject(it).getJSONObject("metadata").getString("name") }
+ .forEach { this.client.customResource(context).delete(client.namespace, it) }
+ }
+
+ /**
+ * Block until all pods with are deleted
+ *
+ * @param [labelName] the label name
+ * @param [labelValue] the value of this label
+ * */
+ fun blockUntilPodsDeleted(matchLabels: MutableMap<String, String>) {
+ while (
+ !this.client
+ .pods()
+ .withLabels(matchLabels)
+ .list()
+ .items
+ .isNullOrEmpty()
+ ) {
+ logger.info { "Wait for pods with label ${matchLabels.toString()} to be deleted." }
+ Thread.sleep(1000)
+ }
+ }
+}
diff --git a/theodolite/src/main/kotlin/theodolite/k8s/TopicManager.kt b/theodolite/src/main/kotlin/theodolite/k8s/TopicManager.kt
index 3bbae82d77dc5b01a5827c7ee713bf2566be1bab..8e83883fc881db0f7e2b1b75b2fb7c7322a11a00 100644
--- a/theodolite/src/main/kotlin/theodolite/k8s/TopicManager.kt
+++ b/theodolite/src/main/kotlin/theodolite/k8s/TopicManager.kt
@@ -4,7 +4,6 @@ import mu.KotlinLogging
import org.apache.kafka.clients.admin.AdminClient
import org.apache.kafka.clients.admin.CreateTopicsResult
import org.apache.kafka.clients.admin.NewTopic
-import org.apache.kafka.common.errors.TopicExistsException
import java.lang.Thread.sleep
private val logger = KotlinLogging.logger {}
@@ -35,7 +34,7 @@ class TopicManager(private val kafkaConfig: Map<String, Any>) {
logger.debug { e } // TODO remove due to attached exception to warn log?
logger.info { "Remove existing topics." }
delete(newTopics.map { topic -> topic.name() }, kafkaAdmin)
- logger.info { "Will retry the topic creation in ${RETRY_TIME/1000} seconds." }
+ logger.info { "Will retry the topic creation in ${RETRY_TIME / 1000} seconds." }
sleep(RETRY_TIME)
retryCreation = true
}
@@ -104,7 +103,7 @@ class TopicManager(private val kafkaConfig: Map<String, Any>) {
if (toDelete.isNullOrEmpty()) {
deleted = true
} else {
- logger.info { "Deletion of Kafka topics failed, will retry in ${RETRY_TIME/1000} seconds." }
+ logger.info { "Deletion of Kafka topics failed, will retry in ${RETRY_TIME / 1000} seconds." }
sleep(RETRY_TIME)
}
}
diff --git a/theodolite/src/main/kotlin/theodolite/model/crd/ExecutionStatus.kt b/theodolite/src/main/kotlin/theodolite/model/crd/ExecutionStatus.kt
index 51b76fcee8fb35c83dca407691833dbb235b29c5..252738959762aa5d0732babc5589c698d7bd4e9f 100644
--- a/theodolite/src/main/kotlin/theodolite/model/crd/ExecutionStatus.kt
+++ b/theodolite/src/main/kotlin/theodolite/model/crd/ExecutionStatus.kt
@@ -5,7 +5,7 @@ import io.fabric8.kubernetes.api.model.KubernetesResource
import io.fabric8.kubernetes.api.model.Namespaced
@JsonDeserialize
-class ExecutionStatus(): KubernetesResource, Namespaced {
+class ExecutionStatus : KubernetesResource, Namespaced {
var executionState: String = ""
var executionDuration: String = "-"
}
\ No newline at end of file
diff --git a/theodolite/src/main/kotlin/theodolite/patcher/ConfigOverrideModifier.kt b/theodolite/src/main/kotlin/theodolite/patcher/ConfigOverrideModifier.kt
new file mode 100644
index 0000000000000000000000000000000000000000..8f77b1b95f3bf5cc9422cda55cb261048cebaeb6
--- /dev/null
+++ b/theodolite/src/main/kotlin/theodolite/patcher/ConfigOverrideModifier.kt
@@ -0,0 +1,39 @@
+package theodolite.patcher
+
+import theodolite.benchmark.BenchmarkExecution
+import theodolite.util.ConfigurationOverride
+import theodolite.util.PatcherDefinition
+
+/**
+ * The ConfigOverrideModifier makes it possible to update the configuration overrides of an execution.
+ *
+ * @property execution execution for which the config overrides should be updated
+ * @property resources list of all resources that should be updated.
+ */
+class ConfigOverrideModifier(val execution: BenchmarkExecution, val resources: List<String>) {
+
+ /**
+ * Adds a [LabelPatcher] to the configOverrides.
+ *
+ * @param labelValue value argument for the label patcher
+ * @param labelName label name argument for the label patcher
+ */
+ fun setAdditionalLabels(
+ labelValue: String,
+ labelName: String
+ ) {
+ val additionalConfigOverrides = mutableListOf<ConfigurationOverride>()
+ resources.forEach {
+ run {
+ val configurationOverride = ConfigurationOverride()
+ configurationOverride.patcher = PatcherDefinition()
+ configurationOverride.patcher.type = "LabelPatcher"
+ configurationOverride.patcher.properties = mutableMapOf("variableName" to labelName)
+ configurationOverride.patcher.resource = it
+ configurationOverride.value = labelValue
+ additionalConfigOverrides.add(configurationOverride)
+ }
+ }
+ execution.configOverrides.addAll(additionalConfigOverrides)
+ }
+}
\ No newline at end of file
diff --git a/theodolite/src/main/kotlin/theodolite/patcher/DataVolumeLoadGeneratorReplicaPatcher.kt b/theodolite/src/main/kotlin/theodolite/patcher/DataVolumeLoadGeneratorReplicaPatcher.kt
new file mode 100644
index 0000000000000000000000000000000000000000..bdc107910edc8ddfb41e7757c775977086a25a26
--- /dev/null
+++ b/theodolite/src/main/kotlin/theodolite/patcher/DataVolumeLoadGeneratorReplicaPatcher.kt
@@ -0,0 +1,38 @@
+package theodolite.patcher
+
+import io.fabric8.kubernetes.api.model.KubernetesResource
+
+/**
+ * The DataVolumeLoadGeneratorReplicaPatcher takes the total load that should be generated
+ * and computes the number of instances needed for this load based on the `maxVolume`
+ * ((load + maxVolume - 1) / maxVolume) and calculates the load per instance
+ * (loadPerInstance = load / instances).
+ * The number of instances are set for the load generator and the given variable is set to the
+ * load per instance.
+ *
+ * @property k8sResource Kubernetes resource to be patched.
+ * @property maxVolume per load generator instance
+ * @property container Container to be patched.
+ * @property variableName Name of the environment variable to be patched.
+ */
+class DataVolumeLoadGeneratorReplicaPatcher(
+ k8sResource: KubernetesResource,
+ private val maxVolume: Int,
+ container: String,
+ variableName: String
+) : AbstractPatcher(k8sResource) {
+
+ private val replicaPatcher = ReplicaPatcher(k8sResource)
+ private val envVarPatcher = EnvVarPatcher(k8sResource, container, variableName)
+
+ override fun <T> patch(value: T) {
+ // calculate number of load generator instances and load per instance
+ val load = Integer.parseInt(value.toString())
+ val loadGenInstances = (load + maxVolume - 1) / maxVolume
+ val loadPerInstance = load / loadGenInstances
+
+ // Patch instance values and load value of generators
+ replicaPatcher.patch(loadGenInstances.toString())
+ envVarPatcher.patch(loadPerInstance.toString())
+ }
+}
diff --git a/theodolite/src/main/kotlin/theodolite/patcher/LabelPatcher.kt b/theodolite/src/main/kotlin/theodolite/patcher/LabelPatcher.kt
index 4fa7fc893cfaf864d935074ff50af8d61f7aac76..2f8c703afa9e826a79f0785abef493d2d448ac74 100644
--- a/theodolite/src/main/kotlin/theodolite/patcher/LabelPatcher.kt
+++ b/theodolite/src/main/kotlin/theodolite/patcher/LabelPatcher.kt
@@ -11,34 +11,34 @@ class LabelPatcher(private val k8sResource: KubernetesResource, val variableName
AbstractPatcher(k8sResource) {
override fun <String> patch(labelValue: String) {
- if(labelValue is kotlin.String){
- when(k8sResource){
+ if (labelValue is kotlin.String) {
+ when (k8sResource) {
is Deployment -> {
- if (k8sResource.metadata.labels == null){
+ if (k8sResource.metadata.labels == null) {
k8sResource.metadata.labels = mutableMapOf()
}
k8sResource.metadata.labels[this.variableName] = labelValue
}
is StatefulSet -> {
- if (k8sResource.metadata.labels == null){
+ if (k8sResource.metadata.labels == null) {
k8sResource.metadata.labels = mutableMapOf()
}
k8sResource.metadata.labels[this.variableName] = labelValue
}
is Service -> {
- if (k8sResource.metadata.labels == null){
+ if (k8sResource.metadata.labels == null) {
k8sResource.metadata.labels = mutableMapOf()
}
k8sResource.metadata.labels[this.variableName] = labelValue
}
is ConfigMap -> {
- if (k8sResource.metadata.labels == null){
+ if (k8sResource.metadata.labels == null) {
k8sResource.metadata.labels = mutableMapOf()
}
k8sResource.metadata.labels[this.variableName] = labelValue
}
- is CustomResource<*,*> -> {
- if (k8sResource.metadata.labels == null){
+ is CustomResource<*, *> -> {
+ if (k8sResource.metadata.labels == null) {
k8sResource.metadata.labels = mutableMapOf()
}
k8sResource.metadata.labels[this.variableName] = labelValue
diff --git a/theodolite/src/main/kotlin/theodolite/patcher/NumNestedGroupsLoadGeneratorReplicaPatcher.kt b/theodolite/src/main/kotlin/theodolite/patcher/NumNestedGroupsLoadGeneratorReplicaPatcher.kt
index 65489a96974ad566fe7cbd88cf6ff7fb49135e1d..c617917e6894c3a30779dd4257a96365ded35481 100644
--- a/theodolite/src/main/kotlin/theodolite/patcher/NumNestedGroupsLoadGeneratorReplicaPatcher.kt
+++ b/theodolite/src/main/kotlin/theodolite/patcher/NumNestedGroupsLoadGeneratorReplicaPatcher.kt
@@ -8,13 +8,14 @@ class NumNestedGroupsLoadGeneratorReplicaPatcher(
private val k8sResource: KubernetesResource,
private val numSensors: String,
private val loadGenMaxRecords: String
- ) :
+) :
AbstractPatcher(k8sResource) {
override fun <String> patch(value: String) {
if (k8sResource is Deployment) {
if (value is kotlin.String) {
- val approxNumSensors = numSensors.toDouble().pow(Integer.parseInt(value).toDouble())
- val loadGenInstances = (approxNumSensors + loadGenMaxRecords.toDouble() - 1) / loadGenMaxRecords.toDouble()
+ val approxNumSensors = numSensors.toDouble().pow(Integer.parseInt(value).toDouble())
+ val loadGenInstances =
+ (approxNumSensors + loadGenMaxRecords.toDouble() - 1) / loadGenMaxRecords.toDouble()
this.k8sResource.spec.replicas = loadGenInstances.toInt()
}
}
diff --git a/theodolite/src/main/kotlin/theodolite/patcher/NumSensorsLoadGeneratorReplicaPatcher.kt b/theodolite/src/main/kotlin/theodolite/patcher/NumSensorsLoadGeneratorReplicaPatcher.kt
index f6a06324e36d7942d3944a492fee263f428376c1..86bb37db3cb9fd0d3bca1690d5eb4e622329a9bc 100644
--- a/theodolite/src/main/kotlin/theodolite/patcher/NumSensorsLoadGeneratorReplicaPatcher.kt
+++ b/theodolite/src/main/kotlin/theodolite/patcher/NumSensorsLoadGeneratorReplicaPatcher.kt
@@ -12,7 +12,8 @@ class NumSensorsLoadGeneratorReplicaPatcher(
override fun <String> patch(value: String) {
if (k8sResource is Deployment) {
if (value is kotlin.String) {
- val loadGenInstances = (Integer.parseInt(value) + loadGenMaxRecords.toInt() - 1) / loadGenMaxRecords.toInt()
+ val loadGenInstances =
+ (Integer.parseInt(value) + loadGenMaxRecords.toInt() - 1) / loadGenMaxRecords.toInt()
this.k8sResource.spec.replicas = loadGenInstances
}
}
diff --git a/theodolite/src/main/kotlin/theodolite/patcher/PatcherDefinitionFactory.kt b/theodolite/src/main/kotlin/theodolite/patcher/PatcherDefinitionFactory.kt
index d5a6f3821d2688651475625506a78efc6061ab82..6a1f993e2ac327ec242a8a5bafc3e6cc43475710 100644
--- a/theodolite/src/main/kotlin/theodolite/patcher/PatcherDefinitionFactory.kt
+++ b/theodolite/src/main/kotlin/theodolite/patcher/PatcherDefinitionFactory.kt
@@ -21,7 +21,7 @@ class PatcherDefinitionFactory {
* value of the requiredType.
*/
fun createPatcherDefinition(requiredType: String, patcherTypes: List<TypeName>): List<PatcherDefinition> {
- return patcherTypes.firstOrNull() { type -> type.typeName == requiredType }
+ return patcherTypes.firstOrNull { type -> type.typeName == requiredType }
?.patchers ?: throw IllegalArgumentException("typeName $requiredType not found.")
}
}
diff --git a/theodolite/src/main/kotlin/theodolite/patcher/PatcherFactory.kt b/theodolite/src/main/kotlin/theodolite/patcher/PatcherFactory.kt
index 45e50113c964d671962fadc718994a29b2da81f4..cdf115a0755e8794075884bf952db3b8d76f1f50 100644
--- a/theodolite/src/main/kotlin/theodolite/patcher/PatcherFactory.kt
+++ b/theodolite/src/main/kotlin/theodolite/patcher/PatcherFactory.kt
@@ -32,7 +32,7 @@ class PatcherFactory {
k8sResources.filter { it.first == patcherDefinition.resource }
.map { resource -> resource.second }
.firstOrNull()
- ?: throw DeploymentFailedException("Could not find resource ${patcherDefinition.resource}")
+ ?: throw InvalidPatcherConfigurationException("Could not find resource ${patcherDefinition.resource}")
return try {
when (patcherDefinition.type) {
@@ -41,44 +41,56 @@ class PatcherFactory {
)
"NumNestedGroupsLoadGeneratorReplicaPatcher" -> NumNestedGroupsLoadGeneratorReplicaPatcher(
k8sResource = resource,
- loadGenMaxRecords = patcherDefinition.properties["loadGenMaxRecords"] !!,
- numSensors = patcherDefinition.properties["numSensors"] !!
+ loadGenMaxRecords = patcherDefinition.properties["loadGenMaxRecords"]!!,
+ numSensors = patcherDefinition.properties["numSensors"]!!
)
"NumSensorsLoadGeneratorReplicaPatcher" -> NumSensorsLoadGeneratorReplicaPatcher(
k8sResource = resource,
- loadGenMaxRecords = patcherDefinition.properties["loadGenMaxRecords"] !!
+ loadGenMaxRecords = patcherDefinition.properties["loadGenMaxRecords"]!!
+ )
+ "DataVolumeLoadGeneratorReplicaPatcher" -> DataVolumeLoadGeneratorReplicaPatcher(
+ k8sResource = resource,
+ maxVolume = patcherDefinition.properties["maxVolume"]!!.toInt(),
+ container = patcherDefinition.properties["container"]!!,
+ variableName = patcherDefinition.properties["variableName"]!!
)
"EnvVarPatcher" -> EnvVarPatcher(
k8sResource = resource,
- container = patcherDefinition.properties["container"] !!,
- variableName = patcherDefinition.properties["variableName"] !!
+ container = patcherDefinition.properties["container"]!!,
+ variableName = patcherDefinition.properties["variableName"]!!
)
"NodeSelectorPatcher" -> NodeSelectorPatcher(
k8sResource = resource,
- variableName = patcherDefinition.properties["variableName"] !!
+ variableName = patcherDefinition.properties["variableName"]!!
)
"ResourceLimitPatcher" -> ResourceLimitPatcher(
k8sResource = resource,
- container = patcherDefinition.properties["container"] !!,
- limitedResource = patcherDefinition.properties["limitedResource"] !!
+ container = patcherDefinition.properties["container"]!!,
+ limitedResource = patcherDefinition.properties["limitedResource"]!!
)
"ResourceRequestPatcher" -> ResourceRequestPatcher(
k8sResource = resource,
- container = patcherDefinition.properties["container"] !!,
- requestedResource = patcherDefinition.properties["requestedResource"] !!
+ container = patcherDefinition.properties["container"]!!,
+ requestedResource = patcherDefinition.properties["requestedResource"]!!
)
"SchedulerNamePatcher" -> SchedulerNamePatcher(
k8sResource = resource
)
"LabelPatcher" -> LabelPatcher(
k8sResource = resource,
- variableName = patcherDefinition.properties["variableName"] !!
+ variableName = patcherDefinition.properties["variableName"]!!
+ )
+ "ImagePatcher" -> ImagePatcher(
+ k8sResource = resource,
+ container = patcherDefinition.properties["container"]!!
)
else -> throw InvalidPatcherConfigurationException("Patcher type ${patcherDefinition.type} not found.")
}
- } catch (e: Exception) {
- throw InvalidPatcherConfigurationException("Could not create patcher with type ${patcherDefinition.type}" +
- " Probably a required patcher argument was not specified." )
+ } catch (e: NullPointerException) {
+ throw InvalidPatcherConfigurationException(
+ "Could not create patcher with type ${patcherDefinition.type}" +
+ " Probably a required patcher argument was not specified.", e
+ )
}
}
}
diff --git a/theodolite/src/main/kotlin/theodolite/patcher/ResourceLimitPatcher.kt b/theodolite/src/main/kotlin/theodolite/patcher/ResourceLimitPatcher.kt
index 1a6fa35a944d00634ec0607b0bff34f4cb9d9b9c..9dcdffa0407dd4fdaf2d9b0a898bcdf6cebe5a8b 100644
--- a/theodolite/src/main/kotlin/theodolite/patcher/ResourceLimitPatcher.kt
+++ b/theodolite/src/main/kotlin/theodolite/patcher/ResourceLimitPatcher.kt
@@ -6,6 +6,7 @@ import io.fabric8.kubernetes.api.model.Quantity
import io.fabric8.kubernetes.api.model.ResourceRequirements
import io.fabric8.kubernetes.api.model.apps.Deployment
import io.fabric8.kubernetes.api.model.apps.StatefulSet
+import theodolite.util.InvalidPatcherConfigurationException
/**
* The Resource limit [Patcher] set resource limits for deployments and statefulSets.
@@ -33,7 +34,7 @@ class ResourceLimitPatcher(
}
}
else -> {
- throw IllegalArgumentException("ResourceLimitPatcher not applicable for $k8sResource")
+ throw InvalidPatcherConfigurationException("ResourceLimitPatcher not applicable for $k8sResource")
}
}
}
diff --git a/theodolite/src/main/kotlin/theodolite/patcher/ResourceRequestPatcher.kt b/theodolite/src/main/kotlin/theodolite/patcher/ResourceRequestPatcher.kt
index 9bf8c3c72f656d326ca3070cd5843778e5cdff42..24cdde40f7f78bd67d115b2dc44f47e180f51ee2 100644
--- a/theodolite/src/main/kotlin/theodolite/patcher/ResourceRequestPatcher.kt
+++ b/theodolite/src/main/kotlin/theodolite/patcher/ResourceRequestPatcher.kt
@@ -6,6 +6,7 @@ import io.fabric8.kubernetes.api.model.Quantity
import io.fabric8.kubernetes.api.model.ResourceRequirements
import io.fabric8.kubernetes.api.model.apps.Deployment
import io.fabric8.kubernetes.api.model.apps.StatefulSet
+import theodolite.util.InvalidPatcherConfigurationException
/**
* The Resource request [Patcher] set resource limits for deployments and statefulSets.
@@ -33,7 +34,7 @@ class ResourceRequestPatcher(
}
}
else -> {
- throw IllegalArgumentException("ResourceRequestPatcher not applicable for $k8sResource")
+ throw InvalidPatcherConfigurationException("ResourceRequestPatcher not applicable for $k8sResource")
}
}
}
diff --git a/theodolite/src/main/kotlin/theodolite/util/DeploymentFailedException.kt b/theodolite/src/main/kotlin/theodolite/util/DeploymentFailedException.kt
index 0e276d7de4e205a75eb309a71a793e70f7565ea4..639a7c86c641cbdcba361410cf5e25fa56dd795f 100644
--- a/theodolite/src/main/kotlin/theodolite/util/DeploymentFailedException.kt
+++ b/theodolite/src/main/kotlin/theodolite/util/DeploymentFailedException.kt
@@ -1,5 +1,4 @@
package theodolite.util
-class DeploymentFailedException(message:String): Exception(message) {
-}
\ No newline at end of file
+class DeploymentFailedException(message: String) : Exception(message)
\ No newline at end of file
diff --git a/theodolite/src/main/kotlin/theodolite/util/ExecutionStateComparator.kt b/theodolite/src/main/kotlin/theodolite/util/ExecutionStateComparator.kt
new file mode 100644
index 0000000000000000000000000000000000000000..66ebe12d6505296682744c10c69f182f07d1a16e
--- /dev/null
+++ b/theodolite/src/main/kotlin/theodolite/util/ExecutionStateComparator.kt
@@ -0,0 +1,19 @@
+package theodolite.util
+
+import theodolite.model.crd.ExecutionCRD
+import theodolite.model.crd.States
+
+class ExecutionStateComparator(private val preferredState: States): Comparator<ExecutionCRD> {
+
+ /**
+ * Simple comparator which can be used to order a list of [ExecutionCRD] such that executions with
+ * status [States.RESTART] are before all other executions.
+ */
+ override fun compare(p0: ExecutionCRD, p1: ExecutionCRD): Int {
+ return when {
+ (p0 == null && p1 == null) -> 0
+ (p0.status.executionState == preferredState.value) -> -1
+ else -> 1
+ }
+ }
+}
\ No newline at end of file
diff --git a/theodolite/src/main/kotlin/theodolite/util/IOHandler.kt b/theodolite/src/main/kotlin/theodolite/util/IOHandler.kt
index 8d379fcf0543257edafd2e45383a02ba0254563d..57032189412d0937e4d77ddbf4354c78ffcc71a3 100644
--- a/theodolite/src/main/kotlin/theodolite/util/IOHandler.kt
+++ b/theodolite/src/main/kotlin/theodolite/util/IOHandler.kt
@@ -22,7 +22,7 @@ class IOHandler {
var resultsFolder: String = System.getenv("RESULTS_FOLDER") ?: ""
val createResultsFolder = System.getenv("CREATE_RESULTS_FOLDER") ?: "false"
- if (resultsFolder != ""){
+ if (resultsFolder != "") {
logger.info { "RESULT_FOLDER: $resultsFolder" }
val directory = File(resultsFolder)
if (!directory.exists()) {
@@ -35,7 +35,7 @@ class IOHandler {
}
resultsFolder += "/"
}
- return resultsFolder
+ return resultsFolder
}
/**
@@ -70,9 +70,9 @@ class IOHandler {
fun writeToCSVFile(fileURL: String, data: List<List<String>>, columns: List<String>) {
val outputFile = File("$fileURL.csv")
PrintWriter(outputFile).use { pw ->
- pw.println(columns.joinToString(separator=","))
+ pw.println(columns.joinToString(separator = ","))
data.forEach {
- pw.println(it.joinToString(separator=","))
+ pw.println(it.joinToString(separator = ","))
}
}
logger.info { "Wrote CSV file: $fileURL to ${outputFile.absolutePath}." }
@@ -87,7 +87,7 @@ class IOHandler {
fun writeStringToTextFile(fileURL: String, data: String) {
val outputFile = File("$fileURL")
outputFile.printWriter().use {
- it.println(data)
+ it.println(data)
}
logger.info { "Wrote txt file: $fileURL to ${outputFile.absolutePath}." }
}
diff --git a/theodolite/src/main/kotlin/theodolite/util/InvalidPatcherConfigurationException.kt b/theodolite/src/main/kotlin/theodolite/util/InvalidPatcherConfigurationException.kt
index c103ef1f35a1b3ffa56dad50c7cf6c1db51eb57f..81ea227d0d9871c2420a414d81749a34b97676b8 100644
--- a/theodolite/src/main/kotlin/theodolite/util/InvalidPatcherConfigurationException.kt
+++ b/theodolite/src/main/kotlin/theodolite/util/InvalidPatcherConfigurationException.kt
@@ -1,5 +1,4 @@
package theodolite.util
-class InvalidPatcherConfigurationException(message:String): Exception(message) {
-}
+class InvalidPatcherConfigurationException(message: String, e: Exception? = null) : Exception(message, e)
diff --git a/theodolite/src/main/kotlin/theodolite/util/PrometheusResponse.kt b/theodolite/src/main/kotlin/theodolite/util/PrometheusResponse.kt
index 846577387c425e920da1c2fca1f972c880e1540a..bf33fcf6104645727a13b92cf3a13d36e04a10c6 100644
--- a/theodolite/src/main/kotlin/theodolite/util/PrometheusResponse.kt
+++ b/theodolite/src/main/kotlin/theodolite/util/PrometheusResponse.kt
@@ -17,8 +17,7 @@ data class PrometheusResponse(
* The data section of the query result contains the information about the resultType and the values itself.
*/
var data: PromData? = null
-)
-{
+) {
/**
* Return the data of the PrometheusResponse as [List] of [List]s of [String]s
* The format of the returned list is: `[[ group, timestamp, value ], [ group, timestamp, value ], ... ]`
diff --git a/theodolite/src/test/kotlin/theodolite/execution/operator/ControllerTest.kt b/theodolite/src/test/kotlin/theodolite/execution/operator/ControllerTest.kt
index 7ed868c7adc4afcd7a6a606d22124c92910ecd89..7350f564a71e2f0cbf640b782f5dbf19cbdc7ecb 100644
--- a/theodolite/src/test/kotlin/theodolite/execution/operator/ControllerTest.kt
+++ b/theodolite/src/test/kotlin/theodolite/execution/operator/ControllerTest.kt
@@ -72,33 +72,35 @@ class ControllerTest {
@Test
@DisplayName("Check namespaced property of benchmarkCRDClient")
- fun testBenchmarkClientNamespaced(){
+ fun testBenchmarkClientNamespaced() {
val method = controller
.javaClass
.getDeclaredMethod("getBenchmarks")
method.isAccessible = true
method.invoke(controller)
- assert(server
- .lastRequest
- .toString()
- .contains("namespaces")
+ assert(
+ server
+ .lastRequest
+ .toString()
+ .contains("namespaces")
)
}
@Test
@DisplayName("Check namespaced property of executionCRDClient")
- fun testExecutionClientNamespaced(){
+ fun testExecutionClientNamespaced() {
val method = controller
.javaClass
.getDeclaredMethod("getNextExecution")
method.isAccessible = true
method.invoke(controller)
- assert(server
- .lastRequest
- .toString()
- .contains("namespaces")
+ assert(
+ server
+ .lastRequest
+ .toString()
+ .contains("namespaces")
)
}
@@ -132,43 +134,4 @@ class ControllerTest {
gson.toJson(result)
)
}
-
- @Test
- fun setAdditionalLabelsTest() {
- val method = controller
- .javaClass
- .getDeclaredMethod(
- "setAdditionalLabels",
- String::class.java,
- String::class.java,
- List::class.java,
- BenchmarkExecution::class.java
- )
- method.isAccessible = true
-
- this.benchmark.appResource = listOf("test-resource.yaml")
-
- method.invoke(
- controller,
- "test-value",
- "test-name",
- this.benchmark.appResource,
- this.execution
- ) as BenchmarkExecution?
-
- assertEquals(
- "test-name",
- this.execution
- .configOverrides.firstOrNull()
- ?.patcher
- ?.properties
- ?.get("variableName")
- )
- assertEquals(
- "test-value",
- this.execution
- .configOverrides.firstOrNull()
- ?.value
- )
- }
}
\ No newline at end of file
diff --git a/theodolite/src/test/kotlin/theodolite/execution/operator/ExecutionCRDummy.kt b/theodolite/src/test/kotlin/theodolite/execution/operator/ExecutionCRDummy.kt
index 2589319299cfa29f95216033ddc806d002f38663..56d46279e8effe1f0b5bf307cd896ebd5b7eb2ee 100644
--- a/theodolite/src/test/kotlin/theodolite/execution/operator/ExecutionCRDummy.kt
+++ b/theodolite/src/test/kotlin/theodolite/execution/operator/ExecutionCRDummy.kt
@@ -15,6 +15,10 @@ class ExecutionCRDummy(name: String, benchmark: String) {
return this.executionCR
}
+ fun getStatus() : ExecutionStatus {
+ return this.executionState
+ }
+
init {
// configure metadata
executionCR.spec = execution
diff --git a/theodolite/src/test/kotlin/theodolite/execution/operator/ExecutionEventHandlerTest.kt b/theodolite/src/test/kotlin/theodolite/execution/operator/ExecutionEventHandlerTest.kt
index bbbb3092cd54a8f3313bb923be3682be50801f39..5598f48a2d291db4eab8563dd3325534f49b2eb6 100644
--- a/theodolite/src/test/kotlin/theodolite/execution/operator/ExecutionEventHandlerTest.kt
+++ b/theodolite/src/test/kotlin/theodolite/execution/operator/ExecutionEventHandlerTest.kt
@@ -39,7 +39,7 @@ class ExecutionEventHandlerTest {
executionStateHandler = ExecutionStateHandler(client = server.client)
)
- this.factory = operator.getExecutionEventHandler(this.controller,server.client)
+ this.factory = operator.getExecutionEventHandler(this.controller, server.client)
this.stateHandler = TheodoliteOperator().getExecutionStateHandler(client = server.client)
this.executionVersion1 = K8sResourceLoader(server.client)
@@ -66,10 +66,11 @@ class ExecutionEventHandlerTest {
factory.startAllRegisteredInformers()
server.lastRequest
// the second request must be namespaced (this is the first `GET` request)
- assert(server
- .lastRequest
- .toString()
- .contains("namespaces")
+ assert(
+ server
+ .lastRequest
+ .toString()
+ .contains("namespaces")
)
}
diff --git a/theodolite/src/test/kotlin/theodolite/execution/operator/StateHandlerTest.kt b/theodolite/src/test/kotlin/theodolite/execution/operator/StateHandlerTest.kt
index de74cf9ac87a8aca7db133a04ef3809c5e5087c2..24cb6c90d8ea222c90a398e12c7a50a2f6058a93 100644
--- a/theodolite/src/test/kotlin/theodolite/execution/operator/StateHandlerTest.kt
+++ b/theodolite/src/test/kotlin/theodolite/execution/operator/StateHandlerTest.kt
@@ -35,23 +35,24 @@ class StateHandlerTest {
fun namespacedTest() {
val handler = ExecutionStateHandler(client = server.client)
handler.getExecutionState("example-execution")
- assert(server
- .lastRequest
- .toString()
- .contains("namespaces")
+ assert(
+ server
+ .lastRequest
+ .toString()
+ .contains("namespaces")
)
}
@Test
@DisplayName("Test empty execution state")
- fun executionWithoutExecutionStatusTest(){
+ fun executionWithoutExecutionStatusTest() {
val handler = ExecutionStateHandler(client = server.client)
assertEquals(States.NO_STATE, handler.getExecutionState("example-execution"))
}
@Test
@DisplayName("Test empty duration state")
- fun executionWithoutDurationStatusTest(){
+ fun executionWithoutDurationStatusTest() {
val handler = ExecutionStateHandler(client = server.client)
assertEquals("-", handler.getDurationState("example-execution"))
}
diff --git a/theodolite/src/test/kotlin/theodolite/patcher/ConfigOverrideModifierTest.kt b/theodolite/src/test/kotlin/theodolite/patcher/ConfigOverrideModifierTest.kt
new file mode 100644
index 0000000000000000000000000000000000000000..739fadd3f02bfd1e60fd67e7afc695bf99e68d31
--- /dev/null
+++ b/theodolite/src/test/kotlin/theodolite/patcher/ConfigOverrideModifierTest.kt
@@ -0,0 +1,57 @@
+package theodolite.patcher
+
+import io.quarkus.test.junit.QuarkusTest
+import org.junit.jupiter.api.Assertions
+import org.junit.jupiter.api.BeforeEach
+import org.junit.jupiter.api.Test
+import theodolite.benchmark.BenchmarkExecution
+import theodolite.benchmark.KubernetesBenchmark
+import theodolite.execution.operator.BenchmarkCRDummy
+import theodolite.execution.operator.ExecutionCRDummy
+
+@QuarkusTest
+class ConfigOverrideModifierTest {
+ private var execution = BenchmarkExecution()
+ private var benchmark = KubernetesBenchmark()
+
+
+ @BeforeEach
+ fun setup() {
+ val execution1 = ExecutionCRDummy(name = "matching-execution", benchmark = "Test-Benchmark")
+ val benchmark1 = BenchmarkCRDummy(name = "Test-Benchmark")
+
+ this.execution = execution1.getCR().spec
+ this.benchmark = benchmark1.getCR().spec
+ }
+
+
+ @Test
+ fun setAdditionalLabelsTest() {
+ this.benchmark.appResource = listOf("test-resource.yaml")
+
+ val modifier = ConfigOverrideModifier(
+ execution = this.execution,
+ resources = this.benchmark.appResource
+ )
+
+ modifier.setAdditionalLabels(
+ labelName = "test-name",
+ labelValue = "test-value"
+ )
+
+ Assertions.assertEquals(
+ "test-name",
+ this.execution
+ .configOverrides.firstOrNull()
+ ?.patcher
+ ?.properties
+ ?.get("variableName")
+ )
+ Assertions.assertEquals(
+ "test-value",
+ this.execution
+ .configOverrides.firstOrNull()
+ ?.value
+ )
+ }
+}
\ No newline at end of file
diff --git a/theodolite/src/test/kotlin/theodolite/util/ExecutionStateComparatorTest.kt b/theodolite/src/test/kotlin/theodolite/util/ExecutionStateComparatorTest.kt
new file mode 100644
index 0000000000000000000000000000000000000000..eec029f3878171eb2fd502bf68f549cfce793f23
--- /dev/null
+++ b/theodolite/src/test/kotlin/theodolite/util/ExecutionStateComparatorTest.kt
@@ -0,0 +1,33 @@
+package theodolite.util
+
+import io.quarkus.test.junit.QuarkusTest
+import org.junit.Rule
+import org.junit.jupiter.api.Assertions
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.BeforeEach
+import org.junit.jupiter.api.Test
+import theodolite.execution.operator.ExecutionCRDummy
+import theodolite.model.crd.ExecutionCRD
+import theodolite.model.crd.States
+
+
+@QuarkusTest
+class ExecutionStateComparatorTest {
+
+ @Test
+ fun testCompare() {
+ val comparator = ExecutionStateComparator(States.RESTART)
+ val execution1 = ExecutionCRDummy("dummy1", "default-benchmark")
+ val execution2 = ExecutionCRDummy("dummy2", "default-benchmark")
+ execution1.getStatus().executionState = States.RESTART.value
+ execution2.getStatus().executionState = States.PENDING.value
+ val list = listOf(execution2.getCR(), execution1.getCR())
+
+
+ assertEquals(
+ list.reversed(),
+ list.sortedWith(comparator)
+ )
+ }
+
+}
\ No newline at end of file
diff --git a/theodolite/src/test/resources/k8s-resource-files/test-execution-update.yaml b/theodolite/src/test/resources/k8s-resource-files/test-execution-update.yaml
index 4ef4fdc007816bb492fbd90d6ddc516a2332cd5e..c075702da218397352f1dc1e5b283534fbb4d718 100644
--- a/theodolite/src/test/resources/k8s-resource-files/test-execution-update.yaml
+++ b/theodolite/src/test/resources/k8s-resource-files/test-execution-update.yaml
@@ -13,11 +13,12 @@ spec:
resourceValues: [1, 2, 3, 4, 5]
slos:
- sloType: "lag trend"
- threshold: 2000
prometheusUrl: "http://prometheus-operated:9090"
- externalSloUrl: "http://localhost:80/evaluate-slope"
offset: 0
- warmup: 60 # in seconds
+ properties:
+ threshold: 2000
+ externalSloUrl: "http://localhost:80/evaluate-slope"
+ warmup: 60 # in seconds
execution:
strategy: "LinearSearch"
duration: 300 # in seconds
diff --git a/theodolite/src/test/resources/k8s-resource-files/test-execution.yaml b/theodolite/src/test/resources/k8s-resource-files/test-execution.yaml
index 4d6ade6ae32b064fd45b3fa508a936645538a543..e12c851da5d8a79f57b1fa59b86239c219370c0f 100644
--- a/theodolite/src/test/resources/k8s-resource-files/test-execution.yaml
+++ b/theodolite/src/test/resources/k8s-resource-files/test-execution.yaml
@@ -13,11 +13,12 @@ spec:
resourceValues: [1, 2, 3, 4, 5]
slos:
- sloType: "lag trend"
- threshold: 2000
prometheusUrl: "http://prometheus-operated:9090"
- externalSloUrl: "http://localhost:80/evaluate-slope"
offset: 0
- warmup: 60 # in seconds
+ properties:
+ threshold: 2000
+ externalSloUrl: "http://localhost:80/evaluate-slope"
+ warmup: 60 # in seconds
execution:
strategy: "LinearSearch"
duration: 300 # in seconds