diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 433facfa99765228af9b753cf88378fc9d939a88..c4b8495224ad81ae2eebe81b0087319422da1969 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -319,7 +319,18 @@ test-slo-checker-lag-trend: tags: - exec-docker script: - - cd slope-evaluator + - cd slo-checker/record-lag + - pip install -r requirements.txt + - cd app + - python -m unittest + +test-slo-checker-dropped-records-kstreams: + stage: test + image: python:3.7-slim + tags: + - exec-docker + script: + - cd slo-checker/dropped-records - pip install -r requirements.txt - cd app - python -m unittest @@ -332,7 +343,7 @@ deploy-slo-checker-lag-trend: - test-slo-checker-lag-trend script: - DOCKER_TAG_NAME=$(echo $CI_COMMIT_REF_SLUG- | sed 's/^master-$//') - - docker build --pull -t theodolite-slo-checker-lag-trend slope-evaluator + - docker build --pull -t theodolite-slo-checker-lag-trend slo-checker/record-lag - "[ ! $CI_COMMIT_TAG ] && docker tag theodolite-slo-checker-lag-trend $CR_HOST/$CR_ORG/theodolite-slo-checker-lag-trend:${DOCKER_TAG_NAME}latest" - "[ $CI_COMMIT_TAG ] && docker tag theodolite-slo-checker-lag-trend $CR_HOST/$CR_ORG/theodolite-slo-checker-lag-trend:$CI_COMMIT_TAG" - echo $CR_PW | docker login $CR_HOST -u $CR_USER --password-stdin @@ -342,7 +353,32 @@ deploy-slo-checker-lag-trend: - if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW && $CI_COMMIT_TAG" when: always - changes: - - slope-evaluator/**/* + - slo-checker/record-lag/**/* + if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW" + when: always + - if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW" + when: manual + allow_failure: true + +deploy-slo-checker-dropped-records-kstreams: + stage: deploy + extends: + - .dind + needs: + - test-slo-checker-dropped-records-kstreams + script: + - DOCKER_TAG_NAME=$(echo $CI_COMMIT_REF_SLUG- | sed 's/^master-$//') + - docker build --pull -t theodolite-slo-checker-dropped-records-kstreams slo-checker/dropped-records + - "[ ! $CI_COMMIT_TAG ] && docker tag theodolite-slo-checker-dropped-records-kstreams $CR_HOST/$CR_ORG/theodolite-slo-checker-dropped-records-kstreams:${DOCKER_TAG_NAME}latest" + - "[ $CI_COMMIT_TAG ] && docker tag theodolite-slo-checker-dropped-records-kstreams $CR_HOST/$CR_ORG/theodolite-slo-checker-dropped-records-kstreams:$CI_COMMIT_TAG" + - echo $CR_PW | docker login $CR_HOST -u $CR_USER --password-stdin + - docker push $CR_HOST/$CR_ORG/theodolite-slo-checker-dropped-records-kstreams + - docker logout + rules: + - if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW && $CI_COMMIT_TAG" + when: always + - changes: + - slo-checker/dropped-records/**/* if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW" when: always - if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW" diff --git a/CITATION.cff b/CITATION.cff index 52e6e13286c0ba0aca34005a4d245d73b9869874..ca94e1c5039d3aeac3a4535767d5217de4960a6f 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -8,7 +8,7 @@ authors: given-names: Wilhelm orcid: "https://orcid.org/0000-0001-6625-4335" title: Theodolite -version: "0.4.0" +version: "0.5.1" repository-code: "https://github.com/cau-se/theodolite" license: "Apache-2.0" doi: "10.1016/j.bdr.2021.100209" diff --git a/README.md b/README.md index f2673f4b9ed0c46987963f8b455e19def802db79..804a193df21f3883ecf9a727af5a743b77a9cceb 100644 --- a/README.md +++ b/README.md @@ -4,20 +4,17 @@ Theodolite is a framework for benchmarking the horizontal and vertical scalability of stream processing engines. It consists of three modules: -## Theodolite Benchmarks - -Theodolite contains 4 application benchmarks, which are based on typical use cases for stream processing within microservices. For each benchmark, a corresponding workload generator is provided. Currently, this repository provides benchmark implementations for Apache Kafka Streams and Apache Flink. The benchmark sources can be found in [Thedolite benchmarks](benchmarks). - - -## Theodolite Execution Framework - -Theodolite aims to benchmark scalability of stream processing engines for real use cases. Microservices that apply stream processing techniques are usually deployed in elastic cloud environments. Hence, Theodolite's cloud-native benchmarking framework deploys its components in a cloud environment, orchestrated by Kubernetes. More information on how to execute scalability benchmarks can be found in [Thedolite execution framework](execution). +## Theodolite Benchmarking Tool +Theodolite aims to benchmark scalability of stream processing engines for real use cases. Microservices that apply stream processing techniques are usually deployed in elastic cloud environments. Hence, Theodolite's cloud-native benchmarking framework deploys its components in a cloud environment, orchestrated by Kubernetes. It is recommended to install Theodolite with the package manager Helm. The Theodolite Helm chart along with instructions how to install it can be found in the [`helm`](helm) directory. ## Theodolite Analysis Tools -Theodolite's benchmarking method creates a *scalability graph* allowing to draw conclusions about the scalability of a stream processing engine or its deployment. A scalability graph shows how resource demand evolves with an increasing workload. Theodolite provides Jupyter notebooks for creating such scalability graphs based on benchmarking results from the execution framework. More information can be found in [Theodolite analysis tool](analysis). +Theodolite's benchmarking method maps load intensities to the resource amounts that are required for processing them. A plot showing how resource demand evolves with an increasing load allows to draw conclusions about the scalability of a stream processing engine or its deployment. Theodolite provides Jupyter notebooks for creating such plots based on benchmarking results from the execution framework. More information can be found in [Theodolite analysis tool](analysis). + +## Theodolite Benchmarks +Theodolite comes with 4 application benchmarks, which are based on typical use cases for stream processing within microservices. For each benchmark, a corresponding load generator is provided. Currently, this repository provides benchmark implementations for Apache Kafka Streams and Apache Flink. The benchmark sources can be found in [Thedolite benchmarks](theodolite-benchmarks). ## How to Cite diff --git a/analysis/demand-metric-plot.ipynb b/analysis/demand-metric-plot.ipynb index 90ef227dbf6a4566760329b615d5f59b4cc2bc25..71e08f0590f819a63b1bdd6bf13b57ac665f65bc 100644 --- a/analysis/demand-metric-plot.ipynb +++ b/analysis/demand-metric-plot.ipynb @@ -1,22 +1,22 @@ { "cells": [ { + "cell_type": "markdown", + "metadata": {}, "source": [ "# Theodolite Analysis - Plotting the Demand Metric\n", "\n", "This notebook creates a plot, showing scalability as a function that maps load intensities to the resources required for processing them. It is able to combine multiple such plots in one figure, for example, to compare multiple systems or configurations.\n", "\n", "The notebook takes a CSV file for each plot mapping load intensities to minimum required resources, computed by the `demand-metric-plot.ipynb` notebook." - ], - "cell_type": "markdown", - "metadata": {} + ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "First, we need to import some libraries, which are required for creating the plots." - ], - "cell_type": "markdown", - "metadata": {} + ] }, { "cell_type": "code", @@ -33,11 +33,11 @@ ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "We need to specify the directory, where the demand CSV files can be found, and a dictionary that maps a system description (e.g. its name) to the corresponding CSV file (prefix). To use Unicode narrow non-breaking spaces in the description format it as `u\"1000\\u202FmCPU\"`." - ], - "cell_type": "markdown", - "metadata": {} + ] }, { "cell_type": "code", @@ -53,11 +53,11 @@ ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "Now, we combie all systems described in `experiments`." - ], - "cell_type": "markdown", - "metadata": {} + ] }, { "cell_type": "code", @@ -71,11 +71,11 @@ ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "We might want to display the mappings before we plot it." - ], - "cell_type": "markdown", - "metadata": {} + ] }, { "cell_type": "code", @@ -87,11 +87,11 @@ ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "The following code creates a MatPlotLib figure showing the scalability plots for all specified systems. You might want to adjust its styling etc. according to your preferences. Make sure to also set a filename." - ], - "cell_type": "markdown", - "metadata": {} + ] }, { "cell_type": "code", @@ -149,27 +149,33 @@ } ], "metadata": { + "file_extension": ".py", + "interpreter": { + "hash": "e9e076445e1891a25f59b525adcc71b09846b3f9cf034ce4147fc161b19af121" + }, + "kernelspec": { + "display_name": "Python 3.8.10 64-bit ('.venv': venv)", + "name": "python3" + }, "language_info": { - "name": "python", "codemirror_mode": { "name": "ipython", "version": 3 }, - "version": "3.8.5-final" + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" }, - "orig_nbformat": 2, - "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "npconvert_exporter": "python", + "orig_nbformat": 2, "pygments_lexer": "ipython3", - "version": 3, - "kernelspec": { - "name": "python37064bitvenvvenv6c432ee1239d4f3cb23f871068b0267d", - "display_name": "Python 3.7.0 64-bit ('.venv': venv)", - "language": "python" - } + "version": 3 }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/analysis/demand-metric.ipynb b/analysis/demand-metric.ipynb index bcea129b7cb07465fa99f32b6f8b2b6115e8a0aa..fbf3ee02960a1e06457eef5dda96cb6d0a1a75ac 100644 --- a/analysis/demand-metric.ipynb +++ b/analysis/demand-metric.ipynb @@ -1,6 +1,8 @@ { "cells": [ { + "cell_type": "markdown", + "metadata": {}, "source": [ "# Theodolite Analysis - Demand Metric\n", "\n", @@ -9,11 +11,11 @@ "Theodolite's *demand* metric is a function, mapping load intensities to the minimum required resources (e.g., instances) that are required to process this load. With this notebook, the *demand* metric function is approximated by a map of tested load intensities to their minimum required resources.\n", "\n", "The final output when running this notebook will be a CSV file, providig this mapping. It can be used to create nice plots of a system's scalability using the `demand-metric-plot.ipynb` notebook." - ], - "cell_type": "markdown", - "metadata": {} + ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "In the following cell, we need to specifiy:\n", "\n", @@ -22,9 +24,7 @@ "* `max_lag_trend_slope`: The maximum tolerable increase in queued messages per second.\n", "* `measurement_dir`: The directory where the measurement data files are to be found.\n", "* `results_dir`: The directory where the computed demand CSV files are to be stored." - ], - "cell_type": "markdown", - "metadata": {} + ] }, { "cell_type": "code", @@ -40,11 +40,11 @@ ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "With the following call, we compute our demand mapping." - ], - "cell_type": "markdown", - "metadata": {} + ] }, { "cell_type": "code", @@ -58,11 +58,11 @@ ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "We might already want to plot a simple visualization here:" - ], - "cell_type": "markdown", - "metadata": {} + ] }, { "cell_type": "code", @@ -74,11 +74,11 @@ ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "Finally we store the results in a CSV file." - ], - "cell_type": "markdown", - "metadata": {} + ] }, { "cell_type": "code", @@ -93,27 +93,33 @@ } ], "metadata": { + "file_extension": ".py", + "interpreter": { + "hash": "e9e076445e1891a25f59b525adcc71b09846b3f9cf034ce4147fc161b19af121" + }, + "kernelspec": { + "display_name": "Python 3.8.10 64-bit ('.venv': venv)", + "name": "python3" + }, "language_info": { - "name": "python", "codemirror_mode": { "name": "ipython", "version": 3 }, - "version": "3.8.5-final" + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" }, - "orig_nbformat": 2, - "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "npconvert_exporter": "python", + "orig_nbformat": 2, "pygments_lexer": "ipython3", - "version": 3, - "kernelspec": { - "name": "python37064bitvenvvenv6c432ee1239d4f3cb23f871068b0267d", - "display_name": "Python 3.7.0 64-bit ('.venv': venv)", - "language": "python" - } + "version": 3 }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/analysis/src/demand.py b/analysis/src/demand.py index dfb20c05af8e9a134eedd2cdb584c961a82369f5..2178ab7c5dc5f7e4c04ebb58d4c14c9bf8b1aeff 100644 --- a/analysis/src/demand.py +++ b/analysis/src/demand.py @@ -1,59 +1,51 @@ import os from datetime import datetime, timedelta, timezone import pandas as pd +from pandas.core.frame import DataFrame from sklearn.linear_model import LinearRegression def demand(exp_id, directory, threshold, warmup_sec): raw_runs = [] - # Compute SL, i.e., lag trend, for each tested configuration - filenames = [filename for filename in os.listdir(directory) if filename.startswith(f"exp{exp_id}") and filename.endswith("totallag.csv")] + # Compute SLI, i.e., lag trend, for each tested configuration + filenames = [filename for filename in os.listdir(directory) if filename.startswith(f"exp{exp_id}") and "lag-trend" in filename and filename.endswith(".csv")] for filename in filenames: - #print(filename) run_params = filename[:-4].split("_") - dim_value = run_params[2] - instances = run_params[3] + dim_value = run_params[1] + instances = run_params[2] df = pd.read_csv(os.path.join(directory, filename)) - #input = df.loc[df['topic'] == "input"] input = df - #print(input) + input['sec_start'] = input.loc[0:, 'timestamp'] - input.iloc[0]['timestamp'] - #print(input) - #print(input.iloc[0, 'timestamp']) + regress = input.loc[input['sec_start'] >= warmup_sec] # Warm-Up - #regress = input - #input.plot(kind='line',x='timestamp',y='value',color='red') - #plt.show() + X = regress.iloc[:, 1].values.reshape(-1, 1) # values converts it into a numpy array + Y = regress.iloc[:, 2].values.reshape(-1, 1) # -1 means that calculate the dimension of rows, but have 1 column - X = regress.iloc[:, 2].values.reshape(-1, 1) # values converts it into a numpy array - Y = regress.iloc[:, 3].values.reshape(-1, 1) # -1 means that calculate the dimension of rows, but have 1 column linear_regressor = LinearRegression() # create object for the class linear_regressor.fit(X, Y) # perform linear regression Y_pred = linear_regressor.predict(X) # make predictions trend_slope = linear_regressor.coef_[0][0] - #print(linear_regressor.coef_) row = {'load': int(dim_value), 'resources': int(instances), 'trend_slope': trend_slope} - #print(row) raw_runs.append(row) runs = pd.DataFrame(raw_runs) - # Set suitable = True if SLOs are met, i.e., lag trend is below threshold - runs["suitable"] = runs.apply(lambda row: row['trend_slope'] < threshold, axis=1) - - # Sort results table (unsure if required) - runs.columns = runs.columns.str.strip() - runs.sort_values(by=["load", "resources"]) + # Group by the load and resources to handle repetitions, and take from the reptitions the median + # for even reptitions, the mean of the two middle values is used + medians = runs.groupby(by=['load', 'resources'], as_index=False).median() - # Filter only suitable configurations - filtered = runs[runs.apply(lambda x: x['suitable'], axis=1)] - - # Compute demand per load intensity - grouped = filtered.groupby(['load'])['resources'].min() - demand_per_load = grouped.to_frame().reset_index() + # Set suitable = True if SLOs are met, i.e., lag trend slope is below threshold + medians["suitable"] = medians.apply(lambda row: row['trend_slope'] < threshold, axis=1) + suitable = medians[medians.apply(lambda x: x['suitable'], axis=1)] + + # Compute minimal demand per load intensity + demand_per_load = suitable.groupby(by=['load'], as_index=False)['resources'].min() + return demand_per_load + diff --git a/codemeta.json b/codemeta.json index 5696996592f63bf8ece23239d8204e0f25b9cce1..a158e30eb7f1ab433779678aba3a1cc3b7e33c80 100644 --- a/codemeta.json +++ b/codemeta.json @@ -5,10 +5,10 @@ "codeRepository": "https://github.com/cau-se/theodolite", "dateCreated": "2020-03-13", "datePublished": "2020-07-27", - "dateModified": "2021-03-18", + "dateModified": "2021-11-12", "downloadUrl": "https://github.com/cau-se/theodolite/releases", "name": "Theodolite", - "version": "0.4.0", + "version": "0.5.1", "description": "Theodolite is a framework for benchmarking the horizontal and vertical scalability of stream processing engines.", "developmentStatus": "active", "referencePublication": "https://doi.org/10.1016/j.bdr.2021.100209", diff --git a/docs/README.md b/docs/README.md index 4fd13bdfc157efe8b3491695bb83972f96a82c5d..eb0848d52ec4235c6325ba0a373ea2628e52a102 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,16 +10,20 @@ permalink: / Theodolite is a framework for benchmarking the horizontal and vertical scalability of stream processing engines. It consists of three modules: -## Theodolite Benchmarks +## Theodolite Benchmarking Tool -Theodolite contains 4 application benchmarks, which are based on typical use cases for stream processing within microservices. For each benchmark, a corresponding workload generator is provided. Currently, this repository provides benchmark implementations for Kafka Streams. +Theodolite aims to benchmark scalability of stream processing engines for real use cases. Microservices that apply stream processing techniques are usually deployed in elastic cloud environments. Hence, Theodolite's cloud-native benchmarking framework deploys its components in a cloud environment, orchestrated by Kubernetes. It is recommended to install Theodolite with the package manager Helm. The Theodolite Helm chart along with instructions how to install it can be found in the [`helm`](helm) directory. +## Theodolite Analysis Tools -## Theodolite Execution Framework +Theodolite's benchmarking method maps load intensities to the resource amounts that are required for processing them. A plot showing how resource demand evolves with an increasing load allows to draw conclusions about the scalability of a stream processing engine or its deployment. Theodolite provides Jupyter notebooks for creating such plots based on benchmarking results from the execution framework. More information can be found in [Theodolite analysis tool](analysis). -Theodolite aims to benchmark scalability of stream processing engines for real use cases. Microservices that apply stream processing techniques are usually deployed in elastic cloud environments. Hence, Theodolite's cloud-native benchmarking framework deploys as components in a cloud environment, orchestrated by Kubernetes. More information on how to execute scalability benchmarks can be found in [Thedolite execution framework](execution). +## Theodolite Benchmarks +Theodolite comes with 4 application benchmarks, which are based on typical use cases for stream processing within microservices. For each benchmark, a corresponding load generator is provided. Currently, this repository provides benchmark implementations for Apache Kafka Streams and Apache Flink. The benchmark sources can be found in [Thedolite benchmarks](theodolite-benchmarks). -## Theodolite Analysis Tools +## How to Cite + +If you use Theodolite, please cite -Theodolite's benchmarking method create a *scalability graph* allowing to draw conclusions about the scalability of a stream processing engine or its deployment. A scalability graph shows how resource demand evolves with an increasing workload. Theodolite provides Jupyter notebooks for creating such scalability graphs based on benchmarking results from the execution framework. More information can be found in [Theodolite analysis tool](analysis). +> Sören Henning and Wilhelm Hasselbring. (2021). Theodolite: Scalability Benchmarking of Distributed Stream Processing Engines in Microservice Architectures. Big Data Research, Volume 25. DOI: [10.1016/j.bdr.2021.100209](https://doi.org/10.1016/j.bdr.2021.100209). arXiv:[2009.00304](https://arxiv.org/abs/2009.00304). diff --git a/docs/crd-benchmark-doc.md b/docs/crd-benchmark-doc.md deleted file mode 100644 index 1e12d6ebd776b11d76241cfb530daeba9e0b5c24..0000000000000000000000000000000000000000 --- a/docs/crd-benchmark-doc.md +++ /dev/null @@ -1,374 +0,0 @@ -# API Reference - -Packages: - -- [theodolite.com/v1](#theodolitecomv1) - -# theodolite.com/v1 - -Resource Types: - -- [benchmark](#benchmark) - - - - -## benchmark -<sup><sup>[↩ Parent](#theodolitecomv1 )</sup></sup> - - - - - - - - -<table> - <thead> - <tr> - <th>Name</th> - <th>Type</th> - <th>Description</th> - <th>Required</th> - </tr> - </thead> - <tbody><tr> - <td><b>apiVersion</b></td> - <td>string</td> - <td>theodolite.com/v1</td> - <td>true</td> - </tr> - <tr> - <td><b>kind</b></td> - <td>string</td> - <td>benchmark</td> - <td>true</td> - </tr> - <tr> - <td><b><a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#objectmeta-v1-meta">metadata</a></b></td> - <td>object</td> - <td>Refer to the Kubernetes API documentation for the fields of the `metadata` field.</td> - <td>true</td> - </tr><tr> - <td><b><a href="#benchmarkspec">spec</a></b></td> - <td>object</td> - <td> - <br/> - </td> - <td>true</td> - </tr></tbody> -</table> - - -### benchmark.spec -<sup><sup>[↩ Parent](#benchmark)</sup></sup> - - - - - -<table> - <thead> - <tr> - <th>Name</th> - <th>Type</th> - <th>Description</th> - <th>Required</th> - </tr> - </thead> - <tbody><tr> - <td><b>name</b></td> - <td>string</td> - <td> - This field exists only for technical reasons and should not be set by the user. The value of the field will be overwritten.<br/> - </td> - <td>false</td> - </tr><tr> - <td><b>appResource</b></td> - <td>[]string</td> - <td> - A list of file names that reference Kubernetes resources that are deployed on the cluster for the system under test (SUT).<br/> - </td> - <td>true</td> - </tr><tr> - <td><b><a href="#benchmarkspeckafkaconfig">kafkaConfig</a></b></td> - <td>object</td> - <td> - Contains the Kafka configuration.<br/> - </td> - <td>true</td> - </tr><tr> - <td><b>loadGenResource</b></td> - <td>[]string</td> - <td> - A list of file names that reference Kubernetes resources that are deployed on the cluster for the load generator.<br/> - </td> - <td>true</td> - </tr><tr> - <td><b><a href="#benchmarkspecloadtypesindex">loadTypes</a></b></td> - <td>[]object</td> - <td> - A list of load types that can be scaled for this benchmark. For each load type the concrete values are defined in the execution object.<br/> - </td> - <td>true</td> - </tr><tr> - <td><b><a href="#benchmarkspecresourcetypesindex">resourceTypes</a></b></td> - <td>[]object</td> - <td> - A list of resource types that can be scaled for this `benchmark` resource. For each resource type the concrete values are defined in the `execution` object.<br/> - </td> - <td>true</td> - </tr></tbody> -</table> - - -### benchmark.spec.kafkaConfig -<sup><sup>[↩ Parent](#benchmarkspec)</sup></sup> - - - -Contains the Kafka configuration. - -<table> - <thead> - <tr> - <th>Name</th> - <th>Type</th> - <th>Description</th> - <th>Required</th> - </tr> - </thead> - <tbody><tr> - <td><b>bootstrapServer</b></td> - <td>string</td> - <td> - The bootstrap servers connection string.<br/> - </td> - <td>true</td> - </tr><tr> - <td><b><a href="#benchmarkspeckafkaconfigtopicsindex">topics</a></b></td> - <td>[]object</td> - <td> - List of topics to be created for each experiment. Alternative theodolite offers the possibility to remove certain topics after each experiment.<br/> - </td> - <td>true</td> - </tr></tbody> -</table> - - -### benchmark.spec.kafkaConfig.topics[index] -<sup><sup>[↩ Parent](#benchmarkspeckafkaconfig)</sup></sup> - - - - - -<table> - <thead> - <tr> - <th>Name</th> - <th>Type</th> - <th>Description</th> - <th>Required</th> - </tr> - </thead> - <tbody><tr> - <td><b>removeOnly</b></td> - <td>boolean</td> - <td> - Determines if this topic should only be deleted after each experiement. For removeOnly topics the name can be a RegEx describing the topic.<br/> - <br/> - <i>Default</i>: false<br/> - </td> - <td>false</td> - </tr><tr> - <td><b>name</b></td> - <td>string</td> - <td> - The name of the topic.<br/> - <br/> - <i>Default</i>: <br/> - </td> - <td>true</td> - </tr><tr> - <td><b>numPartitions</b></td> - <td>integer</td> - <td> - The number of partitions of the topic.<br/> - <br/> - <i>Default</i>: 0<br/> - </td> - <td>true</td> - </tr><tr> - <td><b>replicationFactor</b></td> - <td>integer</td> - <td> - The replication factor of the topic.<br/> - <br/> - <i>Default</i>: 0<br/> - </td> - <td>true</td> - </tr></tbody> -</table> - - -### benchmark.spec.loadTypes[index] -<sup><sup>[↩ Parent](#benchmarkspec)</sup></sup> - - - - - -<table> - <thead> - <tr> - <th>Name</th> - <th>Type</th> - <th>Description</th> - <th>Required</th> - </tr> - </thead> - <tbody><tr> - <td><b><a href="#benchmarkspecloadtypesindexpatchersindex">patchers</a></b></td> - <td>[]object</td> - <td> - List of patchers used to scale this resource type.<br/> - </td> - <td>true</td> - </tr><tr> - <td><b>typeName</b></td> - <td>string</td> - <td> - Name of the load type.<br/> - </td> - <td>true</td> - </tr></tbody> -</table> - - -### benchmark.spec.loadTypes[index].patchers[index] -<sup><sup>[↩ Parent](#benchmarkspecloadtypesindex)</sup></sup> - - - - - -<table> - <thead> - <tr> - <th>Name</th> - <th>Type</th> - <th>Description</th> - <th>Required</th> - </tr> - </thead> - <tbody><tr> - <td><b>properties</b></td> - <td>object</td> - <td> - (Optional) Patcher specific additional arguments.<br/> - <br/> - <i>Default</i>: map[]<br/> - </td> - <td>false</td> - </tr><tr> - <td><b>resource</b></td> - <td>string</td> - <td> - Specifies the Kubernetes resource to be patched.<br/> - <br/> - <i>Default</i>: <br/> - </td> - <td>true</td> - </tr><tr> - <td><b>type</b></td> - <td>string</td> - <td> - Type of the Patcher.<br/> - <br/> - <i>Default</i>: <br/> - </td> - <td>true</td> - </tr></tbody> -</table> - - -### benchmark.spec.resourceTypes[index] -<sup><sup>[↩ Parent](#benchmarkspec)</sup></sup> - - - - - -<table> - <thead> - <tr> - <th>Name</th> - <th>Type</th> - <th>Description</th> - <th>Required</th> - </tr> - </thead> - <tbody><tr> - <td><b><a href="#benchmarkspecresourcetypesindexpatchersindex">patchers</a></b></td> - <td>[]object</td> - <td> - List of patchers used to scale this resource type.<br/> - </td> - <td>true</td> - </tr><tr> - <td><b>typeName</b></td> - <td>string</td> - <td> - Name of the resource type.<br/> - </td> - <td>true</td> - </tr></tbody> -</table> - - -### benchmark.spec.resourceTypes[index].patchers[index] -<sup><sup>[↩ Parent](#benchmarkspecresourcetypesindex)</sup></sup> - - - - - -<table> - <thead> - <tr> - <th>Name</th> - <th>Type</th> - <th>Description</th> - <th>Required</th> - </tr> - </thead> - <tbody><tr> - <td><b>properties</b></td> - <td>object</td> - <td> - (Optional) Patcher specific additional arguments.<br/> - <br/> - <i>Default</i>: map[]<br/> - </td> - <td>false</td> - </tr><tr> - <td><b>resource</b></td> - <td>string</td> - <td> - Specifies the Kubernetes resource to be patched.<br/> - <br/> - <i>Default</i>: <br/> - </td> - <td>true</td> - </tr><tr> - <td><b>type</b></td> - <td>string</td> - <td> - Type of the patcher.<br/> - <br/> - <i>Default</i>: <br/> - </td> - <td>true</td> - </tr></tbody> -</table> \ No newline at end of file diff --git a/docs/crd-docu.md b/docs/crd-docu.md new file mode 100644 index 0000000000000000000000000000000000000000..4002f9b0ec50bcc64822799a0256b30dd37ac9e4 --- /dev/null +++ b/docs/crd-docu.md @@ -0,0 +1,992 @@ +# API Reference + +Packages: + +- [theodolite.com/v1](#theodolitecomv1) + +# theodolite.com/v1 + +Resource Types: + +- [benchmark](#benchmark) + +- [execution](#execution) + + + + +## benchmark +<sup><sup>[↩ Parent](#theodolitecomv1 )</sup></sup> + + + + + + + + +<table> + <thead> + <tr> + <th>Name</th> + <th>Type</th> + <th>Description</th> + <th>Required</th> + </tr> + </thead> + <tbody><tr> + <td><b>apiVersion</b></td> + <td>string</td> + <td>theodolite.com/v1</td> + <td>true</td> + </tr> + <tr> + <td><b>kind</b></td> + <td>string</td> + <td>benchmark</td> + <td>true</td> + </tr> + <tr> + <td><b><a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#objectmeta-v1-meta">metadata</a></b></td> + <td>object</td> + <td>Refer to the Kubernetes API documentation for the fields of the `metadata` field.</td> + <td>true</td> + </tr><tr> + <td><b><a href="#benchmarkspec">spec</a></b></td> + <td>object</td> + <td> + <br/> + </td> + <td>true</td> + </tr></tbody> +</table> + + +### benchmark.spec +<sup><sup>[↩ Parent](#benchmark)</sup></sup> + + + + + +<table> + <thead> + <tr> + <th>Name</th> + <th>Type</th> + <th>Description</th> + <th>Required</th> + </tr> + </thead> + <tbody><tr> + <td><b>name</b></td> + <td>string</td> + <td> + This field exists only for technical reasons and should not be set by the user. The value of the field will be overwritten.<br/> + </td> + <td>false</td> + </tr><tr> + <td><b><a href="#benchmarkspecappresourcesetsindex">appResourceSets</a></b></td> + <td>[]object</td> + <td> + The appResourceSets specifies all Kubernetes resources required to start the sut. A resourceSet can be either a configMap resourceSet or a fileSystem resourceSet.<br/> + </td> + <td>true</td> + </tr><tr> + <td><b><a href="#benchmarkspeckafkaconfig">kafkaConfig</a></b></td> + <td>object</td> + <td> + Contains the Kafka configuration.<br/> + </td> + <td>true</td> + </tr><tr> + <td><b><a href="#benchmarkspecloadgenresourcesetsindex">loadGenResourceSets</a></b></td> + <td>[]object</td> + <td> + The loadGenResourceSets specifies all Kubernetes resources required to start the load generator. A resourceSet can be either a configMap resourceSet or a fileSystem resourceSet.<br/> + </td> + <td>true</td> + </tr><tr> + <td><b><a href="#benchmarkspecloadtypesindex">loadTypes</a></b></td> + <td>[]object</td> + <td> + A list of load types that can be scaled for this benchmark. For each load type the concrete values are defined in the execution object.<br/> + </td> + <td>true</td> + </tr><tr> + <td><b><a href="#benchmarkspecresourcetypesindex">resourceTypes</a></b></td> + <td>[]object</td> + <td> + A list of resource types that can be scaled for this `benchmark` resource. For each resource type the concrete values are defined in the `execution` object.<br/> + </td> + <td>true</td> + </tr></tbody> +</table> + + +### benchmark.spec.appResourceSets[index] +<sup><sup>[↩ Parent](#benchmarkspec)</sup></sup> + + + + + +<table> + <thead> + <tr> + <th>Name</th> + <th>Type</th> + <th>Description</th> + <th>Required</th> + </tr> + </thead> + <tbody><tr> + <td><b><a href="#benchmarkspecappresourcesetsindexconfigmap">configMap</a></b></td> + <td>object</td> + <td> + The configMap resourceSet loads the Kubernetes manifests from an Kubernetes configMap.<br/> + </td> + <td>false</td> + </tr><tr> + <td><b><a href="#benchmarkspecappresourcesetsindexfilesystem">fileSystem</a></b></td> + <td>object</td> + <td> + The fileSystem resourceSet loads the Kubernetes manifests from the filesystem.<br/> + </td> + <td>false</td> + </tr></tbody> +</table> + + +### benchmark.spec.appResourceSets[index].configMap +<sup><sup>[↩ Parent](#benchmarkspecappresourcesetsindex)</sup></sup> + + + +The configMap resourceSet loads the Kubernetes manifests from an Kubernetes configMap. + +<table> + <thead> + <tr> + <th>Name</th> + <th>Type</th> + <th>Description</th> + <th>Required</th> + </tr> + </thead> + <tbody><tr> + <td><b>files</b></td> + <td>[]string</td> + <td> + (Optional) Specifies which files from the configMap should be loaded. If this field is not set, all files are loaded.<br/> + </td> + <td>false</td> + </tr><tr> + <td><b>name</b></td> + <td>string</td> + <td> + The name of the configMap<br/> + </td> + <td>false</td> + </tr></tbody> +</table> + + +### benchmark.spec.appResourceSets[index].fileSystem +<sup><sup>[↩ Parent](#benchmarkspecappresourcesetsindex)</sup></sup> + + + +The fileSystem resourceSet loads the Kubernetes manifests from the filesystem. + +<table> + <thead> + <tr> + <th>Name</th> + <th>Type</th> + <th>Description</th> + <th>Required</th> + </tr> + </thead> + <tbody><tr> + <td><b>files</b></td> + <td>[]string</td> + <td> + (Optional) Specifies which files from the configMap should be loaded. If this field is not set, all files are loaded.<br/> + </td> + <td>false</td> + </tr><tr> + <td><b>path</b></td> + <td>string</td> + <td> + The path to the folder which contains the Kubernetes manifests files.<br/> + </td> + <td>false</td> + </tr></tbody> +</table> + + +### benchmark.spec.kafkaConfig +<sup><sup>[↩ Parent](#benchmarkspec)</sup></sup> + + + +Contains the Kafka configuration. + +<table> + <thead> + <tr> + <th>Name</th> + <th>Type</th> + <th>Description</th> + <th>Required</th> + </tr> + </thead> + <tbody><tr> + <td><b>bootstrapServer</b></td> + <td>string</td> + <td> + The bootstrap servers connection string.<br/> + </td> + <td>true</td> + </tr><tr> + <td><b><a href="#benchmarkspeckafkaconfigtopicsindex">topics</a></b></td> + <td>[]object</td> + <td> + List of topics to be created for each experiment. Alternative theodolite offers the possibility to remove certain topics after each experiment.<br/> + </td> + <td>true</td> + </tr></tbody> +</table> + + +### benchmark.spec.kafkaConfig.topics[index] +<sup><sup>[↩ Parent](#benchmarkspeckafkaconfig)</sup></sup> + + + + + +<table> + <thead> + <tr> + <th>Name</th> + <th>Type</th> + <th>Description</th> + <th>Required</th> + </tr> + </thead> + <tbody><tr> + <td><b>numPartitions</b></td> + <td>integer</td> + <td> + The number of partitions of the topic.<br/> + <br/> + <i>Default</i>: 0<br/> + </td> + <td>false</td> + </tr><tr> + <td><b>removeOnly</b></td> + <td>boolean</td> + <td> + Determines if this topic should only be deleted after each experiement. For removeOnly topics the name can be a RegEx describing the topic.<br/> + <br/> + <i>Default</i>: false<br/> + </td> + <td>false</td> + </tr><tr> + <td><b>replicationFactor</b></td> + <td>integer</td> + <td> + The replication factor of the topic.<br/> + <br/> + <i>Default</i>: 0<br/> + </td> + <td>false</td> + </tr><tr> + <td><b>name</b></td> + <td>string</td> + <td> + The name of the topic.<br/> + <br/> + <i>Default</i>: <br/> + </td> + <td>true</td> + </tr></tbody> +</table> + + +### benchmark.spec.loadGenResourceSets[index] +<sup><sup>[↩ Parent](#benchmarkspec)</sup></sup> + + + + + +<table> + <thead> + <tr> + <th>Name</th> + <th>Type</th> + <th>Description</th> + <th>Required</th> + </tr> + </thead> + <tbody><tr> + <td><b><a href="#benchmarkspecloadgenresourcesetsindexconfigmap">configMap</a></b></td> + <td>object</td> + <td> + The configMap resourceSet loads the Kubernetes manifests from an Kubernetes configMap.<br/> + </td> + <td>false</td> + </tr><tr> + <td><b><a href="#benchmarkspecloadgenresourcesetsindexfilesystem">fileSystem</a></b></td> + <td>object</td> + <td> + The fileSystem resourceSet loads the Kubernetes manifests from the filesystem.<br/> + </td> + <td>false</td> + </tr></tbody> +</table> + + +### benchmark.spec.loadGenResourceSets[index].configMap +<sup><sup>[↩ Parent](#benchmarkspecloadgenresourcesetsindex)</sup></sup> + + + +The configMap resourceSet loads the Kubernetes manifests from an Kubernetes configMap. + +<table> + <thead> + <tr> + <th>Name</th> + <th>Type</th> + <th>Description</th> + <th>Required</th> + </tr> + </thead> + <tbody><tr> + <td><b>files</b></td> + <td>[]string</td> + <td> + (Optional) Specifies which files from the configMap should be loaded. If this field is not set, all files are loaded.<br/> + </td> + <td>false</td> + </tr><tr> + <td><b>name</b></td> + <td>string</td> + <td> + The name of the configMap<br/> + </td> + <td>false</td> + </tr></tbody> +</table> + + +### benchmark.spec.loadGenResourceSets[index].fileSystem +<sup><sup>[↩ Parent](#benchmarkspecloadgenresourcesetsindex)</sup></sup> + + + +The fileSystem resourceSet loads the Kubernetes manifests from the filesystem. + +<table> + <thead> + <tr> + <th>Name</th> + <th>Type</th> + <th>Description</th> + <th>Required</th> + </tr> + </thead> + <tbody><tr> + <td><b>files</b></td> + <td>[]string</td> + <td> + (Optional) Specifies which files from the configMap should be loaded. If this field is not set, all files are loaded.<br/> + </td> + <td>false</td> + </tr><tr> + <td><b>path</b></td> + <td>string</td> + <td> + The path to the folder which contains the Kubernetes manifests files.<br/> + </td> + <td>false</td> + </tr></tbody> +</table> + + +### benchmark.spec.loadTypes[index] +<sup><sup>[↩ Parent](#benchmarkspec)</sup></sup> + + + + + +<table> + <thead> + <tr> + <th>Name</th> + <th>Type</th> + <th>Description</th> + <th>Required</th> + </tr> + </thead> + <tbody><tr> + <td><b><a href="#benchmarkspecloadtypesindexpatchersindex">patchers</a></b></td> + <td>[]object</td> + <td> + List of patchers used to scale this resource type.<br/> + </td> + <td>true</td> + </tr><tr> + <td><b>typeName</b></td> + <td>string</td> + <td> + Name of the load type.<br/> + </td> + <td>true</td> + </tr></tbody> +</table> + + +### benchmark.spec.loadTypes[index].patchers[index] +<sup><sup>[↩ Parent](#benchmarkspecloadtypesindex)</sup></sup> + + + + + +<table> + <thead> + <tr> + <th>Name</th> + <th>Type</th> + <th>Description</th> + <th>Required</th> + </tr> + </thead> + <tbody><tr> + <td><b>properties</b></td> + <td>map[string]string</td> + <td> + (Optional) Patcher specific additional arguments.<br/> + <br/> + <i>Default</i>: map[]<br/> + </td> + <td>false</td> + </tr><tr> + <td><b>resource</b></td> + <td>string</td> + <td> + Specifies the Kubernetes resource to be patched.<br/> + <br/> + <i>Default</i>: <br/> + </td> + <td>true</td> + </tr><tr> + <td><b>type</b></td> + <td>string</td> + <td> + Type of the Patcher.<br/> + <br/> + <i>Default</i>: <br/> + </td> + <td>true</td> + </tr></tbody> +</table> + + +### benchmark.spec.resourceTypes[index] +<sup><sup>[↩ Parent](#benchmarkspec)</sup></sup> + + + + + +<table> + <thead> + <tr> + <th>Name</th> + <th>Type</th> + <th>Description</th> + <th>Required</th> + </tr> + </thead> + <tbody><tr> + <td><b><a href="#benchmarkspecresourcetypesindexpatchersindex">patchers</a></b></td> + <td>[]object</td> + <td> + List of patchers used to scale this resource type.<br/> + </td> + <td>true</td> + </tr><tr> + <td><b>typeName</b></td> + <td>string</td> + <td> + Name of the resource type.<br/> + </td> + <td>true</td> + </tr></tbody> +</table> + + +### benchmark.spec.resourceTypes[index].patchers[index] +<sup><sup>[↩ Parent](#benchmarkspecresourcetypesindex)</sup></sup> + + + + + +<table> + <thead> + <tr> + <th>Name</th> + <th>Type</th> + <th>Description</th> + <th>Required</th> + </tr> + </thead> + <tbody><tr> + <td><b>properties</b></td> + <td>map[string]string</td> + <td> + (Optional) Patcher specific additional arguments.<br/> + <br/> + <i>Default</i>: map[]<br/> + </td> + <td>false</td> + </tr><tr> + <td><b>resource</b></td> + <td>string</td> + <td> + Specifies the Kubernetes resource to be patched.<br/> + <br/> + <i>Default</i>: <br/> + </td> + <td>true</td> + </tr><tr> + <td><b>type</b></td> + <td>string</td> + <td> + Type of the patcher.<br/> + <br/> + <i>Default</i>: <br/> + </td> + <td>true</td> + </tr></tbody> +</table> + +## execution +<sup><sup>[↩ Parent](#theodolitecomv1 )</sup></sup> + + + + + + + + +<table> + <thead> + <tr> + <th>Name</th> + <th>Type</th> + <th>Description</th> + <th>Required</th> + </tr> + </thead> + <tbody><tr> + <td><b>apiVersion</b></td> + <td>string</td> + <td>theodolite.com/v1</td> + <td>true</td> + </tr> + <tr> + <td><b>kind</b></td> + <td>string</td> + <td>execution</td> + <td>true</td> + </tr> + <tr> + <td><b><a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#objectmeta-v1-meta">metadata</a></b></td> + <td>object</td> + <td>Refer to the Kubernetes API documentation for the fields of the `metadata` field.</td> + <td>true</td> + </tr><tr> + <td><b><a href="#executionstatus">status</a></b></td> + <td>object</td> + <td> + <br/> + </td> + <td>false</td> + </tr><tr> + <td><b><a href="#executionspec">spec</a></b></td> + <td>object</td> + <td> + <br/> + </td> + <td>true</td> + </tr></tbody> +</table> + + +### execution.status +<sup><sup>[↩ Parent](#execution)</sup></sup> + + + + + +<table> + <thead> + <tr> + <th>Name</th> + <th>Type</th> + <th>Description</th> + <th>Required</th> + </tr> + </thead> + <tbody><tr> + <td><b>executionDuration</b></td> + <td>string</td> + <td> + Duration of the execution in seconds<br/> + </td> + <td>false</td> + </tr><tr> + <td><b>executionState</b></td> + <td>string</td> + <td> + <br/> + </td> + <td>false</td> + </tr></tbody> +</table> + + +### execution.spec +<sup><sup>[↩ Parent](#execution)</sup></sup> + + + + + +<table> + <thead> + <tr> + <th>Name</th> + <th>Type</th> + <th>Description</th> + <th>Required</th> + </tr> + </thead> + <tbody><tr> + <td><b>name</b></td> + <td>string</td> + <td> + This field exists only for technical reasons and should not be set by the user. The value of the field will be overwritten.<br/> + <br/> + <i>Default</i>: <br/> + </td> + <td>false</td> + </tr><tr> + <td><b>benchmark</b></td> + <td>string</td> + <td> + The name of the benchmark this execution is referring to.<br/> + </td> + <td>true</td> + </tr><tr> + <td><b><a href="#executionspecconfigoverridesindex">configOverrides</a></b></td> + <td>[]object</td> + <td> + List of patchers that are used to override existing configurations.<br/> + </td> + <td>true</td> + </tr><tr> + <td><b><a href="#executionspecexecution">execution</a></b></td> + <td>object</td> + <td> + Defines the overall parameter for the execution.<br/> + </td> + <td>true</td> + </tr><tr> + <td><b><a href="#executionspecload">load</a></b></td> + <td>object</td> + <td> + Specifies the load values that are benchmarked.<br/> + </td> + <td>true</td> + </tr><tr> + <td><b><a href="#executionspecresources">resources</a></b></td> + <td>object</td> + <td> + Specifies the scaling resource that is benchmarked.<br/> + </td> + <td>true</td> + </tr><tr> + <td><b><a href="#executionspecslosindex">slos</a></b></td> + <td>[]object</td> + <td> + List of resource values for the specified resource type.<br/> + </td> + <td>true</td> + </tr></tbody> +</table> + + +### execution.spec.configOverrides[index] +<sup><sup>[↩ Parent](#executionspec)</sup></sup> + + + + + +<table> + <thead> + <tr> + <th>Name</th> + <th>Type</th> + <th>Description</th> + <th>Required</th> + </tr> + </thead> + <tbody><tr> + <td><b><a href="#executionspecconfigoverridesindexpatcher">patcher</a></b></td> + <td>object</td> + <td> + Patcher used to patch a resource<br/> + </td> + <td>false</td> + </tr><tr> + <td><b>value</b></td> + <td>string</td> + <td> + <br/> + </td> + <td>false</td> + </tr></tbody> +</table> + + +### execution.spec.configOverrides[index].patcher +<sup><sup>[↩ Parent](#executionspecconfigoverridesindex)</sup></sup> + + + +Patcher used to patch a resource + +<table> + <thead> + <tr> + <th>Name</th> + <th>Type</th> + <th>Description</th> + <th>Required</th> + </tr> + </thead> + <tbody><tr> + <td><b>properties</b></td> + <td>map[string]string</td> + <td> + (Optional) Patcher specific additional arguments.<br/> + <br/> + <i>Default</i>: map[]<br/> + </td> + <td>false</td> + </tr><tr> + <td><b>resource</b></td> + <td>string</td> + <td> + Specifies the Kubernetes resource to be patched.<br/> + <br/> + <i>Default</i>: <br/> + </td> + <td>true</td> + </tr><tr> + <td><b>type</b></td> + <td>string</td> + <td> + Type of the Patcher.<br/> + <br/> + <i>Default</i>: <br/> + </td> + <td>true</td> + </tr></tbody> +</table> + + +### execution.spec.execution +<sup><sup>[↩ Parent](#executionspec)</sup></sup> + + + +Defines the overall parameter for the execution. + +<table> + <thead> + <tr> + <th>Name</th> + <th>Type</th> + <th>Description</th> + <th>Required</th> + </tr> + </thead> + <tbody><tr> + <td><b>loadGenerationDelay</b></td> + <td>integer</td> + <td> + Seconds to wait between the start of the SUT and the load generator.<br/> + </td> + <td>false</td> + </tr><tr> + <td><b>duration</b></td> + <td>integer</td> + <td> + Defines the duration of each experiment in seconds.<br/> + </td> + <td>true</td> + </tr><tr> + <td><b>repetitions</b></td> + <td>integer</td> + <td> + Numper of repititions for each experiments.<br/> + </td> + <td>true</td> + </tr><tr> + <td><b>restrictions</b></td> + <td>[]string</td> + <td> + List of restriction strategys used to delimit the search space.<br/> + </td> + <td>true</td> + </tr><tr> + <td><b>strategy</b></td> + <td>string</td> + <td> + Defines the used strategy for the execution, either 'LinearSearch' or 'BinarySearch'<br/> + </td> + <td>true</td> + </tr></tbody> +</table> + + +### execution.spec.load +<sup><sup>[↩ Parent](#executionspec)</sup></sup> + + + +Specifies the load values that are benchmarked. + +<table> + <thead> + <tr> + <th>Name</th> + <th>Type</th> + <th>Description</th> + <th>Required</th> + </tr> + </thead> + <tbody><tr> + <td><b>loadType</b></td> + <td>string</td> + <td> + The type of the load. It must match one of the load types specified in the referenced benchmark.<br/> + </td> + <td>true</td> + </tr><tr> + <td><b>loadValues</b></td> + <td>[]integer</td> + <td> + List of load values for the specified load type.<br/> + </td> + <td>true</td> + </tr></tbody> +</table> + + +### execution.spec.resources +<sup><sup>[↩ Parent](#executionspec)</sup></sup> + + + +Specifies the scaling resource that is benchmarked. + +<table> + <thead> + <tr> + <th>Name</th> + <th>Type</th> + <th>Description</th> + <th>Required</th> + </tr> + </thead> + <tbody><tr> + <td><b>resourceType</b></td> + <td>string</td> + <td> + The type of the resource. It must match one of the resource types specified in the referenced benchmark.<br/> + </td> + <td>true</td> + </tr><tr> + <td><b>resourceValues</b></td> + <td>[]integer</td> + <td> + List of resource values for the specified resource type.<br/> + </td> + <td>true</td> + </tr></tbody> +</table> + + +### execution.spec.slos[index] +<sup><sup>[↩ Parent](#executionspec)</sup></sup> + + + + + +<table> + <thead> + <tr> + <th>Name</th> + <th>Type</th> + <th>Description</th> + <th>Required</th> + </tr> + </thead> + <tbody><tr> + <td><b>properties</b></td> + <td>map[string]string</td> + <td> + (Optional) SLO specific additional arguments.<br/> + <br/> + <i>Default</i>: map[]<br/> + </td> + <td>false</td> + </tr><tr> + <td><b>offset</b></td> + <td>integer</td> + <td> + Hours by which the start and end timestamp will be shifted (for different timezones).<br/> + </td> + <td>true</td> + </tr><tr> + <td><b>prometheusUrl</b></td> + <td>string</td> + <td> + Connection string for Promehteus.<br/> + </td> + <td>true</td> + </tr><tr> + <td><b>sloType</b></td> + <td>string</td> + <td> + The type of the SLO. It must match 'lag trend'.<br/> + </td> + <td>true</td> + </tr></tbody> +</table> \ No newline at end of file diff --git a/docs/crd-execution-doc.md b/docs/crd-execution-doc.md deleted file mode 100644 index 21b5cdecabd4a247df9f2c0c8d376ad578f4032f..0000000000000000000000000000000000000000 --- a/docs/crd-execution-doc.md +++ /dev/null @@ -1,438 +0,0 @@ -# API Reference - -Packages: - -- [theodolite.com/v1](#theodolitecomv1) - -# theodolite.com/v1 - -Resource Types: - -- [execution](#execution) - - - - -## execution -<sup><sup>[↩ Parent](#theodolitecomv1 )</sup></sup> - - - - - - - - -<table> - <thead> - <tr> - <th>Name</th> - <th>Type</th> - <th>Description</th> - <th>Required</th> - </tr> - </thead> - <tbody><tr> - <td><b>apiVersion</b></td> - <td>string</td> - <td>theodolite.com/v1</td> - <td>true</td> - </tr> - <tr> - <td><b>kind</b></td> - <td>string</td> - <td>execution</td> - <td>true</td> - </tr> - <tr> - <td><b><a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#objectmeta-v1-meta">metadata</a></b></td> - <td>object</td> - <td>Refer to the Kubernetes API documentation for the fields of the `metadata` field.</td> - <td>true</td> - </tr><tr> - <td><b><a href="#executionstatus">status</a></b></td> - <td>object</td> - <td> - <br/> - </td> - <td>false</td> - </tr><tr> - <td><b><a href="#executionspec">spec</a></b></td> - <td>object</td> - <td> - <br/> - </td> - <td>true</td> - </tr></tbody> -</table> - - -### execution.status -<sup><sup>[↩ Parent](#execution)</sup></sup> - - - - - -<table> - <thead> - <tr> - <th>Name</th> - <th>Type</th> - <th>Description</th> - <th>Required</th> - </tr> - </thead> - <tbody><tr> - <td><b>executionDuration</b></td> - <td>string</td> - <td> - Duration of the execution in seconds<br/> - </td> - <td>false</td> - </tr><tr> - <td><b>executionState</b></td> - <td>string</td> - <td> - <br/> - </td> - <td>false</td> - </tr></tbody> -</table> - - -### execution.spec -<sup><sup>[↩ Parent](#execution)</sup></sup> - - - - - -<table> - <thead> - <tr> - <th>Name</th> - <th>Type</th> - <th>Description</th> - <th>Required</th> - </tr> - </thead> - <tbody><tr> - <td><b>name</b></td> - <td>string</td> - <td> - This field exists only for technical reasons and should not be set by the user. The value of the field will be overwritten.<br/> - <br/> - <i>Default</i>: <br/> - </td> - <td>false</td> - </tr><tr> - <td><b>benchmark</b></td> - <td>string</td> - <td> - The name of the benchmark this execution is referring to.<br/> - </td> - <td>true</td> - </tr><tr> - <td><b><a href="#executionspecconfigoverridesindex">configOverrides</a></b></td> - <td>[]object</td> - <td> - List of patchers that are used to override existing configurations.<br/> - </td> - <td>true</td> - </tr><tr> - <td><b><a href="#executionspecexecution">execution</a></b></td> - <td>object</td> - <td> - Defines the overall parameter for the execution.<br/> - </td> - <td>true</td> - </tr><tr> - <td><b><a href="#executionspecload">load</a></b></td> - <td>object</td> - <td> - Specifies the load values that are benchmarked.<br/> - </td> - <td>true</td> - </tr><tr> - <td><b><a href="#executionspecresources">resources</a></b></td> - <td>object</td> - <td> - Specifies the scaling resource that is benchmarked.<br/> - </td> - <td>true</td> - </tr><tr> - <td><b><a href="#executionspecslosindex">slos</a></b></td> - <td>[]object</td> - <td> - List of resource values for the specified resource type.<br/> - </td> - <td>true</td> - </tr></tbody> -</table> - - -### execution.spec.configOverrides[index] -<sup><sup>[↩ Parent](#executionspec)</sup></sup> - - - - - -<table> - <thead> - <tr> - <th>Name</th> - <th>Type</th> - <th>Description</th> - <th>Required</th> - </tr> - </thead> - <tbody><tr> - <td><b><a href="#executionspecconfigoverridesindexpatcher">patcher</a></b></td> - <td>object</td> - <td> - Patcher used to patch a resource<br/> - </td> - <td>false</td> - </tr><tr> - <td><b>value</b></td> - <td>string</td> - <td> - <br/> - </td> - <td>false</td> - </tr></tbody> -</table> - - -### execution.spec.configOverrides[index].patcher -<sup><sup>[↩ Parent](#executionspecconfigoverridesindex)</sup></sup> - - - -Patcher used to patch a resource - -<table> - <thead> - <tr> - <th>Name</th> - <th>Type</th> - <th>Description</th> - <th>Required</th> - </tr> - </thead> - <tbody><tr> - <td><b>properties</b></td> - <td>object</td> - <td> - (Optional) Patcher specific additional arguments.<br/> - <br/> - <i>Default</i>: map[]<br/> - </td> - <td>false</td> - </tr><tr> - <td><b>resource</b></td> - <td>string</td> - <td> - Specifies the Kubernetes resource to be patched.<br/> - <br/> - <i>Default</i>: <br/> - </td> - <td>true</td> - </tr><tr> - <td><b>type</b></td> - <td>string</td> - <td> - Type of the Patcher.<br/> - <br/> - <i>Default</i>: <br/> - </td> - <td>true</td> - </tr></tbody> -</table> - - -### execution.spec.execution -<sup><sup>[↩ Parent](#executionspec)</sup></sup> - - - -Defines the overall parameter for the execution. - -<table> - <thead> - <tr> - <th>Name</th> - <th>Type</th> - <th>Description</th> - <th>Required</th> - </tr> - </thead> - <tbody><tr> - <td><b>loadGenerationDelay</b></td> - <td>integer</td> - <td> - Seconds to wait between the start of the SUT and the load generator.<br/> - </td> - <td>false</td> - </tr><tr> - <td><b>duration</b></td> - <td>integer</td> - <td> - Defines the duration of each experiment in seconds.<br/> - </td> - <td>true</td> - </tr><tr> - <td><b>repetitions</b></td> - <td>integer</td> - <td> - Numper of repititions for each experiments.<br/> - </td> - <td>true</td> - </tr><tr> - <td><b>restrictions</b></td> - <td>[]string</td> - <td> - List of restriction strategys used to delimit the search space.<br/> - </td> - <td>true</td> - </tr><tr> - <td><b>strategy</b></td> - <td>string</td> - <td> - Defines the used strategy for the execution, either 'LinearSearch' or 'BinarySearch'<br/> - </td> - <td>true</td> - </tr></tbody> -</table> - - -### execution.spec.load -<sup><sup>[↩ Parent](#executionspec)</sup></sup> - - - -Specifies the load values that are benchmarked. - -<table> - <thead> - <tr> - <th>Name</th> - <th>Type</th> - <th>Description</th> - <th>Required</th> - </tr> - </thead> - <tbody><tr> - <td><b>loadType</b></td> - <td>string</td> - <td> - The type of the load. It must match one of the load types specified in the referenced benchmark.<br/> - </td> - <td>true</td> - </tr><tr> - <td><b>loadValues</b></td> - <td>[]integer</td> - <td> - List of load values for the specified load type.<br/> - </td> - <td>true</td> - </tr></tbody> -</table> - - -### execution.spec.resources -<sup><sup>[↩ Parent](#executionspec)</sup></sup> - - - -Specifies the scaling resource that is benchmarked. - -<table> - <thead> - <tr> - <th>Name</th> - <th>Type</th> - <th>Description</th> - <th>Required</th> - </tr> - </thead> - <tbody><tr> - <td><b>resourceType</b></td> - <td>string</td> - <td> - The type of the resource. It must match one of the resource types specified in the referenced benchmark.<br/> - </td> - <td>true</td> - </tr><tr> - <td><b>resourceValues</b></td> - <td>[]integer</td> - <td> - <br/> - </td> - <td>true</td> - </tr></tbody> -</table> - - -### execution.spec.slos[index] -<sup><sup>[↩ Parent](#executionspec)</sup></sup> - - - - - -<table> - <thead> - <tr> - <th>Name</th> - <th>Type</th> - <th>Description</th> - <th>Required</th> - </tr> - </thead> - <tbody><tr> - <td><b>externalSloUrl</b></td> - <td>string</td> - <td> - Connection string for a external slo analysis.<br/> - </td> - <td>true</td> - </tr><tr> - <td><b>offset</b></td> - <td>integer</td> - <td> - Hours by which the start and end timestamp will be shifted (for different timezones).<br/> - </td> - <td>true</td> - </tr><tr> - <td><b>prometheusUrl</b></td> - <td>string</td> - <td> - Connection string for Promehteus.<br/> - </td> - <td>true</td> - </tr><tr> - <td><b>sloType</b></td> - <td>string</td> - <td> - The type of the SLO. It must match 'lag trend'.<br/> - </td> - <td>true</td> - </tr><tr> - <td><b>threshold</b></td> - <td>integer</td> - <td> - The threshold the SUT should meet for a sucessful experiment.<br/> - </td> - <td>true</td> - </tr><tr> - <td><b>warmup</b></td> - <td>integer</td> - <td> - Seconds of time that are ignored in the analysis.<br/> - </td> - <td>true</td> - </tr></tbody> -</table> \ No newline at end of file diff --git a/docs/index.yaml b/docs/index.yaml index 087124d158794e1b48dfc880e26da2c91d78808f..54580ea45f1c678443dae96c7139f53fdac37f60 100644 --- a/docs/index.yaml +++ b/docs/index.yaml @@ -1,6 +1,76 @@ apiVersion: v1 entries: theodolite: + - apiVersion: v2 + appVersion: 0.5.1 + created: "2021-11-12T16:15:01.629937292+01:00" + dependencies: + - condition: grafana.enabled + name: grafana + repository: https://grafana.github.io/helm-charts + version: 6.17.5 + - condition: kube-prometheus-stack.enabled + name: kube-prometheus-stack + repository: https://prometheus-community.github.io/helm-charts + version: 12.0.0 + - condition: cp-helm-charts.enabled + name: cp-helm-charts + repository: https://soerenhenning.github.io/cp-helm-charts + version: 0.6.0 + - condition: kafka-lag-exporter.enabled + name: kafka-lag-exporter + repository: https://lightbend.github.io/kafka-lag-exporter/repo/ + version: 0.6.6 + description: Theodolite is a framework for benchmarking the scalability stream + processing engines. + digest: a67374c4cb2b0e8b2d711468364c6b4a486a910bd1c667dbf3c5614e36e0680c + home: https://cau-se.github.io/theodolite + maintainers: + - email: soeren.henning@email.uni-kiel.de + name: Sören Henning + url: https://www.se.informatik.uni-kiel.de/en/team/soeren-henning-m-sc + name: theodolite + sources: + - https://github.com/cau-se/theodolite + type: application + urls: + - https://github.com/cau-se/theodolite/releases/download/v0.5.1/theodolite-0.5.1.tgz + version: 0.5.1 + - apiVersion: v2 + appVersion: 0.5.0 + created: "2021-11-04T17:45:14.153231798+01:00" + dependencies: + - condition: grafana.enabled + name: grafana + repository: https://grafana.github.io/helm-charts + version: 6.0.0 + - condition: kube-prometheus-stack.enabled + name: kube-prometheus-stack + repository: https://prometheus-community.github.io/helm-charts + version: 12.0.0 + - condition: cp-helm-charts.enabled + name: cp-helm-charts + repository: https://soerenhenning.github.io/cp-helm-charts + version: 0.6.0 + - condition: kafka-lag-exporter.enabled + name: kafka-lag-exporter + repository: https://lightbend.github.io/kafka-lag-exporter/repo/ + version: 0.6.6 + description: Theodolite is a framework for benchmarking the scalability stream + processing engines. + digest: 8a4f218e44341eb8fb09ddc58c6aaa0a14aded685f3423088c21fe0ffc112281 + home: https://cau-se.github.io/theodolite + maintainers: + - email: soeren.henning@email.uni-kiel.de + name: Sören Henning + url: https://www.se.informatik.uni-kiel.de/en/team/soeren-henning-m-sc + name: theodolite + sources: + - https://github.com/cau-se/theodolite + type: application + urls: + - https://github.com/cau-se/theodolite/releases/download/v0.5.0/theodolite-0.5.0.tgz + version: 0.5.0 - apiVersion: v2 appVersion: 0.4.0 created: "2021-03-18T15:50:50.930902088+01:00" @@ -36,4 +106,4 @@ entries: urls: - https://github.com/cau-se/theodolite/releases/download/v0.4.0/theodolite-0.4.0.tgz version: 0.4.0 -generated: "2021-03-18T15:50:50.897801281+01:00" +generated: "2021-11-12T16:15:01.591258889+01:00" diff --git a/docs/release-process.md b/docs/release-process.md index 981306b0762e43eacb29a434cc1e505593548fce..103d8d1ac65472459bcaad648f921240eaf508c8 100644 --- a/docs/release-process.md +++ b/docs/release-process.md @@ -18,8 +18,11 @@ again be merged into master. 3. Update all references to artifacts which are versioned. This includes: - 1. Update all references to Theodolite Docker images to tag `v0.3.1`. These are the Kubernetes resource definitions in -`execution`, the references to *latest* in `run_uc.py`, the Docker Compose files in `theodolite-benchmarks/docker-test` and the example `theodolite.yaml` job. + 1. Update all references to Theodolite Docker images to tag `v0.3.1`. These are: + 1. the default `helm/values.yaml` file, + 2. the example `execution/theodolite.yaml` job, + 3. the Kubernetes benchmark resources in `theodolite-benchmarks/definitions/**/resources` and + 2. the Docker Compose files in `theodolite-benchmarks/docker-test`. 2. Update both, the `version` and the `appVersion` fields, in the Helm `Charts.yaml` file to `0.3.1`. diff --git a/execution/.gitignore b/execution/.gitignore deleted file mode 100644 index bac9a5d1eeb12d9e40d38376904e8fb69c0e5231..0000000000000000000000000000000000000000 --- a/execution/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -exp_counter.txt -results diff --git a/helm/Chart.yaml b/helm/Chart.yaml index b09b4022d6727029311815b3b2b1bfcf4b4d2bd1..0e56a156832ed6f9159f436ec63f825d132e8dd3 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -13,11 +13,11 @@ type: application dependencies: - name: grafana - version: 6.0.0 + version: 6.17.5 repository: https://grafana.github.io/helm-charts condition: grafana.enabled - name: kube-prometheus-stack - version: 12.0.0 + version: 20.0.1 repository: https://prometheus-community.github.io/helm-charts condition: kube-prometheus-stack.enabled - name: cp-helm-charts @@ -25,10 +25,10 @@ dependencies: repository: https://soerenhenning.github.io/cp-helm-charts condition: cp-helm-charts.enabled - name: kafka-lag-exporter - version: 0.6.6 + version: 0.6.7 repository: https://lightbend.github.io/kafka-lag-exporter/repo/ condition: kafka-lag-exporter.enabled -version: 0.5.0-SNAPSHOT +version: 0.6.0-SNAPSHOT -appVersion: 0.5.0-SNAPSHOT +appVersion: 0.6.0-SNAPSHOT diff --git a/helm/README.md b/helm/README.md index c545804aaec8eb8ed91054f1f7ee97dd293816a4..1a3428b5e601de0c6c33f9dab236321e95592c6c 100644 --- a/helm/README.md +++ b/helm/README.md @@ -2,55 +2,47 @@ ## Installation -Install the chart via: +The Theodolite Helm chart with all its dependencies can be installed via: ```sh helm dependencies update . helm install theodolite . ``` -This chart installs requirements to execute benchmarks with Theodolite. +## Customize Installation -Dependencies and subcharts: +As usual, the installation with Helm can be configured by passing a values YAML file: -- Prometheus Operator -- Prometheus -- Grafana (incl. dashboard and data source configuration) -- Kafka -- Zookeeper -- A Kafka client pod - -## Test - -Test the installation: - -```sh -helm test theodolite +``` +helm install theodolite . -f <your-config.yaml> ``` -Our test files are located [here](templates/../../theodolite-chart/templates/tests). Many subcharts have their own tests, these are also executed and are placed in the respective /templates folders. - -Please note: If a test fails, Helm will stop testing. +We provide a minimal configuration, especially suited for development environments, with the `preconfigs/minimal.yaml` +file. -It is possible that the tests are not running successfully at the moment. This is because the Helm tests of the subchart cp-confluent receive a timeout exception. There is an [issue](https://github.com/confluentinc/cp-helm-charts/issues/318) for this problem on GitHub. +Per default, Helm installs the Theodolite CRDs used for the operator. If Theodolite will not be used as operator or if +the CRDs are already installed, you can skip their installation by adding the flag `--skip-crds`. -## Configuration +## Test Installation -In development environments Kubernetes resources are often low. To reduce resource consumption, we provide an `one-broker-value.yaml` file. This file can be used with: +Test the installation with: ```sh -helm install theodolite . -f preconfigs/one-broker-values.yaml +helm test theodolite ``` +Our test files are located [here](templates/tests). Many subcharts have their own tests, which are also executed. +Please note: If a test fails, Helm will stop testing. + ## Uninstall this Chart -To uninstall/delete the `theodolite` deployment: +The Theodolite Helm can easily be removed with: ```sh -helm delete theodolite +helm uninstall theodolite ``` -This command does not remove the CRDs which are created by this chart. Remove them manually with: +Helm does not remove any CRDs created by this chart. You can remove them manually with: ```sh # CRDs from Theodolite @@ -69,9 +61,20 @@ kubectl delete crd thanosrulers.monitoring.coreos.com ## Development -**Hints**: +### Dependencies + +The following 3rd party charts are used by Theodolite: + +- Kube Prometheus Stack (to install the Prometheus Operator, which is used to create a Prometheus instances) +- Grafana (including a dashboard and a data source configuration) +- Confluent Platform (for Kafka and Zookeeper) +- Kafka Lag Exporter (used to collect monitoring data of the Kafka lag) + +### Hints + +#### Grafana -- Grafana configuration: Grafana ConfigMaps contains expressions like {{ topic }}. Helm uses the same syntax for template function. More information [here](https://github.com/helm/helm/issues/2798) +Grafana ConfigMaps contain expressions like `{{ topic }}`. Helm uses the same syntax for template function. More information [here](https://github.com/helm/helm/issues/2798) - Escape braces: {{ "{{" topic }} - Let Helm render the template as raw string: {{ `{{ <config>}}` }} \ No newline at end of file diff --git a/helm/crds/benchmark.yaml b/helm/crds/benchmark.yaml new file mode 120000 index 0000000000000000000000000000000000000000..fb100de7a1407462bfb6488a54b7f70014a58474 --- /dev/null +++ b/helm/crds/benchmark.yaml @@ -0,0 +1 @@ +./../../theodolite/crd/crd-benchmark.yaml \ No newline at end of file diff --git a/helm/crds/execution.yaml b/helm/crds/execution.yaml new file mode 120000 index 0000000000000000000000000000000000000000..62d268c23c391cd7bbfbaffeaee8af1697dc446a --- /dev/null +++ b/helm/crds/execution.yaml @@ -0,0 +1 @@ +./../../theodolite/crd/crd-execution.yaml \ No newline at end of file diff --git a/helm/preconfigs/minimal.yaml b/helm/preconfigs/minimal.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b0828c2f424e8456933dc626a66a199cd60aa5da --- /dev/null +++ b/helm/preconfigs/minimal.yaml @@ -0,0 +1,12 @@ +cp-helm-charts: + cp-zookeeper: + servers: 1 + + cp-kafka: + brokers: 1 + configurationOverrides: + offsets.topic.replication.factor: "1" + +operator: + resultsVolume: + enabled: false diff --git a/helm/preconfigs/oci.yaml b/helm/preconfigs/oci.yaml new file mode 100644 index 0000000000000000000000000000000000000000..35fe5dcf423eed77cb2d3d4298088738125fa9fe --- /dev/null +++ b/helm/preconfigs/oci.yaml @@ -0,0 +1,6 @@ +operator: + resultsVolume: + persistent: + enabled: true + storageClassName: "oci-bv" + size: 50Gi # minimal size in OCI diff --git a/helm/preconfigs/one-broker-values.yaml b/helm/preconfigs/one-broker-values.yaml deleted file mode 100644 index c53c1f1eb8bc7a17f192d70a6f10f8cacc09c98f..0000000000000000000000000000000000000000 --- a/helm/preconfigs/one-broker-values.yaml +++ /dev/null @@ -1,15 +0,0 @@ -cp-helm-charts: - ## ------------------------------------------------------ - ## Zookeeper - ## ------------------------------------------------------ - cp-zookeeper: - servers: 1 # default: 3 - - ## ------------------------------------------------------ - ## Kafka - ## ------------------------------------------------------ - cp-kafka: - brokers: 1 # default: 10 - - configurationOverrides: - offsets.topic.replication.factor: "1" \ No newline at end of file diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl index f59f74d369b64ec89a44cbf2048fda9e844df92b..569cf454a950f3f621c23472f0346c8bbd52229d 100644 --- a/helm/templates/_helpers.tpl +++ b/helm/templates/_helpers.tpl @@ -60,3 +60,10 @@ Create the name of the service account to use {{- default "default" .Values.serviceAccount.name }} {{- end }} {{- end }} + +{{/* +Create the name of the results volume to use +*/}} +{{- define "theodolite.resultsClaimName" -}} +{{- default (printf "%s-results" (include "theodolite.fullname" .)) .Values.operator.resultsVolume.persistent.existingClaim }} +{{- end }} diff --git a/helm/templates/grafana/dashboard-config-map.yaml b/helm/templates/grafana/dashboard-config-map.yaml index 41365e5efefaddc92a9f2f25f867a9d895e4ca3d..0df01b20efa0fb1100fe4b7289b00b3058eb032f 100644 --- a/helm/templates/grafana/dashboard-config-map.yaml +++ b/helm/templates/grafana/dashboard-config-map.yaml @@ -24,7 +24,7 @@ data: "editable": true, "gnetId": null, "graphTooltip": 0, - "id": 2, + "id": 1, "iteration": 1589140028684, "links": [], "panels": [ @@ -1004,4 +1004,4 @@ data: "uid": "dad0CNlZz", "version": 25 }`}} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/helm/templates/grafana/osp-dashboard-config-map.yaml b/helm/templates/grafana/osp-dashboard-config-map.yaml new file mode 100644 index 0000000000000000000000000000000000000000..22685adf8e93d95614bdbb71ef6bc993a009a150 --- /dev/null +++ b/helm/templates/grafana/osp-dashboard-config-map.yaml @@ -0,0 +1,1029 @@ +{{- if .Values.grafana.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "theodolite.fullname" . }}-grafana-scalability-osp + labels: + grafana_dashboard: "2" +data: + osp-dashboard.json: |- + {{`{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 2, + "iteration": 1631777972723, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(cp_kafka_server_brokertopicmetrics_messagesinpersec_topic_ndwflow)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "ndwflow", + "refId": "A" + }, + { + "expr": "sum(cp_kafka_server_brokertopicmetrics_messagesinpersec_topic_ndwspeed)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "ndwspeed", + "refId": "B" + }, + { + "expr": "sum(cp_kafka_server_brokertopicmetrics_messagesinpersec_topic_ndwflow+cp_kafka_server_brokertopicmetrics_messagesinpersec_topic_ndwspeed)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "ndwflow+ndwspeed", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages In Per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(cp_kafka_server_brokertopicmetrics_messagesinpersec_topic_metrics)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Metrics", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages Out Per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(group, topic) (kafka_consumergroup_group_lag >= 0)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{topic}}", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Record Lag", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "count(count (kafka_consumer_consumer_fetch_manager_metrics_records_lag) by(pod))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "instances", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of Instances", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(group,topic) (kafka_consumergroup_group_offset >= 0)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{topic}}", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Records Consumed", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "count by(group,topic) (kafka_consumergroup_group_offset >= 0)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{topic}}", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of Partitions", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "hiddenSeries": false, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(group,topic) (kafka_partition_latest_offset)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{topic}}", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Records Produced (Kafka Lag Exporter)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "count by(job, topic) (kafka_consumer_consumer_fetch_manager_metrics_records_lag)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{topic}}", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of Partitions (Kafka Streams Export)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(job, topic) (kafka_consumer_consumer_fetch_manager_metrics_records_lag)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{topic}}", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Record Lag (Kafka Streams Export)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "hiddenSeries": false, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(group) (kafka_consumergroup_group_lag >= 0)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "total lag", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total Record Lag (Kafka Lag Exporter)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "10s", + "schemaVersion": 22, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "tags": [], + "text": "titan-ccp-aggregation", + "value": "titan-ccp-aggregation" + }, + "datasource": "Prometheus", + "definition": "label_values(kafka_consumer_consumer_fetch_manager_metrics_records_lag, job)", + "hide": 0, + "includeAll": false, + "index": -1, + "label": "Job", + "multi": false, + "name": "Job", + "options": [ + { + "selected": true, + "text": "titan-ccp-aggregation", + "value": "titan-ccp-aggregation" + } + ], + "query": "label_values(kafka_consumer_consumer_fetch_manager_metrics_records_lag, job)", + "refresh": 0, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "OSPBench", + "uid": "01O646v7z", + "variables": { + "list": [] + }, + "version": 1 + }`}} +{{- end }} diff --git a/helm/templates/theodolite/crd-benchmark.yaml b/helm/templates/theodolite/crd-benchmark.yaml deleted file mode 100644 index 084480e1f9e2ef827fb145cd823bbd2f68a20bac..0000000000000000000000000000000000000000 --- a/helm/templates/theodolite/crd-benchmark.yaml +++ /dev/null @@ -1,119 +0,0 @@ -{{- if .Values.operator.benchmarkCRD.create -}} -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: benchmarks.theodolite.com -spec: - group: theodolite.com - names: - kind: benchmark - plural: benchmarks - shortNames: - - bench - versions: - - name: v1 - served: true - storage: true - schema: - openAPIV3Schema: - type: object - required: ["spec"] - properties: - spec: - type: object - required: [] - properties: - name: - type: string - appResource: - type: array - minItems: 1 - items: - type: string - loadGenResource: - type: array - minItems: 1 - items: - type: string - resourceTypes: - type: array - minItems: 1 - items: - type: object - properties: - typeName: - type: string - patchers: - type: array - minItems: 1 - items: - type: object - properties: - type: - type: string - default: "" - resource: - type: string - default: "" - properties: - type: object - additionalProperties: true - x-kubernetes-map-type: "granular" - default: {} - loadTypes: - type: array - minItems: 1 - items: - type: object - properties: - typeName: - type: string - patchers: - type: array - minItems: 1 - items: - type: object - properties: - type: - type: string - default: "" - resource: - type: string - default: "" - properties: - type: object - additionalProperties: true - x-kubernetes-map-type: "granular" - default: {} - kafkaConfig: - type: object - properties: - bootstrapServer: - type: string - topics: - type: array - minItems: 1 - items: - type: object - required: [] - properties: - name: - type: string - default: "" - numPartitions: - type: integer - default: 0 - replicationFactor: - type: integer - default: 0 - removeOnly: - type: boolean - default: false - additionalPrinterColumns: - - name: Age - type: date - jsonPath: .metadata.creationTimestamp - subresources: - status: {} - scope: Namespaced -{{- end }} diff --git a/helm/templates/theodolite/crd-execution.yaml b/helm/templates/theodolite/crd-execution.yaml deleted file mode 100644 index 163835e9b37aca774983d4f019cc61d4bde06510..0000000000000000000000000000000000000000 --- a/helm/templates/theodolite/crd-execution.yaml +++ /dev/null @@ -1,133 +0,0 @@ -{{- if .Values.operator.executionCRD.create -}} -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: executions.theodolite.com -spec: - group: theodolite.com - names: - kind: execution - plural: executions - shortNames: - - exec - versions: - - name: v1 - served: true - storage: true - schema: - openAPIV3Schema: - type: object - required: ["spec"] - properties: - spec: - type: object - required: ["benchmark", "load", "resources", "slos", "execution", "configOverrides"] - properties: - name: - type: string - default: "" - benchmark: - type: string - load: # definition of the load dimension - type: object - required: ["loadType", "loadValues"] - properties: - loadType: - type: string - loadValues: - type: array - items: - type: integer - resources: # definition of the resource dimension - type: object - required: ["resourceType", "resourceValues"] - properties: - resourceType: - type: string - resourceValues: - type: array - items: - type: integer - slos: # def of service level objectives - type: array - items: - type: object - required: ["sloType", "prometheusUrl", "offset"] - properties: - sloType: - description: The type of the SLO. It must match 'lag trend'. - type: string - prometheusUrl: - description: Connection string for Promehteus. - type: string - offset: - description: Hours by which the start and end timestamp will be shifted (for different timezones). - type: integer - properties: - description: (Optional) SLO specific additional arguments. - type: object - additionalProperties: true - x-kubernetes-map-type: "granular" - default: {} - execution: # def execution config - type: object - required: ["strategy", "duration", "repetitions", "restrictions"] - properties: - strategy: - type: string - duration: - type: integer - repetitions: - type: integer - loadGenerationDelay: - type: integer - restrictions: - type: array - items: - type: string - configOverrides: - type: array - items: - type: object - properties: - patcher: - type: object - properties: - type: - type: string - default: "" - resource: - type: string - default: "" - properties: - type: object - additionalProperties: true - x-kubernetes-map-type: "granular" - default: {} - value: - type: string - status: - type: object - properties: - executionState: - description: "" - type: string - executionDuration: - description: "Duration of the execution in seconds" - type: string - additionalPrinterColumns: - - name: STATUS - type: string - description: State of the execution - jsonPath: .status.executionState - - name: Duration - type: string - description: Duration of the execution - jsonPath: .status.executionDuration - - name: Age - type: date - jsonPath: .metadata.creationTimestamp - subresources: - status: {} - scope: Namespaced -{{- end }} diff --git a/helm/templates/theodolite/random-scheduler/deployment.yaml b/helm/templates/theodolite/random-scheduler/deployment.yaml index a1ea535d52d3dce971806dd638a90e9acb81c5d0..55b6e4ad5f8fafccc9623e69ef1df1fccf81ed39 100644 --- a/helm/templates/theodolite/random-scheduler/deployment.yaml +++ b/helm/templates/theodolite/random-scheduler/deployment.yaml @@ -22,8 +22,8 @@ spec: serviceAccount: {{ include "theodolite.fullname" . }}-random-scheduler containers: - name: random-scheduler - image: ghcr.io/cau-se/theodolite-random-scheduler:latest - #imagePullPolicy: Always + image: "{{ .Values.randomScheduler.image }}:{{ .Values.randomScheduler.imageTag }}" + imagePullPolicy: "{{ .Values.randomScheduler.imagePullPolicy }}" env: - name: TARGET_NAMESPACE value: {{ .Release.Namespace }} diff --git a/helm/templates/theodolite/results-volume/pvc.yaml b/helm/templates/theodolite/results-volume/pvc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..26ac56e42889ccaebbc669791ad4d318b8318fec --- /dev/null +++ b/helm/templates/theodolite/results-volume/pvc.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.operator.resultsVolume.persistent.enabled (not .Values.operator.resultsVolume.persistent.existingClaim) -}} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "theodolite.resultsClaimName" . }} +spec: + {{- if .Values.operator.resultsVolume.persistent.storageClassName }} + storageClassName: {{ .Values.operator.resultsVolume.persistent.storageClassName }} + {{- end }} + accessModes: + - ReadWriteOnce + {{- range .Values.operator.resultsVolume.persistent.accessModes }} + - {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.operator.resultsVolume.persistent.size | quote }} +{{- end }} \ No newline at end of file diff --git a/helm/templates/theodolite/role-binding.yaml b/helm/templates/theodolite/role-binding.yaml index 93d8c34e7bc544c3b0c231e986bc58c792cce38e..3b327bb246f9716be0939416db55fc1b2cc5dd70 100644 --- a/helm/templates/theodolite/role-binding.yaml +++ b/helm/templates/theodolite/role-binding.yaml @@ -1,5 +1,5 @@ {{- if .Values.rbac.create -}} -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: {{ include "theodolite.fullname" . }} diff --git a/helm/templates/theodolite/role.yaml b/helm/templates/theodolite/role.yaml index b9924ea4908718c361851ff6137b44a19589a0be..0b496f3ef506e56b74acf5b7c8d0f4edc4f2cd96 100644 --- a/helm/templates/theodolite/role.yaml +++ b/helm/templates/theodolite/role.yaml @@ -68,5 +68,11 @@ rules: - get - create - update + - apiGroups: + - "" + resources: + - events + verbs: + - create {{- end }} {{- end }} \ No newline at end of file diff --git a/helm/templates/theodolite/theodolite-operator.yaml b/helm/templates/theodolite/theodolite-operator.yaml index 3487b9a4fabb16897b302d8be65f065a647ffb16..3d1d186306a5d1103a921b4c39e1425621548db8 100644 --- a/helm/templates/theodolite/theodolite-operator.yaml +++ b/helm/templates/theodolite/theodolite-operator.yaml @@ -31,10 +31,8 @@ spec: - name: RESULTS_FOLDER value: "./results" volumeMounts: - {{- if .Values.operator.resultsVolume.enabled }} - - name: theodolite-pv-storage + - name: theodolite-results-volume mountPath: "/deployments/results" - {{- end }} - name: benchmark-resources-uc1-kstreams mountPath: /deployments/benchmark-resources/uc1-kstreams - name: benchmark-resources-uc2-kstreams @@ -64,7 +62,20 @@ spec: - name: LOG_LEVEL value: INFO {{- end }} - {{- if and .Values.operator.resultsVolume.enabled .Values.operator.resultsVolume.accessSidecar.enabled }} + {{- if .Values.operator.sloChecker.droppedRecordsKStreams.enabled }} + - name: slo-checker-dropped-records-kstreams + image: "{{ .Values.operator.sloChecker.droppedRecordsKStreams.image }}:{{ .Values.operator.sloChecker.droppedRecordsKStreams.imageTag }}" + imagePullPolicy: "{{ .Values.operator.sloChecker.droppedRecordsKStreams.imagePullPolicy }}" + ports: + - containerPort: 8081 + name: analysis + env: + - name: PORT + value: "8081" + - name: LOG_LEVEL + value: INFO + {{- end }} + {{- if .Values.operator.resultsVolume.accessSidecar.enabled }} - name: results-access image: busybox:stable image: "{{ .Values.operator.resultsVolume.accessSidecar.image }}:{{ .Values.operator.resultsVolume.accessSidecar.imageTag }}" @@ -75,14 +86,16 @@ spec: - exec tail -f /dev/null volumeMounts: - mountPath: /results - name: theodolite-pv-storage + name: theodolite-results-volume {{- end }} volumes: - {{- if .Values.operator.resultsVolume.enabled }} - - name: theodolite-pv-storage + - name: theodolite-results-volume + {{- if .Values.operator.resultsVolume.persistent.enabled }} persistentVolumeClaim: - claimName: {{ .Values.operator.resultsVolume.persistentVolumeClaim.name | quote }} - {{- end }} + claimName: {{ include "theodolite.resultsClaimName" . | quote }} + {{- else }} + emptyDir: {} + {{- end }} - name: benchmark-resources-uc1-kstreams configMap: name: benchmark-resources-uc1-kstreams diff --git a/helm/update-index.sh b/helm/update-index.sh index 286724dd87718387df58ed993af417bf0fd4d8ec..66c55bb8b79e18e3d06d156cb1859f2a53078999 100755 --- a/helm/update-index.sh +++ b/helm/update-index.sh @@ -3,7 +3,7 @@ RELEASE_NAME=$1 # Supposed to be equal to tag, e.g., v0.3.0 RELEASE_PATH="https://github.com/cau-se/theodolite/releases/download" -REPO_INDEX="../../docs/index.yaml" +REPO_INDEX="../docs/index.yaml" helm repo index . --url $RELEASE_PATH/$RELEASE_NAME --merge $REPO_INDEX && \ mv index.yaml $REPO_INDEX \ No newline at end of file diff --git a/helm/values.yaml b/helm/values.yaml index 917ec5cdbda7541b56b366aff2412ccf7d856f3b..6c20f4bab35825e66412381b454df5c9dedec0fc 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -155,7 +155,10 @@ cp-helm-charts: ### kafka-lag-exporter: enabled: true + image: + pullPolicy: IfNotPresent nodeSelector: {} + clusters: - name: "theodolite-cp-kafka" bootstrapBrokers: "theodolite-cp-kafka:9092" @@ -250,22 +253,26 @@ operator: imageTag: latest imagePullPolicy: Always - executionCRD: - create: true - benchmarkCRD: - create: true - sloChecker: lagTrend: enabled: true image: ghcr.io/cau-se/theodolite-slo-checker-lag-trend imageTag: latest imagePullPolicy: Always + droppedRecordsKStreams: + enabled: true + image: ghcr.io/cau-se/theodolite-slo-checker-dropped-records-kstreams + imageTag: latest + imagePullPolicy: Always resultsVolume: - enabled: true - persistentVolumeClaim: - name: theodolite-pv-claim + persistent: + enabled: false + # existingClaim: + # storageClassName: + accessModes: + - ReadWriteOnce + size: 1Gi accessSidecar: enabled: true image: busybox @@ -281,6 +288,9 @@ rbac: randomScheduler: enabled: true + image: ghcr.io/cau-se/theodolite-random-scheduler + imageTag: latest + imagePullPolicy: Always rbac: create: true serviceAccount: diff --git a/slope-evaluator/Dockerfile b/slo-checker/dropped-records/Dockerfile similarity index 100% rename from slope-evaluator/Dockerfile rename to slo-checker/dropped-records/Dockerfile diff --git a/slo-checker/dropped-records/README.md b/slo-checker/dropped-records/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3a1ea982a399201143ad50f173c934ff58abbf4a --- /dev/null +++ b/slo-checker/dropped-records/README.md @@ -0,0 +1,80 @@ +# Kafka Streams Dropped Record SLO Evaluator + +## Execution + +For development: + +```sh +uvicorn main:app --reload --port 81 # run this command inside the app/ folder +``` + +## Build the docker image: + +```sh +docker build . -t theodolite-evaluator +``` + +Run the Docker image: + +```sh +docker run -p 80:81 theodolite-evaluator +``` + +## Configuration + +You can set the `HOST` and the `PORT` (and a lot of more parameters) via environment variables. Default is `0.0.0.0:80`. +For more information see the [Gunicorn/FastAPI Docker docs](https://github.com/tiangolo/uvicorn-gunicorn-fastapi-docker#advanced-usage). + +## API Documentation + +The running webserver provides a REST API with the following route: + +* /dropped-records + * Method: POST + * Body: + * results + * metric-metadata + * values + * metadata + * threshold + * warmup + +The body of the request must be a JSON string that satisfies the following conditions: + +* **dropped records**: This property is based on the [Range Vector type](https://www.prometheus.io/docs/prometheus/latest/querying/api/#range-vectors) from Prometheus and must have the following JSON *structure*: + + ```json + { + "results": [ + [ + { + "metric": { + "<label-name>": "<label-value>" + }, + "values": [ + [ + <unix_timestamp>, // 1.634624989695E9 + "<sample_value>" // integer + ] + ] + } + ] + ], + "metadata": { + "threshold": 2000000, + "warmup": 60 + } + } + ``` + +### description + +* results: + * metric-metadata: + * Labels of this metric. The `dropped-records` slo checker does not use labels in the calculation of the service level objective. + * results + * The `<unix_timestamp>` provided as the first element of each element in the "values" array must be the timestamp of the measurement value in seconds (with optional decimal precision) + * The `<sample_value>` must be the measurement value as string. +* metadata: For the calculation of the service level objective require metadata. + * **threshold**: Must be an unsigned integer that specifies the threshold for the SLO evaluation. The SLO is considered fulfilled, if the result value is below the threshold. If the result value is equal or above the threshold, the SLO is considered not fulfilled. + * **warmup**: Specifieds the warmup time in seconds that are ignored for evaluating the SLO. diff --git a/slo-checker/dropped-records/app/main.py b/slo-checker/dropped-records/app/main.py new file mode 100644 index 0000000000000000000000000000000000000000..c1577f9c11ed5a2798ee0b4505ae0739301ab2a8 --- /dev/null +++ b/slo-checker/dropped-records/app/main.py @@ -0,0 +1,32 @@ +from fastapi import FastAPI,Request +import logging +import os +import json +import sys + +app = FastAPI() + +logging.basicConfig(stream=sys.stdout, + format="%(asctime)s %(levelname)s %(name)s: %(message)s") +logger = logging.getLogger("API") + + +if os.getenv('LOG_LEVEL') == 'INFO': + logger.setLevel(logging.INFO) +elif os.getenv('LOG_LEVEL') == 'WARNING': + logger.setLevel(logging.WARNING) +elif os.getenv('LOG_LEVEL') == 'DEBUG': + logger.setLevel(logging.DEBUG) + + +def check_service_level_objective(results, threshold): + return max(results) < threshold + +@app.post("/dropped-records",response_model=bool) +async def evaluate_slope(request: Request): + data = json.loads(await request.body()) + warmup = int(data['results'][0][0]['values'][0][0]) + int(data['metadata']['warmup']) + results = [int(val[1]) if(int(val[0]>=warmup)) else 0 for result in data['results'] for r in result for val in r['values'] ] + return check_service_level_objective(results=results, threshold=data['metadata']["threshold"]) + +logger.info("SLO evaluator is online") \ No newline at end of file diff --git a/slo-checker/dropped-records/app/test.py b/slo-checker/dropped-records/app/test.py new file mode 100644 index 0000000000000000000000000000000000000000..3c657c914002066357d58d88d7f8e4afe920db45 --- /dev/null +++ b/slo-checker/dropped-records/app/test.py @@ -0,0 +1,23 @@ +import unittest +from main import app, check_service_level_objective +import numpy as np +import json +from fastapi.testclient import TestClient + +class TestSloEvaluation(unittest.TestCase): + client = TestClient(app) + + def test_1_rep(self): + with open('../resources/test-1-rep-success.json') as json_file: + data = json.load(json_file) + response = self.client.post("/dropped-records", json=data) + self.assertEquals(response.json(), True) + + def test_check_service_level_objective(self): + list = [ x for x in range(-100, 100) ] + + self.assertEquals(check_service_level_objective(list, 90), False) + self.assertEquals(check_service_level_objective(list, 110), True) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/slope-evaluator/requirements.txt b/slo-checker/dropped-records/requirements.txt similarity index 100% rename from slope-evaluator/requirements.txt rename to slo-checker/dropped-records/requirements.txt diff --git a/slo-checker/dropped-records/resources/test-1-rep-success.json b/slo-checker/dropped-records/resources/test-1-rep-success.json new file mode 100644 index 0000000000000000000000000000000000000000..0964c30fed60e34c1ac4cf6b6b89f81d95a2f0eb --- /dev/null +++ b/slo-checker/dropped-records/resources/test-1-rep-success.json @@ -0,0 +1,273 @@ +{ + "results": [ + [ + { + "metric": { + "job": "titan-ccp-aggregation" + }, + "values": [ + [ + 1.634624674695E9, + "0" + ], + [ + 1.634624679695E9, + "0" + ], + [ + 1.634624684695E9, + "0" + ], + [ + 1.634624689695E9, + "0" + ], + [ + 1.634624694695E9, + "0" + ], + [ + 1.634624699695E9, + "0" + ], + [ + 1.634624704695E9, + "0" + ], + [ + 1.634624709695E9, + "0" + ], + [ + 1.634624714695E9, + "0" + ], + [ + 1.634624719695E9, + "0" + ], + [ + 1.634624724695E9, + "0" + ], + [ + 1.634624729695E9, + "0" + ], + [ + 1.634624734695E9, + "0" + ], + [ + 1.634624739695E9, + "0" + ], + [ + 1.634624744695E9, + "1" + ], + [ + 1.634624749695E9, + "3" + ], + [ + 1.634624754695E9, + "4" + ], + [ + 1.634624759695E9, + "4" + ], + [ + 1.634624764695E9, + "4" + ], + [ + 1.634624769695E9, + "4" + ], + [ + 1.634624774695E9, + "4" + ], + [ + 1.634624779695E9, + "4" + ], + [ + 1.634624784695E9, + "4" + ], + [ + 1.634624789695E9, + "4" + ], + [ + 1.634624794695E9, + "4" + ], + [ + 1.634624799695E9, + "4" + ], + [ + 1.634624804695E9, + "176" + ], + [ + 1.634624809695E9, + "176" + ], + [ + 1.634624814695E9, + "176" + ], + [ + 1.634624819695E9, + "176" + ], + [ + 1.634624824695E9, + "176" + ], + [ + 1.634624829695E9, + "159524" + ], + [ + 1.634624834695E9, + "209870" + ], + [ + 1.634624839695E9, + "278597" + ], + [ + 1.634624844695E9, + "460761" + ], + [ + 1.634624849695E9, + "460761" + ], + [ + 1.634624854695E9, + "460761" + ], + [ + 1.634624859695E9, + "460761" + ], + [ + 1.634624864695E9, + "460761" + ], + [ + 1.634624869695E9, + "606893" + ], + [ + 1.634624874695E9, + "653534" + ], + [ + 1.634624879695E9, + "755796" + ], + [ + 1.634624884695E9, + "919317" + ], + [ + 1.634624889695E9, + "919317" + ], + [ + 1.634624894695E9, + "955926" + ], + [ + 1.634624899695E9, + "955926" + ], + [ + 1.634624904695E9, + "955926" + ], + [ + 1.634624909695E9, + "955926" + ], + [ + 1.634624914695E9, + "955926" + ], + [ + 1.634624919695E9, + "1036530" + ], + [ + 1.634624924695E9, + "1078477" + ], + [ + 1.634624929695E9, + "1194775" + ], + [ + 1.634624934695E9, + "1347755" + ], + [ + 1.634624939695E9, + "1352151" + ], + [ + 1.634624944695E9, + "1360428" + ], + [ + 1.634624949695E9, + "1360428" + ], + [ + 1.634624954695E9, + "1360428" + ], + [ + 1.634624959695E9, + "1360428" + ], + [ + 1.634624964695E9, + "1360428" + ], + [ + 1.634624969695E9, + "1525685" + ], + [ + 1.634624974695E9, + "1689296" + ], + [ + 1.634624979695E9, + "1771358" + ], + [ + 1.634624984695E9, + "1854284" + ], + [ + 1.634624989695E9, + "1854284" + ] + ] + } + ] + ], + "metadata": { + "threshold": 2000000, + "warmup": 60 + } +} \ No newline at end of file diff --git a/slo-checker/record-lag/Dockerfile b/slo-checker/record-lag/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..032b8153a6989ca04631ba553289dacb3620a38d --- /dev/null +++ b/slo-checker/record-lag/Dockerfile @@ -0,0 +1,6 @@ +FROM tiangolo/uvicorn-gunicorn-fastapi:python3.7 + +COPY requirements.txt requirements.txt +RUN pip install -r requirements.txt + +COPY ./app /app \ No newline at end of file diff --git a/slo-checker/record-lag/README.md b/slo-checker/record-lag/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b4882eeaf54aadfb8cbf33a957e6052a7b74123b --- /dev/null +++ b/slo-checker/record-lag/README.md @@ -0,0 +1,80 @@ +# Lag Trend SLO Evaluator + +## Execution + +For development: + +```sh +uvicorn main:app --reload # run this command inside the app/ folder +``` + +## Build the docker image: + +```sh +docker build . -t theodolite-evaluator +``` + +Run the Docker image: + +```sh +docker run -p 80:80 theodolite-evaluator +``` + +## Configuration + +You can set the `HOST` and the `PORT` (and a lot of more parameters) via environment variables. Default is `0.0.0.0:80`. +For more information see the [Gunicorn/FastAPI Docker docs](https://github.com/tiangolo/uvicorn-gunicorn-fastapi-docker#advanced-usage). + +# API Documentation + +The running webserver provides a REST API with the following route: + +* /dropped-records + * Method: POST + * Body: + * results + * metric-metadata + * values + * metadata + * threshold + * warmup + +The body of the request must be a JSON string that satisfies the following conditions: + +* **total_lag**: This property is based on the [Range Vector type](https://www.prometheus.io/docs/prometheus/latest/querying/api/#range-vectors) from Prometheus and must have the following JSON *structure*: + + ```json + { + "results": [ + [ + { + "metric": { + "<label-name>": "<label-value>" + }, + "values": [ + [ + <unix_timestamp>, // 1.634624989695E9 + "<sample_value>" // integer + ] + ] + } + ] + ], + "metadata": { + "threshold": 2000000, + "warmup": 60 + } + } + ``` + +### description + +* results: + * metric-metadata: + * Labels of this metric. The `dropped-records` slo checker does not use labels in the calculation of the service level objective. + * results + * The `<unix_timestamp>` provided as the first element of each element in the "values" array must be the timestamp of the measurement value in seconds (with optional decimal precision) + * The `<sample_value>` must be the measurement value as string. +* metadata: For the calculation of the service level objective require metadata. + * **threshold**: Must be an unsigned integer that specifies the threshold for the SLO evaluation. The SLO is considered fulfilled, if the result value is below the threshold. If the result value is equal or above the threshold, the SLO is considered not fulfilled. + * **warmup**: Specifieds the warmup time in seconds that are ignored for evaluating the SLO. diff --git a/slope-evaluator/app/main.py b/slo-checker/record-lag/app/main.py similarity index 90% rename from slope-evaluator/app/main.py rename to slo-checker/record-lag/app/main.py index 6f6788f0ca84b7710be5b509ca4f0641047e963d..621fa0cfc9c27e809fd92752de93f2795fa32c05 100644 --- a/slope-evaluator/app/main.py +++ b/slo-checker/record-lag/app/main.py @@ -38,7 +38,7 @@ def calculate_slope_trend(results, warmup): err_msg = 'Computing trend slope failed.' logger.exception(err_msg) logger.error('Mark this subexperiment as not successful and continue benchmark.') - return False + return float('inf') logger.info("Computed lag trend slope is '%s'", trend_slope) return trend_slope @@ -49,7 +49,7 @@ def check_service_level_objective(results, threshold): @app.post("/evaluate-slope",response_model=bool) async def evaluate_slope(request: Request): data = json.loads(await request.body()) - results = [calculate_slope_trend(total_lag, data['warmup']) for total_lag in data['total_lags']] - return check_service_level_objective(results=results, threshold=data["threshold"]) + results = [calculate_slope_trend(total_lag, data['metadata']['warmup']) for total_lag in data['results']] + return check_service_level_objective(results=results, threshold=data['metadata']["threshold"]) logger.info("SLO evaluator is online") \ No newline at end of file diff --git a/slope-evaluator/app/test.py b/slo-checker/record-lag/app/test.py similarity index 99% rename from slope-evaluator/app/test.py rename to slo-checker/record-lag/app/test.py index 9b165ea479bb9a552edaba7692df4fd4ef3f4ab4..c8d81f86b16255dcdce5337d8f00e922b98b4f82 100644 --- a/slope-evaluator/app/test.py +++ b/slo-checker/record-lag/app/test.py @@ -17,7 +17,7 @@ class TestSloEvaluation(unittest.TestCase): data = json.load(json_file) response = self.client.post("/evaluate-slope", json=data) self.assertEquals(response.json(), True) - + def test_check_service_level_objective(self): list = [1,2,3,4] self.assertEquals(check_service_level_objective(list, 2), False) diff --git a/slope-evaluator/app/trend_slope_computer.py b/slo-checker/record-lag/app/trend_slope_computer.py similarity index 100% rename from slope-evaluator/app/trend_slope_computer.py rename to slo-checker/record-lag/app/trend_slope_computer.py diff --git a/slo-checker/record-lag/requirements.txt b/slo-checker/record-lag/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..8b6c3863226c2bd5e8bcd7982b2674dee593f192 --- /dev/null +++ b/slo-checker/record-lag/requirements.txt @@ -0,0 +1,5 @@ +fastapi==0.65.2 +scikit-learn==0.20.3 +pandas==1.0.3 +uvicorn +requests diff --git a/slope-evaluator/resources/test-1-rep-success.json b/slo-checker/record-lag/resources/test-1-rep-success.json similarity index 97% rename from slope-evaluator/resources/test-1-rep-success.json rename to slo-checker/record-lag/resources/test-1-rep-success.json index 9e315c707be7b2a874c58fcb1093aa86f7676560..dfe11282720ebfcdd60582b7717da892bc85a923 100644 --- a/slope-evaluator/resources/test-1-rep-success.json +++ b/slo-checker/record-lag/resources/test-1-rep-success.json @@ -1,5 +1,5 @@ { - "total_lags": [ + "results": [ [ { "metric": { @@ -134,6 +134,8 @@ } ] ], - "threshold": 2000, - "warmup": 0 + "metadata": { + "threshold": 2000, + "warmup": 0 + } } \ No newline at end of file diff --git a/slope-evaluator/resources/test-3-rep-success.json b/slo-checker/record-lag/resources/test-3-rep-success.json similarity index 98% rename from slope-evaluator/resources/test-3-rep-success.json rename to slo-checker/record-lag/resources/test-3-rep-success.json index 485966cba40f01e4a646e626914510ba49b707bc..cf483f42f3783aecd1f428ac7bbbe2090c4cade0 100644 --- a/slope-evaluator/resources/test-3-rep-success.json +++ b/slo-checker/record-lag/resources/test-3-rep-success.json @@ -1,5 +1,5 @@ { - "total_lags": [ + "results": [ [ { "metric": { @@ -284,6 +284,8 @@ } ] ], - "threshold": 2000, - "warmup": 0 + "metadata": { + "threshold": 2000, + "warmup": 0 + } } \ No newline at end of file diff --git a/slope-evaluator/README.md b/slope-evaluator/README.md deleted file mode 100644 index cd9e6820ed46452ce44d57d0c7e5cd5ae05e5a3b..0000000000000000000000000000000000000000 --- a/slope-evaluator/README.md +++ /dev/null @@ -1,61 +0,0 @@ -# Lag Trend SLO Evaluator - -## Execution - -For development: - -```sh -uvicorn main:app --reload # run this command inside the app/ folder -``` - -## Build the docker image: - -```sh -docker build . -t theodolite-evaluator -``` - -Run the Docker image: - -```sh -docker run -p 80:80 theodolite-evaluator -``` - -## Configuration - -You can set the `HOST` and the `PORT` (and a lot of more parameters) via environment variables. Default is `0.0.0.0:80`. -For more information see the [Gunicorn/FastAPI Docker docs](https://github.com/tiangolo/uvicorn-gunicorn-fastapi-docker#advanced-usage). - -## API Documentation - -The running webserver provides a REST API with the following route: - -* /evaluate-slope - * Method: POST - * Body: - * total_lags - * threshold - * warmup - -The body of the request must be a JSON string that satisfies the following conditions: - -* **total_lag**: This property is based on the [Range Vector type](https://www.prometheus.io/docs/prometheus/latest/querying/api/#range-vectors) from Prometheus and must have the following JSON structure: - ``` - { - [ - "metric": { - "group": "<label_value>" - }, - "values": [ - [ - <unix_timestamp>, - "<sample_value>" - ] - ] - ] - } - ``` - * The `<label_value>` provided in "metric.group" must be equal to the id of the Kafka consumer group. - * The `<unix_timestamp>` provided as the first element of each element in the "values" array must be the timestamp of the measurement value in seconds (with optional decimal precision) - * The `<sample_value>` must be the measurement value as string. -* **threshold**: Must be an unsigned integer that specifies the threshold for the SLO evaluation. The SLO is considered fulfilled, if the result value is below the threshold. If the result value is equal or above the threshold, the SLO is considered not fulfilled. -* **warmup**: Specifieds the warmup time in seconds that are ignored for evaluating the SLO. \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/install-configmaps.sh b/theodolite-benchmarks/definitions/install-configmaps.sh index 7ddd606a162185993ce402ef4d3b84d8d00eb82c..c5c0283f434d7a4c13da0e93a855cef7e112bbb9 100755 --- a/theodolite-benchmarks/definitions/install-configmaps.sh +++ b/theodolite-benchmarks/definitions/install-configmaps.sh @@ -1,8 +1,17 @@ +# flink kubectl create configmap benchmark-resources-uc1-flink --from-file uc1-flink/resources kubectl create configmap benchmark-resources-uc2-flink --from-file uc2-flink/resources kubectl create configmap benchmark-resources-uc3-flink --from-file uc3-flink/resources kubectl create configmap benchmark-resources-uc4-flink --from-file uc4-flink/resources + +# kafka kubectl create configmap benchmark-resources-uc1-kstreams --from-file uc1-kstreams/resources kubectl create configmap benchmark-resources-uc2-kstreams --from-file uc2-kstreams/resources kubectl create configmap benchmark-resources-uc3-kstreams --from-file uc3-kstreams/resources -kubectl create configmap benchmark-resources-uc4-kstreams --from-file uc4-kstreams/resources \ No newline at end of file +kubectl create configmap benchmark-resources-uc4-kstreams --from-file uc4-kstreams/resources + +# load generator +kubectl create configmap benchmark-resources-uc1-loadgen --from-file uc1-loadGen +kubectl create configmap benchmark-resources-uc2-loadgen --from-file uc2-loadGen +kubectl create configmap benchmark-resources-uc3-loadgen --from-file uc3-loadGen +kubectl create configmap benchmark-resources-uc4-loadgen --from-file uc4-loadGen \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc1-flink/resources/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc1-flink/resources/jobmanager-deployment.yaml index 512d4fe3c786e1b2c44e6ec57fccadf41a2e2eeb..1f328b1cd553c8036e570d28b97795fb2b00ec81 100644 --- a/theodolite-benchmarks/definitions/uc1-flink/resources/jobmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc1-flink/resources/jobmanager-deployment.yaml @@ -14,6 +14,7 @@ spec: app: flink component: jobmanager spec: + terminationGracePeriodSeconds: 0 containers: - name: jobmanager image: ghcr.io/cau-se/theodolite-uc1-flink:latest @@ -90,4 +91,4 @@ spec: emptyDir: {} # - name: job-artifacts-volume # hostPath: -# path: /host/path/to/job/artifacts \ No newline at end of file +# path: /host/path/to/job/artifacts diff --git a/theodolite-benchmarks/definitions/uc1-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc1-flink/resources/taskmanager-deployment.yaml index 7d46554692696b194736df6023eed5686040497d..c2266a4aeb21302262279f147e6512d5264e1dc1 100644 --- a/theodolite-benchmarks/definitions/uc1-flink/resources/taskmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc1-flink/resources/taskmanager-deployment.yaml @@ -14,6 +14,7 @@ spec: app: flink component: taskmanager spec: + terminationGracePeriodSeconds: 0 containers: - name: taskmanager image: ghcr.io/cau-se/theodolite-uc1-flink:latest diff --git a/theodolite-benchmarks/definitions/uc1-flink/uc1-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc1-flink/uc1-flink-benchmark-operator.yaml index 3e16e486446d568d2cac0fe5196ab7446be25978..070a2cd89b64fde8fc462709112e2af1f1e79544 100644 --- a/theodolite-benchmarks/definitions/uc1-flink/uc1-flink-benchmark-operator.yaml +++ b/theodolite-benchmarks/definitions/uc1-flink/uc1-flink-benchmark-operator.yaml @@ -3,17 +3,23 @@ kind: benchmark metadata: name: uc1-flink spec: - appResource: - - "uc1-flink/flink-configuration-configmap.yaml" - - "uc1-flink/taskmanager-deployment.yaml" - - "uc1-flink/taskmanager-service.yaml" - - "uc1-flink/service-monitor.yaml" - - "uc1-flink/jobmanager-service.yaml" - - "uc1-flink/jobmanager-deployment.yaml" - #- "uc1-flink/jobmanager-rest-service.yaml" - loadGenResource: - - "uc1-kstreams/uc1-load-generator-deployment.yaml" - - "uc1-kstreams/uc1-load-generator-service.yaml" + appResourceSets: + - configMap: + name: "benchmark-resources-uc1-flink" + files: + - "uc1-flink/flink-configuration-configmap.yaml" + - "uc1-flink/taskmanager-deployment.yaml" + - "uc1-flink/taskmanager-service.yaml" + - "uc1-flink/service-monitor.yaml" + - "uc1-flink/jobmanager-service.yaml" + - "uc1-flink/jobmanager-deployment.yaml" + #- "uc1-flink/jobmanager-rest-service.yaml" + loadGenResourceSets: + - configMap: + name: "benchmark-resources-uc1-loadgen" + files: + - "uc1-kstreams/uc1-load-generator-deployment.yaml" + - "uc1-kstreams/uc1-load-generator-service.yaml" resourceTypes: - typeName: "Instances" patchers: diff --git a/theodolite-benchmarks/definitions/uc1-kstreams/uc1-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-benchmark-operator.yaml index 8bcc7d5a1d750eed140527e7c6176a881bc9e6e1..283b3baa53b2090b445a379641086f2dc6c7753e 100644 --- a/theodolite-benchmarks/definitions/uc1-kstreams/uc1-benchmark-operator.yaml +++ b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-benchmark-operator.yaml @@ -3,14 +3,20 @@ kind: benchmark metadata: name: uc1-kstreams spec: - appResource: - - "uc1-kstreams/uc1-kstreams-deployment.yaml" - - "uc1-kstreams/uc1-kstreams-service.yaml" - - "uc1-kstreams/uc1-jmx-configmap.yaml" - - "uc1-kstreams/uc1-service-monitor.yaml" - loadGenResource: - - "uc1-kstreams/uc1-load-generator-deployment.yaml" - - "uc1-kstreams/uc1-load-generator-service.yaml" + appResourceSets: + - configMap: + name: "benchmark-resources-uc1-kstreams" + files: + - "uc1-kstreams/uc1-kstreams-deployment.yaml" + - "uc1-kstreams/uc1-kstreams-service.yaml" + - "uc1-kstreams/uc1-jmx-configmap.yaml" + - "uc1-kstreams/uc1-service-monitor.yaml" + loadGenResourceSets: + - configMap: + name: "benchmark-resources-uc1-loadgen" + files: + - "uc1-kstreams/uc1-load-generator-deployment.yaml" + - "uc1-kstreams/uc1-load-generator-service.yaml" resourceTypes: - typeName: "Instances" patchers: diff --git a/theodolite-benchmarks/definitions/uc1-kstreams/uc1-benchmark-standalone.yaml b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-benchmark-standalone.yaml index 871210c213b70070e2aae8f4986058763be74b7e..b90edf5e4dcae0a4f338fb5edb90c73f34b3d14b 100644 --- a/theodolite-benchmarks/definitions/uc1-kstreams/uc1-benchmark-standalone.yaml +++ b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-benchmark-standalone.yaml @@ -1,12 +1,18 @@ name: "uc1-kstreams" -appResource: - - "uc1-kstreams-deployment.yaml" - - "uc1-kstreams-service.yaml" - - "uc1-jmx-configmap.yaml" - - "uc1-service-monitor.yaml" -loadGenResource: - - "uc1-load-generator-deployment.yaml" - - "uc1-load-generator-service.yaml" +appResourceSet: + - configMap: + name: "benchmark-resources-uc1-kstreams" + files: + - "uc1-kstreams/uc1-kstreams-deployment.yaml" + - "uc1-kstreams/uc1-kstreams-service.yaml" + - "uc1-kstreams/uc1-jmx-configmap.yaml" + - "uc1-kstreams/uc1-service-monitor.yaml" +loadGenResourceSet: + - configMap: + name: "benchmark-resources-uc1-loadgen" + files: + - "uc1-kstreams/uc1-load-generator-deployment.yaml" + - "uc1-kstreams/uc1-load-generator-service.yaml" resourceTypes: - typeName: "Instances" patchers: diff --git a/theodolite-benchmarks/definitions/uc1-kstreams/resources/uc1-load-generator-deployment.yaml b/theodolite-benchmarks/definitions/uc1-loadGen/uc1-load-generator-deployment.yaml similarity index 100% rename from theodolite-benchmarks/definitions/uc1-kstreams/resources/uc1-load-generator-deployment.yaml rename to theodolite-benchmarks/definitions/uc1-loadGen/uc1-load-generator-deployment.yaml diff --git a/theodolite-benchmarks/definitions/uc1-kstreams/resources/uc1-load-generator-service.yaml b/theodolite-benchmarks/definitions/uc1-loadGen/uc1-load-generator-service.yaml similarity index 100% rename from theodolite-benchmarks/definitions/uc1-kstreams/resources/uc1-load-generator-service.yaml rename to theodolite-benchmarks/definitions/uc1-loadGen/uc1-load-generator-service.yaml diff --git a/theodolite-benchmarks/definitions/uc2-flink/resources/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc2-flink/resources/jobmanager-deployment.yaml index cece4286d49a3f6ff139ca7f1e01c647acd5d9f3..87ea174f71c592bbffab4e5fc9ce6e3963596b9c 100644 --- a/theodolite-benchmarks/definitions/uc2-flink/resources/jobmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc2-flink/resources/jobmanager-deployment.yaml @@ -14,6 +14,7 @@ spec: app: flink component: jobmanager spec: + terminationGracePeriodSeconds: 0 containers: - name: jobmanager image: ghcr.io/cau-se/theodolite-uc2-flink:latest @@ -90,4 +91,4 @@ spec: emptyDir: {} # - name: job-artifacts-volume # hostPath: -# path: /host/path/to/job/artifacts \ No newline at end of file +# path: /host/path/to/job/artifacts diff --git a/theodolite-benchmarks/definitions/uc2-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc2-flink/resources/taskmanager-deployment.yaml index c0a10f65aae92e4ac1fd8fb92bae97794c142232..c37df972a334a4a0e27f0420030f99f1dff15b53 100644 --- a/theodolite-benchmarks/definitions/uc2-flink/resources/taskmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc2-flink/resources/taskmanager-deployment.yaml @@ -14,6 +14,7 @@ spec: app: flink component: taskmanager spec: + terminationGracePeriodSeconds: 0 containers: - name: taskmanager image: ghcr.io/cau-se/theodolite-uc2-flink:latest diff --git a/theodolite-benchmarks/definitions/uc2-flink/uc2-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc2-flink/uc2-flink-benchmark-operator.yaml index ab335cc4f9f8413dd7871fc2184491235aad67e4..dcbdf57efe7397d2a7b646b6388550e205f39957 100644 --- a/theodolite-benchmarks/definitions/uc2-flink/uc2-flink-benchmark-operator.yaml +++ b/theodolite-benchmarks/definitions/uc2-flink/uc2-flink-benchmark-operator.yaml @@ -3,17 +3,23 @@ kind: benchmark metadata: name: uc2-flink spec: - appResource: - - "uc2-flink/flink-configuration-configmap.yaml" - - "uc2-flink/taskmanager-deployment.yaml" - - "uc2-flink/taskmanager-service.yaml" - - "uc2-flink/service-monitor.yaml" - - "uc2-flink/jobmanager-service.yaml" - - "uc2-flink/jobmanager-deployment.yaml" - #- "uc2-flink/jobmanager-rest-service.yaml" - loadGenResource: - - "uc2-kstreams/uc2-load-generator-deployment.yaml" - - "uc2-kstreams/uc2-load-generator-service.yaml" + appResourceSets: + - configMap: + name: "benchmark-resources-uc2-flink" + files: + - "uc2-flink/flink-configuration-configmap.yaml" + - "uc2-flink/taskmanager-deployment.yaml" + - "uc2-flink/taskmanager-service.yaml" + - "uc2-flink/service-monitor.yaml" + - "uc2-flink/jobmanager-service.yaml" + - "uc2-flink/jobmanager-deployment.yaml" + #- "uc2-flink/jobmanager-rest-service.yaml" + loadGenResourceSets: + - configMap: + name: "benchmark-resources-uc2-loadgen" + files: + - "uc2-kstreams/uc2-load-generator-deployment.yaml" + - "uc2-kstreams/uc2-load-generator-service.yaml" resourceTypes: - typeName: "Instances" patchers: diff --git a/theodolite-benchmarks/definitions/uc2-kstreams/uc2-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-benchmark-operator.yaml index c71318c95ceb9d462f64e19b135ce59b43b03099..aaf479ae89329cabb1180925bb8984b25f40aa48 100644 --- a/theodolite-benchmarks/definitions/uc2-kstreams/uc2-benchmark-operator.yaml +++ b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-benchmark-operator.yaml @@ -3,14 +3,20 @@ kind: benchmark metadata: name: uc2-kstreams spec: - appResource: - - "uc2-kstreams/uc2-kstreams-deployment.yaml" - - "uc2-kstreams/uc2-kstreams-service.yaml" - - "uc2-kstreams/uc2-jmx-configmap.yaml" - - "uc2-kstreams/uc2-service-monitor.yaml" - loadGenResource: - - "uc2-kstreams/uc2-load-generator-deployment.yaml" - - "uc2-kstreams/uc2-load-generator-service.yaml" + appResourceSets: + - configMap: + name: "benchmark-resources-uc2-kstreams" + files: + - "uc2-kstreams/uc2-kstreams-deployment.yaml" + - "uc2-kstreams/uc2-kstreams-service.yaml" + - "uc2-kstreams/uc2-jmx-configmap.yaml" + - "uc2-kstreams/uc2-service-monitor.yaml" + loadGenResourceSets: + - configMap: + name: "benchmark-resources-uc2-loadgen" + files: + - "uc2-kstreams/uc2-load-generator-deployment.yaml" + - "uc2-kstreams/uc2-load-generator-service.yaml" resourceTypes: - typeName: "Instances" patchers: diff --git a/theodolite-benchmarks/definitions/uc2-kstreams/uc2-benchmark-standalone.yaml b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-benchmark-standalone.yaml index 48269b38a086f074ead80964df3bd4633742743e..b0dbd6bf26b1751add491836a76580e5ce980611 100644 --- a/theodolite-benchmarks/definitions/uc2-kstreams/uc2-benchmark-standalone.yaml +++ b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-benchmark-standalone.yaml @@ -1,12 +1,18 @@ name: "uc2-kstreams" -appResource: - - "uc2-kstreams-deployment.yaml" - - "uc2-kstreams-service.yaml" - - "uc2-jmx-configmap.yaml" - - "uc2-service-monitor.yaml" -loadGenResource: - - "uc2-load-generator-deployment.yaml" - - "uc2-load-generator-service.yaml" +appResourceSets: + - configMap: + name: "benchmark-resources-uc2-kstreams" + files: + - "uc2-kstreams/uc2-kstreams-deployment.yaml" + - "uc2-kstreams/uc2-kstreams-service.yaml" + - "uc2-kstreams/uc2-jmx-configmap.yaml" + - "uc2-kstreams/uc2-service-monitor.yaml" +loadGenResourceSets: + - configMap: + name: "benchmark-resources-uc2-loadgen" + files: + - "uc2-kstreams/uc2-load-generator-deployment.yaml" + - "uc2-kstreams/uc2-load-generator-service.yaml" resourceTypes: - typeName: "Instances" patchers: diff --git a/theodolite-benchmarks/definitions/uc2-kstreams/resources/uc2-load-generator-deployment.yaml b/theodolite-benchmarks/definitions/uc2-loadGen/uc2-load-generator-deployment.yaml similarity index 100% rename from theodolite-benchmarks/definitions/uc2-kstreams/resources/uc2-load-generator-deployment.yaml rename to theodolite-benchmarks/definitions/uc2-loadGen/uc2-load-generator-deployment.yaml diff --git a/theodolite-benchmarks/definitions/uc2-kstreams/resources/uc2-load-generator-service.yaml b/theodolite-benchmarks/definitions/uc2-loadGen/uc2-load-generator-service.yaml similarity index 100% rename from theodolite-benchmarks/definitions/uc2-kstreams/resources/uc2-load-generator-service.yaml rename to theodolite-benchmarks/definitions/uc2-loadGen/uc2-load-generator-service.yaml diff --git a/theodolite-benchmarks/definitions/uc3-flink/resources/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc3-flink/resources/jobmanager-deployment.yaml index 33bf1f1121a9764785db7a504799314a7ed40cf3..d01123b13fe2d63637ee4000051091a99bad0546 100644 --- a/theodolite-benchmarks/definitions/uc3-flink/resources/jobmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc3-flink/resources/jobmanager-deployment.yaml @@ -14,6 +14,7 @@ spec: app: flink component: jobmanager spec: + terminationGracePeriodSeconds: 0 containers: - name: jobmanager image: ghcr.io/cau-se/theodolite-uc3-flink:latest @@ -90,4 +91,4 @@ spec: emptyDir: {} # - name: job-artifacts-volume # hostPath: -# path: /host/path/to/job/artifacts \ No newline at end of file +# path: /host/path/to/job/artifacts diff --git a/theodolite-benchmarks/definitions/uc3-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc3-flink/resources/taskmanager-deployment.yaml index 8f70b7308429f79cfd8f8bda7a7a96e2bc8d8689..495f97817e43d692c30fe898c4ef3118cae682d7 100644 --- a/theodolite-benchmarks/definitions/uc3-flink/resources/taskmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc3-flink/resources/taskmanager-deployment.yaml @@ -14,6 +14,7 @@ spec: app: flink component: taskmanager spec: + terminationGracePeriodSeconds: 0 containers: - name: taskmanager image: ghcr.io/cau-se/theodolite-uc3-flink:latest diff --git a/theodolite-benchmarks/definitions/uc3-flink/uc3-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc3-flink/uc3-flink-benchmark-operator.yaml index bc881d83d622f49409c4b2023b7fd9445038c2e8..7ba17f8e4dee447e7adda0f7c3d0e16eb01f35c5 100644 --- a/theodolite-benchmarks/definitions/uc3-flink/uc3-flink-benchmark-operator.yaml +++ b/theodolite-benchmarks/definitions/uc3-flink/uc3-flink-benchmark-operator.yaml @@ -3,17 +3,23 @@ kind: benchmark metadata: name: uc3-flink spec: - appResource: - - "uc3-flink/flink-configuration-configmap.yaml" - - "uc3-flink/taskmanager-deployment.yaml" - - "uc3-flink/taskmanager-service.yaml" - - "uc3-flink/service-monitor.yaml" - - "uc3-flink/jobmanager-service.yaml" - - "uc3-flink/jobmanager-deployment.yaml" - #- "uc3-flink/jobmanager-rest-service.yaml" - loadGenResource: - - "uc3-kstreams/uc3-load-generator-deployment.yaml" - - "uc3-kstreams/uc3-load-generator-service.yaml" + appResourceSets: + - configMap: + name: "benchmark-resources-uc3-flink" + files: + - "uc3-flink/flink-configuration-configmap.yaml" + - "uc3-flink/taskmanager-deployment.yaml" + - "uc3-flink/taskmanager-service.yaml" + - "uc3-flink/service-monitor.yaml" + - "uc3-flink/jobmanager-service.yaml" + - "uc3-flink/jobmanager-deployment.yaml" + #- "uc3-flink/jobmanager-rest-service.yaml" + loadGenResourceSets: + - configMap: + name: "benchmark-resources-uc3-loadgen" + files: + - "uc3-kstreams/uc3-load-generator-deployment.yaml" + - "uc3-kstreams/uc3-load-generator-service.yaml" resourceTypes: - typeName: "Instances" patchers: diff --git a/theodolite-benchmarks/definitions/uc3-kstreams/uc3-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-benchmark-operator.yaml index cc10020b17c7a31b61f5fdaf6963be9fda75865f..ef49aac8bcfbe5d3798982689fc73b540d0aade7 100644 --- a/theodolite-benchmarks/definitions/uc3-kstreams/uc3-benchmark-operator.yaml +++ b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-benchmark-operator.yaml @@ -3,14 +3,20 @@ kind: benchmark metadata: name: uc3-kstreams spec: - appResource: - - "uc3-kstreams/uc3-kstreams-deployment.yaml" - - "uc3-kstreams/uc3-kstreams-service.yaml" - - "uc3-kstreams/uc3-jmx-configmap.yaml" - - "uc3-kstreams/uc3-service-monitor.yaml" - loadGenResource: - - "uc3-kstreams/uc3-load-generator-deployment.yaml" - - "uc3-kstreams/uc3-load-generator-service.yaml" + appResourceSets: + - configMap: + name: "benchmark-resources-uc3-kstreams" + files: + - "uc3-kstreams/uc3-kstreams-deployment.yaml" + - "uc3-kstreams/uc3-kstreams-service.yaml" + - "uc3-kstreams/uc3-jmx-configmap.yaml" + - "uc3-kstreams/uc3-service-monitor.yaml" + loadGenResourceSets: + - configMap: + name: "benchmark-resources-uc3-loadgen" + files: + - "uc3-kstreams/uc3-load-generator-deployment.yaml" + - "uc3-kstreams/uc3-load-generator-service.yaml" resourceTypes: - typeName: "Instances" patchers: diff --git a/theodolite-benchmarks/definitions/uc3-kstreams/uc3-benchmark-standalone.yaml b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-benchmark-standalone.yaml index 0c2311388f1dfa87e8a182eb8399020bc83ae4ce..8bb0b6733bc474d224def09cf22b0315c050b5dd 100644 --- a/theodolite-benchmarks/definitions/uc3-kstreams/uc3-benchmark-standalone.yaml +++ b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-benchmark-standalone.yaml @@ -1,12 +1,18 @@ name: "uc3-kstreams" -appResource: - - "uc3-kstreams-deployment.yaml" - - "uc3-kstreams-service.yaml" - - "uc3-jmx-configmap.yaml" - - "uc3-service-monitor.yaml" -loadGenResource: - - "uc3-load-generator-deployment.yaml" - - "uc3-load-generator-service.yaml" +appResourceSets: + - configMap: + name: "benchmark-resources-uc3-kstreams" + files: + - "uc3-kstreams/uc3-kstreams-deployment.yaml" + - "uc3-kstreams/uc3-kstreams-service.yaml" + - "uc3-kstreams/uc3-jmx-configmap.yaml" + - "uc3-kstreams/uc3-service-monitor.yaml" +loadGenResourceSets: + - configMap: + name: "benchmark-resources-uc3-loadgen" + files: + - "uc3-kstreams/uc3-load-generator-deployment.yaml" + - "uc3-kstreams/uc3-load-generator-service.yaml" resourceTypes: - typeName: "Instances" patchers: diff --git a/theodolite-benchmarks/definitions/uc3-kstreams/resources/uc3-load-generator-deployment.yaml b/theodolite-benchmarks/definitions/uc3-loadGen/uc3-load-generator-deployment.yaml similarity index 100% rename from theodolite-benchmarks/definitions/uc3-kstreams/resources/uc3-load-generator-deployment.yaml rename to theodolite-benchmarks/definitions/uc3-loadGen/uc3-load-generator-deployment.yaml diff --git a/theodolite-benchmarks/definitions/uc3-kstreams/resources/uc3-load-generator-service.yaml b/theodolite-benchmarks/definitions/uc3-loadGen/uc3-load-generator-service.yaml similarity index 100% rename from theodolite-benchmarks/definitions/uc3-kstreams/resources/uc3-load-generator-service.yaml rename to theodolite-benchmarks/definitions/uc3-loadGen/uc3-load-generator-service.yaml diff --git a/theodolite-benchmarks/definitions/uc4-flink/resources/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc4-flink/resources/jobmanager-deployment.yaml index b6533a2c4355e227a16aeface2080253bce19958..032499ea498f8155fd80e42ec4cbdd850498b217 100644 --- a/theodolite-benchmarks/definitions/uc4-flink/resources/jobmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc4-flink/resources/jobmanager-deployment.yaml @@ -14,6 +14,7 @@ spec: app: flink component: jobmanager spec: + terminationGracePeriodSeconds: 0 containers: - name: jobmanager image: ghcr.io/cau-se/theodolite-uc4-flink:latest @@ -90,4 +91,4 @@ spec: emptyDir: {} # - name: job-artifacts-volume # hostPath: -# path: /host/path/to/job/artifacts \ No newline at end of file +# path: /host/path/to/job/artifacts diff --git a/theodolite-benchmarks/definitions/uc4-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc4-flink/resources/taskmanager-deployment.yaml index 7363b013b21ad29b481e449113ccf31538505634..7af13f20b6b2edf3c8878adf4f381dc1c1add115 100644 --- a/theodolite-benchmarks/definitions/uc4-flink/resources/taskmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc4-flink/resources/taskmanager-deployment.yaml @@ -14,6 +14,7 @@ spec: app: flink component: taskmanager spec: + terminationGracePeriodSeconds: 0 containers: - name: taskmanager image: ghcr.io/cau-se/theodolite-uc4-flink:latest diff --git a/theodolite-benchmarks/definitions/uc4-flink/uc4-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc4-flink/uc4-flink-benchmark-operator.yaml index 9979f540ae343a065d8ca483fa9c5934fc550a46..a6e427af267602a5a862b34270d44c299282ba55 100644 --- a/theodolite-benchmarks/definitions/uc4-flink/uc4-flink-benchmark-operator.yaml +++ b/theodolite-benchmarks/definitions/uc4-flink/uc4-flink-benchmark-operator.yaml @@ -3,17 +3,23 @@ kind: benchmark metadata: name: uc4-flink spec: - appResource: - - "uc4-flink/flink-configuration-configmap.yaml" - - "uc4-flink/taskmanager-deployment.yaml" - - "uc4-flink/taskmanager-service.yaml" - - "uc4-flink/service-monitor.yaml" - - "uc4-flink/jobmanager-service.yaml" - - "uc4-flink/jobmanager-deployment.yaml" - #- "uc4-flink/jobmanager-rest-service.yaml" - loadGenResource: - - "uc4-kstreams/uc4-load-generator-deployment.yaml" - - "uc4-kstreams/uc4-load-generator-service.yaml" + appResourceSets: + - configMap: + name: "benchmark-resources-uc4-flink" + files: + - "uc4-flink/flink-configuration-configmap.yaml" + - "uc4-flink/taskmanager-deployment.yaml" + - "uc4-flink/taskmanager-service.yaml" + - "uc4-flink/service-monitor.yaml" + - "uc4-flink/jobmanager-service.yaml" + - "uc4-flink/jobmanager-deployment.yaml" + #- "uc4-flink/jobmanager-rest-service.yaml" + loadGenResourceSets: + - configMap: + name: "benchmark-resources-uc4-loadgen" + files: + - "uc4-kstreams/uc4-load-generator-deployment.yaml" + - "uc4-kstreams/uc4-load-generator-service.yaml" resourceTypes: - typeName: "Instances" patchers: diff --git a/theodolite-benchmarks/definitions/uc4-kstreams/uc4-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-benchmark-operator.yaml index 61cdfeeea5298d05cca5f5a44cdb4bdf9f108b7c..828ab9321b85b6c186a89f54586c312bace0fd53 100644 --- a/theodolite-benchmarks/definitions/uc4-kstreams/uc4-benchmark-operator.yaml +++ b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-benchmark-operator.yaml @@ -3,14 +3,20 @@ kind: benchmark metadata: name: uc4-kstreams spec: - appResource: - - "uc4-kstreams/uc4-kstreams-deployment.yaml" - - "uc4-kstreams/uc4-kstreams-service.yaml" - - "uc4-kstreams/uc4-jmx-configmap.yaml" - - "uc4-kstreams/uc4-service-monitor.yaml" - loadGenResource: - - "uc4-kstreams/uc4-load-generator-deployment.yaml" - - "uc4-kstreams/uc4-load-generator-service.yaml" + appResourceSets: + - configMap: + name: "benchmark-resources-uc4-kstreams" + files: + - "uc4-kstreams/uc4-kstreams-deployment.yaml" + - "uc4-kstreams/uc4-kstreams-service.yaml" + - "uc4-kstreams/uc4-jmx-configmap.yaml" + - "uc4-kstreams/uc4-service-monitor.yaml" + loadGenResourceSets: + - configMap: + name: "benchmark-resources-uc4-loadgen" + files: + - "uc4-kstreams/uc4-load-generator-deployment.yaml" + - "uc4-kstreams/uc4-load-generator-service.yaml" resourceTypes: - typeName: "Instances" patchers: diff --git a/theodolite-benchmarks/definitions/uc4-kstreams/uc4-benchmark-standalone.yaml b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-benchmark-standalone.yaml index ec7a67db4ea24547bc23d5c57e7b907ba489859c..ab112cdcb1adcc1cb8a1a4fd455af5d4aec888a2 100644 --- a/theodolite-benchmarks/definitions/uc4-kstreams/uc4-benchmark-standalone.yaml +++ b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-benchmark-standalone.yaml @@ -1,12 +1,18 @@ name: "uc4-kstreams" -appResource: - - "uc4-kstreams-deployment.yaml" - - "uc4-kstreams-service.yaml" - - "uc4-jmx-configmap.yaml" - - "uc4-service-monitor.yaml" -loadGenResource: - - "uc4-load-generator-deployment.yaml" - - "uc4-load-generator-service.yaml" +appResourceSets: + - configMap: + name: "benchmark-resources-uc4-kstreams" + files: + - "uc4-kstreams/uc4-kstreams-deployment.yaml" + - "uc4-kstreams/uc4-kstreams-service.yaml" + - "uc4-kstreams/uc4-jmx-configmap.yaml" + - "uc4-kstreams/uc4-service-monitor.yaml" +loadGenResourceSets: + - configMap: + name: "benchmark-resources-uc4-loadgen" + files: + - "uc4-kstreams/uc4-load-generator-deployment.yaml" + - "uc4-kstreams/uc4-load-generator-service.yaml" resourceTypes: - typeName: "Instances" patchers: diff --git a/theodolite-benchmarks/definitions/uc4-kstreams/resources/uc4-load-generator-deployment.yaml b/theodolite-benchmarks/definitions/uc4-loadGen/uc4-load-generator-deployment.yaml similarity index 100% rename from theodolite-benchmarks/definitions/uc4-kstreams/resources/uc4-load-generator-deployment.yaml rename to theodolite-benchmarks/definitions/uc4-loadGen/uc4-load-generator-deployment.yaml diff --git a/theodolite-benchmarks/definitions/uc4-kstreams/resources/uc4-load-generator-service.yaml b/theodolite-benchmarks/definitions/uc4-loadGen/uc4-load-generator-service.yaml similarity index 100% rename from theodolite-benchmarks/definitions/uc4-kstreams/resources/uc4-load-generator-service.yaml rename to theodolite-benchmarks/definitions/uc4-loadGen/uc4-load-generator-service.yaml diff --git a/theodolite-benchmarks/kstreams-commons/build.gradle b/theodolite-benchmarks/kstreams-commons/build.gradle index e177aa5c4770c1a77fc21084a7766741fdb9bdec..c5a880acd4377056cc0b0f06b33a2d74c9f87c4e 100644 --- a/theodolite-benchmarks/kstreams-commons/build.gradle +++ b/theodolite-benchmarks/kstreams-commons/build.gradle @@ -7,6 +7,9 @@ repositories { maven { url "https://oss.sonatype.org/content/repositories/snapshots/" } + maven { + url 'https://packages.confluent.io/maven/' + } } dependencies { diff --git a/theodolite/README.md b/theodolite/README.md index 26efe96b4756316971378be810f69f6138613958..e0d7b9b46473a877d982e912cb85e2455e09be79 100644 --- a/theodolite/README.md +++ b/theodolite/README.md @@ -2,13 +2,13 @@ This project uses Quarkus, the Supersonic Subatomic Java Framework. -If you want to learn more about Quarkus, please visit its website: https://quarkus.io/ . +If you want to learn more about Quarkus, please visit its website: <https://quarkus.io/> . ## Running the application in dev mode You can run your application in dev mode using: -```shell script +```sh ./gradlew quarkusDev ``` @@ -16,7 +16,7 @@ You can run your application in dev mode using: The application can be packaged using: -```shell script +```sh ./gradlew build ``` @@ -25,7 +25,7 @@ an _über-jar_ as the dependencies are copied into the `build/lib` directory. If you want to build an _über-jar_, execute the following command: -```shell script +```sh ./gradlew build -Dquarkus.package.type=uber-jar ``` @@ -33,92 +33,108 @@ The application is now runnable using `java -jar build/theodolite-1.0.0-SNAPSHOT ## Creating a native executable +It is recommended to use the native GraalVM images to create executable jars from Theodolite. For more information please visit the [Native Image guide](https://www.graalvm.org/reference-manual/native-image/). + You can create a native executable using: -```shell script +```sh ./gradlew build -Dquarkus.package.type=native ``` Or, if you don't have GraalVM installed, you can run the native executable build in a container using: -```shell script +```sh ./gradlew build -Dquarkus.package.type=native -Dquarkus.native.container-build=true ``` You can then execute your native executable with: ```./build/theodolite-1.0.0-SNAPSHOT-runner``` -If you want to learn more about building native executables, please consult https://quarkus.io/guides/gradle-tooling. +If you want to learn more about building native executables, please consult <https://quarkus.io/guides/gradle-tooling>. ## Build docker images For the jvm version use: -```shell script +```sh ./gradlew build docker build -f src/main/docker/Dockerfile.jvm -t theodolite-jvm . ``` For the native image version use: -```shell script +```sh ./gradlew build -Dquarkus.package.type=native docker build -f src/main/docker/Dockerfile.native -t theodolite-native . ``` -## Execute docker images: +## Execute docker images Remember to set the environment variables first. Jvm version: -```shell script +```sh docker run -i --rm theodolite-jvm ``` Native image version: -```shell script +```sh docker run -i --rm theodolite-native ``` ## Environment variables -**Production:** (Docker-Container) +**Execution in Docker**: | Variables name | Default value |Usage | | -----------------------------|:----------------------------------:| ------------:| | `NAMESPACE` | `default` |Determines the namespace of the Theodolite will be executed in. Used in the KubernetesBenchmark| -| `THEODOLITE_EXECUTION` | `./config/BenchmarkExecution.yaml`|The complete path to the benchmarkExecution file. Used in the TheodoliteYamlExecutor. | -| `THEODOLITE_BENCHMARK_TYPE` | `./config/BenchmarkType.yaml` |The complete path to the benchmarkType file. Used in the TheodoliteYamlExecutor.| -| `THEODOLITE_APP_RESOURCES` | `./config` |The path under which the yamls for the resources for the subexperiments are found. Used in the KubernetesBenchmark| -| `MODE` | `yaml-executor` | Defines the mode of operation: either `yaml-executor` or `operator`| +| `THEODOLITE_EXECUTION` | `execution/execution.yaml` |The complete path to the benchmarkExecution file. Used in the TheodoliteYamlExecutor. | +| `THEODOLITE_BENCHMARK_TYPE` | `benchmark/benchmark.yaml` |The complete path to the benchmarkType file. Used in the TheodoliteYamlExecutor.| +| `THEODOLITE_APP_RESOURCES` | `./benchmark-resources` |The path under which the yamls for the resources for the subexperiments are found. Used in the KubernetesBenchmark| +| `MODE` | `standalone` |Defines the mode of operation: either `standalone` or `operator` -**Development:** (local via Intellij) +**Execution in IntelliJ**: When running Theodolite from within IntelliJ via -[Run Configurations](https://www.jetbrains.com/help/idea/work-with-gradle-tasks.html#gradle_run_config), set the * -Environment variables* field to: +[Run Configurations](https://www.jetbrains.com/help/idea/work-with-gradle-tasks.html#gradle_run_config), set the *Environment variables* field to: + +Set the following environment variables to run the example in the `standalone` mode within the IDE: + +```sh +THEODOLITE_BENCHMARK=./../../../../examples/standalone/example-benchmark.yaml;THEODOLITE_EXECUTION=./../../../../examples/standalone/example-execution.yaml;THEODOLITE_APP_RESOURCES=./../../../../examples/resources; +``` + +Alternative: +``` sh +export THEODOLITE_BENCHMARK=./../../../../examples/standalone/example-benchmark.yaml +export THEODOLITE_APP_RESOURCES=./../../../../examples/resources; +export THEODOLITE_EXECUTION=./../../../../examples/standalone/example-execution.yaml +./gradlew quarkusDev ``` -NAMESPACE=default;THEODOLITE_BENCHMARK=./../../../../config/BenchmarkType.yaml;THEODOLITE_APP_RESOURCES=./../../../../config;THEODOLITE_EXECUTION=./../../../../config/BenchmarkExecution.yaml;MODE=operator + +Set the following environment variables to run the example in the `operator` mode within the IDE: + +```sh +THEODOLITE_APP_RESOURCES=./../../../../examples/resources;MODE=operator ``` Alternative: ``` sh -export NAMESPACE=default -export THEODOLITE_BENCHMARK=./../../../../config/BenchmarkType.yaml -export THEODOLITE_APP_RESOURCES=./../../../../config -export THEODOLITE_EXECUTION=./../../../../config/BenchmarkExecution.yaml +export THEODOLITE_APP_RESOURCES=./../../../../examples/resources; export MODE=operator ./gradlew quarkusDev - ``` -#### Install Detekt Code analysis Plugin +Additionally, the benchmark and execution resources must be installed. + +### Install Detekt Code analysis Plugin -Install https://plugins.jetbrains.com/plugin/10761-detekt +Install <https://plugins.jetbrains.com/plugin/10761-detekt> - Install the plugin - Navigate to Settings/Preferences -> Tools -> Detekt @@ -127,7 +143,7 @@ Install https://plugins.jetbrains.com/plugin/10761-detekt -> detekt issues will be annotated on-the-fly while coding -**ingore Failures in build:** add +**ingore Failures in build**: add ```ignoreFailures = true``` diff --git a/theodolite/crd/crd-benchmark.yaml b/theodolite/crd/crd-benchmark.yaml index 65504d6ec23245a1206f8cc22fdb8f186b3cf297..1fa085a1af9a8d7a199a4dc00e258ba8a6921edc 100644 --- a/theodolite/crd/crd-benchmark.yaml +++ b/theodolite/crd/crd-benchmark.yaml @@ -20,24 +20,11 @@ spec: properties: spec: type: object - required: ["appResource", "loadGenResource", "resourceTypes", "loadTypes", "kafkaConfig"] + required: ["appResourceSets", "loadGenResourceSets", "resourceTypes", "loadTypes", "kafkaConfig"] properties: name: description: This field exists only for technical reasons and should not be set by the user. The value of the field will be overwritten. type: string - default: "" - appResource: - description: A list of file names that reference Kubernetes resources that are deployed on the cluster for the system under test (SUT). - type: array - minItems: 1 - items: - type: string - loadGenResource: - description: A list of file names that reference Kubernetes resources that are deployed on the cluster for the load generator. - type: array - minItems: 1 - items: - type: string resourceTypes: description: A list of resource types that can be scaled for this `benchmark` resource. For each resource type the concrete values are defined in the `execution` object. type: array @@ -51,7 +38,6 @@ spec: type: string patchers: description: List of patchers used to scale this resource type. - examples: test test test type: array minItems: 1 items: @@ -119,7 +105,7 @@ spec: minItems: 1 items: type: object - required: ["name", "numPartitions", "replicationFactor"] + required: ["name"] properties: name: description: The name of the topic. @@ -137,11 +123,77 @@ spec: description: Determines if this topic should only be deleted after each experiement. For removeOnly topics the name can be a RegEx describing the topic. type: boolean default: false + appResourceSets: + description: The appResourceSets specifies all Kubernetes resources required to start the sut. A resourceSet can be either a configMap resourceSet or a fileSystem resourceSet. + type: array + items: + type: object + oneOf: + - required: [configMap] + - required: [fileSystem] + properties: + configMap: + description: The configMap resourceSet loads the Kubernetes manifests from an Kubernetes configMap. + type: object + properties: + name: + description: The name of the configMap + type: string + files: + description: (Optional) Specifies which files from the configMap should be loaded. If this field is not set, all files are loaded. + type: array + items: + type: string + fileSystem: + description: The fileSystem resourceSet loads the Kubernetes manifests from the filesystem. + type: object + properties: + path: + description: The path to the folder which contains the Kubernetes manifests files. + type: string + files: + description: (Optional) Specifies which files from the configMap should be loaded. If this field is not set, all files are loaded. + type: array + items: + type: string + loadGenResourceSets: + description: The loadGenResourceSets specifies all Kubernetes resources required to start the load generator. A resourceSet can be either a configMap resourceSet or a fileSystem resourceSet. + type: array + items: + type: object + oneOf: + - required: [configMap] + - required: [fileSystem] + properties: + configMap: + description: The configMap resourceSet loads the Kubernetes manifests from an Kubernetes configMap. + type: object + properties: + name: + description: The name of the configMap + type: string + files: + description: (Optional) Specifies which files from the configMap should be loaded. If this field is not set, all files are loaded. + type: array + items: + type: string + fileSystem: + description: The fileSystem resourceSet loads the Kubernetes manifests from the filesystem. + type: object + properties: + path: + description: The path to the folder which contains the Kubernetes manifests files. + type: string + files: + description: (Optional) Specifies which files from the configMap should be loaded. If this field is not set, all files are loaded. + type: array + items: + type: string status: type: object properties: resourceSets: - description: todo + description: The status of a Benchmark indicates whether all resources are available to start the benchmark or not. type: string additionalPrinterColumns: - name: Age diff --git a/theodolite/examples/operator/example-benchmark.yaml b/theodolite/examples/operator/example-benchmark.yaml index 5fc004e3b510dca3365a9d1b053ad3f69416a3db..c10c01bd87937a7741199c3e1d0ddb497ce2a036 100644 --- a/theodolite/examples/operator/example-benchmark.yaml +++ b/theodolite/examples/operator/example-benchmark.yaml @@ -3,14 +3,17 @@ kind: benchmark metadata: name: uc1-kstreams spec: - appResource: - - "uc1-kstreams-deployment.yaml" - - "aggregation-service.yaml" - - "jmx-configmap.yaml" - - "uc1-service-monitor.yaml" - loadGenResource: - - "uc1-load-generator-deployment.yaml" - - "uc1-load-generator-service.yaml" + appResourceSets: + - configMap: + name: "example-configmap" + files: + - "uc1-kstreams-deployment.yaml" + loadGenResourceSets: + - configMap: + name: "example-configmap" + files: + - uc1-load-generator-service.yaml + - uc1-load-generator-deployment.yaml resourceTypes: - typeName: "Instances" patchers: @@ -36,5 +39,26 @@ spec: replicationFactor: 1 - name: "theodolite-.*" removeOnly: True - numPartitions: 40 - replicationFactor: 1 \ No newline at end of file + appResourceSets: + # - fileSystem: + # path: ./../../../../../../config + # files: + # - "uc1-kstreams-deployment.yaml" + # - "aggregation-service.yaml" + # - "jmx-configmap.yaml" + # - "uc1-service-monitor.yaml" + - configMap: + name: "example-configmap" + files: + - "uc1-kstreams-deployment.yaml" + loadGenResourceSets: + # - fileSystem: + # path: ./../../../../../../config + # files: + # - uc1-load-generator-service.yaml + # - uc1-load-generator-deployment.yaml + - configMap: + name: "example-configmap" + files: + - uc1-load-generator-service.yaml + - uc1-load-generator-deployment.yaml diff --git a/theodolite/examples/operator/example-configmap.yaml b/theodolite/examples/operator/example-configmap.yaml new file mode 100644 index 0000000000000000000000000000000000000000..210ce32d3fc0f75b9ffce874d1fa0a1ea9bdc3cd --- /dev/null +++ b/theodolite/examples/operator/example-configmap.yaml @@ -0,0 +1,87 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: example-configmap +data: + uc1-kstreams-deployment.yaml: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: titan-ccp-aggregation + spec: + selector: + matchLabels: + app: titan-ccp-aggregation + replicas: 1 + template: + metadata: + labels: + app: titan-ccp-aggregation + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: uc-application + image: ghcr.io/cau-se/theodolite-uc1-kstreams-app:latest + env: + - name: KAFKA_BOOTSTRAP_SERVERS + value: "theodolite-cp-kafka:9092" + - name: SCHEMA_REGISTRY_URL + value: "http://theodolite-cp-schema-registry:8081" + - name: JAVA_OPTS + value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555" + - name: COMMIT_INTERVAL_MS # Set as default for the applications + value: "100" + resources: + limits: + memory: 4Gi + cpu: 1000m + uc1-load-generator-deployment.yaml: | + apiVersion: apps/v1 + kind: Deployment + metadata: + name: titan-ccp-load-generator + spec: + selector: + matchLabels: + app: titan-ccp-load-generator + replicas: 1 + template: + metadata: + labels: + app: titan-ccp-load-generator + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: workload-generator + image: ghcr.io/cau-se/theodolite-uc1-workload-generator:latest + ports: + - containerPort: 5701 + name: coordination + env: + - name: KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: KUBERNETES_DNS_NAME + value: "titan-ccp-load-generator.$(KUBERNETES_NAMESPACE).svc.cluster.local" + - name: KAFKA_BOOTSTRAP_SERVERS + value: "theodolite-cp-kafka:9092" + - name: SCHEMA_REGISTRY_URL + value: "http://theodolite-cp-schema-registry:8081" + uc1-load-generator-service.yaml: | + apiVersion: v1 + kind: Service + metadata: + name: titan-ccp-load-generator + labels: + app: titan-ccp-load-generator + spec: + type: ClusterIP + clusterIP: None + selector: + app: titan-ccp-load-generator + ports: + - name: coordination + port: 5701 + targetPort: 5701 + protocol: TCP \ No newline at end of file diff --git a/theodolite/examples/resources/uc1-kstreams-deployment.yaml b/theodolite/examples/resources/uc1-kstreams-deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fdd1ff867ac83beb10856baec53569c88169232e --- /dev/null +++ b/theodolite/examples/resources/uc1-kstreams-deployment.yaml @@ -0,0 +1,34 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: titan-ccp-aggregation +spec: + selector: + matchLabels: + app: titan-ccp-aggregation + replicas: 1 + template: + metadata: + labels: + app: titan-ccp-aggregation + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: uc-application + image: ghcr.io/cau-se/theodolite-uc1-kstreams-app:latest + ports: + - containerPort: 5555 + name: jmx + env: + - name: KAFKA_BOOTSTRAP_SERVERS + value: "theodolite-cp-kafka:9092" + - name: SCHEMA_REGISTRY_URL + value: "http://theodolite-cp-schema-registry:8081" + - name: JAVA_OPTS + value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555" + - name: COMMIT_INTERVAL_MS # Set as default for the applications + value: "100" + resources: + limits: + memory: 4Gi + cpu: 1000m \ No newline at end of file diff --git a/theodolite/examples/resources/uc1-load-generator-deployment.yaml b/theodolite/examples/resources/uc1-load-generator-deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9f9ccc6ae39407bb1f027e1e23cb152944b869e0 --- /dev/null +++ b/theodolite/examples/resources/uc1-load-generator-deployment.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: titan-ccp-load-generator +spec: + selector: + matchLabels: + app: titan-ccp-load-generator + replicas: 1 + template: + metadata: + labels: + app: titan-ccp-load-generator + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: workload-generator + image: ghcr.io/cau-se/theodolite-uc1-workload-generator:latest + ports: + - containerPort: 5701 + name: coordination + env: + - name: KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: KUBERNETES_DNS_NAME + value: "titan-ccp-load-generator.$(KUBERNETES_NAMESPACE).svc.cluster.local" + - name: KAFKA_BOOTSTRAP_SERVERS + value: "theodolite-cp-kafka:9092" + - name: SCHEMA_REGISTRY_URL + value: "http://theodolite-cp-schema-registry:8081" diff --git a/theodolite/examples/resources/uc1-load-generator-service.yaml b/theodolite/examples/resources/uc1-load-generator-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f8b26b3f6dece427f9c1ad4db94e351b042749b3 --- /dev/null +++ b/theodolite/examples/resources/uc1-load-generator-service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: titan-ccp-load-generator + labels: + app: titan-ccp-load-generator +spec: + type: ClusterIP + clusterIP: None + selector: + app: titan-ccp-load-generator + ports: + - name: coordination + port: 5701 + targetPort: 5701 + protocol: TCP diff --git a/theodolite/examples/standalone/example-benchmark.yaml b/theodolite/examples/standalone/example-benchmark.yaml index 83edce93834ca9b8eef5606c1e5884ce40bdd7d8..9c00dd0fde10e7fb2a9424a243bdbf4d0ba86980 100644 --- a/theodolite/examples/standalone/example-benchmark.yaml +++ b/theodolite/examples/standalone/example-benchmark.yaml @@ -1,12 +1,15 @@ name: "uc1-kstreams" -appResource: - - "uc1-kstreams-deployment.yaml" - - "aggregation-service.yaml" - - "jmx-configmap.yaml" - - "uc1-service-monitor.yaml" -loadGenResource: - - "uc1-load-generator-deployment.yaml" - - "uc1-load-generator-service.yaml" +appResourceSets: + - configMap: + name: "example-configmap" + files: + - "uc1-kstreams-deployment.yaml" +loadGenResourceSets: + - configMap: + name: "example-configmap" + files: + - uc1-load-generator-service.yaml + - uc1-load-generator-deployment.yaml resourceTypes: - typeName: "Instances" patchers: @@ -25,7 +28,7 @@ loadTypes: properties: loadGenMaxRecords: "15000" kafkaConfig: - bootstrapServer: "localhost:31290" + bootstrapServer: "theodolite-cp-kafka:9092" topics: - name: "input" numPartitions: 40 diff --git a/theodolite/src/main/kotlin/theodolite/benchmark/ConfigMapResourceSet.kt b/theodolite/src/main/kotlin/theodolite/benchmark/ConfigMapResourceSet.kt new file mode 100644 index 0000000000000000000000000000000000000000..1f89549fb4eabb86fa1a766627abe3bf7c1b3cc6 --- /dev/null +++ b/theodolite/src/main/kotlin/theodolite/benchmark/ConfigMapResourceSet.kt @@ -0,0 +1,74 @@ +package theodolite.benchmark + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize +import io.fabric8.kubernetes.api.model.KubernetesResource +import io.fabric8.kubernetes.client.KubernetesClientException +import io.fabric8.kubernetes.client.NamespacedKubernetesClient +import io.quarkus.runtime.annotations.RegisterForReflection +import mu.KotlinLogging +import theodolite.k8s.resourceLoader.K8sResourceLoaderFromString +import theodolite.util.DeploymentFailedException +import theodolite.util.YamlParserFromString +import java.lang.IllegalArgumentException +import java.lang.IllegalStateException + +private val logger = KotlinLogging.logger {} + +@RegisterForReflection +@JsonDeserialize +class ConfigMapResourceSet: ResourceSet, KubernetesResource { + lateinit var name: String + lateinit var files: List<String> // load all files, iff files is not set + + @OptIn(ExperimentalStdlibApi::class) + override fun getResourceSet(client: NamespacedKubernetesClient): Collection<Pair<String, KubernetesResource>> { + val loader = K8sResourceLoaderFromString(client) + + logger.info {"use namespace: ${client.namespace} in configmap resource set" } + + var resources: Map<String, String> + + try { + resources = client + .configMaps() + .withName(name) + .get() + .data + .filter { it.key.endsWith(".yaml") } // consider only yaml files, e.g. ignore readme files + } catch (e: KubernetesClientException) { + throw DeploymentFailedException("can not find or read configmap: $name", e) + } catch (e: IllegalStateException) { + throw DeploymentFailedException("can not find configmap or data section is null $name", e) + } + + if (::files.isInitialized){ + resources = resources + .filter { files.contains(it.key) } + } + + return try { + resources + .map { Pair( + getKind(resource = it.value), + it) } + .map { + Pair( + it.second.key, + loader.loadK8sResource(it.first, it.second.value)) } + } catch (e: IllegalArgumentException) { + throw DeploymentFailedException("Can not creat resource set from specified configmap", e) + } + + } + + private fun getKind(resource: String): String { + val parser = YamlParserFromString() + val resourceAsMap = parser.parse(resource, HashMap<String, String>()::class.java) + + return try { + resourceAsMap?.get("kind") !! + } catch (e: NullPointerException) { + throw DeploymentFailedException( "Could not find field kind of Kubernetes resource: ${resourceAsMap?.get("name")}", e) + } + } +} \ No newline at end of file diff --git a/theodolite/src/main/kotlin/theodolite/benchmark/FileSystemResourceSet.kt b/theodolite/src/main/kotlin/theodolite/benchmark/FileSystemResourceSet.kt new file mode 100644 index 0000000000000000000000000000000000000000..92df1bec3cd6f21b1f830e73b466f70e37a9f4c8 --- /dev/null +++ b/theodolite/src/main/kotlin/theodolite/benchmark/FileSystemResourceSet.kt @@ -0,0 +1,66 @@ +package theodolite.benchmark + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize +import io.fabric8.kubernetes.api.model.KubernetesResource +import io.fabric8.kubernetes.client.DefaultKubernetesClient +import io.fabric8.kubernetes.client.NamespacedKubernetesClient +import io.quarkus.runtime.annotations.RegisterForReflection +import mu.KotlinLogging +import theodolite.k8s.resourceLoader.K8sResourceLoaderFromFile +import theodolite.util.DeploymentFailedException +import theodolite.util.YamlParserFromFile +import java.io.File +import java.io.FileNotFoundException +import java.lang.IllegalArgumentException + +private val logger = KotlinLogging.logger {} + +@RegisterForReflection +@JsonDeserialize +class FileSystemResourceSet: ResourceSet, KubernetesResource { + lateinit var path: String + lateinit var files: List<String> + + override fun getResourceSet(client: NamespacedKubernetesClient): Collection<Pair<String, KubernetesResource>> { + + //if files is set ... + if(::files.isInitialized){ + return files + .map { loadSingleResource(resourceURL = it, client = client) } + } + + return try { + File(path) + .list() !! + .filter { it.endsWith(".yaml") } // consider only yaml files, e.g. ignore readme files + .map { + loadSingleResource(resourceURL = it, client = client) + } + } catch (e: NullPointerException) { + throw DeploymentFailedException("Could not load files located in $path", e) + } + } + + private fun loadSingleResource(resourceURL: String, client: NamespacedKubernetesClient): Pair<String, KubernetesResource> { + val parser = YamlParserFromFile() + val loader = K8sResourceLoaderFromFile(client) + val resourcePath = "$path/$resourceURL" + lateinit var kind: String + + try { + kind = parser.parse(resourcePath, HashMap<String, String>()::class.java)?.get("kind")!! + } catch (e: NullPointerException) { + throw DeploymentFailedException("Can not get Kind from resource $resourcePath", e) + } catch (e: FileNotFoundException){ + throw DeploymentFailedException("File $resourcePath not found", e) + + } + + return try { + val k8sResource = loader.loadK8sResource(kind, resourcePath) + Pair(resourceURL, k8sResource) + } catch (e: IllegalArgumentException) { + throw DeploymentFailedException("Could not load resource: $resourcePath", e) + } + } +} \ No newline at end of file diff --git a/theodolite/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt b/theodolite/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt index 125fdfd8adbedf85267bb87ac241af371245c2ad..cbdaab0d3158990ceff781045134638e8782989f 100644 --- a/theodolite/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt +++ b/theodolite/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt @@ -5,10 +5,11 @@ import io.fabric8.kubernetes.api.model.KubernetesResource import io.fabric8.kubernetes.client.DefaultKubernetesClient import io.quarkus.runtime.annotations.RegisterForReflection import mu.KotlinLogging -import theodolite.k8s.K8sResourceLoader +import theodolite.k8s.resourceLoader.K8sResourceLoader import theodolite.patcher.PatcherFactory import theodolite.util.* + private val logger = KotlinLogging.logger {} private var DEFAULT_NAMESPACE = "default" @@ -34,32 +35,20 @@ private var DEFAULT_THEODOLITE_APP_RESOURCES = "./benchmark-resources" @RegisterForReflection class KubernetesBenchmark : KubernetesResource, Benchmark { lateinit var name: String - lateinit var appResource: List<String> - lateinit var loadGenResource: List<String> lateinit var resourceTypes: List<TypeName> lateinit var loadTypes: List<TypeName> lateinit var kafkaConfig: KafkaConfig + lateinit var appResourceSets: List<ResourceSets> + lateinit var loadGenResourceSets: List<ResourceSets> var namespace = System.getenv("NAMESPACE") ?: DEFAULT_NAMESPACE - /** * Loads [KubernetesResource]s. - * It first loads them via the [YamlParser] to check for their concrete type and afterwards initializes them using + * It first loads them via the [YamlParserFromFile] to check for their concrete type and afterwards initializes them using * the [K8sResourceLoader] */ - fun loadKubernetesResources(resources: List<String>): List<Pair<String, KubernetesResource>> { - val path = System.getenv("THEODOLITE_APP_RESOURCES") ?: DEFAULT_THEODOLITE_APP_RESOURCES - logger.info { "Using $path as resource path." } - - val parser = YamlParser() - val loader = K8sResourceLoader(DefaultKubernetesClient()) - return resources - .map { resource -> - val resourcePath = "$path/$resource" - val kind = parser.parse(resourcePath, HashMap<String, String>()::class.java)?.get("kind")!! - val k8sResource = loader.loadK8sResource(kind, resourcePath) - Pair(resource, k8sResource) - } + fun loadKubernetesResources(resourceSet: List<ResourceSets>): Collection<Pair<String, KubernetesResource>> { + return resourceSet.flatMap { it.loadResourceSet(DefaultKubernetesClient().inNamespace(namespace)) } } /** @@ -80,8 +69,8 @@ class KubernetesBenchmark : KubernetesResource, Benchmark { ): BenchmarkDeployment { logger.info { "Using $namespace as namespace." } - val appResources = loadKubernetesResources(this.appResource) - val loadGenResources = loadKubernetesResources(this.loadGenResource) + val appResources = loadKubernetesResources(this.appResourceSets) + val loadGenResources = loadKubernetesResources(this.loadGenResourceSets) val patcherFactory = PatcherFactory() diff --git a/theodolite/src/main/kotlin/theodolite/benchmark/ResourceSet.kt b/theodolite/src/main/kotlin/theodolite/benchmark/ResourceSet.kt new file mode 100644 index 0000000000000000000000000000000000000000..19fc85845ae99c7a5e4f7369db4b6cd383c3131b --- /dev/null +++ b/theodolite/src/main/kotlin/theodolite/benchmark/ResourceSet.kt @@ -0,0 +1,13 @@ +package theodolite.benchmark + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize +import io.fabric8.kubernetes.api.model.KubernetesResource +import io.fabric8.kubernetes.client.NamespacedKubernetesClient +import io.quarkus.runtime.annotations.RegisterForReflection + +@RegisterForReflection +@JsonDeserialize +interface ResourceSet: KubernetesResource { + + fun getResourceSet(client: NamespacedKubernetesClient): Collection<Pair<String, KubernetesResource>> +} \ No newline at end of file diff --git a/theodolite/src/main/kotlin/theodolite/benchmark/ResourceSets.kt b/theodolite/src/main/kotlin/theodolite/benchmark/ResourceSets.kt new file mode 100644 index 0000000000000000000000000000000000000000..19c0808093f6729978f98940641031d8e425c349 --- /dev/null +++ b/theodolite/src/main/kotlin/theodolite/benchmark/ResourceSets.kt @@ -0,0 +1,31 @@ +package theodolite.benchmark + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.databind.annotation.JsonDeserialize +import io.fabric8.kubernetes.api.model.KubernetesResource +import io.fabric8.kubernetes.client.NamespacedKubernetesClient +import io.quarkus.runtime.annotations.RegisterForReflection +import mu.KotlinLogging +import theodolite.util.DeploymentFailedException + +@JsonDeserialize +@RegisterForReflection +@JsonInclude(JsonInclude.Include.NON_NULL) +class ResourceSets: KubernetesResource { + @JsonProperty("configMap") + lateinit var configMap: ConfigMapResourceSet + + @JsonProperty("fileSystem") + lateinit var fileSystem: FileSystemResourceSet + + fun loadResourceSet(client: NamespacedKubernetesClient): Collection<Pair<String, KubernetesResource>> { + return if (::configMap.isInitialized) { + configMap.getResourceSet(client= client) + } else if (::fileSystem.isInitialized) { + fileSystem.getResourceSet(client= client ) + } else { + throw DeploymentFailedException("could not load resourceSet.") + } + } +} \ No newline at end of file diff --git a/theodolite/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt b/theodolite/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt index 777ea1c0ed43ac3af244dc0aaf770c69c11718cf..281c68e318784ee8206473cd014f814b3f5152a9 100644 --- a/theodolite/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt +++ b/theodolite/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt @@ -2,6 +2,7 @@ package theodolite.evaluation import mu.KotlinLogging import theodolite.benchmark.BenchmarkExecution +import theodolite.util.EvaluationFailedException import theodolite.util.IOHandler import theodolite.util.LoadDimension import theodolite.util.Resource @@ -12,7 +13,6 @@ import java.util.* import java.util.regex.Pattern private val logger = KotlinLogging.logger {} -private val RECORD_LAG_QUERY = "sum by(group)(kafka_consumergroup_group_lag >= 0)" /** * Contains the analysis. Fetches a metric from Prometheus, documents it, and evaluates it. @@ -37,7 +37,7 @@ class AnalysisExecutor( * @return true if the experiment succeeded. */ fun analyze(load: LoadDimension, res: Resource, executionIntervals: List<Pair<Instant, Instant>>): Boolean { - var result = false + var result: Boolean var repetitionCounter = 1 try { @@ -50,7 +50,7 @@ class AnalysisExecutor( fetcher.fetchMetric( start = interval.first, end = interval.second, - query = RECORD_LAG_QUERY + query = SloConfigHandler.getQueryString(sloType = slo.sloType) ) } @@ -58,7 +58,7 @@ class AnalysisExecutor( ioHandler.writeToCSVFile( fileURL = "${fileURL}_${repetitionCounter++}", data = data.getResultAsList(), - columns = listOf("group", "timestamp", "value") + columns = listOf("labels", "timestamp", "value") ) } @@ -71,8 +71,7 @@ class AnalysisExecutor( result = sloChecker.evaluate(prometheusData) } catch (e: Exception) { - // TODO(throw exception in order to make it possible to mark an experiment as unsuccessfully) - logger.error { "Evaluation failed for resource '${res.get()}' and load '${load.get()}'. Error: $e" } + throw EvaluationFailedException("Evaluation failed for resource '${res.get()}' and load '${load.get()} ", e) } return result } diff --git a/theodolite/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt b/theodolite/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt index 448a2a05f8dbeb1aef153895360bfb40e7275224..d646286b70bc5880df1f603afdc2bda22bcc3259 100644 --- a/theodolite/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt +++ b/theodolite/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt @@ -36,13 +36,12 @@ class ExternalSloChecker( */ override fun evaluate(fetchedData: List<PrometheusResponse>): Boolean { var counter = 0 - val data = Gson().toJson( - mapOf( - "total_lags" to fetchedData.map { it.data?.result }, - "threshold" to threshold, - "warmup" to warmup - ) - ) + val data = SloJson.Builder() + .results(fetchedData.map { it.data?.result }) + .addMetadata("threshold", threshold) + .addMetadata( "warmup", warmup) + .build() + .toJson() while (counter < RETRIES) { val result = post(externalSlopeURL, data = data, timeout = TIMEOUT) diff --git a/theodolite/src/main/kotlin/theodolite/evaluation/MetricFetcher.kt b/theodolite/src/main/kotlin/theodolite/evaluation/MetricFetcher.kt index 833d7d1e16c2fbc91b58817b319a7d02af7f5b2b..e54d79fe0f95b9f6079bd4295a74e81250b73a90 100644 --- a/theodolite/src/main/kotlin/theodolite/evaluation/MetricFetcher.kt +++ b/theodolite/src/main/kotlin/theodolite/evaluation/MetricFetcher.kt @@ -53,8 +53,7 @@ class MetricFetcher(private val prometheusURL: String, private val offset: Durat } else { val values = parseValues(response) if (values.data?.result.isNullOrEmpty()) { - logger.error { "Empty query result: $values between $start and $end for query $query." } - throw NoSuchFieldException() + throw NoSuchFieldException("Empty query result: $values between $start and $end for query $query.") } return parseValues(response) } diff --git a/theodolite/src/main/kotlin/theodolite/evaluation/SloCheckerFactory.kt b/theodolite/src/main/kotlin/theodolite/evaluation/SloCheckerFactory.kt index 76b158a580102e209b13e247864dd7481b557638..64f9110cd931feef41dc65f88d6623e82f4e03a2 100644 --- a/theodolite/src/main/kotlin/theodolite/evaluation/SloCheckerFactory.kt +++ b/theodolite/src/main/kotlin/theodolite/evaluation/SloCheckerFactory.kt @@ -22,13 +22,13 @@ class SloCheckerFactory { * - `warmup`: time from the beginning to skip in the analysis. * * - * ### `lag trend percent` + * ### `lag trend ratio` * Creates an [ExternalSloChecker] with defined parameters. - * The required threshold is computed using a percentage and the load of the experiment. + * The required threshold is computed using a ratio and the load of the experiment. * * The properties map needs the following fields: * - `externalSlopeURL`: Url to the concrete SLO checker service. - * - `percent`: of the executed load that is accepted for the slope. + * - `ratio`: of the executed load that is accepted for the slope. * - `warmup`: time from the beginning to skip in the analysis. * * @param sloType Type of the [SloChecker]. @@ -43,25 +43,23 @@ class SloCheckerFactory { properties: MutableMap<String, String>, load: LoadDimension ): SloChecker { - return when (sloType) { - "lag trend" -> ExternalSloChecker( + return when (sloType.toLowerCase()) { + SloTypes.LAG_TREND.value, SloTypes.DROPPED_RECORDS.value -> ExternalSloChecker( externalSlopeURL = properties["externalSloUrl"] ?: throw IllegalArgumentException("externalSloUrl expected"), threshold = properties["threshold"]?.toInt() ?: throw IllegalArgumentException("threshold expected"), warmup = properties["warmup"]?.toInt() ?: throw IllegalArgumentException("warmup expected") ) - "lag trend percent" -> { - if (!properties["loadType"].equals("NumSensors")) { - throw IllegalArgumentException("Percent Threshold is only allowed with load type NumSensors") - } - var thresholdPercent = - properties["percent"]?.toDouble() - ?: throw IllegalArgumentException("percent for threshold expected") - if (thresholdPercent < 0.0 || thresholdPercent > 1.0) { - throw IllegalArgumentException("Threshold percent need to be an Double in the range between 0.0 and 1.0 (inclusive)") + + SloTypes.LAG_TREND_RATIO.value, SloTypes.DROPPED_RECORDS_RATIO.value -> { + val thresholdRatio = + properties["ratio"]?.toDouble() + ?: throw IllegalArgumentException("ratio for threshold expected") + if (thresholdRatio < 0.0) { + throw IllegalArgumentException("Threshold ratio needs to be an Double greater or equal 0.0") } // cast to int, as rounding is not really necessary - var threshold = (load.get() * thresholdPercent).toInt() + val threshold = (load.get() * thresholdRatio).toInt() ExternalSloChecker( externalSlopeURL = properties["externalSloUrl"] diff --git a/theodolite/src/main/kotlin/theodolite/evaluation/SloConfigHandler.kt b/theodolite/src/main/kotlin/theodolite/evaluation/SloConfigHandler.kt new file mode 100644 index 0000000000000000000000000000000000000000..93929218c822030ff065dafb19cce1fbaa69a179 --- /dev/null +++ b/theodolite/src/main/kotlin/theodolite/evaluation/SloConfigHandler.kt @@ -0,0 +1,20 @@ +package theodolite.evaluation + +import theodolite.util.InvalidPatcherConfigurationException +import javax.enterprise.context.ApplicationScoped + +private const val CONSUMER_LAG_QUERY = "sum by(group)(kafka_consumergroup_group_lag >= 0)" +private const val DROPPED_RECORDS_QUERY = "sum by(job) (kafka_streams_stream_task_metrics_dropped_records_total>=0)" + +@ApplicationScoped +class SloConfigHandler() { + companion object { + fun getQueryString(sloType: String): String { + return when (sloType.toLowerCase()) { + SloTypes.LAG_TREND.value, SloTypes.LAG_TREND_RATIO.value -> CONSUMER_LAG_QUERY + SloTypes.DROPPED_RECORDS.value, SloTypes.DROPPED_RECORDS_RATIO.value -> DROPPED_RECORDS_QUERY + else -> throw InvalidPatcherConfigurationException("Could not find Prometheus query string for slo type $sloType") + } + } + } +} \ No newline at end of file diff --git a/theodolite/src/main/kotlin/theodolite/evaluation/SloJson.kt b/theodolite/src/main/kotlin/theodolite/evaluation/SloJson.kt new file mode 100644 index 0000000000000000000000000000000000000000..fc9fe17b255dbb5ae68881538d8d2a50a191edb1 --- /dev/null +++ b/theodolite/src/main/kotlin/theodolite/evaluation/SloJson.kt @@ -0,0 +1,63 @@ +package theodolite.evaluation + +import com.google.gson.Gson +import theodolite.util.PromResult + +class SloJson private constructor( + val results: List<List<PromResult>?>? = null, + var metadata: MutableMap<String, Any>? = null +) { + + data class Builder( + var results:List<List<PromResult>?>? = null, + var metadata: MutableMap<String, Any>? = null + ) { + + /** + * Set the results + * + * @param results list of prometheus results + */ + fun results(results: List<List<PromResult>?>) = apply { this.results = results } + + /** + * Add metadata as key value pairs + * + * @param key key of the metadata to be added + * @param value value of the metadata to be added + */ + fun addMetadata(key: String, value: String) = apply { + if (this.metadata.isNullOrEmpty()) { + this.metadata = mutableMapOf(key to value) + } else { + this.metadata!![key] = value + } + } + + /** + * Add metadata as key value pairs + * + * @param key key of the metadata to be added + * @param value value of the metadata to be added + */ + fun addMetadata(key: String, value: Int) = apply { + if (this.metadata.isNullOrEmpty()) { + this.metadata = mutableMapOf(key to value) + } else { + this.metadata!![key] = value + } + } + + fun build() = SloJson( + results = results, + metadata = metadata + ) + } + + fun toJson(): String { + return Gson().toJson(mapOf( + "results" to this.results, + "metadata" to this.metadata + )) + } +} \ No newline at end of file diff --git a/theodolite/src/main/kotlin/theodolite/evaluation/SloTypes.kt b/theodolite/src/main/kotlin/theodolite/evaluation/SloTypes.kt new file mode 100644 index 0000000000000000000000000000000000000000..ac9de35861b0bd9c012bfb0b8cfcb2e1aa5aed68 --- /dev/null +++ b/theodolite/src/main/kotlin/theodolite/evaluation/SloTypes.kt @@ -0,0 +1,10 @@ +package theodolite.evaluation + +enum class SloTypes(val value: String) { + LAG_TREND("lag trend"), + LAG_TREND_RATIO("lag trend ratio"), + DROPPED_RECORDS("dropped records"), + DROPPED_RECORDS_RATIO("dropped records ratio") + + +} \ No newline at end of file diff --git a/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt b/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt index e7b511d8c83b5abccece1204aad2a4a9ecfdfd26..3238f447be06ce6486bb7f6ca1758700f36ba558 100644 --- a/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt +++ b/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt @@ -25,11 +25,12 @@ abstract class BenchmarkExecutor( val results: Results, val executionDuration: Duration, val configurationOverrides: List<ConfigurationOverride?>, - val slo: BenchmarkExecution.Slo, + val slos: List<BenchmarkExecution.Slo>, val repetitions: Int, val executionId: Int, val loadGenerationDelay: Long, - val afterTeardownDelay: Long + val afterTeardownDelay: Long, + val executionName: String ) { var run: AtomicBoolean = AtomicBoolean(true) diff --git a/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt b/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt index c54d1878d4957a0b8ec6a8fdfb18ec6342d7bfc1..2e938be3a6e503a5e7e3f94c18a9454e173db5b0 100644 --- a/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt +++ b/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt @@ -5,10 +5,8 @@ import mu.KotlinLogging import theodolite.benchmark.Benchmark import theodolite.benchmark.BenchmarkExecution import theodolite.evaluation.AnalysisExecutor -import theodolite.util.ConfigurationOverride -import theodolite.util.LoadDimension -import theodolite.util.Resource -import theodolite.util.Results +import theodolite.execution.operator.EventCreator +import theodolite.util.* import java.time.Duration import java.time.Instant @@ -20,29 +18,34 @@ class BenchmarkExecutorImpl( results: Results, executionDuration: Duration, configurationOverrides: List<ConfigurationOverride?>, - slo: BenchmarkExecution.Slo, + slos: List<BenchmarkExecution.Slo>, repetitions: Int, executionId: Int, loadGenerationDelay: Long, - afterTeardownDelay: Long + afterTeardownDelay: Long, + executionName: String ) : BenchmarkExecutor( benchmark, results, executionDuration, configurationOverrides, - slo, + slos, repetitions, executionId, loadGenerationDelay, - afterTeardownDelay + afterTeardownDelay, + executionName ) { + private val eventCreator = EventCreator() + private val mode = Configuration.EXECUTION_MODE + override fun runExperiment(load: LoadDimension, res: Resource): Boolean { var result = false val executionIntervals: MutableList<Pair<Instant, Instant>> = ArrayList() for (i in 1.rangeTo(repetitions)) { - logger.info { "Run repetition $i/$repetitions" } if (this.run.get()) { + logger.info { "Run repetition $i/$repetitions" } executionIntervals.add(runSingleExperiment(load, res)) } else { break @@ -53,14 +56,23 @@ class BenchmarkExecutorImpl( * Analyse the experiment, if [run] is true, otherwise the experiment was canceled by the user. */ if (this.run.get()) { - result = AnalysisExecutor(slo = slo, executionId = executionId) - .analyze( - load = load, - res = res, - executionIntervals = executionIntervals - ) + val experimentResults = slos.map { + AnalysisExecutor(slo = it, executionId = executionId) + .analyze( + load = load, + res = res, + executionIntervals = executionIntervals + ) + } + + result = (false !in experimentResults) this.results.setResult(Pair(load, res), result) } + + if(!this.run.get()) { + throw ExecutionFailedException("The execution was interrupted") + } + return result } @@ -73,22 +85,49 @@ class BenchmarkExecutorImpl( this.afterTeardownDelay ) val from = Instant.now() - // TODO(restructure try catch in order to throw exceptions if there are significant problems by running a experiment) + try { benchmarkDeployment.setup() this.waitAndLog() + if (mode == ExecutionModes.OPERATOR.value) { + eventCreator.createEvent( + executionName = executionName, + type = "NORMAL", + reason = "Start experiment", + message = "load: ${load.get()}, resources: ${res.get()}") + } } catch (e: Exception) { - logger.error { "Error while setup experiment." } - logger.error { "Error is: $e" } this.run.set(false) + + if (mode == ExecutionModes.OPERATOR.value) { + eventCreator.createEvent( + executionName = executionName, + type = "WARNING", + reason = "Start experiment failed", + message = "load: ${load.get()}, resources: ${res.get()}") + } + throw ExecutionFailedException("Error during setup the experiment", e) } val to = Instant.now() try { benchmarkDeployment.teardown() + if (mode == ExecutionModes.OPERATOR.value) { + eventCreator.createEvent( + executionName = executionName, + type = "NORMAL", + reason = "Stop experiment", + message = "Teardown complete") + } } catch (e: Exception) { - logger.warn { "Error while tearing down the benchmark deployment." } - logger.debug { "Teardown failed, caused by: $e" } + if (mode == ExecutionModes.OPERATOR.value) { + eventCreator.createEvent( + executionName = executionName, + type = "WARNING", + reason = "Stop experiment failed", + message = "Teardown failed: ${e.message}") + } + throw ExecutionFailedException("Error during teardown the experiment", e) } return Pair(from, to) } -} +} \ No newline at end of file diff --git a/theodolite/src/main/kotlin/theodolite/execution/ExecutionModes.kt b/theodolite/src/main/kotlin/theodolite/execution/ExecutionModes.kt new file mode 100644 index 0000000000000000000000000000000000000000..bf947be01b534fd000d3967f0b72ef25978d4110 --- /dev/null +++ b/theodolite/src/main/kotlin/theodolite/execution/ExecutionModes.kt @@ -0,0 +1,7 @@ +package theodolite.execution + +enum class ExecutionModes(val value: String) { + OPERATOR("operator"), + YAML_EXECUTOR("yaml-executor"), + STANDALONE("standalone") +} \ No newline at end of file diff --git a/theodolite/src/main/kotlin/theodolite/execution/Main.kt b/theodolite/src/main/kotlin/theodolite/execution/Main.kt index 7d5fca859422a194e81468d9766a9e7ba29fb998..11f696ddd739e987e92ecec724390948714d898b 100644 --- a/theodolite/src/main/kotlin/theodolite/execution/Main.kt +++ b/theodolite/src/main/kotlin/theodolite/execution/Main.kt @@ -3,6 +3,7 @@ package theodolite.execution import io.quarkus.runtime.annotations.QuarkusMain import mu.KotlinLogging import theodolite.execution.operator.TheodoliteOperator +import theodolite.util.Configuration import kotlin.system.exitProcess private val logger = KotlinLogging.logger {} @@ -13,13 +14,12 @@ object Main { @JvmStatic fun main(args: Array<String>) { - val mode = System.getenv("MODE") ?: "standalone" + val mode = Configuration.EXECUTION_MODE logger.info { "Start Theodolite with mode $mode" } - when (mode) { - "standalone" -> TheodoliteStandalone().start() - "yaml-executor" -> TheodoliteStandalone().start() // TODO remove (#209) - "operator" -> TheodoliteOperator().start() + when (mode.toLowerCase()) { + ExecutionModes.STANDALONE.value, ExecutionModes.YAML_EXECUTOR.value -> TheodoliteStandalone().start() // TODO remove standalone (#209) + ExecutionModes.OPERATOR.value -> TheodoliteOperator().start() else -> { logger.error { "MODE $mode not found" } exitProcess(1) diff --git a/theodolite/src/main/kotlin/theodolite/execution/Shutdown.kt b/theodolite/src/main/kotlin/theodolite/execution/Shutdown.kt index e795ada3e3bcb2dba19f1e088f426f38a824f4a7..6dedc94af864269d7d15929c69ec54aa384fc8e3 100644 --- a/theodolite/src/main/kotlin/theodolite/execution/Shutdown.kt +++ b/theodolite/src/main/kotlin/theodolite/execution/Shutdown.kt @@ -34,16 +34,15 @@ class Shutdown(private val benchmarkExecution: BenchmarkExecution, private val b afterTeardownDelay = 5L ) deployment.teardown() + logger.info { + "Finished teardown of all benchmark resources." + } } catch (e: Exception) { - // TODO(throw exception in order to make it possible to mark an experiment as unsuccessfully) logger.warn { "Could not delete all specified resources from Kubernetes. " + "This could be the case, if not all resources are deployed and running." } } - logger.info { - "Finished teardown of all benchmark resources." - } } } diff --git a/theodolite/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt b/theodolite/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt index bc9371763c30e4cef913a368b64e9989e7f2286b..a5a4904f8ea8de152932333a1b8302f9539e260b 100644 --- a/theodolite/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt +++ b/theodolite/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt @@ -55,20 +55,18 @@ class TheodoliteExecutor( this.kubernetesBenchmark.loadTypes ) - // Add load type to check if the percentage lag trend is applicable - config.slos.forEach { it.properties["loadType"] = config.load.loadType } - executor = BenchmarkExecutorImpl( benchmark = kubernetesBenchmark, results = results, executionDuration = executionDuration, configurationOverrides = config.configOverrides, - slo = config.slos[0], + slos = config.slos, repetitions = config.execution.repetitions, executionId = config.executionId, loadGenerationDelay = config.execution.loadGenerationDelay, - afterTeardownDelay = config.execution.afterTeardownDelay + afterTeardownDelay = config.execution.afterTeardownDelay, + executionName = config.name ) if (config.load.loadValues != config.load.loadValues.sorted()) { @@ -118,23 +116,26 @@ class TheodoliteExecutor( val ioHandler = IOHandler() val resultsFolder = ioHandler.getResultFolderURL() this.config.executionId = getAndIncrementExecutionID(resultsFolder + "expID.txt") - ioHandler.writeToJSONFile(this.config, "$resultsFolder${this.config.executionId}-execution-configuration") + ioHandler.writeToJSONFile(this.config, "${resultsFolder}exp${this.config.executionId}-execution-configuration") ioHandler.writeToJSONFile( kubernetesBenchmark, - "$resultsFolder${this.config.executionId}-benchmark-configuration" + "${resultsFolder}exp${this.config.executionId}-benchmark-configuration" ) val config = buildConfig() // execute benchmarks for each load - for (load in config.loads) { - if (executor.run.get()) { - config.compositeStrategy.findSuitableResource(load, config.resources) + try { + for (load in config.loads) { + if (executor.run.get()) { + config.compositeStrategy.findSuitableResource(load, config.resources) + } } + } finally { + ioHandler.writeToJSONFile( + config.compositeStrategy.benchmarkExecutor.results, + "${resultsFolder}exp${this.config.executionId}-result" + ) } - ioHandler.writeToJSONFile( - config.compositeStrategy.benchmarkExecutor.results, - "$resultsFolder${this.config.executionId}-result" - ) } private fun getAndIncrementExecutionID(fileURL: String): Int { diff --git a/theodolite/src/main/kotlin/theodolite/execution/TheodoliteStandalone.kt b/theodolite/src/main/kotlin/theodolite/execution/TheodoliteStandalone.kt index 76fd7f707a3e190ff6c61052ae4b5aaf50459418..1bbf3e01f461a19dbe588aedd41be63b84c86162 100644 --- a/theodolite/src/main/kotlin/theodolite/execution/TheodoliteStandalone.kt +++ b/theodolite/src/main/kotlin/theodolite/execution/TheodoliteStandalone.kt @@ -3,7 +3,9 @@ package theodolite.execution import mu.KotlinLogging import theodolite.benchmark.BenchmarkExecution import theodolite.benchmark.KubernetesBenchmark -import theodolite.util.YamlParser +import theodolite.util.YamlParserFromFile +import theodolite.util.EvaluationFailedException +import theodolite.util.ExecutionFailedException import kotlin.concurrent.thread import kotlin.system.exitProcess @@ -26,7 +28,7 @@ private val logger = KotlinLogging.logger {} * @constructor Create empty Theodolite yaml executor */ class TheodoliteStandalone { - private val parser = YamlParser() + private val parser = YamlParserFromFile() fun start() { logger.info { "Theodolite started" } @@ -49,8 +51,14 @@ class TheodoliteStandalone { val shutdown = thread(start = false) { Shutdown(benchmarkExecution, benchmark).run() } Runtime.getRuntime().addShutdownHook(shutdown) - val executor = TheodoliteExecutor(benchmarkExecution, benchmark) - executor.run() + try { + TheodoliteExecutor(benchmarkExecution, benchmark).run() + } catch (e: EvaluationFailedException) { + logger.error { "Evaluation failed with error: ${e.message}" } + }catch (e: ExecutionFailedException) { + logger.error { "Execution failed with error: ${e.message}" } + } + logger.info { "Theodolite finished" } Runtime.getRuntime().removeShutdownHook(shutdown) exitProcess(0) diff --git a/theodolite/src/main/kotlin/theodolite/execution/operator/AbstractStateHandler.kt b/theodolite/src/main/kotlin/theodolite/execution/operator/AbstractStateHandler.kt index 9d7436526f18081c7130870956d8a5eea5fc8997..0b5d6040bdea1316f8fb55bcc3f204c5443f6eee 100644 --- a/theodolite/src/main/kotlin/theodolite/execution/operator/AbstractStateHandler.kt +++ b/theodolite/src/main/kotlin/theodolite/execution/operator/AbstractStateHandler.kt @@ -4,10 +4,13 @@ import io.fabric8.kubernetes.api.model.HasMetadata import io.fabric8.kubernetes.api.model.KubernetesResourceList import io.fabric8.kubernetes.api.model.Namespaced import io.fabric8.kubernetes.client.CustomResource +import io.fabric8.kubernetes.client.KubernetesClientException import io.fabric8.kubernetes.client.NamespacedKubernetesClient import io.fabric8.kubernetes.client.dsl.MixedOperation import io.fabric8.kubernetes.client.dsl.Resource +import mu.KotlinLogging import java.lang.Thread.sleep +private val logger = KotlinLogging.logger {} abstract class AbstractStateHandler<T, L, D>( private val client: NamespacedKubernetesClient, @@ -20,11 +23,15 @@ abstract class AbstractStateHandler<T, L, D>( @Synchronized override fun setState(resourceName: String, f: (T) -> T?) { - this.crdClient - .list().items - .filter { it.metadata.name == resourceName } - .map { customResource -> f(customResource) } - .forEach { this.crdClient.updateStatus(it) } + try { + this.crdClient + .list().items + .filter { it.metadata.name == resourceName } + .map { customResource -> f(customResource) } + .forEach { this.crdClient.updateStatus(it) } + } catch (e: KubernetesClientException) { + logger.warn { "Status cannot be set for resource $resourceName" } + } } @Synchronized diff --git a/theodolite/src/main/kotlin/theodolite/execution/operator/ClusterSetup.kt b/theodolite/src/main/kotlin/theodolite/execution/operator/ClusterSetup.kt index c3a2b7b25ed71e797c45d8b497bad6cad15e21e8..6987372f96a6d956378a928011be9b5406590a16 100644 --- a/theodolite/src/main/kotlin/theodolite/execution/operator/ClusterSetup.kt +++ b/theodolite/src/main/kotlin/theodolite/execution/operator/ClusterSetup.kt @@ -54,11 +54,8 @@ class ClusterSetup( benchmark.spec.name = benchmark.metadata.name Shutdown(execution.spec, benchmark.spec).start() } else { - logger.error { - "Execution with state ${States.RUNNING.value} was found, but no corresponding benchmark. " + - "Could not initialize cluster." - } - throw IllegalStateException("Cluster state is invalid, required Benchmark for running execution not found.") + throw IllegalStateException("Execution with state ${States.RUNNING.value} was found, but no corresponding benchmark. " + + "Could not initialize cluster.") } } } diff --git a/theodolite/src/main/kotlin/theodolite/execution/operator/EventCreator.kt b/theodolite/src/main/kotlin/theodolite/execution/operator/EventCreator.kt new file mode 100644 index 0000000000000000000000000000000000000000..fab098ebd5fe765a455d787ddb7fcbfbb6c9ffc7 --- /dev/null +++ b/theodolite/src/main/kotlin/theodolite/execution/operator/EventCreator.kt @@ -0,0 +1,60 @@ +package theodolite.execution.operator + +import io.fabric8.kubernetes.api.model.EventBuilder +import io.fabric8.kubernetes.api.model.EventSource +import io.fabric8.kubernetes.api.model.ObjectReference +import io.fabric8.kubernetes.client.DefaultKubernetesClient +import io.fabric8.kubernetes.client.NamespacedKubernetesClient +import mu.KotlinLogging +import theodolite.util.Configuration +import java.time.Instant +import java.util.* +import kotlin.NoSuchElementException +private val logger = KotlinLogging.logger {} + +class EventCreator { + val client: NamespacedKubernetesClient = DefaultKubernetesClient().inNamespace(Configuration.NAMESPACE) + + fun createEvent(executionName: String, type: String, message: String, reason: String) { + val uuid = UUID.randomUUID().toString() + try { + val objectRef = buildObjectReference(executionName) + val event = EventBuilder() + .withNewMetadata() + .withName(uuid) + .endMetadata() + .withMessage(message) + .withReason(reason) + .withType(type) + .withFirstTimestamp(Instant.now().toString()) // TODO change datetime format + .build() + + val source = EventSource() + source.component = Configuration.COMPONENT_NAME + event.source = source + + event.involvedObject = objectRef + client.v1().events().inNamespace(Configuration.NAMESPACE).createOrReplace(event) + } catch (e: NoSuchElementException) { + logger.warn {"Could not create event: type: $type, message: $message, reason: $reason, no corresponding execution found."} + } + } + + private fun buildObjectReference(executionName: String): ObjectReference { + val exec = TheodoliteOperator() + .getExecutionClient(client = client) + .list() + .items + .first{it.metadata.name == executionName} + + val objectRef = ObjectReference() + objectRef.apiVersion = exec.apiVersion + objectRef.kind = exec.kind + objectRef.uid = exec.metadata.uid + objectRef.name = exec.metadata.name + objectRef.namespace = exec.metadata.namespace + objectRef.resourceVersion = exec.metadata.resourceVersion + + return objectRef + } +} \ No newline at end of file diff --git a/theodolite/src/main/kotlin/theodolite/execution/operator/ExecutionEventHandler.kt b/theodolite/src/main/kotlin/theodolite/execution/operator/ExecutionEventHandler.kt index 62c1ddd4eecb41aecde7000eb048455c95cab949..1209195ee09cebe382f010f38e955dea1c860cd1 100644 --- a/theodolite/src/main/kotlin/theodolite/execution/operator/ExecutionEventHandler.kt +++ b/theodolite/src/main/kotlin/theodolite/execution/operator/ExecutionEventHandler.kt @@ -80,7 +80,7 @@ class ExecutionHandler( override fun onDelete(execution: ExecutionCRD, b: Boolean) { logger.info { "Delete execution ${execution.metadata.name}" } if (execution.status.executionState == States.RUNNING.value - && this.controller.isExecutionRunning(execution.spec.name) + && this.controller.isExecutionRunning(execution.metadata.name) ) { this.controller.stop() } diff --git a/theodolite/src/main/kotlin/theodolite/execution/operator/TheodoliteController.kt b/theodolite/src/main/kotlin/theodolite/execution/operator/TheodoliteController.kt index 882cd9c597d0d0240a77e08267fad1fa76006768..0226911c9007fd9e2e1c685df73ed6dc455cd3a7 100644 --- a/theodolite/src/main/kotlin/theodolite/execution/operator/TheodoliteController.kt +++ b/theodolite/src/main/kotlin/theodolite/execution/operator/TheodoliteController.kt @@ -5,9 +5,11 @@ import io.fabric8.kubernetes.client.dsl.Resource import mu.KotlinLogging import theodolite.benchmark.BenchmarkExecution import theodolite.benchmark.KubernetesBenchmark +import theodolite.execution.ExecutionModes import theodolite.execution.TheodoliteExecutor import theodolite.model.crd.* import theodolite.patcher.ConfigOverrideModifier +import theodolite.util.ExecutionFailedException import theodolite.util.ExecutionStateComparator import java.lang.Thread.sleep @@ -69,9 +71,11 @@ class TheodoliteController( * @see BenchmarkExecution */ private fun runExecution(execution: BenchmarkExecution, benchmark: KubernetesBenchmark) { - val modifier = ConfigOverrideModifier( + try { + val modifier = ConfigOverrideModifier( execution = execution, - resources = benchmark.appResource + benchmark.loadGenResource + resources = benchmark.loadKubernetesResources(benchmark.appResourceSets).map { it.first } + + benchmark.loadKubernetesResources(benchmark.loadGenResourceSets).map { it.first } ) modifier.setAdditionalLabels( labelValue = execution.name, @@ -89,7 +93,6 @@ class TheodoliteController( executionStateHandler.setExecutionState(execution.name, States.RUNNING) executionStateHandler.startDurationStateTimer(execution.name) - try { executor = TheodoliteExecutor(execution, benchmark) executor.run() when (executionStateHandler.getExecutionState(execution.name)) { @@ -97,9 +100,18 @@ class TheodoliteController( States.RUNNING -> { executionStateHandler.setExecutionState(execution.name, States.FINISHED) logger.info { "Execution of ${execution.name} is finally stopped." } + } + else -> { + executionStateHandler.setExecutionState(execution.name, States.FAILURE) + logger.warn { "Unexpected execution state, set state to ${States.FAILURE.value}" } } } } catch (e: Exception) { + EventCreator().createEvent( + executionName = execution.name, + type = "WARNING", + reason = "Execution failed", + message = "An error occurs while executing: ${e.message}") logger.error { "Failure while executing execution ${execution.name} with benchmark ${benchmark.name}." } logger.error { "Problem is: $e" } executionStateHandler.setExecutionState(execution.name, States.FAILURE) diff --git a/theodolite/src/main/kotlin/theodolite/execution/operator/TheodoliteOperator.kt b/theodolite/src/main/kotlin/theodolite/execution/operator/TheodoliteOperator.kt index 3a7b37cc0612dd319814e6d99d450a8e3c217f8e..4850a44fdddba117178e29d3170f44a95df646e7 100644 --- a/theodolite/src/main/kotlin/theodolite/execution/operator/TheodoliteOperator.kt +++ b/theodolite/src/main/kotlin/theodolite/execution/operator/TheodoliteOperator.kt @@ -11,6 +11,7 @@ import theodolite.model.crd.BenchmarkCRD import theodolite.model.crd.BenchmarkExecutionList import theodolite.model.crd.ExecutionCRD import theodolite.model.crd.KubernetesBenchmarkList +import theodolite.util.Configuration private const val DEFAULT_NAMESPACE = "default" @@ -27,7 +28,7 @@ private val logger = KotlinLogging.logger {} * **See Also:** [Kubernetes Operator Pattern](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) */ class TheodoliteOperator { - private val namespace = System.getenv("NAMESPACE") ?: DEFAULT_NAMESPACE + private val namespace = Configuration.NAMESPACE private val client: NamespacedKubernetesClient = DefaultKubernetesClient().inNamespace(namespace) private lateinit var controller: TheodoliteController @@ -38,7 +39,7 @@ class TheodoliteOperator { fun start() { LeaderElector( client = client, - name = "theodolite-operator" // TODO(make leaslock name configurable via env var) + name = Configuration.COMPONENT_NAME ) .getLeadership(::startOperator) } @@ -127,7 +128,7 @@ class TheodoliteOperator { return this.controller } - private fun getExecutionClient(client: NamespacedKubernetesClient): MixedOperation< + fun getExecutionClient(client: NamespacedKubernetesClient): MixedOperation< ExecutionCRD, BenchmarkExecutionList, Resource<ExecutionCRD>> { @@ -137,7 +138,7 @@ class TheodoliteOperator { ) } - private fun getBenchmarkClient(client: NamespacedKubernetesClient): MixedOperation< + fun getBenchmarkClient(client: NamespacedKubernetesClient): MixedOperation< BenchmarkCRD, KubernetesBenchmarkList, Resource<BenchmarkCRD>> { diff --git a/theodolite/src/main/kotlin/theodolite/k8s/CustomResourceWrapper.kt b/theodolite/src/main/kotlin/theodolite/k8s/CustomResourceWrapper.kt index ab355677ec53216072fb58a170610aa5f12dba28..797ed88389947d66aa626ba2ef3fdf6732f8369d 100644 --- a/theodolite/src/main/kotlin/theodolite/k8s/CustomResourceWrapper.kt +++ b/theodolite/src/main/kotlin/theodolite/k8s/CustomResourceWrapper.kt @@ -8,7 +8,7 @@ import mu.KotlinLogging private val logger = KotlinLogging.logger {} class CustomResourceWrapper( - private val crAsMap: Map<String, String>, + val crAsMap: Map<String, String>, private val context: CustomResourceDefinitionContext ) : KubernetesResource { /** @@ -33,7 +33,7 @@ class CustomResourceWrapper( client.customResource(this.context) .delete(client.configuration.namespace, this.getName()) } catch (e: Exception) { - logger.warn { "Could not delete service monitor" } + logger.warn { "Could not delete custom resource" } } } diff --git a/theodolite/src/main/kotlin/theodolite/k8s/K8sManager.kt b/theodolite/src/main/kotlin/theodolite/k8s/K8sManager.kt index abeb1c514d100fc3a12bd8f210e89d65eff9b2cf..389d5eefad556df502c218862e2f253ef8ad2100 100644 --- a/theodolite/src/main/kotlin/theodolite/k8s/K8sManager.kt +++ b/theodolite/src/main/kotlin/theodolite/k8s/K8sManager.kt @@ -46,8 +46,7 @@ class K8sManager(private val client: NamespacedKubernetesClient) { this.client.apps().deployments().delete(resource) ResourceByLabelHandler(client = client) .blockUntilPodsDeleted( - labelName = "app", - labelValue = resource.spec.selector.matchLabels["app"]!! + matchLabels = resource.spec.selector.matchLabels ) logger.info { "Deployment '${resource.metadata.name}' deleted." } } @@ -59,8 +58,7 @@ class K8sManager(private val client: NamespacedKubernetesClient) { this.client.apps().statefulSets().delete(resource) ResourceByLabelHandler(client = client) .blockUntilPodsDeleted( - labelName = "app", - labelValue = resource.spec.selector.matchLabels["app"]!! + matchLabels = resource.spec.selector.matchLabels ) logger.info { "StatefulSet '$resource.metadata.name' deleted." } } diff --git a/theodolite/src/main/kotlin/theodolite/k8s/K8sResourceLoader.kt b/theodolite/src/main/kotlin/theodolite/k8s/K8sResourceLoader.kt deleted file mode 100644 index faae5ade28deb579df6a463007cbdfbc9cc7706e..0000000000000000000000000000000000000000 --- a/theodolite/src/main/kotlin/theodolite/k8s/K8sResourceLoader.kt +++ /dev/null @@ -1,156 +0,0 @@ -package theodolite.k8s - -import io.fabric8.kubernetes.api.model.ConfigMap -import io.fabric8.kubernetes.api.model.KubernetesResource -import io.fabric8.kubernetes.api.model.Service -import io.fabric8.kubernetes.api.model.apps.Deployment -import io.fabric8.kubernetes.client.NamespacedKubernetesClient -import io.fabric8.kubernetes.client.dsl.base.CustomResourceDefinitionContext -import mu.KotlinLogging -import theodolite.util.YamlParser - -private val logger = KotlinLogging.logger {} - -/** - * Used to load different Kubernetes resources. - * Supports: Deployments, Services, ConfigMaps, and CustomResources. - * @param client KubernetesClient used to deploy or remove. - */ -class K8sResourceLoader(private val client: NamespacedKubernetesClient) { - - /** - * Parses a Service from a service yaml - * @param path of the yaml file - * @return Service from fabric8 - */ - private fun loadService(path: String): Service { - return loadGenericResource(path) { client.services().load(it).get() } - } - - - /** - * Parses a CustomResource from a yaml - * @param path of the yaml file - * @param context specific crd context for this custom resource - * @return CustomResourceWrapper from fabric8 - */ - private fun loadCustomResourceWrapper( - path: String, - context: CustomResourceDefinitionContext - ): CustomResourceWrapper { - return loadGenericResource(path) { - CustomResourceWrapper( - YamlParser().parse( - path, - HashMap<String, String>()::class.java - )!!, - context - ) - } - } - - private fun loadServiceMonitor(path: String): CustomResourceWrapper { - val context = K8sContextFactory().create( - api = "v1", - scope = "Namespaced", - group = "monitoring.coreos.com", - plural = "servicemonitors" - ) - return loadCustomResourceWrapper(path, context) - } - - private fun loadExecution(path: String): KubernetesResource { - val context = K8sContextFactory().create( - api = "v1", - scope = "Namespaced", - group = "theodolite.com", - plural = "executions" - ) - return loadCustomResourceWrapper(path, context) - } - - private fun loadBenchmark(path: String): KubernetesResource { - val context = K8sContextFactory().create( - api = "v1", - scope = "Namespaced", - group = "theodolite.com", - plural = "benchmarks" - ) - return loadCustomResourceWrapper(path, context) - } - - - /** - * Parses a Deployment from a Deployment yaml - * @param path of the yaml file - * @return Deployment from fabric8 - */ - private fun loadDeployment(path: String): Deployment { - return loadGenericResource(path) { client.apps().deployments().load(it).get() } - } - - /** - * Parses a ConfigMap from a ConfigMap yaml - * @param path of the yaml file - * @return ConfigMap from fabric8 - */ - private fun loadConfigmap(path: String): ConfigMap { - return loadGenericResource(path) { client.configMaps().load(it).get() } - } - - /** - * Parses a StatefulSet from a StatefulSet yaml - * @param path of the yaml file - * @return StatefulSet from fabric8 - */ - private fun loadStatefulSet(path: String): KubernetesResource { - return loadGenericResource(path) { client.apps().statefulSets().load(it).get() } - - } - - /** - * Generic helper function to load a resource. - * @param path of the resource - * @param f function that is applied to the resource. - * @throws IllegalArgumentException If the resource could not be loaded. - */ - private fun <T> loadGenericResource(path: String, f: (String) -> T): T { - var resource: T? = null - - try { - resource = f(path) - } catch (e: Exception) { - logger.warn { "You potentially misspelled the path: $path" } - logger.warn { e } - } - - if (resource == null) { - throw IllegalArgumentException("The Resource at path: $path could not be loaded") - } - return resource - } - - /** - * Factory function used to load different k8s resources from a path. - * Supported kinds are: Deployments, Services, ServiceMonitors, ConfigMaps and CustomResources. - * Uses CustomResource as default if Kind is not supported. - * @param kind of the resource. CustomResource as default. - * @param path of the resource to be loaded. - * @throws Exception if the resource could not be loaded. - */ - fun loadK8sResource(kind: String, path: String): KubernetesResource { - return when (kind) { - "Deployment" -> loadDeployment(path) - "Service" -> loadService(path) - "ServiceMonitor" -> loadServiceMonitor(path) - "ConfigMap" -> loadConfigmap(path) - "StatefulSet" -> loadStatefulSet(path) - "Execution" -> loadExecution(path) - "Benchmark" -> loadBenchmark(path) - else -> { - logger.error { "Error during loading of unspecified resource Kind" } - throw IllegalArgumentException("error while loading resource with kind: $kind") - } - } - } -} diff --git a/theodolite/src/main/kotlin/theodolite/k8s/ResourceByLabelHandler.kt b/theodolite/src/main/kotlin/theodolite/k8s/ResourceByLabelHandler.kt index 9f3754c54f4b1eeb018b55787974179647f726b6..28a72c8947bffe7b57203cacf2460d7080fa7b51 100644 --- a/theodolite/src/main/kotlin/theodolite/k8s/ResourceByLabelHandler.kt +++ b/theodolite/src/main/kotlin/theodolite/k8s/ResourceByLabelHandler.kt @@ -99,16 +99,16 @@ class ResourceByLabelHandler(private val client: NamespacedKubernetesClient) { * @param [labelName] the label name * @param [labelValue] the value of this label * */ - fun blockUntilPodsDeleted(labelName: String, labelValue: String) { + fun blockUntilPodsDeleted(matchLabels: MutableMap<String, String>) { while ( !this.client .pods() - .withLabel("$labelName=$labelValue") + .withLabels(matchLabels) .list() .items .isNullOrEmpty() ) { - logger.info { "Wait for pods with label $labelName=$labelValue to be deleted." } + logger.info { "Wait for pods with label ${matchLabels.toString()} to be deleted." } Thread.sleep(1000) } } diff --git a/theodolite/src/main/kotlin/theodolite/k8s/TopicManager.kt b/theodolite/src/main/kotlin/theodolite/k8s/TopicManager.kt index 8e83883fc881db0f7e2b1b75b2fb7c7322a11a00..f2afd71f6e4b4cf8e7106a8fc8a9bd113d9f36e6 100644 --- a/theodolite/src/main/kotlin/theodolite/k8s/TopicManager.kt +++ b/theodolite/src/main/kotlin/theodolite/k8s/TopicManager.kt @@ -30,8 +30,7 @@ class TopicManager(private val kafkaConfig: Map<String, Any>) { result = kafkaAdmin.createTopics(newTopics) result.all().get() // wait for the future to be completed } catch (e: Exception) { // TopicExistsException - logger.warn(e) { "Error during topic creation." } - logger.debug { e } // TODO remove due to attached exception to warn log? + logger.warn { "Error during topic creation. Error is: ${e.message}" } logger.info { "Remove existing topics." } delete(newTopics.map { topic -> topic.name() }, kafkaAdmin) logger.info { "Will retry the topic creation in ${RETRY_TIME / 1000} seconds." } @@ -94,7 +93,7 @@ class TopicManager(private val kafkaConfig: Map<String, Any>) { }" } } catch (e: Exception) { - logger.error(e) { "Error while removing topics: $e" } + logger.error { "Error while removing topics: ${e.message}" } logger.info { "Existing topics are: ${kafkaAdmin.listTopics().names().get()}." } } diff --git a/theodolite/src/main/kotlin/theodolite/k8s/resourceLoader/AbstractK8sLoader.kt b/theodolite/src/main/kotlin/theodolite/k8s/resourceLoader/AbstractK8sLoader.kt new file mode 100644 index 0000000000000000000000000000000000000000..862de14e2a7a4721e15215b0a1389e14f943fe24 --- /dev/null +++ b/theodolite/src/main/kotlin/theodolite/k8s/resourceLoader/AbstractK8sLoader.kt @@ -0,0 +1,73 @@ +package theodolite.k8s.resourceLoader + +import io.fabric8.kubernetes.api.model.KubernetesResource +import mu.KotlinLogging +import theodolite.k8s.K8sContextFactory + +private val logger = KotlinLogging.logger {} + +abstract class AbstractK8sLoader: K8sResourceLoader { + + fun loadK8sResource(kind: String, resourceString: String): KubernetesResource { + return when (kind.replaceFirst(kind[0],kind[0].toUpperCase())) { + "Deployment" -> loadDeployment(resourceString) + "Service" -> loadService(resourceString) + "ServiceMonitor" -> loadServiceMonitor(resourceString) + "ConfigMap" -> loadConfigmap(resourceString) + "StatefulSet" -> loadStatefulSet(resourceString) + "Execution" -> loadExecution(resourceString) + "Benchmark" -> loadBenchmark(resourceString) + else -> { + logger.error { "Error during loading of unspecified resource Kind $kind" } + throw java.lang.IllegalArgumentException("error while loading resource with kind: $kind") + } + } + } + + fun <T> loadGenericResource(resourceString: String, f: (String) -> T): T { + var resource: T? = null + + try { + resource = f(resourceString) + } catch (e: Exception) { + logger.warn { e } + } + + if (resource == null) { + throw IllegalArgumentException("The Resource: $resourceString could not be loaded") + } + return resource + } + + + + override fun loadServiceMonitor(resource: String): KubernetesResource { + val context = K8sContextFactory().create( + api = "v1", + scope = "Namespaced", + group = "monitoring.coreos.com", + plural = "servicemonitors" + ) + return loadCustomResourceWrapper(resource, context) + } + + override fun loadExecution(resource: String): KubernetesResource { + val context = K8sContextFactory().create( + api = "v1", + scope = "Namespaced", + group = "theodolite.com", + plural = "executions" + ) + return loadCustomResourceWrapper(resource, context) + } + + override fun loadBenchmark(resource: String): KubernetesResource { + val context = K8sContextFactory().create( + api = "v1", + scope = "Namespaced", + group = "theodolite.com", + plural = "benchmarks" + ) + return loadCustomResourceWrapper(resource, context) + } +} \ No newline at end of file diff --git a/theodolite/src/main/kotlin/theodolite/k8s/resourceLoader/K8sResourceLoader.kt b/theodolite/src/main/kotlin/theodolite/k8s/resourceLoader/K8sResourceLoader.kt new file mode 100644 index 0000000000000000000000000000000000000000..c123ab2958132cb43ad188136f738b561e91310b --- /dev/null +++ b/theodolite/src/main/kotlin/theodolite/k8s/resourceLoader/K8sResourceLoader.kt @@ -0,0 +1,15 @@ +package theodolite.k8s.resourceLoader + +import io.fabric8.kubernetes.api.model.KubernetesResource +import io.fabric8.kubernetes.client.dsl.base.CustomResourceDefinitionContext + +interface K8sResourceLoader { + fun loadDeployment(resource: String): KubernetesResource + fun loadService(resource: String): KubernetesResource + fun loadStatefulSet(resource: String): KubernetesResource + fun loadExecution(resource: String): KubernetesResource + fun loadBenchmark(resource: String): KubernetesResource + fun loadConfigmap(resource: String): KubernetesResource + fun loadServiceMonitor(resource: String): KubernetesResource + fun loadCustomResourceWrapper(resource: String, context: CustomResourceDefinitionContext): KubernetesResource +} \ No newline at end of file diff --git a/theodolite/src/main/kotlin/theodolite/k8s/resourceLoader/K8sResourceLoaderFromFile.kt b/theodolite/src/main/kotlin/theodolite/k8s/resourceLoader/K8sResourceLoaderFromFile.kt new file mode 100644 index 0000000000000000000000000000000000000000..08f34e1d67c9821c9f9a07a49f4ba8683a072611 --- /dev/null +++ b/theodolite/src/main/kotlin/theodolite/k8s/resourceLoader/K8sResourceLoaderFromFile.kt @@ -0,0 +1,75 @@ +package theodolite.k8s.resourceLoader + +import io.fabric8.kubernetes.api.model.ConfigMap +import io.fabric8.kubernetes.api.model.KubernetesResource +import io.fabric8.kubernetes.api.model.Service +import io.fabric8.kubernetes.api.model.apps.Deployment +import io.fabric8.kubernetes.client.NamespacedKubernetesClient +import io.fabric8.kubernetes.client.dsl.base.CustomResourceDefinitionContext +import theodolite.k8s.CustomResourceWrapper +import theodolite.util.YamlParserFromFile + +/** + * Used to load different Kubernetes resources. + * Supports: Deployments, Services, ConfigMaps, and CustomResources. + * @param client KubernetesClient used to deploy or remove. + */ +class K8sResourceLoaderFromFile(private val client: NamespacedKubernetesClient): AbstractK8sLoader(), + K8sResourceLoader { + + /** + * Parses a Service from a service yaml + * @param resource of the yaml file + * @return Service from fabric8 + */ + override fun loadService(resource: String): Service { + return loadGenericResource(resource) { x: String -> client.services().load(x).get() } + } + + + /** + * Parses a CustomResource from a yaml + * @param path of the yaml file + * @param context specific crd context for this custom resource + * @return CustomResourceWrapper from fabric8 + */ + override fun loadCustomResourceWrapper(resource: String, context: CustomResourceDefinitionContext): CustomResourceWrapper { + return loadGenericResource(resource) { + CustomResourceWrapper( + YamlParserFromFile().parse( + resource, + HashMap<String, String>()::class.java + )!!, + context + ) + } + } + + /** + * Parses a Deployment from a Deployment yaml + * @param resource of the yaml file + * @return Deployment from fabric8 + */ + override fun loadDeployment(resource: String): Deployment { + return loadGenericResource(resource) { x: String -> client.apps().deployments().load(x).get() } + } + + /** + * Parses a ConfigMap from a ConfigMap yaml + * @param resource of the yaml file + * @return ConfigMap from fabric8 + */ + override fun loadConfigmap(resource: String): ConfigMap { + return loadGenericResource(resource) { x: String -> client.configMaps().load(x).get() } + } + + /** + * Parses a StatefulSet from a StatefulSet yaml + * @param resource of the yaml file + * @return StatefulSet from fabric8 + */ + override fun loadStatefulSet(resource: String): KubernetesResource { + return loadGenericResource(resource) { x: String -> client.apps().statefulSets().load(x).get() } + + } +} diff --git a/theodolite/src/main/kotlin/theodolite/k8s/resourceLoader/K8sResourceLoaderFromString.kt b/theodolite/src/main/kotlin/theodolite/k8s/resourceLoader/K8sResourceLoaderFromString.kt new file mode 100644 index 0000000000000000000000000000000000000000..e9611aaa82870dfb676820029cf42c5aab63d672 --- /dev/null +++ b/theodolite/src/main/kotlin/theodolite/k8s/resourceLoader/K8sResourceLoaderFromString.kt @@ -0,0 +1,60 @@ +package theodolite.k8s.resourceLoader + +import io.fabric8.kubernetes.api.model.ConfigMap +import io.fabric8.kubernetes.api.model.KubernetesResource +import io.fabric8.kubernetes.api.model.apps.Deployment +import io.fabric8.kubernetes.client.NamespacedKubernetesClient +import io.fabric8.kubernetes.client.dsl.base.CustomResourceDefinitionContext +import theodolite.k8s.CustomResourceWrapper +import theodolite.util.YamlParserFromString +import java.io.ByteArrayInputStream + +class K8sResourceLoaderFromString(private val client: NamespacedKubernetesClient): AbstractK8sLoader(), + K8sResourceLoader { + + @OptIn(ExperimentalStdlibApi::class) + override fun loadService(resource: String): KubernetesResource { + return loadGenericResource(resource) { x: String -> + val stream = ByteArrayInputStream(x.encodeToByteArray()) + client.services().load(stream).get() } + } + + @OptIn(ExperimentalStdlibApi::class) + override fun loadDeployment(resource: String): Deployment { + return loadGenericResource(resource) { x: String -> + val stream = ByteArrayInputStream(x.encodeToByteArray()) + client.apps().deployments().load(stream).get() } + } + + @OptIn(ExperimentalStdlibApi::class) + override fun loadConfigmap(resource: String): ConfigMap { + return loadGenericResource(resource) { x: String -> + val stream = ByteArrayInputStream(x.encodeToByteArray()) + client.configMaps().load(stream).get() } + } + + @OptIn(ExperimentalStdlibApi::class) + override fun loadStatefulSet(resource: String): KubernetesResource { + return loadGenericResource(resource) { x: String -> + val stream = ByteArrayInputStream(x.encodeToByteArray()) + client.apps().statefulSets().load(stream).get() } + } + + /** + * Parses a CustomResource from a yaml + * @param resource of the yaml file + * @param context specific crd context for this custom resource + * @return CustomResourceWrapper from fabric8 + */ + override fun loadCustomResourceWrapper(resource: String, context: CustomResourceDefinitionContext): CustomResourceWrapper { + return loadGenericResource(resource) { + CustomResourceWrapper( + YamlParserFromString().parse( + resource, + HashMap<String, String>()::class.java + )!!, + context + ) + } + } +} \ No newline at end of file diff --git a/theodolite/src/main/kotlin/theodolite/patcher/DataVolumeLoadGeneratorReplicaPatcher.kt b/theodolite/src/main/kotlin/theodolite/patcher/DataVolumeLoadGeneratorReplicaPatcher.kt new file mode 100644 index 0000000000000000000000000000000000000000..bdc107910edc8ddfb41e7757c775977086a25a26 --- /dev/null +++ b/theodolite/src/main/kotlin/theodolite/patcher/DataVolumeLoadGeneratorReplicaPatcher.kt @@ -0,0 +1,38 @@ +package theodolite.patcher + +import io.fabric8.kubernetes.api.model.KubernetesResource + +/** + * The DataVolumeLoadGeneratorReplicaPatcher takes the total load that should be generated + * and computes the number of instances needed for this load based on the `maxVolume` + * ((load + maxVolume - 1) / maxVolume) and calculates the load per instance + * (loadPerInstance = load / instances). + * The number of instances are set for the load generator and the given variable is set to the + * load per instance. + * + * @property k8sResource Kubernetes resource to be patched. + * @property maxVolume per load generator instance + * @property container Container to be patched. + * @property variableName Name of the environment variable to be patched. + */ +class DataVolumeLoadGeneratorReplicaPatcher( + k8sResource: KubernetesResource, + private val maxVolume: Int, + container: String, + variableName: String +) : AbstractPatcher(k8sResource) { + + private val replicaPatcher = ReplicaPatcher(k8sResource) + private val envVarPatcher = EnvVarPatcher(k8sResource, container, variableName) + + override fun <T> patch(value: T) { + // calculate number of load generator instances and load per instance + val load = Integer.parseInt(value.toString()) + val loadGenInstances = (load + maxVolume - 1) / maxVolume + val loadPerInstance = load / loadGenInstances + + // Patch instance values and load value of generators + replicaPatcher.patch(loadGenInstances.toString()) + envVarPatcher.patch(loadPerInstance.toString()) + } +} diff --git a/theodolite/src/main/kotlin/theodolite/patcher/PatcherFactory.kt b/theodolite/src/main/kotlin/theodolite/patcher/PatcherFactory.kt index 29723355ce23810c709fe4537242d1fd7e195d25..ebad5de74a6b819dbf7887dfad91faac37ed5074 100644 --- a/theodolite/src/main/kotlin/theodolite/patcher/PatcherFactory.kt +++ b/theodolite/src/main/kotlin/theodolite/patcher/PatcherFactory.kt @@ -26,13 +26,13 @@ class PatcherFactory { */ fun createPatcher( patcherDefinition: PatcherDefinition, - k8sResources: List<Pair<String, KubernetesResource>> + k8sResources: Collection<Pair<String, KubernetesResource>> ): Patcher { val resource = k8sResources.filter { it.first == patcherDefinition.resource } .map { resource -> resource.second } .firstOrNull() - ?: throw DeploymentFailedException("Could not find resource ${patcherDefinition.resource}") + ?: throw InvalidPatcherConfigurationException("Could not find resource ${patcherDefinition.resource}") return try { when (patcherDefinition.type) { @@ -48,6 +48,12 @@ class PatcherFactory { k8sResource = resource, loadGenMaxRecords = patcherDefinition.properties["loadGenMaxRecords"]!! ) + "DataVolumeLoadGeneratorReplicaPatcher" -> DataVolumeLoadGeneratorReplicaPatcher( + k8sResource = resource, + maxVolume = patcherDefinition.properties["maxVolume"]!!.toInt(), + container = patcherDefinition.properties["container"]!!, + variableName = patcherDefinition.properties["variableName"]!! + ) "EnvVarPatcher" -> EnvVarPatcher( k8sResource = resource, container = patcherDefinition.properties["container"]!!, @@ -80,10 +86,10 @@ class PatcherFactory { ) else -> throw InvalidPatcherConfigurationException("Patcher type ${patcherDefinition.type} not found.") } - } catch (e: Exception) { + } catch (e: NullPointerException) { throw InvalidPatcherConfigurationException( "Could not create patcher with type ${patcherDefinition.type}" + - " Probably a required patcher argument was not specified." + " Probably a required patcher argument was not specified.", e ) } } diff --git a/theodolite/src/main/kotlin/theodolite/util/Configuration.kt b/theodolite/src/main/kotlin/theodolite/util/Configuration.kt new file mode 100644 index 0000000000000000000000000000000000000000..dac3b943e69bd7e208d318f2a788275f19db11e4 --- /dev/null +++ b/theodolite/src/main/kotlin/theodolite/util/Configuration.kt @@ -0,0 +1,18 @@ +package theodolite.util + +import theodolite.execution.ExecutionModes + +// Defaults +private const val DEFAULT_NAMESPACE = "default" +private const val DEFAULT_COMPONENT_NAME = "theodolite-operator" + + +class Configuration( +) { + companion object { + val NAMESPACE = System.getenv("NAMESPACE") ?: DEFAULT_NAMESPACE + val COMPONENT_NAME = System.getenv("COMPONENT_NAME") ?: DEFAULT_COMPONENT_NAME + val EXECUTION_MODE = System.getenv("MODE") ?: ExecutionModes.STANDALONE.value + } + +} diff --git a/theodolite/src/main/kotlin/theodolite/util/DeploymentFailedException.kt b/theodolite/src/main/kotlin/theodolite/util/DeploymentFailedException.kt index 639a7c86c641cbdcba361410cf5e25fa56dd795f..9f4caedf3db1e09dca7924bf0035c6ace0b835d7 100644 --- a/theodolite/src/main/kotlin/theodolite/util/DeploymentFailedException.kt +++ b/theodolite/src/main/kotlin/theodolite/util/DeploymentFailedException.kt @@ -1,4 +1,4 @@ package theodolite.util -class DeploymentFailedException(message: String) : Exception(message) \ No newline at end of file +open class DeploymentFailedException(message: String, e: Exception? = null) : TheodoliteException(message,e) diff --git a/theodolite/src/main/kotlin/theodolite/util/EvaluationFailedException.kt b/theodolite/src/main/kotlin/theodolite/util/EvaluationFailedException.kt new file mode 100644 index 0000000000000000000000000000000000000000..c67ed7ffd79afc733a97dae05c3203f8e78722ea --- /dev/null +++ b/theodolite/src/main/kotlin/theodolite/util/EvaluationFailedException.kt @@ -0,0 +1,4 @@ +package theodolite.util + +class EvaluationFailedException(message: String, e: Exception? = null) : ExecutionFailedException(message,e) { +} diff --git a/theodolite/src/main/kotlin/theodolite/util/ExecutionFailedException.kt b/theodolite/src/main/kotlin/theodolite/util/ExecutionFailedException.kt new file mode 100644 index 0000000000000000000000000000000000000000..6566a451a3e273214f59962531b6bd17b33a850d --- /dev/null +++ b/theodolite/src/main/kotlin/theodolite/util/ExecutionFailedException.kt @@ -0,0 +1,4 @@ +package theodolite.util + +open class ExecutionFailedException(message: String, e: Exception? = null) : TheodoliteException(message,e) { +} \ No newline at end of file diff --git a/theodolite/src/main/kotlin/theodolite/util/InvalidPatcherConfigurationException.kt b/theodolite/src/main/kotlin/theodolite/util/InvalidPatcherConfigurationException.kt index e8ecd11d524f5c365149ac0b37c7b985812f8c4b..d02948ad341207051c4653ba9400ac0ffe5b03aa 100644 --- a/theodolite/src/main/kotlin/theodolite/util/InvalidPatcherConfigurationException.kt +++ b/theodolite/src/main/kotlin/theodolite/util/InvalidPatcherConfigurationException.kt @@ -1,4 +1,3 @@ package theodolite.util -class InvalidPatcherConfigurationException(message: String) : Exception(message) - +class InvalidPatcherConfigurationException(message: String, e: Exception? = null) : DeploymentFailedException(message,e) diff --git a/theodolite/src/main/kotlin/theodolite/util/PrometheusResponse.kt b/theodolite/src/main/kotlin/theodolite/util/PrometheusResponse.kt index bf33fcf6104645727a13b92cf3a13d36e04a10c6..9b0b0dd4e0a5a48072ca576e874cb850c5f8df3b 100644 --- a/theodolite/src/main/kotlin/theodolite/util/PrometheusResponse.kt +++ b/theodolite/src/main/kotlin/theodolite/util/PrometheusResponse.kt @@ -23,7 +23,7 @@ data class PrometheusResponse( * The format of the returned list is: `[[ group, timestamp, value ], [ group, timestamp, value ], ... ]` */ fun getResultAsList(): List<List<String>> { - val group = data?.result?.get(0)?.metric?.group.toString() + val group = data?.result?.get(0)?.metric?.toString()!! val values = data?.result?.get(0)?.values val result = mutableListOf<List<String>>() @@ -64,18 +64,9 @@ data class PromResult( /** * Label of the metric */ - var metric: PromMetric? = null, + var metric: Map<String, String>? = null, /** * Values of the metric (e.g. [ [ <unix_time>, "<sample_value>" ], ... ]) */ var values: List<Any>? = null -) - -/** - * Corresponds to the metric field in the range-vector result format of a Prometheus range-query response. - */ -@RegisterForReflection -data class PromMetric( - var group: String? = null -) - +) \ No newline at end of file diff --git a/theodolite/src/main/kotlin/theodolite/util/TheodoliteException.kt b/theodolite/src/main/kotlin/theodolite/util/TheodoliteException.kt new file mode 100644 index 0000000000000000000000000000000000000000..fc7453bae6aaa4c5c526eee72c006562ea887eb5 --- /dev/null +++ b/theodolite/src/main/kotlin/theodolite/util/TheodoliteException.kt @@ -0,0 +1,3 @@ +package theodolite.util + +open class TheodoliteException (message: String, e: Exception? = null) : Exception(message,e) \ No newline at end of file diff --git a/theodolite/src/main/kotlin/theodolite/util/YamlParser.kt b/theodolite/src/main/kotlin/theodolite/util/YamlParserFromFile.kt similarity index 92% rename from theodolite/src/main/kotlin/theodolite/util/YamlParser.kt rename to theodolite/src/main/kotlin/theodolite/util/YamlParserFromFile.kt index ce69894e4145372aef07286ae315d11631a4df3f..ae36349e628621bb7ad287d8cf557fbefa3ff5c5 100644 --- a/theodolite/src/main/kotlin/theodolite/util/YamlParser.kt +++ b/theodolite/src/main/kotlin/theodolite/util/YamlParserFromFile.kt @@ -9,7 +9,7 @@ import java.io.InputStream /** * The YamlParser parses a YAML file */ -class YamlParser : Parser { +class YamlParserFromFile : Parser { override fun <T> parse(path: String, E: Class<T>): T? { val input: InputStream = FileInputStream(File(path)) val parser = Yaml(Constructor(E)) diff --git a/theodolite/src/main/kotlin/theodolite/util/YamlParserFromString.kt b/theodolite/src/main/kotlin/theodolite/util/YamlParserFromString.kt new file mode 100644 index 0000000000000000000000000000000000000000..61db189ee99fa5fe36113b0fdecf589ad1114852 --- /dev/null +++ b/theodolite/src/main/kotlin/theodolite/util/YamlParserFromString.kt @@ -0,0 +1,17 @@ +package theodolite.util + +import org.yaml.snakeyaml.Yaml +import org.yaml.snakeyaml.constructor.Constructor +import java.io.File +import java.io.FileInputStream +import java.io.InputStream + +/** + * The YamlParser parses a YAML string + */ +class YamlParserFromString : Parser { + override fun <T> parse(fileString: String, E: Class<T>): T? { + val parser = Yaml(Constructor(E)) + return parser.loadAs(fileString, E) + } +} diff --git a/theodolite/src/test/kotlin/theodolite/CompositeStrategyTest.kt b/theodolite/src/test/kotlin/theodolite/CompositeStrategyTest.kt index 49131352cfe517a382ddd7aa1be09d3fbe317466..580d9e747bde687a91ffb1bce2e7c9dfb6f166a2 100644 --- a/theodolite/src/test/kotlin/theodolite/CompositeStrategyTest.kt +++ b/theodolite/src/test/kotlin/theodolite/CompositeStrategyTest.kt @@ -31,7 +31,7 @@ class CompositeStrategyTest { val results = Results() val benchmark = TestBenchmark() val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo() - val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, sloChecker, 0, 0, 5) + val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, listOf(sloChecker), 0, 0, 5) val linearSearch = LinearSearch(benchmarkExecutor) val lowerBoundRestriction = LowerBoundRestriction(results) val strategy = @@ -65,7 +65,7 @@ class CompositeStrategyTest { val benchmark = TestBenchmark() val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo() val benchmarkExecutorImpl = - TestBenchmarkExecutorImpl(mockResults, benchmark, results, sloChecker, 0, 0, 0) + TestBenchmarkExecutorImpl(mockResults, benchmark, results, listOf(sloChecker), 0, 0, 0) val binarySearch = BinarySearch(benchmarkExecutorImpl) val lowerBoundRestriction = LowerBoundRestriction(results) val strategy = @@ -98,7 +98,7 @@ class CompositeStrategyTest { val results = Results() val benchmark = TestBenchmark() val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo() - val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, sloChecker, 0, 0, 0) + val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, listOf(sloChecker), 0, 0, 0) val binarySearch = BinarySearch(benchmarkExecutor) val lowerBoundRestriction = LowerBoundRestriction(results) val strategy = diff --git a/theodolite/src/test/kotlin/theodolite/ResourceLimitPatcherTest.kt b/theodolite/src/test/kotlin/theodolite/ResourceLimitPatcherTest.kt index e88192dd7fe4393494a4fb76bd74d1123bd75f1d..46758583172c3fcd6417e17ff5bab85f8659734b 100644 --- a/theodolite/src/test/kotlin/theodolite/ResourceLimitPatcherTest.kt +++ b/theodolite/src/test/kotlin/theodolite/ResourceLimitPatcherTest.kt @@ -5,7 +5,7 @@ import io.fabric8.kubernetes.client.DefaultKubernetesClient import io.quarkus.test.junit.QuarkusTest import io.smallrye.common.constraint.Assert.assertTrue import org.junit.jupiter.api.Test -import theodolite.k8s.K8sResourceLoader +import theodolite.k8s.resourceLoader.K8sResourceLoaderFromFile import theodolite.patcher.PatcherFactory import theodolite.util.PatcherDefinition @@ -22,7 +22,7 @@ import theodolite.util.PatcherDefinition @QuarkusTest class ResourceLimitPatcherTest { val testPath = "./src/test/resources/" - val loader = K8sResourceLoader(DefaultKubernetesClient().inNamespace("")) + val loader = K8sResourceLoaderFromFile(DefaultKubernetesClient().inNamespace("")) val patcherFactory = PatcherFactory() fun applyTest(fileName: String) { diff --git a/theodolite/src/test/kotlin/theodolite/ResourceRequestPatcherTest.kt b/theodolite/src/test/kotlin/theodolite/ResourceRequestPatcherTest.kt index 2af6c632567bf47e150a74808ab009bd0bc0598a..8794d4dc2d67b8af78f4fa409c727f882922d0b8 100644 --- a/theodolite/src/test/kotlin/theodolite/ResourceRequestPatcherTest.kt +++ b/theodolite/src/test/kotlin/theodolite/ResourceRequestPatcherTest.kt @@ -5,7 +5,7 @@ import io.fabric8.kubernetes.client.DefaultKubernetesClient import io.quarkus.test.junit.QuarkusTest import io.smallrye.common.constraint.Assert.assertTrue import org.junit.jupiter.api.Test -import theodolite.k8s.K8sResourceLoader +import theodolite.k8s.resourceLoader.K8sResourceLoaderFromFile import theodolite.patcher.PatcherFactory import theodolite.util.PatcherDefinition @@ -22,7 +22,7 @@ import theodolite.util.PatcherDefinition @QuarkusTest class ResourceRequestPatcherTest { val testPath = "./src/test/resources/" - val loader = K8sResourceLoader(DefaultKubernetesClient().inNamespace("")) + val loader = K8sResourceLoaderFromFile(DefaultKubernetesClient().inNamespace("")) val patcherFactory = PatcherFactory() fun applyTest(fileName: String) { diff --git a/theodolite/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt b/theodolite/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt index cbd2d5926d61b0bfd4de6fab0c14422ddf88f190..2efddc48cb93a0870d1716c58a7018145c16e2ff 100644 --- a/theodolite/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt +++ b/theodolite/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt @@ -12,7 +12,7 @@ class TestBenchmarkExecutorImpl( private val mockResults: Array<Array<Boolean>>, benchmark: Benchmark, results: Results, - slo: BenchmarkExecution.Slo, + slo: List<BenchmarkExecution.Slo>, executionId: Int, loadGenerationDelay: Long, afterTeardownDelay: Long @@ -22,11 +22,12 @@ class TestBenchmarkExecutorImpl( results, executionDuration = Duration.ofSeconds(1), configurationOverrides = emptyList(), - slo = slo, + slos = slo, repetitions = 1, executionId = executionId, loadGenerationDelay = loadGenerationDelay, - afterTeardownDelay = afterTeardownDelay + afterTeardownDelay = afterTeardownDelay, + executionName = "test-execution" ) { override fun runExperiment(load: LoadDimension, res: Resource): Boolean { diff --git a/theodolite/src/test/kotlin/theodolite/benchmark/ConfigMapResourceSetTest.kt b/theodolite/src/test/kotlin/theodolite/benchmark/ConfigMapResourceSetTest.kt new file mode 100644 index 0000000000000000000000000000000000000000..2cc8f931418e28ae8841b592f93df8d88440cf3c --- /dev/null +++ b/theodolite/src/test/kotlin/theodolite/benchmark/ConfigMapResourceSetTest.kt @@ -0,0 +1,226 @@ +package theodolite.benchmark + +import com.google.gson.Gson +import io.fabric8.kubernetes.api.model.* +import io.fabric8.kubernetes.api.model.apps.Deployment +import io.fabric8.kubernetes.api.model.apps.DeploymentBuilder +import io.fabric8.kubernetes.api.model.apps.StatefulSet +import io.fabric8.kubernetes.api.model.apps.StatefulSetBuilder +import io.fabric8.kubernetes.client.server.mock.KubernetesServer +import io.quarkus.test.junit.QuarkusTest +import io.smallrye.common.constraint.Assert.assertTrue +import junit.framework.Assert.assertEquals +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import theodolite.k8s.CustomResourceWrapper +import theodolite.k8s.resourceLoader.K8sResourceLoaderFromFile +import theodolite.util.DeploymentFailedException + +private val testResourcePath = "./src/test/resources/k8s-resource-files/" + +@QuarkusTest +class ConfigMapResourceSetTest { + private val server = KubernetesServer(false, true) + + @BeforeEach + fun setUp() { + server.before() + } + + @AfterEach + fun tearDown() { + server.after() + } + + fun deployAndGetResource(resource: String): Collection<Pair<String, KubernetesResource>> { + val configMap1 = ConfigMapBuilder() + .withNewMetadata().withName("test-configmap").endMetadata() + .addToData("test-resource.yaml",resource) + .build() + + server.client.configMaps().createOrReplace(configMap1) + + val resourceSet = ConfigMapResourceSet() + resourceSet.name = "test-configmap" + + return resourceSet.getResourceSet(server.client) + } + + + @Test + fun testLoadDeployment() { + val resourceBuilder = DeploymentBuilder() + resourceBuilder.withNewSpec().endSpec() + resourceBuilder.withNewMetadata().endMetadata() + val resource = resourceBuilder.build() + resource.metadata.name = "test-deployment" + + val createdResource = deployAndGetResource(resource = Gson().toJson(resource)) + assertEquals(1, createdResource.size) + assertTrue(createdResource.toMutableSet().first().second is Deployment) + assertTrue(createdResource.toMutableSet().first().second.toString().contains(other = resource.metadata.name)) + } + + @Test + fun testLoadStateFulSet() { + val resourceBuilder = StatefulSetBuilder() + resourceBuilder.withNewSpec().endSpec() + resourceBuilder.withNewMetadata().endMetadata() + val resource = resourceBuilder.build() + resource.metadata.name = "test-resource" + + val createdResource = deployAndGetResource(resource = Gson().toJson(resource)) + assertEquals(1, createdResource.size) + assertTrue(createdResource.toMutableSet().first().second is StatefulSet) + assertTrue(createdResource.toMutableSet().first().second.toString().contains(other = resource.metadata.name)) + } + + @Test + fun testLoadService() { + val resourceBuilder = ServiceBuilder() + resourceBuilder.withNewSpec().endSpec() + resourceBuilder.withNewMetadata().endMetadata() + val resource = resourceBuilder.build() + resource.metadata.name = "test-resource" + + val createdResource = deployAndGetResource(resource = Gson().toJson(resource)) + assertEquals(1, createdResource.size) + assertTrue(createdResource.toMutableSet().first().second is Service) + assertTrue(createdResource.toMutableSet().first().second.toString().contains(other = resource.metadata.name)) + } + + @Test + fun testLoadConfigMap() { + val resourceBuilder = ConfigMapBuilder() + resourceBuilder.withNewMetadata().endMetadata() + val resource = resourceBuilder.build() + resource.metadata.name = "test-resource" + + val createdResource = deployAndGetResource(resource = Gson().toJson(resource)) + assertEquals(1, createdResource.size) + assertTrue(createdResource.toMutableSet().first().second is ConfigMap) + assertTrue(createdResource.toMutableSet().first().second.toString().contains(other = resource.metadata.name)) + } + + @Test + fun testLoadExecution() { + val loader = K8sResourceLoaderFromFile(server.client) + val resource = loader.loadK8sResource("Execution", testResourcePath + "test-execution.yaml") as CustomResourceWrapper + val createdResource = deployAndGetResource(resource = Gson().toJson(resource.crAsMap)) + + assertEquals(1, createdResource.size) + assertTrue(createdResource.toMutableSet().first().second is CustomResourceWrapper) + + val loadedResource = createdResource.toMutableSet().first().second + if (loadedResource is CustomResourceWrapper){ + assertTrue(loadedResource.getName() == "example-execution") + } + } + + @Test + fun testLoadBenchmark() { + val loader = K8sResourceLoaderFromFile(server.client) + val resource = loader.loadK8sResource("Benchmark", testResourcePath + "test-benchmark.yaml") as CustomResourceWrapper + val createdResource = deployAndGetResource(resource = Gson().toJson(resource.crAsMap)) + + assertEquals(1, createdResource.size) + assertTrue(createdResource.toMutableSet().first().second is CustomResourceWrapper) + + val loadedResource = createdResource.toMutableSet().first().second + if (loadedResource is CustomResourceWrapper){ + assertTrue(loadedResource.getName() == "example-benchmark") + } + } + + @Test + fun testLoadServiceMonitor() { + val loader = K8sResourceLoaderFromFile(server.client) + val resource = loader.loadK8sResource("ServiceMonitor", testResourcePath + "test-service-monitor.yaml") as CustomResourceWrapper + val createdResource = deployAndGetResource(resource = Gson().toJson(resource.crAsMap)) + + assertEquals(1, createdResource.size) + assertTrue(createdResource.toMutableSet().first().second is CustomResourceWrapper) + + val loadedResource = createdResource.toMutableSet().first().second + if (loadedResource is CustomResourceWrapper){ + assertTrue(loadedResource.getName() == "test-service-monitor") + } + } + + @Test + fun testMultipleFiles(){ + val resourceBuilder = DeploymentBuilder() + resourceBuilder.withNewSpec().endSpec() + resourceBuilder.withNewMetadata().endMetadata() + val resource = resourceBuilder.build() + resource.metadata.name = "test-deployment" + + val resourceBuilder1 = ConfigMapBuilder() + resourceBuilder1.withNewMetadata().endMetadata() + val resource1 = resourceBuilder1.build() + resource1.metadata.name = "test-configmap" + + val configMap1 = ConfigMapBuilder() + .withNewMetadata().withName("test-configmap").endMetadata() + .addToData("test-deployment.yaml",Gson().toJson(resource)) + .addToData("test-configmap.yaml",Gson().toJson(resource1)) + .build() + + server.client.configMaps().createOrReplace(configMap1) + + val resourceSet = ConfigMapResourceSet() + resourceSet.name = "test-configmap" + + val createdResourcesSet = resourceSet.getResourceSet(server.client) + + assertEquals(2,createdResourcesSet.size ) + assert(createdResourcesSet.toMutableList()[0].second is Deployment) + assert(createdResourcesSet.toMutableList()[1].second is ConfigMap) + } + + @Test + fun testFileIsSet(){ + val resourceBuilder = DeploymentBuilder() + resourceBuilder.withNewSpec().endSpec() + resourceBuilder.withNewMetadata().endMetadata() + val resource = resourceBuilder.build() + resource.metadata.name = "test-deployment" + + val resourceBuilder1 = ConfigMapBuilder() + resourceBuilder1.withNewMetadata().endMetadata() + val resource1 = resourceBuilder1.build() + resource1.metadata.name = "test-configmap" + + val configMap1 = ConfigMapBuilder() + .withNewMetadata().withName("test-configmap").endMetadata() + .addToData("test-deployment.yaml",Gson().toJson(resource)) + .addToData("test-configmap.yaml",Gson().toJson(resource1)) + .build() + + server.client.configMaps().createOrReplace(configMap1) + + val resourceSet = ConfigMapResourceSet() + resourceSet.name = "test-configmap" + resourceSet.files = listOf("test-deployment.yaml") + + val createdResourcesSet = resourceSet.getResourceSet(server.client) + + assertEquals(1,createdResourcesSet.size ) + assert(createdResourcesSet.toMutableSet().first().second is Deployment) + } + + + @Test() + fun testConfigMapNotExist() { + val resourceSet = ConfigMapResourceSet() + resourceSet.name = "test-configmap1" + lateinit var ex: Exception + try { + resourceSet.getResourceSet(server.client) + } catch (e: Exception) { + ex = e + } + assertTrue(ex is DeploymentFailedException) + } +} \ No newline at end of file diff --git a/theodolite/src/test/kotlin/theodolite/benchmark/FileSystemResourceSetTest.kt b/theodolite/src/test/kotlin/theodolite/benchmark/FileSystemResourceSetTest.kt new file mode 100644 index 0000000000000000000000000000000000000000..59ad2be3248f67442ce352788f8b94b26f3b6b90 --- /dev/null +++ b/theodolite/src/test/kotlin/theodolite/benchmark/FileSystemResourceSetTest.kt @@ -0,0 +1,115 @@ +package theodolite.benchmark + +import io.fabric8.kubernetes.api.model.ConfigMap +import io.fabric8.kubernetes.api.model.Service +import io.fabric8.kubernetes.api.model.apps.Deployment +import io.fabric8.kubernetes.api.model.apps.StatefulSet +import io.fabric8.kubernetes.client.server.mock.KubernetesServer +import io.smallrye.common.constraint.Assert.assertTrue +import junit.framework.Assert.assertEquals +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import theodolite.k8s.CustomResourceWrapper +import theodolite.util.DeploymentFailedException +import java.lang.IllegalStateException + +private val testResourcePath = "./src/test/resources/k8s-resource-files/" + +class FileSystemResourceSetTest { + + private val server = KubernetesServer(false, true) + + @BeforeEach + fun setUp() { + server.before() + } + + @AfterEach + fun tearDown() { + server.after() + } + + @Test + fun testLoadDeployment() { + val resourceSet = FileSystemResourceSet() + resourceSet.path = testResourcePath + resourceSet.files = listOf("test-deployment.yaml") + assertEquals(1,resourceSet.getResourceSet(server.client).size) + assertTrue(resourceSet.getResourceSet(server.client).toMutableSet().first().second is Deployment) + } + + @Test + fun testLoadService() { + val resourceSet = FileSystemResourceSet() + resourceSet.path = testResourcePath + resourceSet.files = listOf("test-service.yaml") + assertEquals(1,resourceSet.getResourceSet(server.client).size) + assertTrue(resourceSet.getResourceSet(server.client).toMutableSet().first().second is Service) + } + + @Test + fun testLoadStatefulSet() { + val resourceSet = FileSystemResourceSet() + resourceSet.path = testResourcePath + resourceSet.files = listOf("test-statefulset.yaml") + assertEquals(1,resourceSet.getResourceSet(server.client).size) + assertTrue(resourceSet.getResourceSet(server.client).toMutableSet().first().second is StatefulSet) + } + + @Test + fun testLoadConfigMap() { + val resourceSet = FileSystemResourceSet() + resourceSet.path = testResourcePath + resourceSet.files = listOf("test-configmap.yaml") + assertEquals(1,resourceSet.getResourceSet(server.client).size) + assertTrue(resourceSet.getResourceSet(server.client).toMutableSet().first().second is ConfigMap) + } + + @Test + fun testLoadServiceMonitor() { + val resourceSet = FileSystemResourceSet() + resourceSet.path = testResourcePath + resourceSet.files = listOf("test-service-monitor.yaml") + assertEquals(1,resourceSet.getResourceSet(server.client).size) + assertTrue(resourceSet.getResourceSet(server.client).toMutableSet().first().second is CustomResourceWrapper) + } + + @Test + fun testLoadBenchmark() { + val resourceSet = FileSystemResourceSet() + resourceSet.path = testResourcePath + resourceSet.files = listOf("test-benchmark.yaml") + assertEquals(1,resourceSet.getResourceSet(server.client).size) + assertTrue(resourceSet.getResourceSet(server.client).toMutableSet().first().second is CustomResourceWrapper) + } + + @Test + fun testLoadExecution() { + val resourceSet = FileSystemResourceSet() + resourceSet.path = testResourcePath + resourceSet.files = listOf("test-execution.yaml") + assertEquals(1,resourceSet.getResourceSet(server.client).size) + assertTrue(resourceSet.getResourceSet(server.client).toMutableSet().first().second is CustomResourceWrapper) + } + + @Test + fun testFilesNotSet() { + val resourceSet = FileSystemResourceSet() + resourceSet.path = testResourcePath + assertEquals(9,resourceSet.getResourceSet(server.client).size) + } + + @Test + fun testWrongPath() { + val resourceSet = FileSystemResourceSet() + resourceSet.path = "/abc/not-exist" + lateinit var ex: Exception + try { + resourceSet.getResourceSet(server.client) + } catch (e: Exception) { + println(e) + ex = e + } + assertTrue(ex is DeploymentFailedException) } +} \ No newline at end of file diff --git a/theodolite/src/test/kotlin/theodolite/execution/operator/BenchmarkCRDummy.kt b/theodolite/src/test/kotlin/theodolite/execution/operator/BenchmarkCRDummy.kt index f3fd06a16e38439a2a694b415edc4d8b332ffd4d..24192282a7407daf60390660a5858e15640207f3 100644 --- a/theodolite/src/test/kotlin/theodolite/execution/operator/BenchmarkCRDummy.kt +++ b/theodolite/src/test/kotlin/theodolite/execution/operator/BenchmarkCRDummy.kt @@ -24,8 +24,9 @@ class BenchmarkCRDummy(name: String) { benchmarkCR.kind = "Benchmark" benchmarkCR.apiVersion = "v1" - benchmark.appResource = emptyList() - benchmark.loadGenResource = emptyList() + benchmark.appResourceSets = emptyList() + benchmark.loadGenResourceSets = emptyList() + benchmark.resourceTypes = emptyList() benchmark.loadTypes = emptyList() benchmark.kafkaConfig = kafkaConfig diff --git a/theodolite/src/test/kotlin/theodolite/execution/operator/ControllerTest.kt b/theodolite/src/test/kotlin/theodolite/execution/operator/ControllerTest.kt index 79159bfe20c886e21751fd033ff14314ce321f16..b0b72fe4b9cf43434c0f90fa0a2dbd226fddfa66 100644 --- a/theodolite/src/test/kotlin/theodolite/execution/operator/ControllerTest.kt +++ b/theodolite/src/test/kotlin/theodolite/execution/operator/ControllerTest.kt @@ -2,6 +2,7 @@ package theodolite.execution.operator import com.google.gson.Gson import com.google.gson.GsonBuilder +import io.fabric8.kubernetes.api.model.Service import io.fabric8.kubernetes.client.CustomResourceList import io.fabric8.kubernetes.client.server.mock.KubernetesServer import io.quarkus.test.junit.QuarkusTest diff --git a/theodolite/src/test/kotlin/theodolite/execution/operator/ExecutionEventHandlerTest.kt b/theodolite/src/test/kotlin/theodolite/execution/operator/ExecutionEventHandlerTest.kt index 77e2f364edd7c92d6acb9570e50e1cf6106e96d1..45dbf92d0797b69912e3fe70a02cad8a82cae7df 100644 --- a/theodolite/src/test/kotlin/theodolite/execution/operator/ExecutionEventHandlerTest.kt +++ b/theodolite/src/test/kotlin/theodolite/execution/operator/ExecutionEventHandlerTest.kt @@ -10,7 +10,7 @@ import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.DisplayName import org.junit.jupiter.api.Test import theodolite.k8s.K8sManager -import theodolite.k8s.K8sResourceLoader +import theodolite.k8s.resourceLoader.K8sResourceLoaderFromFile import theodolite.model.crd.States import java.lang.Thread.sleep @@ -43,10 +43,10 @@ class ExecutionEventHandlerTest { this.factory = operator.getExecutionEventHandler(this.controller, server.client) this.stateHandler = TheodoliteOperator().getExecutionStateHandler(client = server.client) - this.executionVersion1 = K8sResourceLoader(server.client) + this.executionVersion1 = K8sResourceLoaderFromFile(server.client) .loadK8sResource("Execution", testResourcePath + "test-execution.yaml") - this.executionVersion2 = K8sResourceLoader(server.client) + this.executionVersion2 = K8sResourceLoaderFromFile(server.client) .loadK8sResource("Execution", testResourcePath + "test-execution-update.yaml") this.stateHandler = operator.getExecutionStateHandler(server.client) diff --git a/theodolite/src/test/kotlin/theodolite/execution/operator/StateHandlerTest.kt b/theodolite/src/test/kotlin/theodolite/execution/operator/StateHandlerTest.kt index 24cb6c90d8ea222c90a398e12c7a50a2f6058a93..b435b47fddcb58d6444e1fc31304bd355a9e7783 100644 --- a/theodolite/src/test/kotlin/theodolite/execution/operator/StateHandlerTest.kt +++ b/theodolite/src/test/kotlin/theodolite/execution/operator/StateHandlerTest.kt @@ -8,7 +8,7 @@ import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.DisplayName import org.junit.jupiter.api.Test import theodolite.k8s.K8sManager -import theodolite.k8s.K8sResourceLoader +import theodolite.k8s.resourceLoader.K8sResourceLoaderFromFile import theodolite.model.crd.States import java.time.Duration @@ -19,7 +19,7 @@ class StateHandlerTest { @BeforeEach fun setUp() { server.before() - val executionResource = K8sResourceLoader(server.client) + val executionResource = K8sResourceLoaderFromFile(server.client) .loadK8sResource("Execution", testResourcePath + "test-execution.yaml") K8sManager(server.client).deploy(executionResource) diff --git a/theodolite/src/test/kotlin/theodolite/k8s/K8sManagerTest.kt b/theodolite/src/test/kotlin/theodolite/k8s/K8sManagerTest.kt index dc2bf016994d79b1021bebdc751102e291d60682..7c69618de03f730f5b6f1cb83c5df544e2cd120c 100644 --- a/theodolite/src/test/kotlin/theodolite/k8s/K8sManagerTest.kt +++ b/theodolite/src/test/kotlin/theodolite/k8s/K8sManagerTest.kt @@ -15,6 +15,7 @@ import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.DisplayName import org.junit.jupiter.api.Test +import theodolite.k8s.resourceLoader.K8sResourceLoaderFromFile private val logger = KotlinLogging.logger {} @@ -125,7 +126,7 @@ class K8sManagerTest { @DisplayName("Test handling of custom resources") fun handleCustomResourcesTest() { val manager = K8sManager(server.client) - val servicemonitor = K8sResourceLoader(server.client) + val servicemonitor = K8sResourceLoaderFromFile(server.client) .loadK8sResource("ServiceMonitor", testResourcePath + "test-service-monitor.yaml") val serviceMonitorContext = K8sContextFactory().create( diff --git a/theodolite/src/test/kotlin/theodolite/k8s/K8sResourceLoaderTest.kt b/theodolite/src/test/kotlin/theodolite/k8s/K8sResourceLoaderTest.kt index 7c2aa50007274ff9b4d49f1c0cc05ae45a37d323..4a41dac8b27b9d4ddcfb9915f759b14ea4eaba4a 100644 --- a/theodolite/src/test/kotlin/theodolite/k8s/K8sResourceLoaderTest.kt +++ b/theodolite/src/test/kotlin/theodolite/k8s/K8sResourceLoaderTest.kt @@ -12,6 +12,7 @@ import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.DisplayName import org.junit.jupiter.api.Test +import theodolite.k8s.resourceLoader.K8sResourceLoaderFromFile @QuarkusTest class K8sResourceLoaderTest { @@ -31,7 +32,7 @@ class K8sResourceLoaderTest { @Test @DisplayName("Test loading of Deployments") fun loadDeploymentTest() { - val loader = K8sResourceLoader(server.client) + val loader = K8sResourceLoaderFromFile(server.client) val resource = loader.loadK8sResource("Deployment", testResourcePath + "test-deployment.yaml") assertTrue(resource is Deployment) @@ -41,7 +42,7 @@ class K8sResourceLoaderTest { @Test @DisplayName("Test loading of StatefulSet") fun loadStatefulSetTest() { - val loader = K8sResourceLoader(server.client) + val loader = K8sResourceLoaderFromFile(server.client) val resource = loader.loadK8sResource("StatefulSet", testResourcePath + "test-statefulset.yaml") assertTrue(resource is StatefulSet) @@ -51,7 +52,7 @@ class K8sResourceLoaderTest { @Test @DisplayName("Test loading of Service") fun loadServiceTest() { - val loader = K8sResourceLoader(server.client) + val loader = K8sResourceLoaderFromFile(server.client) val resource = loader.loadK8sResource("Service", testResourcePath + "test-service.yaml") assertTrue(resource is Service) @@ -61,7 +62,7 @@ class K8sResourceLoaderTest { @Test @DisplayName("Test loading of ConfigMap") fun loadConfigMapTest() { - val loader = K8sResourceLoader(server.client) + val loader = K8sResourceLoaderFromFile(server.client) val resource = loader.loadK8sResource("ConfigMap", testResourcePath + "test-configmap.yaml") assertTrue(resource is ConfigMap) @@ -71,7 +72,7 @@ class K8sResourceLoaderTest { @Test @DisplayName("Test loading of ServiceMonitors") fun loadServiceMonitorTest() { - val loader = K8sResourceLoader(server.client) + val loader = K8sResourceLoaderFromFile(server.client) val resource = loader.loadK8sResource("ServiceMonitor", testResourcePath + "test-service-monitor.yaml") assertTrue(resource is CustomResourceWrapper) @@ -84,7 +85,7 @@ class K8sResourceLoaderTest { @Test @DisplayName("Test loading of Executions") fun loadExecutionTest() { - val loader = K8sResourceLoader(server.client) + val loader = K8sResourceLoaderFromFile(server.client) val resource = loader.loadK8sResource("Execution", testResourcePath + "test-execution.yaml") assertTrue(resource is CustomResourceWrapper) @@ -97,7 +98,7 @@ class K8sResourceLoaderTest { @Test @DisplayName("Test loading of Benchmarks") fun loadBenchmarkTest() { - val loader = K8sResourceLoader(server.client) + val loader = K8sResourceLoaderFromFile(server.client) val resource = loader.loadK8sResource("Benchmark", testResourcePath + "test-benchmark.yaml") assertTrue(resource is CustomResourceWrapper) diff --git a/theodolite/src/test/kotlin/theodolite/patcher/ConfigOverrideModifierTest.kt b/theodolite/src/test/kotlin/theodolite/patcher/ConfigOverrideModifierTest.kt index 739fadd3f02bfd1e60fd67e7afc695bf99e68d31..1db1122e1caa5a783159ecaba849b99963e3c2a9 100644 --- a/theodolite/src/test/kotlin/theodolite/patcher/ConfigOverrideModifierTest.kt +++ b/theodolite/src/test/kotlin/theodolite/patcher/ConfigOverrideModifierTest.kt @@ -27,11 +27,10 @@ class ConfigOverrideModifierTest { @Test fun setAdditionalLabelsTest() { - this.benchmark.appResource = listOf("test-resource.yaml") val modifier = ConfigOverrideModifier( execution = this.execution, - resources = this.benchmark.appResource + resources = listOf("test-resource.yaml") ) modifier.setAdditionalLabels( diff --git a/theodolite/src/test/resources/k8s-resource-files/test-execution-1.yaml b/theodolite/src/test/resources/k8s-resource-files/test-execution-1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1407a9952b7454053d204454841d51cfb4d7dbf4 --- /dev/null +++ b/theodolite/src/test/resources/k8s-resource-files/test-execution-1.yaml @@ -0,0 +1,28 @@ +apiVersion: theodolite.com/v1 +kind: Execution +metadata: + name: example-execution +spec: + name: test + benchmark: "uc1-kstreams" + load: + loadType: "NumSensors" + loadValues: [25000, 50000, 75000, 100000, 125000, 150000] + resources: + resourceType: "Instances" + resourceValues: [1, 2, 3, 4, 5] + slos: + - sloType: "lag trend" + threshold: 2000 + prometheusUrl: "http://prometheus-operated:9090" + externalSloUrl: "http://localhost:80/evaluate-slope" + offset: 0 + warmup: 60 # in seconds + execution: + strategy: "LinearSearch" + duration: 300 # in seconds + repetitions: 1 + loadGenerationDelay: 30 # in seconds + restrictions: + - "LowerBound" + configOverrides: []