diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9a6ccb8e17f9b01e751e0631f7a9fd2488f0b56b..48d3681dab13b31ade355d9a1f13704cdc2e9c2e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -143,6 +143,7 @@ test-benchmarks: - build-benchmarks script: ./gradlew test --continue artifacts: + when: always reports: junit: - "theodolite-benchmarks/**/build/test-results/test/TEST-*.xml" @@ -396,6 +397,7 @@ test-theodolite: #- build-theodolite-native script: ./gradlew test --stacktrace artifacts: + when: always reports: junit: - "theodolite/**/build/test-results/test/TEST-*.xml" diff --git a/CITATION.cff b/CITATION.cff index ab95efe7b82bcc8e2fb3228376f4cfc1efac05bc..07c2dcee319f73604f95414b987f8ed5274f7e82 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -8,7 +8,7 @@ authors: given-names: Wilhelm orcid: "https://orcid.org/0000-0001-6625-4335" title: Theodolite -version: "0.6.0" +version: "0.6.1" repository-code: "https://github.com/cau-se/theodolite" license: "Apache-2.0" doi: "10.1016/j.bdr.2021.100209" diff --git a/binder/requirements.txt b/binder/requirements.txt new file mode 120000 index 0000000000000000000000000000000000000000..6de15663a8c83876719aa07d6cb09b5a7b71df21 --- /dev/null +++ b/binder/requirements.txt @@ -0,0 +1 @@ +../analysis/requirements.txt \ No newline at end of file diff --git a/codemeta.json b/codemeta.json index 32b490f588ac38f610447e3eea579c6584d75c8f..2a190092b96adb3462c011e49db3c160d639d6fe 100644 --- a/codemeta.json +++ b/codemeta.json @@ -5,10 +5,10 @@ "codeRepository": "https://github.com/cau-se/theodolite", "dateCreated": "2020-03-13", "datePublished": "2020-07-27", - "dateModified": "2022-01-12", + "dateModified": "2022-01-17", "downloadUrl": "https://github.com/cau-se/theodolite/releases", "name": "Theodolite", - "version": "0.6.0", + "version": "0.6.1", "description": "Theodolite is a framework for benchmarking the horizontal and vertical scalability of cloud-native applications.", "developmentStatus": "active", "relatedLink": [ diff --git a/docs/creating-an-execution.md b/docs/creating-an-execution.md index e70893e7ea4364bfbb30465df95273703ec7f43b..263d630ff2db82927c72d2c2482fcddc09705bfc 100644 --- a/docs/creating-an-execution.md +++ b/docs/creating-an-execution.md @@ -58,7 +58,29 @@ As a Benchmark may define multiple supported load and resource types, an Executi ## Definition of SLOs SLOs provide a way to quantify whether a certain load intensity can be handled by a certain amount of provisioned resources. -An Execution must at least specify one SLO to be checked. +In Theodolite, SLO are evaluated by requesting monitoring data from Prometheus and analyzing it in a benchmark-specific way. +An Execution must at least define one SLO to be checked. + +A good choice to get started is defining an SLO of type `generic`: + +```yaml +- sloType: "generic" + prometheusUrl: "http://prometheus-operated:9090" + offset: 0 + properties: + externalSloUrl: "http://localhost:8082" + promQLQuery: "sum by(job) (kafka_streams_stream_task_metrics_dropped_records_total>=0)" + warmup: 60 # in seconds + queryAggregation: max + repetitionAggregation: median + operator: lte + threshold: 1000 +``` + +All you have to do is to define a [PromQL query](https://prometheus.io/docs/prometheus/latest/querying/basics/) describing which metrics should be requested (`promQLQuery`) and how the resulting time series should be evaluated. With `queryAggregation` you specify how the resulting time series is aggregated to a single value and `repetitionAggregation` describes how the results of multiple repetitions are aggregated. Possible values are +`mean`, `median`, `mode`, `sum`, `count`, `max`, `min`, `std`, `var`, `skew`, `kurt` as well as percentiles such as `p99` or `p99.9`. The result of aggregation all repetitions is checked against `threshold`. This check is performed using an `operator`, which describes that the result must be "less than" (`lt`), "less than equal" (`lte`), "greater than" (`gt`) or "greater than equal" (`gte`) to the threshold. + +In case you need to evaluate monitoring data in a more flexible fashion, you can also change the value of `externalSloUrl` to your custom SLO checker. Have a look at the source code of the [generic SLO checker](https://github.com/cau-se/theodolite/tree/master/slo-checker/generic) to get started. ## Experimental Setup @@ -72,7 +94,7 @@ The experimental setup can be configured by: ## Configuration Overrides -In cases where only small modifications of a system under test should be benchmarked, it is not necessarily required to [create a new benchmark](creating-a-benchmark). +In cases where only small modifications of a system under test should be benchmarked, it is not necessary to [create a new benchmark](creating-a-benchmark). Instead, also Executions allow to do small reconfigurations, such as switching on or off a specific Pod scheduler. This is done by defining `configOverrides` in the Execution. Each override consists of a patcher, defining which Kubernetes resource should be patched in which way, and a value the patcher is applied with. diff --git a/docs/index.yaml b/docs/index.yaml index 995d7523e8e47914a59ee99aad07d25a86322a0c..185ff1b0616b760c647a809006c48bf26c554490 100644 --- a/docs/index.yaml +++ b/docs/index.yaml @@ -1,6 +1,41 @@ apiVersion: v1 entries: theodolite: + - apiVersion: v2 + appVersion: 0.6.1 + created: "2022-01-18T10:40:00.557347616+01:00" + dependencies: + - condition: grafana.enabled + name: grafana + repository: https://grafana.github.io/helm-charts + version: 6.17.5 + - condition: kube-prometheus-stack.enabled + name: kube-prometheus-stack + repository: https://prometheus-community.github.io/helm-charts + version: 20.0.1 + - condition: cp-helm-charts.enabled + name: cp-helm-charts + repository: https://soerenhenning.github.io/cp-helm-charts + version: 0.6.0 + - condition: kafka-lag-exporter.enabled + name: kafka-lag-exporter + repository: https://lightbend.github.io/kafka-lag-exporter/repo/ + version: 0.6.7 + description: Theodolite is a framework for benchmarking the horizontal and vertical + scalability of cloud-native applications. + digest: 4896111999375c248d7dda0bdff090c155f464b79416decc0e0b47dc6710b5c7 + home: https://www.theodolite.rocks + maintainers: + - email: soeren.henning@email.uni-kiel.de + name: Sören Henning + url: https://www.se.informatik.uni-kiel.de/en/team/soeren-henning-m-sc + name: theodolite + sources: + - https://github.com/cau-se/theodolite + type: application + urls: + - https://github.com/cau-se/theodolite/releases/download/v0.6.1/theodolite-0.6.1.tgz + version: 0.6.1 - apiVersion: v2 appVersion: 0.6.0 created: "2022-01-12T13:53:08.413006558+01:00" @@ -141,4 +176,4 @@ entries: urls: - https://github.com/cau-se/theodolite/releases/download/v0.4.0/theodolite-0.4.0.tgz version: 0.4.0 -generated: "2022-01-12T13:53:08.367695997+01:00" +generated: "2022-01-18T10:40:00.486387187+01:00" diff --git a/docs/running-benchmarks.md b/docs/running-benchmarks.md index eda817d28b6a10b2e2f33e6986a3b018e089beff..7da1c7e5f8385a2818ae587b4c3ab3715a6c2bb2 100644 --- a/docs/running-benchmarks.md +++ b/docs/running-benchmarks.md @@ -11,6 +11,7 @@ Running scalability benchmarks with Theodolite involves the following steps: 1. [Deploying a benchmark to Kubernetes](#deploying-a-benchmark) 1. [Creating an execution](#creating-an-execution), which describes the experimental setup for running the benchmark 1. [Accessing benchmark results](#accessing-benchmark-results) +1. [Analyzing benchmark results](#analyzing-benchmark-results) with Theodolite's Jupyter notebooks ## Deploying a Benchmark @@ -131,3 +132,32 @@ For installations without persistence, but also as an alternative for installati ```sh kubectl cp $(kubectl get pod -l app=theodolite -o jsonpath="{.items[0].metadata.name}"):/results . -c results-access ``` + +## Analyzing Benchmark Results + +Theodolite comes with Jupyter notebooks for analyzing and visualizing benchmark execution results. +The easiest way to use them is at MyBinder: + +[Launch Notebooks](https://mybinder.org/v2/gh/cau-se/theodolite/HEAD?labpath=analysis){: .btn .btn-primary } +{: .text-center } + +Alternatively, you can also [run these notebook locally](https://github.com/cau-se/theodolite/tree/master/analysis), for example, with Docker or Visual Studio Code. + +The notebooks allow to compute a scalability function using its *demand* metric and to visualize multiple such functions in plots: + +### Computing the *demand* metric with `demand-metric.ipynb` (optional) + +After finishing a benchmark execution, Theodolite creates a `exp<id>_demand.csv` file. It maps the tested load intensities to the minimal required resources for that load. If the monitoring data collected during benchmark execution should be analyzed in more detail, the `demand-metric.ipynb` notebook can be used. + +Theodolite stores monitoring data for each conducted SLO experiment in `exp<id>_<load>_<resources>_<slo-slug>_<rep>.csv` files, where `<id>` is the ID of an execution, `<load>` the corresponding load intensity value, `<resources>` the resources value, `<slo-slug>` the [name of the SLO](creating-an-execution.html#definition-of-slos) and `<rep>` the repetition counter. +The `demand-metric.ipynb` notebook reads these files and generates a new CSV file mapping load intensities to the minimal required resources. The format of this file corresponds to the original `exp<id>_demand.csv` file created when running the benchmark, but allows, for example, to evaluate different warm-up periods. + +Currently, the `demand-metric.ipynb` notebook only supports benchmarks with the *lag trend SLO* out-of-the-box, but can easily be adjusted to perform any other type of analysis. + +### Plotting benchmark results with the *demand* metric with `demand-metric-plot.ipynb` + +The `demand-metric-plot.ipynb` takes one or multiple `exp<id>_demand.csv` files as input and visualize them together in a plot. +Input files can either be taken directly from Theodolite, or created from the `demand-metric.ipynb` notebooks. + +All plotting code is only intended to serve as a template. Adjust it as needed to change colors, labels, formatting, etc. as needed. +Please refer to the official docs of [MatPlotLib](https://matplotlib.org/) and the [ggplot](https://matplotlib.org/stable/gallery/style_sheets/ggplot.html) style, which are used to generate the plots. diff --git a/helm/preconfigs/minimal.yaml b/helm/preconfigs/minimal.yaml index b0828c2f424e8456933dc626a66a199cd60aa5da..80a83f06cc9838e01f812e730932b9b79d947161 100644 --- a/helm/preconfigs/minimal.yaml +++ b/helm/preconfigs/minimal.yaml @@ -8,5 +8,8 @@ cp-helm-charts: offsets.topic.replication.factor: "1" operator: + sloChecker: + droppedRecordsKStreams: + enabled: false resultsVolume: enabled: false diff --git a/helm/templates/theodolite/theodolite-operator.yaml b/helm/templates/theodolite/theodolite-operator.yaml index c7ced880cbbfbb9795ef59156ea1df7d5b860ec6..ff9c7e4de87c703af3350f7d9c797a5a53e2e675 100644 --- a/helm/templates/theodolite/theodolite-operator.yaml +++ b/helm/templates/theodolite/theodolite-operator.yaml @@ -31,6 +31,19 @@ spec: volumeMounts: - name: theodolite-results-volume mountPath: "/deployments/results" + {{- if .Values.operator.sloChecker.droppedRecordsKStreams.enabled }} + - name: slo-checker-generic + image: "{{ .Values.operator.sloChecker.generic.image }}:{{ .Values.operator.sloChecker.generic.imageTag }}" + imagePullPolicy: "{{ .Values.operator.sloChecker.generic.imagePullPolicy }}" + ports: + - containerPort: 8082 + name: analysis + env: + - name: PORT + value: "8082" + - name: LOG_LEVEL + value: INFO + {{- end }} {{- if .Values.operator.sloChecker.lagTrend.enabled }} - name: lag-trend-slo-checker image: "{{ .Values.operator.sloChecker.lagTrend.image }}:{{ .Values.operator.sloChecker.lagTrend.imageTag }}" diff --git a/helm/values.yaml b/helm/values.yaml index 1e57b42c485eb20a5525f25cfc0ef616e65a325c..ba58b040974886518ab111d668cb0db1140b2eb8 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -256,6 +256,11 @@ operator: nodeSelector: {} sloChecker: + generic: + enabled: true + image: ghcr.io/cau-se/theodolite-slo-checker-generic + imageTag: latest + imagePullPolicy: Always lagTrend: enabled: true image: ghcr.io/cau-se/theodolite-slo-checker-lag-trend diff --git a/slo-checker/generic/app/main.py b/slo-checker/generic/app/main.py index e55c478c5df5c7e8ff7d26289cd99f9a82b725fc..f36c8739da00128ad94feb1f2d7871df7e2ff137 100644 --- a/slo-checker/generic/app/main.py +++ b/slo-checker/generic/app/main.py @@ -57,15 +57,14 @@ def check_result(result, operator: str, threshold): @app.post("/",response_model=bool) async def check_slo(request: Request): data = json.loads(await request.body()) + logger.info('Received request with metadata: %s', data['metadata']) + warmup = int(data['metadata']['warmup']) query_aggregation = get_aggr_func(data['metadata']['queryAggregation']) rep_aggregation = get_aggr_func(data['metadata']['repetitionAggregation']) operator = data['metadata']['operator'] threshold = int(data['metadata']['threshold']) - for r in data["results"]: - aggr_query(r[0]["values"], warmup, query_aggregation) - query_results = [aggr_query(r[0]["values"], warmup, query_aggregation) for r in data["results"]] result = pd.DataFrame(query_results).aggregate(rep_aggregation).at[0] return check_result(result, operator, threshold) diff --git a/theodolite/src/main/kotlin/theodolite/evaluation/SloCheckerFactory.kt b/theodolite/src/main/kotlin/theodolite/evaluation/SloCheckerFactory.kt index c2514469925bcfc20c15377e93963df04a3b91f6..f57cebfcb13d0e86919ec15a0a479d1258e318a6 100644 --- a/theodolite/src/main/kotlin/theodolite/evaluation/SloCheckerFactory.kt +++ b/theodolite/src/main/kotlin/theodolite/evaluation/SloCheckerFactory.kt @@ -50,12 +50,11 @@ class SloCheckerFactory { // TODO validate property contents metadata = mapOf( "warmup" to (properties["warmup"]?.toInt() ?: throw IllegalArgumentException("warmup expected")), - "queryAggregation" to (properties["warmup"]?.toInt() + "queryAggregation" to (properties["queryAggregation"] ?: throw IllegalArgumentException("queryAggregation expected")), - "repetitionAggregation" to (properties["warmup"]?.toInt() + "repetitionAggregation" to (properties["repetitionAggregation"] ?: throw IllegalArgumentException("repetitionAggregation expected")), - "operator" to (properties["warmup"]?.toInt() - ?: throw IllegalArgumentException("operator expected")), + "operator" to (properties["operator"] ?: throw IllegalArgumentException("operator expected")), "threshold" to (properties["threshold"]?.toInt() ?: throw IllegalArgumentException("threshold expected")) ) diff --git a/theodolite/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt b/theodolite/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt index 315d1cf1afe7fd2ffbfc1c437d725d4dff29f637..8596576e0a7984c32b6dabf90c6bbf06961d2bb1 100644 --- a/theodolite/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt +++ b/theodolite/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt @@ -137,6 +137,12 @@ class TheodoliteExecutor( config.compositeStrategy.benchmarkExecutor.results, "${resultsFolder}exp${this.config.executionId}-result" ) + // Create expXYZ_demand.csv file + ioHandler.writeToCSVFile( + "${resultsFolder}exp${this.config.executionId}_demand", + calculateDemandMetric(config.loads, config.compositeStrategy.benchmarkExecutor.results), + listOf("load","resources") + ) } kubernetesBenchmark.teardownInfrastructure() } @@ -151,4 +157,8 @@ class TheodoliteExecutor( return executionID } + private fun calculateDemandMetric(loadDimensions: List<LoadDimension>, results: Results): List<List<String>> { + return loadDimensions.map { listOf(it.get().toString(), results.getMinRequiredInstances(it).get().toString()) } + } + }