diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4e1a3b37b8cccad0718f16a45e6dcbb5bccc7b2b..0b8a87bb35ddfabf966e59b9e43af639e0b7fe41 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -33,18 +33,18 @@ default: entrypoint: [""] script: - mkdir -p /kaniko/.docker - - echo "{\"auths\":{\"${CR_HOST}\":{\"auth\":\"$(printf "%s:%s" "${CR_USER}" "${CR_PW}" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json + - echo "{\"auths\":{\"${CR_PUBLIC_HOST}\":{\"auth\":\"$(printf "%s:%s" "${CR_PUBLIC_USER}" "${CR_PUBLIC_PW}" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json - > if [ $IMAGE_TAG ]; then - KANIKO_D="$KANIKO_D -d $CR_HOST/$CR_ORG/$IMAGE_NAME:$IMAGE_TAG" + KANIKO_D="$KANIKO_D -d $CR_PUBLIC_HOST/$CR_PUBLIC_ORG/$IMAGE_NAME:$IMAGE_TAG" export PUBLISHED_IMAGE_TAG=$IMAGE_TAG elif [ $CI_COMMIT_TAG ]; then - KANIKO_D="$KANIKO_D -d $CR_HOST/$CR_ORG/$IMAGE_NAME:$CI_COMMIT_TAG" + KANIKO_D="$KANIKO_D -d $CR_PUBLIC_HOST/$CR_PUBLIC_ORG/$IMAGE_NAME:$CI_COMMIT_TAG" export PUBLISHED_IMAGE_TAG=$CI_COMMIT_TAG else - DOCKER_TAG_NAME=$(echo $CI_COMMIT_REF_SLUG- | sed 's/^master-$//') - KANIKO_D="$KANIKO_D -d $CR_HOST/$CR_ORG/$IMAGE_NAME:${DOCKER_TAG_NAME}latest" - KANIKO_D="$KANIKO_D -d $CR_HOST/$CR_ORG/$IMAGE_NAME:$DOCKER_TAG_NAME$CI_COMMIT_SHORT_SHA" + DOCKER_TAG_NAME=$(echo $CI_COMMIT_REF_SLUG- | sed 's/^main-$//') + KANIKO_D="$KANIKO_D -d $CR_PUBLIC_HOST/$CR_PUBLIC_ORG/$IMAGE_NAME:${DOCKER_TAG_NAME}latest" + KANIKO_D="$KANIKO_D -d $CR_PUBLIC_HOST/$CR_PUBLIC_ORG/$IMAGE_NAME:$DOCKER_TAG_NAME$CI_COMMIT_SHORT_SHA" export PUBLISHED_IMAGE_TAG=$DOCKER_TAG_NAME$CI_COMMIT_SHORT_SHA fi - "[ $DOCKERFILE ] && KANIKO_DOCKERFILE=\"--dockerfile $DOCKERFILE\"" @@ -269,8 +269,8 @@ spotbugs-benchmarks: - theodolite-benchmarks/* - theodolite-benchmarks/$JAVA_PROJECT_NAME/**/* - theodolite-benchmarks/{commons,$JAVA_PROJECT_DEPS}/**/* - if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW && $IMAGE_NAME && $JAVA_PROJECT_NAME && $JAVA_PROJECT_DEPS" - - if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW && $IMAGE_NAME && $JAVA_PROJECT_NAME" + if: "$CR_PUBLIC_HOST && $CR_PUBLIC_ORG && $CR_PUBLIC_USER && $CR_PUBLIC_PW && $IMAGE_NAME && $JAVA_PROJECT_NAME && $JAVA_PROJECT_DEPS" + - if: "$CR_PUBLIC_HOST && $CR_PUBLIC_ORG && $CR_PUBLIC_USER && $CR_PUBLIC_PW && $IMAGE_NAME && $JAVA_PROJECT_NAME" when: manual allow_failure: true @@ -469,8 +469,8 @@ deploy-http-bridge: - changes: - theodolite-benchmarks/* - theodolite-benchmarks/{$JAVA_PROJECT_DEPS}/**/* - if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW && $DOCKER_COMPOSE_DIR && $JAVA_PROJECT_DEPS" - - if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW && $DOCKER_COMPOSE_DIR && $JAVA_PROJECT_DEPS" + if: "$CR_PUBLIC_HOST && $CR_PUBLIC_ORG && $CR_PUBLIC_USER && $CR_PUBLIC_PW && $DOCKER_COMPOSE_DIR && $JAVA_PROJECT_DEPS" + - if: "$CR_PUBLIC_HOST && $CR_PUBLIC_ORG && $CR_PUBLIC_USER && $CR_PUBLIC_PW && $DOCKER_COMPOSE_DIR && $JAVA_PROJECT_DEPS" when: manual allow_failure: true @@ -736,8 +736,8 @@ deploy-theodolite: rules: - changes: - theodolite/**/* - if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW" - - if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW" + if: "$CR_PUBLIC_HOST && $CR_PUBLIC_ORG && $CR_PUBLIC_USER && $CR_PUBLIC_PW" + - if: "$CR_PUBLIC_HOST && $CR_PUBLIC_ORG && $CR_PUBLIC_USER && $CR_PUBLIC_PW" when: manual allow_failure: true @@ -805,8 +805,8 @@ deploy-slo-checker-lag-trend: rules: - changes: - slo-checker/record-lag/**/* - if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW" - - if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW" + if: "$CR_PUBLIC_HOST && $CR_PUBLIC_ORG && $CR_PUBLIC_USER && $CR_PUBLIC_PW" + - if: "$CR_PUBLIC_HOST && $CR_PUBLIC_ORG && $CR_PUBLIC_USER && $CR_PUBLIC_PW" when: manual allow_failure: true @@ -823,8 +823,8 @@ deploy-slo-checker-dropped-records-kstreams: rules: - changes: - slo-checker/dropped-records/**/* - if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW" - - if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW" + if: "$CR_PUBLIC_HOST && $CR_PUBLIC_ORG && $CR_PUBLIC_USER && $CR_PUBLIC_PW" + - if: "$CR_PUBLIC_HOST && $CR_PUBLIC_ORG && $CR_PUBLIC_USER && $CR_PUBLIC_PW" when: manual allow_failure: true @@ -841,8 +841,8 @@ deploy-slo-checker-generic: rules: - changes: - slo-checker/generic/**/* - if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW" - - if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW" + if: "$CR_PUBLIC_HOST && $CR_PUBLIC_ORG && $CR_PUBLIC_USER && $CR_PUBLIC_PW" + - if: "$CR_PUBLIC_HOST && $CR_PUBLIC_ORG && $CR_PUBLIC_USER && $CR_PUBLIC_PW" when: manual allow_failure: true @@ -861,8 +861,8 @@ deploy-random-scheduler: rules: - changes: - execution/infrastructure/random-scheduler/**/* - if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW" - - if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW" + if: "$CR_PUBLIC_HOST && $CR_PUBLIC_ORG && $CR_PUBLIC_USER && $CR_PUBLIC_PW" + - if: "$CR_PUBLIC_HOST && $CR_PUBLIC_ORG && $CR_PUBLIC_USER && $CR_PUBLIC_PW" when: manual allow_failure: true @@ -880,8 +880,8 @@ deploy-buildimage-docker-compose-jq: rules: - changes: - buildimages/docker-compose-jq/Dockerfile - if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW" - - if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW && $CI_PIPELINE_SOURCE == 'web'" + if: "$CR_PUBLIC_HOST && $CR_PUBLIC_ORG && $CR_PUBLIC_USER && $CR_PUBLIC_PW" + - if: "$CR_PUBLIC_HOST && $CR_PUBLIC_ORG && $CR_PUBLIC_USER && $CR_PUBLIC_PW && $CI_PIPELINE_SOURCE == 'web'" when: manual allow_failure: true @@ -901,7 +901,7 @@ deploy-buildimage-k3d-helm: rules: - changes: - buildimages/k3d-helm/Dockerfile - if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW" - - if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW && $CI_PIPELINE_SOURCE == 'web'" + if: "$CR_PUBLIC_HOST && $CR_PUBLIC_ORG && $CR_PUBLIC_USER && $CR_PUBLIC_PW" + - if: "$CR_PUBLIC_HOST && $CR_PUBLIC_ORG && $CR_PUBLIC_USER && $CR_PUBLIC_PW && $CI_PIPELINE_SOURCE == 'web'" when: manual allow_failure: true diff --git a/CITATION.cff b/CITATION.cff index 160146c844b1d128299617ae8d93ac4af77e4ca0..0ebea8019c94aea0c294396f7430e38e81ceef6a 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -8,7 +8,7 @@ authors: given-names: Wilhelm orcid: "https://orcid.org/0000-0001-6625-4335" title: Theodolite -version: "0.7.0" +version: "0.8.0" repository-code: "https://github.com/cau-se/theodolite" license: "Apache-2.0" doi: "10.1016/j.bdr.2021.100209" diff --git a/codemeta.json b/codemeta.json index fa2eb23a956d9ad2525beee1b5ca343845c38d2d..5a4c6e8bcb11191e1d5b775cb30f889817d73c69 100644 --- a/codemeta.json +++ b/codemeta.json @@ -5,10 +5,10 @@ "codeRepository": "https://github.com/cau-se/theodolite", "dateCreated": "2020-03-13", "datePublished": "2020-07-27", - "dateModified": "2022-05-11", + "dateModified": "2022-07-18", "downloadUrl": "https://github.com/cau-se/theodolite/releases", "name": "Theodolite", - "version": "0.7.0", + "version": "0.8.0", "description": "Theodolite is a framework for benchmarking the horizontal and vertical scalability of cloud-native applications.", "developmentStatus": "active", "relatedLink": [ diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock index 4acb2ba79d5cde699cf9dd4d379bf17c3c93e068..3bbf25983bd87f91386bf7c4722ce177ede4cdcb 100644 --- a/docs/Gemfile.lock +++ b/docs/Gemfile.lock @@ -273,7 +273,7 @@ GEM thread_safe (0.3.6) typhoeus (1.4.0) ethon (>= 0.9.0) - tzinfo (1.2.9) + tzinfo (1.2.10) thread_safe (~> 0.1) unf (0.1.4) unf_ext diff --git a/docs/api-reference/crds.md b/docs/api-reference/crds.md index 9a42d0b5dee3f211ebbc2ea82cb2ab7976dd44d8..6a0562651c7ea5677f6c3ebb532d84d175f587c7 100644 --- a/docs/api-reference/crds.md +++ b/docs/api-reference/crds.md @@ -2285,13 +2285,6 @@ Contains the Kafka configuration. The name of the benchmark this execution is referring to.<br/> </td> <td>true</td> - </tr><tr> - <td><b><a href="#executionspecconfigoverridesindex">configOverrides</a></b></td> - <td>[]object</td> - <td> - List of patchers that are used to override existing configurations.<br/> - </td> - <td>true</td> </tr><tr> <td><b><a href="#executionspecexecution">execution</a></b></td> <td>object</td> @@ -2300,7 +2293,7 @@ Contains the Kafka configuration. </td> <td>true</td> </tr><tr> - <td><b><a href="#executionspecloads">loads</a></b></td> + <td><b><a href="#executionspecload">load</a></b></td> <td>object</td> <td> Specifies the load values that are benchmarked.<br/> @@ -2314,100 +2307,28 @@ Contains the Kafka configuration. </td> <td>true</td> </tr><tr> - <td><b>name</b></td> - <td>string</td> - <td> - This field exists only for technical reasons and should not be set by the user. The value of the field will be overwritten.<br/> - <br/> - <i>Default</i>: <br/> - </td> - <td>false</td> - </tr><tr> - <td><b><a href="#executionspecslosindex">slos</a></b></td> + <td><b><a href="#executionspecconfigoverridesindex">configOverrides</a></b></td> <td>[]object</td> <td> - List of SLOs with their properties, which differ from the benchmark definition.<br/> - </td> - <td>false</td> - </tr></tbody> -</table> - - -### execution.spec.configOverrides[index] -<sup><sup>[↩ Parent](#executionspec)</sup></sup> - - - - - -<table> - <thead> - <tr> - <th>Name</th> - <th>Type</th> - <th>Description</th> - <th>Required</th> - </tr> - </thead> - <tbody><tr> - <td><b><a href="#executionspecconfigoverridesindexpatcher">patcher</a></b></td> - <td>object</td> - <td> - Patcher used to patch a resource<br/> - </td> - <td>false</td> - </tr><tr> - <td><b>value</b></td> - <td>string</td> - <td> + List of patchers that are used to override existing configurations.<br/> <br/> + <i>Default</i>: []<br/> </td> <td>false</td> - </tr></tbody> -</table> - - -### execution.spec.configOverrides[index].patcher -<sup><sup>[↩ Parent](#executionspecconfigoverridesindex)</sup></sup> - - - -Patcher used to patch a resource - -<table> - <thead> - <tr> - <th>Name</th> - <th>Type</th> - <th>Description</th> - <th>Required</th> - </tr> - </thead> - <tbody><tr> - <td><b>resource</b></td> - <td>string</td> - <td> - Specifies the Kubernetes resource to be patched.<br/> - <br/> - <i>Default</i>: <br/> - </td> - <td>true</td> </tr><tr> - <td><b>type</b></td> + <td><b>name</b></td> <td>string</td> <td> - Type of the Patcher.<br/> + This field exists only for technical reasons and should not be set by the user. The value of the field will be overwritten.<br/> <br/> <i>Default</i>: <br/> </td> - <td>true</td> + <td>false</td> </tr><tr> - <td><b>properties</b></td> - <td>map[string]string</td> + <td><b><a href="#executionspecslosindex">slos</a></b></td> + <td>[]object</td> <td> - (Optional) Patcher specific additional arguments.<br/> - <br/> - <i>Default</i>: map[]<br/> + List of SLOs with their properties, which differ from the benchmark definition.<br/> </td> <td>false</td> </tr></tbody> @@ -2517,7 +2438,7 @@ Defines the used strategy for the execution, either 'LinearSearch', 'BinarySearc </table> -### execution.spec.loads +### execution.spec.load <sup><sup>[↩ Parent](#executionspec)</sup></sup> @@ -2585,6 +2506,87 @@ Specifies the scaling resource that is benchmarked. </table> +### execution.spec.configOverrides[index] +<sup><sup>[↩ Parent](#executionspec)</sup></sup> + + + + + +<table> + <thead> + <tr> + <th>Name</th> + <th>Type</th> + <th>Description</th> + <th>Required</th> + </tr> + </thead> + <tbody><tr> + <td><b><a href="#executionspecconfigoverridesindexpatcher">patcher</a></b></td> + <td>object</td> + <td> + Patcher used to patch a resource<br/> + </td> + <td>false</td> + </tr><tr> + <td><b>value</b></td> + <td>string</td> + <td> + <br/> + </td> + <td>false</td> + </tr></tbody> +</table> + + +### execution.spec.configOverrides[index].patcher +<sup><sup>[↩ Parent](#executionspecconfigoverridesindex)</sup></sup> + + + +Patcher used to patch a resource + +<table> + <thead> + <tr> + <th>Name</th> + <th>Type</th> + <th>Description</th> + <th>Required</th> + </tr> + </thead> + <tbody><tr> + <td><b>resource</b></td> + <td>string</td> + <td> + Specifies the Kubernetes resource to be patched.<br/> + <br/> + <i>Default</i>: <br/> + </td> + <td>true</td> + </tr><tr> + <td><b>type</b></td> + <td>string</td> + <td> + Type of the Patcher.<br/> + <br/> + <i>Default</i>: <br/> + </td> + <td>true</td> + </tr><tr> + <td><b>properties</b></td> + <td>map[string]string</td> + <td> + (Optional) Patcher specific additional arguments.<br/> + <br/> + <i>Default</i>: map[]<br/> + </td> + <td>false</td> + </tr></tbody> +</table> + + ### execution.spec.slos[index] <sup><sup>[↩ Parent](#executionspec)</sup></sup> diff --git a/docs/concepts/benchmarks-and-executions.md b/docs/concepts/benchmarks-and-executions.md index 149658f7bbca472a4f6adad90ffdc16c08ba5e96..004d76422dcf3b0af43bc6447b788c54aefcce56 100644 --- a/docs/concepts/benchmarks-and-executions.md +++ b/docs/concepts/benchmarks-and-executions.md @@ -42,3 +42,7 @@ In contrast to benchmarks, execution have a life-cycle. They can be planned, executed, or aborted. Each execution of a benchmark is represented by an individual entity. This supports repeatability as executions can be archived and shared. + +## Further Reading + +S. Henning and W. Hasselbring. “[A Configurable Method for Benchmarking Scalability of Cloud-Native Applications](https://doi.org/10.1007/s10664-022-10162-1)”. In: *Empirical Software Engineering* 27. 2022. DOI: [10.1007/s10664-022-10162-1](https://doi.org/10.1007/s10664-022-10162-1). diff --git a/docs/concepts/metrics.md b/docs/concepts/metrics.md index 775af41ff1138fa822942f4e054089e773a01993..21b3519124183d0ef3e1377ba24ec537983326b3 100644 --- a/docs/concepts/metrics.md +++ b/docs/concepts/metrics.md @@ -64,5 +64,4 @@ $$ ## Further Reading -S. Henning and W. Hasselbring. “[How to Measure Scalability of Distributed Stream Processing Engines?](https://research.spec.org/icpe_proceedings/2021/companion/p85.pdf)” In: International Conference on Performance Engineering. 2021. DOI: [10.1145/3447545.3451190](https://doi.org/10.1145/3447545.3451190). -<!-- TODO replace with EMSE paper one day--> +S. Henning and W. Hasselbring. “[A Configurable Method for Benchmarking Scalability of Cloud-Native Applications](https://doi.org/10.1007/s10664-022-10162-1)”. In: *Empirical Software Engineering* 27. 2022. DOI: [10.1007/s10664-022-10162-1](https://doi.org/10.1007/s10664-022-10162-1). diff --git a/docs/concepts/search-strategies.md b/docs/concepts/search-strategies.md index 03c14675432e670830e48473955fb37d83204502..41b6f84dcce24f97aca5c90e8aaf4305bc52221c 100644 --- a/docs/concepts/search-strategies.md +++ b/docs/concepts/search-strategies.md @@ -35,4 +35,6 @@ The lower bound restriction is a search strategy that uses the results of alread For the resource demand, it starts searching (with another strategy) beginning from the minimal required resources of all lower load intensities. The lower bound restriction is based on the assumption that with increasing load intensity, the resource demand never decreases. -<!-- further reading in EMSE paper --> \ No newline at end of file +## Further Reading + +S. Henning and W. Hasselbring. “[A Configurable Method for Benchmarking Scalability of Cloud-Native Applications](https://doi.org/10.1007/s10664-022-10162-1)”. In: *Empirical Software Engineering* 27. 2022. DOI: [10.1007/s10664-022-10162-1](https://doi.org/10.1007/s10664-022-10162-1). diff --git a/docs/creating-a-benchmark.md b/docs/creating-a-benchmark.md index 06b9e17ecaf5f1f25c719495a204d95ae7e09785..9aba9a71ae1f94abf61b7707b0e94d524b431a84 100644 --- a/docs/creating-a-benchmark.md +++ b/docs/creating-a-benchmark.md @@ -29,6 +29,11 @@ spec: files: - uc1-load-generator-service.yaml - uc1-load-generator-deployment.yaml + resourceTypes: + - typeName: "Instances" + patchers: + - type: "ReplicaPatcher" + resource: "uc1-kstreams-deployment.yaml" loadTypes: - typeName: "NumSensors" patchers: @@ -41,6 +46,15 @@ spec: resource: "uc1-load-generator-deployment.yaml" properties: loadGenMaxRecords: "150000" + slos: + - name: "lag trend" + sloType: "lag trend" + prometheusUrl: "http://prometheus-operated:9090" + offset: 0 + properties: + threshold: 3000 + externalSloUrl: "http://localhost:80/evaluate-slope" + warmup: 60 # in seconds kafkaConfig: bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092" topics: @@ -60,8 +74,6 @@ Infrastructure resources live over the entire duration of a benchmark run. They ### Resources -#### ConfigMap - The recommended way to link Kubernetes resources files from a Benchmark is by bundling them in one or multiple ConfigMaps and refer to that ConfigMap from `sut.resources`, `loadGenerator.resources` or `infrastructure.resources`. To create a ConfigMap from all the Kubernetes resources in a directory run: @@ -79,21 +91,13 @@ configMap: - example-service.yaml ``` -#### Filesystem - -Alternatively, resources can also be read from the filesystem, Theodolite has access to. This usually requires that the Benchmark resources are available in a volume, which is mounted into the Theodolite container. - -```yaml -filesystem: - path: example/path/to/files - files: - - example-deployment.yaml - - example-service.yaml -``` - ### Actions Sometimes it is not sufficient to just define resources that are created and deleted when running a benchmark. Instead, it might be necessary to define certain actions that will be executed before running or after stopping the benchmark. +Theodolite supports *actions*, which can run before (`beforeActions`) or after `afterActions` all `sut`, `loadGenerator` or `infrastructure` resources are deployed. +Theodolite provides two types of actions: + +#### Exec Actions Theodolite allows to execute commands on running pods. This is similar to `kubectl exec` or Kubernetes' [container lifecycle handlers](https://kubernetes.io/docs/tasks/configure-pod-container/attach-handler-lifecycle-event/). Theodolite actions can run before (`beforeActions`) or after `afterActions` all `sut`, `loadGenerator` or `infrastructure` resources are deployed. For example, the following actions will create a file in a pod with label `app: logger` before the SUT is started and delete if after the SUT is stopped: @@ -102,26 +106,48 @@ For example, the following actions will create a file in a pod with label `app: sut: resources: # ... beforeActions: - - selector: - pod: - matchLabels: - app: logger - exec: + - exec: + selector: + pod: + matchLabels: + app: logger + container: logger # optional command: ["touch", "file-used-by-logger.txt"] timeoutSeconds: 90 afterActions: - - selector: - pod: - matchLabels: - app: logger - exec: + - exec: + selector: + pod: + matchLabels: + app: logger + container: logger # optional command: [ "rm", "file-used-by-logger.txt" ] timeoutSeconds: 90 ``` Theodolite checks if all referenced pods are available for the specified actions. That means these pods must either be defined in `infrastructure` or already deployed in the cluster. If not all referenced pods are available, the benchmark will not be set as `Ready`. Consequently, an action cannot be executed on a pod that is defined as an SUT or load generator resource. -*Note: Actions should be used sparingly. While it is possible to define entire benchmarks imperatively as actions, it is considered better practice to define as much as possible using declarative, native Kubernetes resource files.* +*Note: Exec actions should be used sparingly. While it is possible to define entire benchmarks imperatively as actions, it is considered better practice to define as much as possible using declarative, native Kubernetes resource files.* + +#### Delete Actions + +Sometimes it is required to delete Kubernetes resources before or after running a benchmark. +This is typically the case for resources that are automatically created while running a benchmark. +For example, Kafka Streams creates internal Kafka topics. When using the [Strimzi](https://strimzi.io/) Kafka operator, we can delete these topics by deleting the corresponding Kafka topic resource. + +As shown in the following example, delete actions select the resources to be deleted by specifying their *apiVersion*, *kind* and a [regular expression](https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html) for their name. + +```yaml + sut: + resources: # ... + beforeActions: + - delete: + selector: + apiVersion: kafka.strimzi.io/v1beta2 + kind: KafkaTopic + nameRegex: ^some-internal-topic-.* +``` + <!-- A Benchmark refers to other Kubernetes resources (e.g., Deployments, Services, ConfigMaps), which describe the system under test, the load generator and infrastructure components such as a middleware used in the benchmark. To manage those resources, Theodolite needs to have access to them. This is done by bundling resources in ConfigMaps. @@ -140,6 +166,33 @@ See the [patcher API reference](api-reference/patchers) for an overview of avail If a benchmark is [executed by an Execution](running-benchmarks), these patchers are used to configure SUT and load generator according to the [load and resource values](creating-an-execution) set in the Execution. +## Service Level Objectives SLOs + +SLOs provide a way to quantify whether a certain load intensity can be handled by a certain amount of provisioned resources. +In Theodolite, SLOs are evaluated by requesting monitoring data from Prometheus and analyzing it in a benchmark-specific way. +An Execution must at least define one SLO to be checked. + +A good choice to get started is defining an SLO of type `generic`: + +```yaml +- sloType: "generic" + prometheusUrl: "http://prometheus-operated:9090" + offset: 0 + properties: + externalSloUrl: "http://localhost:8082" + promQLQuery: "sum by(job) (kafka_streams_stream_task_metrics_dropped_records_total>=0)" + warmup: 60 # in seconds + queryAggregation: max + repetitionAggregation: median + operator: lte + threshold: 1000 +``` + +All you have to do is to define a [PromQL query](https://prometheus.io/docs/prometheus/latest/querying/basics/) describing which metrics should be requested (`promQLQuery`) and how the resulting time series should be evaluated. With `queryAggregation` you specify how the resulting time series is aggregated to a single value and `repetitionAggregation` describes how the results of multiple repetitions are aggregated. Possible values are +`mean`, `median`, `mode`, `sum`, `count`, `max`, `min`, `std`, `var`, `skew`, `kurt` as well as percentiles such as `p99` or `p99.9`. The result of aggregation all repetitions is checked against `threshold`. This check is performed using an `operator`, which describes that the result must be "less than" (`lt`), "less than equal" (`lte`), "greater than" (`gt`) or "greater than equal" (`gte`) to the threshold. + +In case you need to evaluate monitoring data in a more flexible fashion, you can also change the value of `externalSloUrl` to your custom SLO checker. Have a look at the source code of the [generic SLO checker](https://github.com/cau-se/theodolite/tree/main/slo-checker/generic) to get started. + ## Kafka Configuration Theodolite allows to automatically create and remove Kafka topics for each SLO experiment by setting a `kafkaConfig`. diff --git a/docs/creating-an-execution.md b/docs/creating-an-execution.md index e33019a574ec7d4a3486c9ea7778efeb6e959260..3a109d903397778da9dfecfe53e80849463156b6 100644 --- a/docs/creating-an-execution.md +++ b/docs/creating-an-execution.md @@ -16,7 +16,7 @@ kind: execution metadata: name: theodolite-example-execution spec: - benchmark: "uc1-kstreams" + benchmark: "example-benchmark" load: loadType: "NumSensors" loadValues: [25000, 50000] @@ -24,20 +24,19 @@ spec: resourceType: "Instances" resourceValues: [1, 2] slos: - - sloType: "lag trend" - prometheusUrl: "http://prometheus-operated:9090" - offset: 0 + - name: "lag trend" properties: threshold: 2000 - externalSloUrl: "http://localhost:80/evaluate-slope" - warmup: 60 # in seconds execution: - strategy: "LinearSearch" + metric: "demand" + strategy: + name: "RestrictionSearch" + restrictions: + - "LowerBound" + searchStrategy: "LinearSearch" duration: 300 # in seconds repetitions: 1 loadGenerationDelay: 30 # in seconds - restrictions: - - "LowerBound" configOverrides: - patcher: type: "SchedulerNamePatcher" @@ -53,41 +52,20 @@ Similar to [Kubernetes Jobs](https://kubernetes.io/docs/concepts/workloads/contr An Execution always refers to a Benchmark. For the Execution to run, the Benchmark must be registered with Kubernetes and it must be in state *Ready*. If this is not the case, the Execution will remain in state *Pending*. -As a Benchmark may define multiple supported load and resource types, an Execution has to pick exactly one of each by its name. Additionally, it defines the set of load values and resource values the benchmark should be executed with. Both these values are represented as integers, which are interpreted in a [Benchmark-specific way](creating-a-benchmark) to configure the SUT and load generator. +## Selecting Load Type, Resource Type and SLOs -## Definition of SLOs - -SLOs provide a way to quantify whether a certain load intensity can be handled by a certain amount of provisioned resources. -In Theodolite, SLO are evaluated by requesting monitoring data from Prometheus and analyzing it in a benchmark-specific way. -An Execution must at least define one SLO to be checked. - -A good choice to get started is defining an SLO of type `generic`: - -```yaml -- sloType: "generic" - prometheusUrl: "http://prometheus-operated:9090" - offset: 0 - properties: - externalSloUrl: "http://localhost:8082" - promQLQuery: "sum by(job) (kafka_streams_stream_task_metrics_dropped_records_total>=0)" - warmup: 60 # in seconds - queryAggregation: max - repetitionAggregation: median - operator: lte - threshold: 1000 -``` - -All you have to do is to define a [PromQL query](https://prometheus.io/docs/prometheus/latest/querying/basics/) describing which metrics should be requested (`promQLQuery`) and how the resulting time series should be evaluated. With `queryAggregation` you specify how the resulting time series is aggregated to a single value and `repetitionAggregation` describes how the results of multiple repetitions are aggregated. Possible values are -`mean`, `median`, `mode`, `sum`, `count`, `max`, `min`, `std`, `var`, `skew`, `kurt` as well as percentiles such as `p99` or `p99.9`. The result of aggregation all repetitions is checked against `threshold`. This check is performed using an `operator`, which describes that the result must be "less than" (`lt`), "less than equal" (`lte`), "greater than" (`gt`) or "greater than equal" (`gte`) to the threshold. - -In case you need to evaluate monitoring data in a more flexible fashion, you can also change the value of `externalSloUrl` to your custom SLO checker. Have a look at the source code of the [generic SLO checker](https://github.com/cau-se/theodolite/tree/master/slo-checker/generic) to get started. +As a Benchmark may define multiple supported load and resource types, an Execution has to pick exactly one of each by its name. Additionally, it defines the set of load values and resource values the benchmark should be executed with. +Both these values are represented as integers, which are interpreted in a [Benchmark-specific way](creating-a-benchmark#load-and-resource-types) to configure the SUT and load generator. +Similarly, an Execution must select a subset of the [SLOs defined in the Benchmark](creating-a-benchmark#service-level-objectives-slos). Additionally, these SLOs can be configured by their `properties`. +<!-- TODO: What happpens if slos are not set? --> ## Experimental Setup According to Theodolite's measurement method, isolated SLO experiments are performed for different combinations of load intensity and resource amounts. The experimental setup can be configured by: -* A search strategy (`strategy`), which determines which load and resource combinations should be tested. Supported values are `FullSearch`, `LinearSearch` and `BinarySearch`. Additionally, a `restrictions` can be set to `LowerBound`. +* A [scalability metric](concepts/metrics) (`metric`). Supported values are `demand` and `capacity`, with `demand` being the default. +* A [search strategy](concepts/search-strategies) (`strategy`), which determines which load and resource combinations should be tested. Supported values are `FullSearch`, `LinearSearch` and `BinarySearch`. Additionally, a `restrictions` can be set to `LowerBound`. * The `duration` per SLO experiment in seconds. * The number of repetitions (`repetitions`) for each SLO experiment. * A `loadGenerationDelay`, specifying the time in seconds before the load generation starts. diff --git a/docs/development/release-process.md b/docs/development/release-process.md index 21f913fbb8626d141d1df49db808fe0b36a01462..c7ef78ad6b2a2485fb715afea2dd0a5021e1b7a0 100644 --- a/docs/development/release-process.md +++ b/docs/development/release-process.md @@ -13,7 +13,7 @@ We assume that we are creating the release `v0.3.1`. Please make sure to adjust the following steps according to the release, you are actually performing. 1. Create a new branch `v0.3` if it does not already exist. This branch will never -again be merged into master. +again be merged into main. 2. Checkout the `v0.3` branch. @@ -43,7 +43,7 @@ again be merged into master. 8. Create *releases* on GitLab and GitHub. Upload the generated Helm package to these releases via the UIs of GitLab and GitHub. -9. Switch to the `master` branch. +9. Switch to the `main` branch. 10. Re-run `./update-index.sh v0.3.1` to include the latest release in the *upstream* Helm repository. You can now delete the packaged Helm chart. @@ -57,4 +57,4 @@ again be merged into master. 4. Update the `CITATION.cff` file according to Step 3. -12. Commit these changes to the `master` branch. +12. Commit these changes to the `main` branch. diff --git a/docs/drafts/filesystem.md b/docs/drafts/filesystem.md new file mode 100644 index 0000000000000000000000000000000000000000..d45d9f8b09c7e5f01099d8d5edc1929d8360b9ea --- /dev/null +++ b/docs/drafts/filesystem.md @@ -0,0 +1,13 @@ +## Creating a Benchmark with Filesystem Resources + +#### Filesystem + +Alternatively, resources can also be read from the filesystem, Theodolite has access to. This usually requires that the Benchmark resources are available in a volume, which is mounted into the Theodolite container. + +```yaml +filesystem: + path: example/path/to/files + files: + - example-deployment.yaml + - example-service.yaml +``` diff --git a/docs/index.yaml b/docs/index.yaml index 956bb83b19ebbae4cddc4da6f07a0d937cf3dc2d..34c68818579e8cf4a486b0a8e49712e1281fa4c7 100644 --- a/docs/index.yaml +++ b/docs/index.yaml @@ -1,6 +1,42 @@ apiVersion: v1 entries: theodolite: + - apiVersion: v2 + appVersion: 0.8.0 + created: "2022-07-18T17:48:21.205921939+02:00" + dependencies: + - condition: grafana.enabled + name: grafana + repository: https://grafana.github.io/helm-charts + version: 6.17.5 + - condition: kube-prometheus-stack.enabled + name: kube-prometheus-stack + repository: https://prometheus-community.github.io/helm-charts + version: 20.0.1 + - condition: cp-helm-charts.enabled + name: cp-helm-charts + repository: https://soerenhenning.github.io/cp-helm-charts + version: 0.6.0 + - condition: strimzi.enabled + name: strimzi-kafka-operator + repository: https://strimzi.io/charts/ + version: 0.28.0 + description: Theodolite is a framework for benchmarking the horizontal and vertical + scalability of cloud-native applications. + digest: 2537d37ae9467a967f50d9231040ee7babb1a05cf55cb8ca216162f91299730a + home: https://www.theodolite.rocks + icon: https://www.theodolite.rocks/assets/logo/theodolite-stacked-transparent.svg + maintainers: + - email: soeren.henning@email.uni-kiel.de + name: Sören Henning + url: https://www.se.informatik.uni-kiel.de/en/team/soeren-henning-m-sc + name: theodolite + sources: + - https://github.com/cau-se/theodolite + type: application + urls: + - https://github.com/cau-se/theodolite/releases/download/v0.8.0/theodolite-0.8.0.tgz + version: 0.8.0 - apiVersion: v2 appVersion: 0.7.0 created: "2022-05-11T13:49:02.491041789+02:00" @@ -351,4 +387,4 @@ entries: urls: - https://github.com/cau-se/theodolite/releases/download/v0.4.0/theodolite-0.4.0.tgz version: 0.4.0 -generated: "2022-05-11T13:49:02.423487026+02:00" +generated: "2022-07-18T17:48:21.163757427+02:00" diff --git a/docs/installation.md b/docs/installation.md index a01cfd14dff18d61f9d0e7de434fcaf9971280eb..f73a25926917ae0edf00a8f75fdb33fb342fe9bb 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -10,7 +10,7 @@ The easiest option to install Theodolite is using [Helm](https://helm.sh). To install Theodolite with all its dependencies run: ```sh -helm repo add theodolite https://cau-se.github.io/theodolite +helm repo add theodolite https://www.theodolite.rocks helm repo update helm install theodolite theodolite/theodolite ``` @@ -26,11 +26,11 @@ As usual, the installation via Helm can be configured by passing a values YAML f helm install theodolite theodolite/theodolite --values <your-config.yaml> ``` -For this purpose the [default values file](https://github.com/cau-se/theodolite/blob/master/helm/values.yaml) can serve as a template for your custom configuration. +For this purpose the [default values file](https://github.com/cau-se/theodolite/blob/main/helm/values.yaml) can serve as a template for your custom configuration. ### Minimal setup -For Kubernetes clusters with limited resources such as on local developer installations, we provide a [minimal values file](https://github.com/cau-se/theodolite/blob/master/helm/preconfigs/minimal.yaml). +For Kubernetes clusters with limited resources such as on local developer installations, we provide a [minimal values file](https://github.com/cau-se/theodolite/blob/main/helm/preconfigs/minimal.yaml). ### Persisting results @@ -38,11 +38,6 @@ To store the results of benchmark executions in a [PersistentVolume](https://kub You can also use an existing PersistentVolumeClaim by setting `operator.resultsVolume.persistent.existingClaim`. If persistence is not enabled, all results will be gone upon pod termination. - -### Standalone mode - -Per default, Theodolite is installed in operator mode, which allows to run and manage benchmarks through the Kubernetes API. For running Theodolite in standalone mode, it is sufficient to disable the operator by setting `operator.enabled` to `false`. Additionally, you might want to add the command line argument `--skip-crds`. With these settings, only Theodolite's dependencies as well as resources to get the necessary permissions are installed. - ### Random scheduler Installation of the random scheduler can be enabled via `randomScheduler.enabled`. Please note that the random scheduler is neither required in operator mode nor in standalone mode. However, it has to be installed if benchmark executions should use random scheduling. @@ -56,19 +51,6 @@ In cases, where you need to install multiple Theodolite instances, it's best to *Note that for meaningful results, usually only one benchmark should be executed at a time.* -### Installation with a release name other than `theodolite` - -When using another release name than `theodolite`, make sure to adjust the Confluent Schema Registry configuration of you `values.yaml` accordingly: - -```yaml -cp-helm-charts: - cp-schema-registry: - kafka: - bootstrapServers: <your-release-name>-kafka-kafka-bootstrap:9092 -``` - -This seems unfortunately to be necessary as Helm does not let us inject values into dependency charts. - ## Test the Installation @@ -113,5 +95,4 @@ kubectl delete crd kafkas.kafka.strimzi.io kubectl delete crd kafkatopics.kafka.strimzi.io kubectl delete crd kafkausers.kafka.strimzi.io kubectl delete crd strimzipodsets.core.strimzi.io - ``` diff --git a/docs/project-info.md b/docs/project-info.md index 923df7fd898a8e2a595857a616e4485d10b30e23..36019019dcaf486fb0befb06b3550fafd41f8855 100644 --- a/docs/project-info.md +++ b/docs/project-info.md @@ -20,11 +20,14 @@ Theodolite's internal development including issue boards, merge requests and ext ## Contributors * [Sören Henning](https://www.se.informatik.uni-kiel.de/en/team/soeren-henning-m-sc) (Maintainer) +* [Marcel Becker](https://www.linkedin.com/in/marcel-becker-11b39b246) * [Jan Bensien](https://oceanrep.geomar.de/id/eprint/52342/) +* [Nico Biernat](https://github.com/NicoBiernat) * [Lorenz Boguhn](https://github.com/lorenzboguhn) * [Simon Ehrenstein](https://github.com/sehrenstein) * [Willi Hasselbring](https://www.se.informatik.uni-kiel.de/en/team/prof.-dr.-wilhelm-willi-hasselbring) * [Tobias Pfandzelter](https://pfandzelter.com/) +* [Julia Rossow](https://www.linkedin.com/in/julia-rossow/) * [Björn Vonheiden](https://github.com/bvonheid) * [Benedikt Wetzel](https://github.com/benediktwetzel) * [Max Wiedenhöft](https://www.linkedin.com/in/maxwiedenhoeft/) diff --git a/docs/publications.md b/docs/publications.md index 41847690180166fac371357846cdf13eb7766863..f87a363f49c00d685abd3d51d78f0365805c5281 100644 --- a/docs/publications.md +++ b/docs/publications.md @@ -8,10 +8,10 @@ nav_order: 9 Below you can find a list of publications that are directly related to Theodolite: -* S. Henning and W. Hasselbring. “A Configurable Method for Benchmarking Scalability of Cloud-Native Applications”. In: *Empirical Software Engineering* (2022). In press. DOI: 10. -1007/s10664-022-10162-1. +* S. Henning and W. Hasselbring. “[A Configurable Method for Benchmarking Scalability of Cloud-Native Applications](https://doi.org/10.1007/s10664-022-10162-1)”. In: *Empirical Software Engineering* 27. 2022. DOI: [10.1007/s10664-022-10162-1](https://doi.org/10.1007/s10664-022-10162-1). * T. Pfandzelter, S. Henning, T. Schirmer, W. Hasselbring, and D. Bermbach. “[Streaming vs. Functions: A Cost Perspective on Cloud Event Processing](https://arxiv.org/pdf/2204.11509.pdf)”. In: *IEEE International Conference on Cloud Engineering*. 2022. In press. * S. Henning and W. Hasselbring. “Demo Paper: Benchmarking Scalability of Cloud-Native Applications with Theodolite”. In: *IEEE International Conference on Cloud Engineering*. 2022. In press. * S. Henning, B. Wetzel, and W. Hasselbring. “[Reproducible Benchmarking of Cloud-Native Applications With the Kubernetes Operator Pattern](http://ceur-ws.org/Vol-3043/short5.pdf)”. In: *Symposium on Software Performance*. 2021. * S. Henning and W. Hasselbring. “[How to Measure Scalability of Distributed Stream Processing Engines?](https://research.spec.org/icpe_proceedings/2021/companion/p85.pdf)” In: *Companion of the ACM/SPEC International Conference on Performance Engineering*. 2021. DOI: [10.1145/3447545.3451190](https://doi.org/10.1145/3447545.3451190). -* S. Henning and W. Hasselbring. “[Theodolite: Scalability Benchmarking of Distributed Stream Processing Engines in Microservice Architectures](https://arxiv.org/abs/2009.00304)”. In: *Big Data Research* 25 (2021). DOI: [10.1016/j.bdr.2021.100209](https://doi.org/10.1016/j.bdr.2021.100209). +* S. Henning and W. Hasselbring. “[Theodolite: Scalability Benchmarking of Distributed Stream Processing Engines in Microservice Architectures](https://arxiv.org/abs/2009.00304)”. In: *Big Data Research* 25. 2021. DOI: [10.1016/j.bdr.2021.100209](https://doi.org/10.1016/j.bdr.2021.100209). +* S. Henning and W. Hasselbring. “[Toward Efficient Scalability Benchmarking of Event-Driven Microservice Architectures at Large Scale](https://www.performance-symposium.org/fileadmin/user_upload/palladio-conference/2020/Papers/SSP2020_paper_14.pdf)”. In: *Softwaretechnik-Trends* 40 (3) (Symposium on Software Performance). 2020. diff --git a/docs/quickstart.md b/docs/quickstart.md index 6effc0cb288c01d28267b12e360c603f17641d32..755b1baf5b809cd4b9acc61a2406bbbe3dcfd061 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -12,63 +12,63 @@ All you need to get started is access to a Kubernetes cluster plus kubectl and H 1. To install Theodolite with all its dependencies run: - ```sh - helm repo add theodolite https://cau-se.github.io/theodolite - helm repo update - helm install theodolite theodolite/theodolite -f https://raw.githubusercontent.com/cau-se/theodolite/master/helm/preconfigs/minimal.yaml - ``` + ```sh + helm repo add theodolite https://www.theodolite.rocks + helm repo update + helm install theodolite theodolite/theodolite -f https://raw.githubusercontent.com/cau-se/theodolite/main/helm/preconfigs/minimal.yaml + ``` 1. Get the Theodolite examples from the [Theodolite repository](https://github.com/cau-se/theodolite) and `cd` into its example directory: - ```sh - git clone https://github.com/cau-se/theodolite.git - cd theodolite/theodolite/examples/operator/ - ``` + ```sh + git clone https://github.com/cau-se/theodolite.git + cd theodolite/theodolite/examples/operator/ + ``` 1. Deploy the example Benchmark and package its associated Kubernetes resources in a ConfigMap: - ```sh - kubectl apply -f example-benchmark.yaml - kubectl apply -f example-configmap.yaml - ``` + ```sh + kubectl apply -f example-benchmark.yaml + kubectl apply -f example-configmap.yaml + ``` 1. Verify that the Benchmark has been deployed successfully: - ```sh - kubectl get benchmarks - ``` + ```sh + kubectl get benchmarks + ``` - The output is similar to this: + The output is similar to this: - ``` - NAME AGE STATUS - example-benchmark 81s Ready - ``` + ``` + NAME AGE STATUS + example-benchmark 81s Ready + ``` 1. Run the Benchmark by deploying an Execution: - ```sh - kubectl apply -f example-execution.yaml - ``` + ```sh + kubectl apply -f example-execution.yaml + ``` 1. Verify that the Executions is running: - ```sh - kubectl get executions - ``` + ```sh + kubectl get executions + ``` - The output is similar to this: + The output is similar to this: - ``` - NAME STATUS DURATION AGE - theodolite-example-execution Running 13s 14s - ``` + ``` + NAME STATUS DURATION AGE + theodolite-example-execution Running 13s 14s + ``` - Theodolite provides additional information on the current status of an Execution by producing Kubernetes events. To see them: + Theodolite provides additional information on the current status of an Execution by producing Kubernetes events. To see them: - ```sh - kubectl describe execution theodolite-example-execution - ``` + ```sh + kubectl describe execution theodolite-example-execution + ``` ## Next Steps diff --git a/docs/running-benchmarks.md b/docs/running-benchmarks.md index c4459183870d945d159551e54aab10cce7ea372e..5a9ab53501c2af6a38fd4cb310292337924e74de 100644 --- a/docs/running-benchmarks.md +++ b/docs/running-benchmarks.md @@ -59,7 +59,7 @@ To run a benchmark, an Execution YAML file needs to be created such as the follo apiVersion: theodolite.rocks/v1beta1 kind: execution metadata: - name: theodolite-example-execution # (1) give your execution a name + name: theodolite-example-execution # (1) give a name to your execution spec: benchmark: "uc1-kstreams" # (2) refer to the benchmark to be run load: @@ -68,21 +68,18 @@ spec: resources: resourceType: "Instances" # (5) chose one of the benchmark's resource types resourceValues: [1, 2] # (6) select a set of resource amounts - slos: # (7) set your SLOs - - sloType: "lag trend" - prometheusUrl: "http://prometheus-operated:9090" - offset: 0 + slos: + - name: "lag trend" properties: threshold: 2000 - externalSloUrl: "http://localhost:80/evaluate-slope" - warmup: 60 # in seconds execution: - strategy: "LinearSearch" # (8) chose a search strategy - restrictions: ["LowerBound"] # (9) add restrictions for the strategy - duration: 300 # (10) set the experiment duration in seconds - repetitions: 1 # (11) set the number of repetitions - loadGenerationDelay: 30 # (12) configure a delay before load generation - configOverrides: [] + strategy: + name: "RestrictionSearch" # (8) chose a search strategy + restrictions: ["LowerBound"] # (9) configure the search strategy + searchStrategy: "LinearSearch" # (10) configure the search strategy (cont.) + duration: 300 # (11) set the experiment duration in seconds + repetitions: 1 # (12) set the number of repetitions + loadGenerationDelay: 30 # (13) configure a delay before load generation ``` See [Creating an Execution](creating-an-execution) for a more detailed explanation on how to create Executions. @@ -141,7 +138,7 @@ The easiest way to use them is at MyBinder: [Launch Notebooks](https://mybinder.org/v2/gh/cau-se/theodolite/HEAD?labpath=analysis){: .btn .btn-primary } {: .text-center } -Alternatively, you can also [run these notebook locally](https://github.com/cau-se/theodolite/tree/master/analysis), for example, with Docker or Visual Studio Code. +Alternatively, you can also [run these notebook locally](https://github.com/cau-se/theodolite/tree/main/analysis), for example, with Docker or Visual Studio Code. The notebooks allow to compute a scalability function using Theodolite's *demand* metric and to visualize multiple such functions in plots: @@ -149,7 +146,7 @@ The notebooks allow to compute a scalability function using Theodolite's *demand After finishing a benchmark execution, Theodolite creates a `exp<id>_demand.csv` file. It maps the tested load intensities to the minimal required resources for that load. If the monitoring data collected during benchmark execution should be analyzed in more detail, the `demand-metric.ipynb` notebook can be used. -Theodolite stores monitoring data for each conducted SLO experiment in `exp<id>_<load>_<resources>_<slo-slug>_<rep>.csv` files, where `<id>` is the ID of an execution, `<load>` the corresponding load intensity value, `<resources>` the resources value, `<slo-slug>` the [name of the SLO](creating-an-execution.html#definition-of-slos) and `<rep>` the repetition counter. +Theodolite stores monitoring data for each conducted SLO experiment in `exp<id>_<load>_<resources>_<slo-slug>_<rep>.csv` files, where `<id>` is the ID of an execution, `<load>` the corresponding load intensity value, `<resources>` the resources value, `<slo-slug>` the [name of the SLO](creating-a-benchmark#service-level-objectives-slos) and `<rep>` the repetition counter. The `demand-metric.ipynb` notebook reads these files and generates a new CSV file mapping load intensities to the minimal required resources. The format of this file corresponds to the original `exp<id>_demand.csv` file created when running the benchmark, but allows, for example, to evaluate different warm-up periods. Currently, the `demand-metric.ipynb` notebook only supports benchmarks with the *lag trend SLO* out-of-the-box, but can easily be adjusted to perform any other type of analysis. diff --git a/docs/theodolite-benchmarks/load-generator.md b/docs/theodolite-benchmarks/load-generator.md index a41c97d52f62f399c9289a15a64991d0fed228ce..ff2e2ecc6afe1fff642cd9be2f04a32b6cd62aa9 100644 --- a/docs/theodolite-benchmarks/load-generator.md +++ b/docs/theodolite-benchmarks/load-generator.md @@ -22,7 +22,7 @@ docker run -it ghcr.io/cau-se/theodolite-uc1-workload-generator ### Message format -Messages generated by the load generators represent a single measurement of [active power](https://en.wikipedia.org/wiki/AC_power#Active,_reactive,_apparent,_and_complex_power_in_sinusoidal_steady-state). The corresponding message type is specified as [`ActivePowerRecords`](https://github.com/cau-se/titan-ccp-common/blob/master/src/main/avro/ActivePower.avdl) +Messages generated by the load generators represent a single measurement of [active power](https://en.wikipedia.org/wiki/AC_power#Active,_reactive,_apparent,_and_complex_power_in_sinusoidal_steady-state). The corresponding message type is specified as [`ActivePowerRecords`](https://github.com/cau-se/theodolite/blob/main/theodolite-benchmarks/commons/src/main/avro/ActivePower.avdl) defined with Avro. It consists of an identifier for simulated power sensor, a timestamp in epoch milliseconds and the actual measured (simulated) value in watts. When sending generated records via Apache Kafka, these records are serialized with the [Confluent Schema Registry](https://docs.confluent.io/platform/current/schema-registry). @@ -43,7 +43,7 @@ The prebuilt container images can be configured with the following environment v | Environment Variable | Description | Default | |:----|:----|:----| | `BOOTSTRAP_SERVER` | Address (`hostname:port`) of another load generator instance to form a cluster with. Can also be this instance. | `localhost:5701` | -| `KUBERNETES_DNS_NAME` | Kubernetes service name to discover other load generators to form a cluster with. Must be a fully qualified domain name (FQDN), e.g., something like `<service>.<namespace>.svc.cluster.local`. * Requires `BOOTSTRAP_SERVER` not to be set. | | +| `KUBERNETES_DNS_NAME` | Kubernetes service name to discover other load generators to form a cluster with. Must be a fully qualified domain name (FQDN), e.g., something like `<service>.<namespace>.svc.cluster.local`. Requires `BOOTSTRAP_SERVER` not to be set. | | | `PORT` | Port used for for coordination among load generator instances. | 5701 | | `PORT_AUTO_INCREMENT` | If set to true and the specified PORT is already used, use the next higher one. Useful if multiple instances should run on the same host, without configuring each instance individually. | true | | `CLUSTER_NAME_PREFIX` | Only required if unrelated load generators form a cluster. | theodolite-load-generation | @@ -66,11 +66,11 @@ The prebuilt container images can be configured with the following environment v | `THREADS` | Number of worker threads used to generate the load. | 4 | | `DISABLE_DNS_CACHING` | Set to `true` to disable DNS caching by the underlying JVM. You might want to do so when generating load via HTTP that should be sent to different target instances. | `false` | -Please note that there are some additional configuration options for benchmark [UC4's load generator](hhttps://github.com/cau-se/theodolite/blob/master/theodolite-benchmarks/uc4-load-generator/src/main/java/rocks/theodolite/benchmarks/uc4/loadgenerator/LoadGenerator.java). +Please note that there are some additional configuration options for benchmark [UC4's load generator](hhttps://github.com/cau-se/theodolite/blob/main/theodolite-benchmarks/uc4-load-generator/src/main/java/rocks/theodolite/benchmarks/uc4/loadgenerator/LoadGenerator.java). ## Creating a custom load generator -To create a custom load generator, you need to import the [load-generator-commons](https://github.com/cau-se/theodolite/tree/master/theodolite-benchmarks/load-generator-commons) project. You can then create an instance of the `LoadGenerator` populated with a default configuration, adjust it as desired, and start it by calling its `run` method: +To create a custom load generator, you need to import the [load-generator-commons](https://github.com/cau-se/theodolite/tree/main/theodolite-benchmarks/load-generator-commons) project. You can then create an instance of the `LoadGenerator` populated with a default configuration, adjust it as desired, and start it by calling its `run` method: ```java LoadGenerator loadGenerator = new LoadGenerator.fromDefaults() diff --git a/docs/troubleshooting-execution.md b/docs/troubleshooting-execution.md new file mode 100644 index 0000000000000000000000000000000000000000..479ba93bd515b8654f2a644176686a39b5a27c43 --- /dev/null +++ b/docs/troubleshooting-execution.md @@ -0,0 +1,46 @@ +--- +title: Troubleshooting +has_children: false +parent: Running Benchmarks +nav_order: 10 +--- + +# Troubleshooting Execution + +## Verify your Installation + +First you should make sure that all Theodolite pods are up and running by typing: + +```sh +kubectl get pods +``` + +If pods are still creating or failing to start, you should simply wait a bit longer or investigate this further. + +If Theodolite is [installed via Helm](installation.md), you can also use `helm test` to verify that everything is installed as expected. Just run the following command (replace `theodolite` by your release name) and wait for all tests to be completed: + +```sh +helm test theodolite +``` + +## Getting Execution Events + +Theodolite produces Kubernetes events, which you can view by running: + +```sh +kubectl describe execution <your-execution-name> +``` + +## Looking the Operator Logs + +If you cannot figure out why your benchmark execution fails, you might want to have look at the operator logs: + +```sh +kubectl logs -l app=theodolite -c theodolite +``` + +You might also add the `-f` flag to follow the logs. + +## Contact the Maintainers + +Do not hesitate to [ask for help](project-info#getting-help) if you are still having problems. Attaching Theodolite's logs to your request is likely to help us solving your problem. diff --git a/helm/Chart.yaml b/helm/Chart.yaml index d890cf35119294faac3590b8beacbe18c61859b1..f78629b3674cfda290f97b48c8cc183476b1e891 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -31,6 +31,6 @@ dependencies: condition: strimzi.enabled -version: 0.8.0-SNAPSHOT +version: 0.9.0-SNAPSHOT -appVersion: 0.8.0-SNAPSHOT +appVersion: 0.9.0-SNAPSHOT diff --git a/helm/benchmark-definitions/test/test-example-execution.yaml b/helm/benchmark-definitions/test/test-example-execution.yaml index f63365bed95c0a1e46a361b6948b47c814a32c25..6d6ae70398e6e707224a62593f2f1f05bfc26823 100644 --- a/helm/benchmark-definitions/test/test-example-execution.yaml +++ b/helm/benchmark-definitions/test/test-example-execution.yaml @@ -4,7 +4,7 @@ metadata: name: theodolite-test-execution spec: benchmark: "example-benchmark" - loads: + load: loadType: "NumSensors" loadValues: [100] resources: diff --git a/helm/templates/prometheus/prometheus.yaml b/helm/templates/prometheus/prometheus.yaml index 196d68487824d7d8e130c56d11cec2687304d7e6..b0279821840a26bc735060db6255e089bcc4f347 100644 --- a/helm/templates/prometheus/prometheus.yaml +++ b/helm/templates/prometheus/prometheus.yaml @@ -5,6 +5,10 @@ metadata: name: {{ template "theodolite.fullname" . }}-prometheus spec: serviceAccountName: {{ template "theodolite.fullname" . }}-prometheus + {{- with .Values.prometheus.podMetadata }} + podMetadata: + {{- toYaml . | nindent 8 }} + {{- end}} podMonitorSelector: {} serviceMonitorSelector: {} resources: diff --git a/helm/templates/theodolite/theodolite-operator.yaml b/helm/templates/theodolite/theodolite-operator.yaml index f2669686eada049d33c5c88169d8d2ec3af84261..c8714fc3249db86efe64123d8f7092b0619ce3c7 100644 --- a/helm/templates/theodolite/theodolite-operator.yaml +++ b/helm/templates/theodolite/theodolite-operator.yaml @@ -12,6 +12,10 @@ spec: metadata: labels: app: {{ include "theodolite.fullname" . }} + {{- with .Values.operator.podAnnotations }} + annotations: + {{ toYaml . | indent 2 }} + {{- end }} spec: terminationGracePeriodSeconds: 0 serviceAccountName: {{ include "theodolite.serviceAccountName" . }} diff --git a/helm/values.yaml b/helm/values.yaml index 995408344d69352136c0569056a24e0f8dec4c10..1fe1938b9d2257eba4e49b581815937e1a400719 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -62,7 +62,7 @@ grafana: ### cp-helm-charts: - enabled: true + enabled: false ## ------------------------------------------------------ ## Zookeeper ## ------------------------------------------------------ @@ -285,6 +285,7 @@ kube-prometheus-stack: ### prometheus: enabled: true + podMetadata: {} nodeSelector: {} serviceAccount: @@ -306,6 +307,8 @@ operator: nodeSelector: {} + podAnnotations: {} + sloChecker: generic: enabled: true @@ -318,7 +321,7 @@ operator: imageTag: latest imagePullPolicy: Always droppedRecordsKStreams: - enabled: true + enabled: false image: ghcr.io/cau-se/theodolite-slo-checker-dropped-records-kstreams imageTag: latest imagePullPolicy: Always diff --git a/theodolite-benchmarks/definitions/uc1-hazelcastjet/resources/uc1-hazelcastjet-deployment.yaml b/theodolite-benchmarks/definitions/uc1-hazelcastjet/resources/uc1-hazelcastjet-deployment.yaml index 6432679b05264150ccccc84490c2badb466d9beb..28cf4761d44a4d98c1d2b889f9c6a679ec96304a 100644 --- a/theodolite-benchmarks/definitions/uc1-hazelcastjet/resources/uc1-hazelcastjet-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc1-hazelcastjet/resources/uc1-hazelcastjet-deployment.yaml @@ -15,8 +15,7 @@ spec: terminationGracePeriodSeconds: 0 containers: - name: uc-application - image: uc1-hazelcastjet - imagePullPolicy: "Never" + image: ghcr.io/cau-se/theodolite-uc1-hazelcastjet:latest env: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" diff --git a/theodolite-benchmarks/definitions/uc2-hazelcastjet/resources/uc2-hazelcastjet-deployment.yaml b/theodolite-benchmarks/definitions/uc2-hazelcastjet/resources/uc2-hazelcastjet-deployment.yaml index fddfa7512a41c75321e49272a41824383b4906e8..15eb8baa4219253270066d68cfa9cb4166eafd48 100644 --- a/theodolite-benchmarks/definitions/uc2-hazelcastjet/resources/uc2-hazelcastjet-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc2-hazelcastjet/resources/uc2-hazelcastjet-deployment.yaml @@ -15,8 +15,7 @@ spec: terminationGracePeriodSeconds: 0 containers: - name: uc-application - image: uc2-hazelcastjet - imagePullPolicy: "Never" + image: ghcr.io/cau-se/theodolite-uc2-hazelcastjet:latest env: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" diff --git a/theodolite-benchmarks/definitions/uc3-hazelcastjet/resources/uc3-hazelcastjet-deployment.yaml b/theodolite-benchmarks/definitions/uc3-hazelcastjet/resources/uc3-hazelcastjet-deployment.yaml index f1c6cb366417e3ba8daf4cdcd7454b890d3c45e0..491bee67b531d983ec3dfb2b008dc0b50994c30c 100644 --- a/theodolite-benchmarks/definitions/uc3-hazelcastjet/resources/uc3-hazelcastjet-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc3-hazelcastjet/resources/uc3-hazelcastjet-deployment.yaml @@ -15,8 +15,7 @@ spec: terminationGracePeriodSeconds: 0 containers: - name: uc-application - image: uc3-hazelcastjet - imagePullPolicy: "Never" + image: ghcr.io/cau-se/theodolite-uc3-hazelcastjet:latest env: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" diff --git a/theodolite-benchmarks/definitions/uc4-hazelcastjet/resources/uc4-hazelcastjet-deployment.yaml b/theodolite-benchmarks/definitions/uc4-hazelcastjet/resources/uc4-hazelcastjet-deployment.yaml index ab872b889877d7787e30a2142e1c2c5b564b5235..1c5272cb6adf05579f79125f300836740450e361 100644 --- a/theodolite-benchmarks/definitions/uc4-hazelcastjet/resources/uc4-hazelcastjet-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc4-hazelcastjet/resources/uc4-hazelcastjet-deployment.yaml @@ -15,8 +15,7 @@ spec: terminationGracePeriodSeconds: 0 containers: - name: uc-application - image: uc4-hazelcastjet - imagePullPolicy: "Never" + image: ghcr.io/cau-se/theodolite-uc4-hazelcastjet:latest env: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" diff --git a/theodolite/README.md b/theodolite/README.md index 49019813c43e0b19e32e35703ca294b2b5c54cb0..de46ff6152027402f2ea56986ad6e4dd8d393a2d 100644 --- a/theodolite/README.md +++ b/theodolite/README.md @@ -51,7 +51,7 @@ Or, if you don't have GraalVM installed, you can run the native executable build ``` You can then execute your native executable with: -```./build/theodolite-0.8.0-SNAPSHOT-runner``` +```./build/theodolite-0.9.0-SNAPSHOT-runner``` If you want to learn more about building native executables, please consult https://quarkus.io/guides/gradle-tooling. diff --git a/theodolite/build.gradle b/theodolite/build.gradle index 2ca866416674985194135def55b782102505f3c7..fec37943df2546a4b675085db2f064bf54585cd2 100644 --- a/theodolite/build.gradle +++ b/theodolite/build.gradle @@ -38,7 +38,7 @@ dependencies { } group 'theodolite' -version '0.8.0-SNAPSHOT' +version '0.9.0-SNAPSHOT' java { sourceCompatibility = JavaVersion.VERSION_11 diff --git a/theodolite/crd/crd-execution.yaml b/theodolite/crd/crd-execution.yaml index fd618e0189a47b3829792d355febe00a2ce36bad..89f4c99a86cb5d349ba865440116e0c75b49bfcc 100644 --- a/theodolite/crd/crd-execution.yaml +++ b/theodolite/crd/crd-execution.yaml @@ -20,7 +20,7 @@ spec: properties: spec: type: object - required: ["benchmark", "loads", "resources", "execution", "configOverrides"] + required: ["benchmark", "load", "resources", "execution"] properties: name: description: This field exists only for technical reasons and should not be set by the user. The value of the field will be overwritten. @@ -29,7 +29,7 @@ spec: benchmark: description: The name of the benchmark this execution is referring to. type: string - loads: # definition of the load dimension + load: # definition of the load dimension description: Specifies the load values that are benchmarked. type: object required: ["loadType", "loadValues"] @@ -130,6 +130,7 @@ spec: default: {} value: type: string + default: [] status: type: object properties: diff --git a/theodolite/examples/operator/example-execution.yaml b/theodolite/examples/operator/example-execution.yaml index 1d889398e1d018ed9b496ad568f79fd8e38aed44..d0c47eaa4bc9e1f600d37021b46a3caaec478f91 100644 --- a/theodolite/examples/operator/example-execution.yaml +++ b/theodolite/examples/operator/example-execution.yaml @@ -4,7 +4,7 @@ metadata: name: theodolite-example-execution spec: benchmark: "example-benchmark" - loads: + load: loadType: "NumSensors" loadValues: [25000, 50000, 75000, 100000, 125000, 150000] resources: diff --git a/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/TheodoliteExecutor.kt b/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/TheodoliteExecutor.kt index 69615522ba9bbd5ef0944528eacbf1dce318caf9..9df19388015d336018766fd5e8ae182c655be936 100644 --- a/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/TheodoliteExecutor.kt +++ b/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/TheodoliteExecutor.kt @@ -58,7 +58,7 @@ class TheodoliteExecutor( val loadDimensionPatcherDefinition = PatcherDefinitionFactory().createPatcherDefinition( - benchmarkExecution.loads.loadType, + benchmarkExecution.load.loadType, this.benchmark.loadTypes ) @@ -81,11 +81,11 @@ class TheodoliteExecutor( waitForResourcesEnabled = this.benchmark.waitForResourcesEnabled ) - if (benchmarkExecution.loads.loadValues != benchmarkExecution.loads.loadValues.sorted()) { - benchmarkExecution.loads.loadValues = benchmarkExecution.loads.loadValues.sorted() + if (benchmarkExecution.load.loadValues != benchmarkExecution.load.loadValues.sorted()) { + benchmarkExecution.load.loadValues = benchmarkExecution.load.loadValues.sorted() logger.info { "Load values are not sorted correctly, Theodolite sorts them in ascending order." + - "New order is: ${benchmarkExecution.loads.loadValues}" + "New order is: ${benchmarkExecution.load.loadValues}" } } @@ -98,7 +98,7 @@ class TheodoliteExecutor( } return Config( - loads = benchmarkExecution.loads.loadValues, + loads = benchmarkExecution.load.loadValues, resources = benchmarkExecution.resources.resourceValues, searchStrategy = strategyFactory.createSearchStrategy(experimentRunner, benchmarkExecution.execution.strategy.name, benchmarkExecution.execution.strategy.searchStrategy, benchmarkExecution.execution.strategy.restrictions, diff --git a/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/model/BenchmarkExecution.kt b/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/model/BenchmarkExecution.kt index 167423ec911cd740b0ee0246e8512dde8402f1e9..9c02b7e2743012776db9c117804cab054a274321 100644 --- a/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/model/BenchmarkExecution.kt +++ b/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/model/BenchmarkExecution.kt @@ -12,7 +12,7 @@ import kotlin.properties.Delegates * A BenchmarkExecution consists of: * - A [name]. * - The [benchmark] that should be executed. - * - The [loads]s that should be checked in the benchmark. + * - The [load]s that should be checked in the benchmark. * - The [resources] that should be checked in the benchmark. * - The [slos] further restrict the Benchmark SLOs for the evaluation of the experiments. * - An [execution] that encapsulates: the strategy, the duration, and the restrictions @@ -28,7 +28,7 @@ class BenchmarkExecution : KubernetesResource { var executionId: Int = 0 lateinit var name: String lateinit var benchmark: String - lateinit var loads: LoadDefinition + lateinit var load: LoadDefinition lateinit var resources: ResourceDefinition lateinit var slos: List<SloConfiguration> lateinit var execution: Execution diff --git a/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/model/crd/ExecutionStatus.kt b/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/model/crd/ExecutionStatus.kt index 6bec7197ddde61185ca37b3e0e96f471a910a0aa..d994da2c0da51580e8d5460f5df423782988d04a 100644 --- a/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/model/crd/ExecutionStatus.kt +++ b/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/model/crd/ExecutionStatus.kt @@ -47,7 +47,7 @@ class ExecutionStatus( private fun JavaDuration.toK8sString(): String { return when { - this <= JavaDuration.ofSeconds(2) -> "${this.toSeconds()}s" + this <= JavaDuration.ofMinutes(2) -> "${this.toSeconds()}s" this < JavaDuration.ofMinutes(99) -> "${this.toMinutes()}m" this < JavaDuration.ofHours(99) -> "${this.toHours()}h" else -> "${this.toDays()}d + ${this.minusDays(this.toDays()).toHours()}h" diff --git a/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/operator/BenchmarkStateChecker.kt b/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/operator/BenchmarkStateChecker.kt index c2c7db6cd6c5d6a02132353228714c3a3b19ec80..c8be7dbfa2632c531c8da46d7bb09e7510a1d9c0 100644 --- a/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/operator/BenchmarkStateChecker.kt +++ b/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/operator/BenchmarkStateChecker.kt @@ -195,9 +195,9 @@ class BenchmarkStateChecker( } -private fun <K, V> Map<K, V>.containsMatchLabels(matchLabels: Map<V, V>): Boolean { +private fun <K, V> Map<K, V>.containsMatchLabels(matchLabels: Map<K, V>): Boolean { for (kv in matchLabels) { - if (kv.value != this[kv.key as K]) { + if (kv.value != this[kv.key]) { return false } } diff --git a/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/operator/ExecutionEventHandler.kt b/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/operator/ExecutionEventHandler.kt index 58120d25d7e26daab015c01fece85cf72344bffa..884606b13783d85ae41c1a8f4c1e5a1980c0e1b0 100644 --- a/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/operator/ExecutionEventHandler.kt +++ b/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/operator/ExecutionEventHandler.kt @@ -34,7 +34,7 @@ class ExecutionEventHandler( override fun onAdd(execution: ExecutionCRD) { logger.info { "Add execution ${execution.metadata.name}." } execution.spec.name = execution.metadata.name - when (this.stateHandler.getExecutionState(execution.metadata.name)) { + when (val currentState = this.stateHandler.getExecutionState(execution.metadata.name)) { ExecutionState.NO_STATE -> this.stateHandler.setExecutionState(execution.spec.name, ExecutionState.PENDING) ExecutionState.RUNNING -> { this.stateHandler.setExecutionState(execution.spec.name, ExecutionState.RESTART) @@ -42,6 +42,9 @@ class ExecutionEventHandler( this.controller.stop(restart = true) } } + else -> { + logger.info { "ExecutionState '$currentState' is not handled." } + } } } diff --git a/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/operator/TheodoliteOperator.kt b/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/operator/TheodoliteOperator.kt index 4d639fdb98ea4a890a734f8db0deca62d83fb508..9e9a0eb5aab376b2eece21864df639e2a27a1715 100644 --- a/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/operator/TheodoliteOperator.kt +++ b/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/operator/TheodoliteOperator.kt @@ -136,7 +136,7 @@ class TheodoliteOperator(private val client: NamespacedKubernetesClient) { ExecutionCRD, BenchmarkExecutionList, Resource<ExecutionCRD>> { - return this.client.customResources( + return client.resources( ExecutionCRD::class.java, BenchmarkExecutionList::class.java ) @@ -146,7 +146,7 @@ class TheodoliteOperator(private val client: NamespacedKubernetesClient) { BenchmarkCRD, KubernetesBenchmarkList, Resource<BenchmarkCRD>> { - return this.client.customResources( + return client.resources( BenchmarkCRD::class.java, KubernetesBenchmarkList::class.java ) diff --git a/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/patcher/AbstractPatcher.kt b/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/patcher/AbstractPatcher.kt index a20a26b351e730de60497ac014b3aba855ac01f5..0ed2a5e5a8c96761074f998721489f0ddd2db1ac 100644 --- a/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/patcher/AbstractPatcher.kt +++ b/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/patcher/AbstractPatcher.kt @@ -27,4 +27,4 @@ abstract class AbstractPatcher : Patcher { abstract fun patchSingleResource(resource: HasMetadata, value: String): HasMetadata -} \ No newline at end of file +} diff --git a/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/patcher/AbstractResourcePatcher.kt b/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/patcher/AbstractResourcePatcher.kt new file mode 100644 index 0000000000000000000000000000000000000000..daa873f17e4dba97b896fd455121a0e1c01e7b81 --- /dev/null +++ b/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/patcher/AbstractResourcePatcher.kt @@ -0,0 +1,63 @@ +package rocks.theodolite.kubernetes.patcher + +import io.fabric8.kubernetes.api.model.Container +import io.fabric8.kubernetes.api.model.HasMetadata +import io.fabric8.kubernetes.api.model.Quantity +import io.fabric8.kubernetes.api.model.apps.Deployment +import io.fabric8.kubernetes.api.model.apps.StatefulSet +import mu.KotlinLogging + +private val logger = KotlinLogging.logger {} + +/** + * Abstract [Patcher] to set resource limits or requests to Deployments and StatefulSets. + * + * @param container Container to be patched. + * @param requiredResource The resource to be requested or limited (e.g., **cpu** or **memory**) + * @param format Format add to the provided value (e.g., `GBi` or `m`, see [Quantity]). + * @param factor A factor to multiply the provided value with. + */ +abstract class AbstractResourcePatcher( + private val container: String, + protected val requiredResource: String, + private val format: String? = null, + private val factor: Int? = null +) : AbstractPatcher() { + + override fun patchSingleResource(resource: HasMetadata, value: String): HasMetadata { + when (resource) { + is Deployment -> { + resource.spec.template.spec.containers.filter { it.name == container }.forEach { + setLimits(it, value) + } + } + is StatefulSet -> { + resource.spec.template.spec.containers.filter { it.name == container }.forEach { + setLimits(it, value) + } + } + else -> { + throw InvalidPatcherConfigurationException("ResourceLimitPatcher is not applicable for $resource.") + } + } + return resource + } + + private fun setLimits(container: Container, value: String) { + val quantity = if (this.format != null || this.factor != null) { + val amountAsInt = value.toIntOrNull()?.times(this.factor ?: 1) + if (amountAsInt == null) { + logger.warn { "Patcher value cannot be parsed as Int. Ignoring quantity format and factor." } + Quantity(value) + } else { + Quantity(amountAsInt.toString(), format ?: "") + } + } else { + Quantity(value) + } + + setLimits(container, quantity) + } + + abstract fun setLimits(container: Container, quantity: Quantity) +} \ No newline at end of file diff --git a/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/patcher/PatcherFactory.kt b/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/patcher/PatcherFactory.kt index a4dcf68d2b4ec12facb26755e9f63e298725e195..b5513b5b796c4ab8119db2a2da17a7cbe7231e57 100644 --- a/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/patcher/PatcherFactory.kt +++ b/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/patcher/PatcherFactory.kt @@ -50,11 +50,15 @@ class PatcherFactory { ) "ResourceLimitPatcher" -> ResourceLimitPatcher( container = patcherDefinition.properties["container"]!!, - limitedResource = patcherDefinition.properties["limitedResource"]!! + limitedResource = patcherDefinition.properties["limitedResource"]!!, + format = patcherDefinition.properties["format"], + factor = patcherDefinition.properties["factor"]?.toInt() ) "ResourceRequestPatcher" -> ResourceRequestPatcher( container = patcherDefinition.properties["container"]!!, - requestedResource = patcherDefinition.properties["requestedResource"]!! + requestedResource = patcherDefinition.properties["requestedResource"]!!, + format = patcherDefinition.properties["format"], + factor = patcherDefinition.properties["factor"]?.toInt() ) "SchedulerNamePatcher" -> SchedulerNamePatcher() "LabelPatcher" -> LabelPatcher( @@ -73,7 +77,7 @@ class PatcherFactory { "ServiceSelectorPatcher" -> ServiceSelectorPatcher( variableName = patcherDefinition.properties["label"]!! ) - "rocks.theodolite.kubernetes.patcher.VolumesConfigMapPatcher" -> VolumesConfigMapPatcher( + "VolumesConfigMapPatcher" -> VolumesConfigMapPatcher( volumeName = patcherDefinition.properties["volumeName"]!! ) else -> throw InvalidPatcherConfigurationException("Patcher type ${patcherDefinition.type} not found.") diff --git a/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/patcher/ResourceLimitPatcher.kt b/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/patcher/ResourceLimitPatcher.kt index c8064c605fbd8c65d97d9fbd8ee24fd49ad893da..c1ae22f00a8fde16aedef6b70ea098cbdd895dd4 100644 --- a/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/patcher/ResourceLimitPatcher.kt +++ b/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/patcher/ResourceLimitPatcher.kt @@ -1,57 +1,43 @@ package rocks.theodolite.kubernetes.patcher -import io.fabric8.kubernetes.api.model.* -import io.fabric8.kubernetes.api.model.apps.Deployment -import io.fabric8.kubernetes.api.model.apps.StatefulSet +import io.fabric8.kubernetes.api.model.Container +import io.fabric8.kubernetes.api.model.Quantity +import io.fabric8.kubernetes.api.model.ResourceRequirements /** * The Resource limit [Patcher] set resource limits for deployments and statefulSets. + * The Resource limit [Patcher] sets resource limits for Deployments and StatefulSets. * - * @param k8sResource Kubernetes resource to be patched. * @param container Container to be patched. - * @param limitedResource The resource to be limited (e.g. **cpu or memory**) + * @param limitedResource The resource to be limited (e.g., **cpu** or **memory**) + * @param format Format add to the provided value (e.g., `GBi` or `m`, see [Quantity]). + * @param factor A factor to multiply the provided value with. */ class ResourceLimitPatcher( - private val container: String, - private val limitedResource: String -) : AbstractPatcher() { - - override fun patchSingleResource(resource: HasMetadata, value: String): HasMetadata { - when (resource) { - is Deployment -> { - resource.spec.template.spec.containers.filter { it.name == container }.forEach { - setLimits(it, value) - } + container: String, + limitedResource: String, + format: String? = null, + factor: Int? = null +) : AbstractResourcePatcher( + container = container, + requiredResource = limitedResource, + format = format, + factor = factor +) { + override fun setLimits(container: Container, quantity: Quantity) { + when { + container.resources == null -> { + val resource = ResourceRequirements() + resource.limits = mapOf(requiredResource to quantity) + container.resources = resource } - is StatefulSet -> { - resource.spec.template.spec.containers.filter { it.name == container }.forEach { - setLimits(it, value) - } + container.resources.limits.isEmpty() -> { + container.resources.limits = mapOf(requiredResource to quantity) } else -> { - throw InvalidPatcherConfigurationException("ResourceLimitPatcher is not applicable for $resource.") + container.resources.limits[requiredResource] = quantity } } - return resource } - - private fun setLimits(container: Container, value: String) { - when { - container.resources == null -> { - val resource = ResourceRequirements() - resource.limits = mapOf(limitedResource to Quantity(value)) - container.resources = resource - } - container.resources.limits.isEmpty() -> { - container.resources.limits = mapOf(limitedResource to Quantity(value)) - } - else -> { - val values = mutableMapOf<String, Quantity>() - container.resources.limits.forEach { entry -> values[entry.key] = entry.value } - values[limitedResource] = Quantity(value) - container.resources.limits = values - } - } - } } diff --git a/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/patcher/ResourceRequestPatcher.kt b/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/patcher/ResourceRequestPatcher.kt index 7d9cd8b748d9e41d5506508259452032b1228015..a6b037ddeb3a1fb3d7aa2c1f2579cada81922d9a 100644 --- a/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/patcher/ResourceRequestPatcher.kt +++ b/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/patcher/ResourceRequestPatcher.kt @@ -1,55 +1,44 @@ package rocks.theodolite.kubernetes.patcher -import io.fabric8.kubernetes.api.model.* -import io.fabric8.kubernetes.api.model.apps.Deployment -import io.fabric8.kubernetes.api.model.apps.StatefulSet +import io.fabric8.kubernetes.api.model.Container +import io.fabric8.kubernetes.api.model.Quantity +import io.fabric8.kubernetes.api.model.ResourceRequirements + /** - * The Resource request [Patcher] set resource limits for deployments and statefulSets. + * The Resource request [Patcher] sets resource requests for Deployments and StatefulSets. * * @param container Container to be patched. - * @param requestedResource The resource to be requested (e.g. **cpu or memory**) + * @param requestedResource The resource to be requested (e.g., **cpu** or **memory**) + * @param format Format add to the provided value (e.g., `GBi` or `m`, see [Quantity]). + * @param factor A factor to multiply the provided value with. */ class ResourceRequestPatcher( - private val container: String, - private val requestedResource: String -) : AbstractPatcher() { - + container: String, + requestedResource: String, + format: String? = null, + factor: Int? = null +) : AbstractResourcePatcher( + container = container, + requiredResource = requestedResource, + format = format, + factor = factor +) { - override fun patchSingleResource(resource: HasMetadata, value: String): HasMetadata { - when (resource) { - is Deployment -> { - resource.spec.template.spec.containers.filter { it.name == container }.forEach { - setRequests(it, value) - } - } - is StatefulSet -> { - resource.spec.template.spec.containers.filter { it.name == container }.forEach { - setRequests(it, value) - } - } - else -> { - throw InvalidPatcherConfigurationException("ResourceRequestPatcher is not applicable for $resource.") - } - } - return resource - } - private fun setRequests(container: Container, value: String) { + override fun setLimits(container: Container, quantity: Quantity) { when { container.resources == null -> { val resource = ResourceRequirements() - resource.requests = mapOf(requestedResource to Quantity(value)) + resource.requests = mapOf(requiredResource to quantity) container.resources = resource } container.resources.requests.isEmpty() -> { - container.resources.requests = mapOf(requestedResource to Quantity(value)) + container.resources.requests = mapOf(requiredResource to quantity) } else -> { - val values = mutableMapOf<String, Quantity>() - container.resources.requests.forEach { entry -> values[entry.key] = entry.value } - values[requestedResource] = Quantity(value) - container.resources.requests = values + container.resources.requests[requiredResource] = quantity } } } + } diff --git a/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/slo/PrometheusResponse.kt b/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/slo/PrometheusResponse.kt index 5222a78bde342fea4a94c69bf1e42e132d0bc706..7a3ac237a88390f85b9d49d3f8eba7447621f0f0 100644 --- a/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/slo/PrometheusResponse.kt +++ b/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/slo/PrometheusResponse.kt @@ -31,8 +31,8 @@ data class PrometheusResponse( for (value in values) { val valueList = value as List<*> val timestamp = (valueList[0] as Double).toLong().toString() - val value = valueList[1].toString() - result.add(listOf(group, timestamp, value)) + val resultValue = valueList[1].toString() + result.add(listOf(group, timestamp, resultValue)) } } return Collections.unmodifiableList(result) diff --git a/theodolite/src/test/kotlin/rocks/theodolite/core/ResultsTest.kt b/theodolite/src/test/kotlin/rocks/theodolite/core/ResultsTest.kt index 2dbeb44b90f780975af884028335a7e398c7cfdc..42bc5883f7ebc50c5dde01555b034433b8fa055e 100644 --- a/theodolite/src/test/kotlin/rocks/theodolite/core/ResultsTest.kt +++ b/theodolite/src/test/kotlin/rocks/theodolite/core/ResultsTest.kt @@ -62,7 +62,6 @@ internal class ResultsTest { assertEquals(20000, maxRequiredInstances) } - @Test fun testGetMaxBenchmarkedLoadWhenAllSuccessfulCapacity() { val results = Results(Metric.from("capacity")) diff --git a/theodolite/src/test/kotlin/rocks/theodolite/kubernetes/model/crd/CRDExecutionTest.kt b/theodolite/src/test/kotlin/rocks/theodolite/kubernetes/model/crd/CRDExecutionTest.kt index a7de76acfe7aac9b92628e87b9911599a13ab438..4b9176693fcb9183796af1c30094088bacbd2c44 100644 --- a/theodolite/src/test/kotlin/rocks/theodolite/kubernetes/model/crd/CRDExecutionTest.kt +++ b/theodolite/src/test/kotlin/rocks/theodolite/kubernetes/model/crd/CRDExecutionTest.kt @@ -69,8 +69,8 @@ internal class CRDExecutionTest { assertEquals("uc1-kstreams", execution.benchmark) assertEquals(mutableListOf<ConfigurationOverride?>(), execution.configOverrides) - assertEquals("NumSensors", execution.loads.loadType) - assertEquals(listOf(25000, 50000, 75000, 100000, 125000, 150000),execution.loads.loadValues) + assertEquals("NumSensors", execution.load.loadType) + assertEquals(listOf(25000, 50000, 75000, 100000, 125000, 150000), execution.load.loadValues) assertEquals("Instances", execution.resources.resourceType) assertEquals(listOf(1, 2, 3, 4, 5), execution.resources.resourceValues) diff --git a/theodolite/src/test/kotlin/rocks/theodolite/kubernetes/model/crd/ExecutionCRDummy.kt b/theodolite/src/test/kotlin/rocks/theodolite/kubernetes/model/crd/ExecutionCRDummy.kt index 871471ee941f5cf2d254fb2bd70556f161d8d4de..8181404a51b68afbf04a8bd83346053d6550e192 100644 --- a/theodolite/src/test/kotlin/rocks/theodolite/kubernetes/model/crd/ExecutionCRDummy.kt +++ b/theodolite/src/test/kotlin/rocks/theodolite/kubernetes/model/crd/ExecutionCRDummy.kt @@ -33,12 +33,12 @@ class ExecutionCRDummy(name: String, benchmark: String) { resourceDef.resourceType = "" resourceDef.resourceValues = emptyList() - val strat = BenchmarkExecution.Strategy() - strat.name = "" - strat.restrictions = emptyList() - strat.guessStrategy = "" - strat.searchStrategy = "" - + val strategy = BenchmarkExecution.Strategy().apply { + this.name = "" + this.restrictions = emptyList() + this.guessStrategy = "" + this.searchStrategy = "" + } val exec = BenchmarkExecution.Execution() exec.afterTeardownDelay = 0 @@ -46,11 +46,11 @@ class ExecutionCRDummy(name: String, benchmark: String) { exec.loadGenerationDelay = 0 exec.repetitions = 1 exec.metric = "" - exec.strategy = strat + exec.strategy = strategy execution.benchmark = benchmark - execution.loads = loadType + execution.load = loadType execution.resources = resourceDef execution.slos = emptyList() execution.execution = exec diff --git a/theodolite/src/test/kotlin/rocks/theodolite/kubernetes/model/crd/ExecutionStatusTest.kt b/theodolite/src/test/kotlin/rocks/theodolite/kubernetes/model/crd/ExecutionStatusTest.kt index 4c9326ac2e99dd7dd9707d4db25cb2e9e360ddf9..69750a2341cbf2590546ad99cd85d660e797f209 100644 --- a/theodolite/src/test/kotlin/rocks/theodolite/kubernetes/model/crd/ExecutionStatusTest.kt +++ b/theodolite/src/test/kotlin/rocks/theodolite/kubernetes/model/crd/ExecutionStatusTest.kt @@ -101,7 +101,7 @@ internal class ExecutionStatusTest { } @Test - fun testDurationSerialization() { + fun testMinutesDurationSerialization() { val objectMapper = ObjectMapper() val executionStatus = ExecutionStatus() val startInstant = Instant.parse("2022-01-02T18:59:20.492103Z") @@ -114,6 +114,20 @@ internal class ExecutionStatusTest { assertEquals("15m", jsonField.asText()) } + @Test + fun testSecondsDurationSerialization() { + val objectMapper = ObjectMapper() + val executionStatus = ExecutionStatus() + val startInstant = Instant.parse("2022-01-02T18:59:20.492103Z") + executionStatus.startTime = MicroTime(startInstant.toString()) + executionStatus.completionTime = MicroTime(startInstant.plus(Duration.ofSeconds(45)).toString()) + val jsonString = objectMapper.writeValueAsString(executionStatus) + val json = objectMapper.readTree(jsonString) + val jsonField = json.get("executionDuration") + assertTrue(jsonField.isTextual) + assertEquals("45s", jsonField.asText()) + } + @Test fun testNotStartedDurationSerialization() { val objectMapper = ObjectMapper() diff --git a/theodolite/src/test/kotlin/rocks/theodolite/kubernetes/operator/ControllerTest.kt b/theodolite/src/test/kotlin/rocks/theodolite/kubernetes/operator/ControllerTest.kt index bdb7cc6ce4c0a6044878496eed3f99efe105c17d..301580bf3f84faaef8df22555d22a33dff0d2acd 100644 --- a/theodolite/src/test/kotlin/rocks/theodolite/kubernetes/operator/ControllerTest.kt +++ b/theodolite/src/test/kotlin/rocks/theodolite/kubernetes/operator/ControllerTest.kt @@ -113,6 +113,7 @@ class ControllerTest { .getDeclaredMethod("getBenchmarks") method.isAccessible = true + @Suppress("UNCHECKED_CAST") val result = method.invoke(controller) as List<BenchmarkCRD> assertEquals(2, result.size) diff --git a/theodolite/src/test/kotlin/rocks/theodolite/kubernetes/patcher/ResourceLimitPatcherTest.kt b/theodolite/src/test/kotlin/rocks/theodolite/kubernetes/patcher/ResourceLimitPatcherTest.kt index b0af74d1e207ee10fac548f27267356711943dd0..86de40e207950657a1e6ebf1114dff51f1c2222e 100644 --- a/theodolite/src/test/kotlin/rocks/theodolite/kubernetes/patcher/ResourceLimitPatcherTest.kt +++ b/theodolite/src/test/kotlin/rocks/theodolite/kubernetes/patcher/ResourceLimitPatcherTest.kt @@ -1,27 +1,15 @@ package rocks.theodolite.kubernetes.patcher -import io.fabric8.kubernetes.api.model.HasMetadata +import io.fabric8.kubernetes.api.model.apps.Deployment import io.fabric8.kubernetes.client.server.mock.KubernetesServer import io.quarkus.test.junit.QuarkusTest import io.quarkus.test.kubernetes.client.KubernetesTestServer import io.quarkus.test.kubernetes.client.WithKubernetesTestServer -import org.junit.jupiter.api.Assertions.assertTrue -import org.junit.jupiter.api.Disabled +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test - -/** - * Resource patcher test - * - * This class tested 4 scenarios for the ResourceLimitPatcher and the ResourceRequestPatcher. - * The different test cases specifies four possible situations: - * Case 1: In the given YAML declaration memory and cpu are defined - * Case 2: In the given YAML declaration only cpu is defined - * Case 3: In the given YAML declaration only memory is defined - * Case 4: In the given YAML declaration neither `Resource Request` nor `Request Limit` is defined - */ @QuarkusTest @WithKubernetesTestServer -@Disabled class ResourceLimitPatcherTest { @KubernetesTestServer @@ -30,14 +18,14 @@ class ResourceLimitPatcherTest { fun applyTest(fileName: String) { val cpuValue = "50m" val memValue = "3Gi" - val k8sResource = server.client.apps().deployments().load(javaClass.getResourceAsStream(fileName)).get() + val k8sResource = getDeployment(fileName) val defCPU = PatcherDefinition() defCPU.resource = "/cpu-memory-deployment.yaml" defCPU.type = "ResourceLimitPatcher" defCPU.properties = mapOf( "limitedResource" to "cpu", - "container" to "application" + "container" to "uc-application" ) val defMEM = PatcherDefinition() @@ -48,13 +36,13 @@ class ResourceLimitPatcherTest { "container" to "uc-application" ) - PatchHandler.patchResource(mutableMapOf(Pair("cpu-memory-deployment.yaml", listOf(k8sResource as HasMetadata))), defCPU, cpuValue) - PatchHandler.patchResource(mutableMapOf(Pair("cpu-memory-deployment.yaml", listOf(k8sResource as HasMetadata))), defMEM, memValue) - - k8sResource.spec.template.spec.containers.filter { it.name == defCPU.properties["container"]!! } + val firstPatched = PatchHandler.patchResource(mutableMapOf(Pair("/cpu-memory-deployment.yaml", listOf(k8sResource))), defCPU, cpuValue) + val finalPatched = PatchHandler.patchResource(mutableMapOf(Pair("/cpu-memory-deployment.yaml", firstPatched)), defMEM, memValue) + assertEquals(1, finalPatched.size) + (finalPatched[0] as Deployment).spec.template.spec.containers.filter { it.name == defCPU.properties["container"]!! } .forEach { - assertTrue(it.resources.limits["cpu"].toString() == cpuValue) - assertTrue(it.resources.limits["memory"].toString() == memValue) + assertEquals(cpuValue, it.resources.limits["cpu"].toString()) + assertEquals(memValue, it.resources.limits["memory"].toString()) } } @@ -81,4 +69,80 @@ class ResourceLimitPatcherTest { // Case 4: In the given YAML declaration neither `Resource Request` nor `Request Limit` is defined applyTest("/no-resources-deployment.yaml") } + + @Test + fun testWithNoFactorSet() { + val initialDeployment = getDeployment("/cpu-memory-deployment.yaml") + val patchedDeployments = ResourceLimitPatcher( + "uc-application", + "memory" + ).patch(listOf(initialDeployment), "1Gi") + assertEquals(1, patchedDeployments.size) + val patchedDeployment = patchedDeployments[0] as Deployment + + val containers = patchedDeployment.spec.template.spec.containers.filter { it.name == "uc-application" } + assertEquals(1, containers.size) + containers.forEach { + assertEquals("1Gi", it.resources.limits["memory"].toString()) + } + } + + @Test + fun testWithFormatSet() { + val initialDeployment = getDeployment("/cpu-memory-deployment.yaml") + val patchedDeployments = ResourceLimitPatcher( + "uc-application", + "memory", + format = "GBi" + ).patch(listOf(initialDeployment), "2") + assertEquals(1, patchedDeployments.size) + val patchedDeployment = patchedDeployments[0] as Deployment + + val containers = patchedDeployment.spec.template.spec.containers.filter { it.name == "uc-application" } + assertEquals(1, containers.size) + containers.forEach { + assertEquals("2GBi", it.resources.limits["memory"].toString()) + } + } + + @Test + fun testWithFactorSet() { + val initialDeployment = getDeployment("/cpu-memory-deployment.yaml") + val patchedDeployments = ResourceLimitPatcher( + "uc-application", + "memory", + factor = 4000 + ).patch(listOf(initialDeployment), "2") + assertEquals(1, patchedDeployments.size) + val patchedDeployment = patchedDeployments[0] as Deployment + + val containers = patchedDeployment.spec.template.spec.containers.filter { it.name == "uc-application" } + assertEquals(1, containers.size) + containers.forEach { + assertEquals("8000", it.resources.limits["memory"].toString()) + } + } + + @Test + fun testWithFactorAndFormatSet() { + val initialDeployment = getDeployment("/cpu-memory-deployment.yaml") + val patchedDeployments = ResourceLimitPatcher( + "uc-application", + "memory", + format = "GBi", + factor = 4, + ).patch(listOf(initialDeployment), "2") + assertEquals(1, patchedDeployments.size) + val patchedDeployment = patchedDeployments[0] as Deployment + + val containers = patchedDeployment.spec.template.spec.containers.filter { it.name == "uc-application" } + assertEquals(1, containers.size) + containers.forEach { + assertEquals("8GBi", it.resources.limits["memory"].toString()) + } + } + + private fun getDeployment(fileName: String): Deployment { + return server.client.apps().deployments().load(javaClass.getResourceAsStream(fileName)).get() + } } diff --git a/theodolite/src/test/kotlin/rocks/theodolite/kubernetes/patcher/ResourceRequestPatcherTest.kt b/theodolite/src/test/kotlin/rocks/theodolite/kubernetes/patcher/ResourceRequestPatcherTest.kt index a076e541e742e97ffa95dccff925892dd63ff17a..415d92f59f8ca0fd5a331ae0fa5bc71b5ef4f506 100644 --- a/theodolite/src/test/kotlin/rocks/theodolite/kubernetes/patcher/ResourceRequestPatcherTest.kt +++ b/theodolite/src/test/kotlin/rocks/theodolite/kubernetes/patcher/ResourceRequestPatcherTest.kt @@ -1,22 +1,12 @@ package rocks.theodolite.kubernetes.patcher +import io.fabric8.kubernetes.api.model.apps.Deployment import io.fabric8.kubernetes.client.server.mock.KubernetesServer import io.quarkus.test.junit.QuarkusTest import io.quarkus.test.kubernetes.client.KubernetesTestServer import io.quarkus.test.kubernetes.client.WithKubernetesTestServer -import io.smallrye.common.constraint.Assert.assertTrue +import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test - -/** - * Resource patcher test - * - * This class tested 4 scenarios for the ResourceLimitPatcher and the ResourceRequestPatcher. - * The different test cases specifies four possible situations: - * Case 1: In the given YAML declaration memory and cpu are defined - * Case 2: In the given YAML declaration only cpu is defined - * Case 3: In the given YAML declaration only memory is defined - * Case 4: In the given YAML declaration neither `Resource Request` nor `Request Limit` is defined - */ @QuarkusTest @WithKubernetesTestServer class ResourceRequestPatcherTest { @@ -27,14 +17,14 @@ class ResourceRequestPatcherTest { fun applyTest(fileName: String) { val cpuValue = "50m" val memValue = "3Gi" - val k8sResource = server.client.apps().deployments().load(javaClass.getResourceAsStream(fileName)).get() + val k8sResource = getDeployment(fileName) val defCPU = PatcherDefinition() defCPU.resource = "/cpu-memory-deployment.yaml" defCPU.type = "ResourceRequestPatcher" defCPU.properties = mapOf( "requestedResource" to "cpu", - "container" to "application" + "container" to "uc-application" ) val defMEM = PatcherDefinition() @@ -42,16 +32,16 @@ class ResourceRequestPatcherTest { defMEM.type = "ResourceRequestPatcher" defMEM.properties = mapOf( "requestedResource" to "memory", - "container" to "application" + "container" to "uc-application" ) - PatchHandler.patchResource(mutableMapOf(Pair("/cpu-memory-deployment.yaml", listOf(k8sResource))), defCPU, cpuValue) - PatchHandler.patchResource(mutableMapOf(Pair("/cpu-memory-deployment.yaml", listOf(k8sResource))), defMEM, memValue) - - k8sResource.spec.template.spec.containers.filter { it.name == defCPU.properties["container"]!! } + val firstPatched = PatchHandler.patchResource(mutableMapOf(Pair("/cpu-memory-deployment.yaml", listOf(k8sResource))), defCPU, cpuValue) + val finalPatched = PatchHandler.patchResource(mutableMapOf(Pair("/cpu-memory-deployment.yaml", firstPatched)), defMEM, memValue) + assertEquals(1, finalPatched.size) + (finalPatched[0] as Deployment).spec.template.spec.containers.filter { it.name == defCPU.properties["container"]!! } .forEach { - assertTrue(it.resources.requests["cpu"].toString() == cpuValue) - assertTrue(it.resources.requests["memory"].toString() == memValue) + assertEquals(cpuValue, it.resources.requests["cpu"].toString()) + assertEquals(memValue, it.resources.requests["memory"].toString()) } } @@ -78,4 +68,81 @@ class ResourceRequestPatcherTest { // Case 4: In the given YAML declaration neither `Resource Request` nor `Request Limit` is defined applyTest("/no-resources-deployment.yaml") } + + @Test + fun testWithNoFactorSet() { + val initialDeployment = getDeployment("/cpu-memory-deployment.yaml") + val patchedDeployments = ResourceRequestPatcher( + "uc-application", + "memory" + ).patch(listOf(initialDeployment), "1Gi") + assertEquals(1, patchedDeployments.size) + val patchedDeployment = patchedDeployments[0] as Deployment + + val containers = patchedDeployment.spec.template.spec.containers.filter { it.name == "uc-application" } + assertEquals(1, containers.size) + containers.forEach { + assertEquals("1Gi", it.resources.requests["memory"].toString()) + } + } + + @Test + fun testWithFormatSet() { + val initialDeployment = getDeployment("/cpu-memory-deployment.yaml") + val patchedDeployments = ResourceRequestPatcher( + "uc-application", + "memory", + format = "GBi" + ).patch(listOf(initialDeployment), "2") + assertEquals(1, patchedDeployments.size) + val patchedDeployment = patchedDeployments[0] as Deployment + + val containers = patchedDeployment.spec.template.spec.containers.filter { it.name == "uc-application" } + assertEquals(1, containers.size) + containers.forEach { + assertEquals("2GBi", it.resources.requests["memory"].toString()) + } + } + + @Test + fun testWithFactorSet() { + val initialDeployment = getDeployment("/cpu-memory-deployment.yaml") + val patchedDeployments = ResourceRequestPatcher( + "uc-application", + "memory", + factor = 4000 + ).patch(listOf(initialDeployment), "2") + assertEquals(1, patchedDeployments.size) + val patchedDeployment = patchedDeployments[0] as Deployment + + val containers = patchedDeployment.spec.template.spec.containers.filter { it.name == "uc-application" } + assertEquals(1, containers.size) + containers.forEach { + assertEquals("8000", it.resources.requests["memory"].toString()) + } + } + + @Test + fun testWithFactorAndFormatSet() { + val initialDeployment = getDeployment("/cpu-memory-deployment.yaml") + val patchedDeployments = ResourceRequestPatcher( + "uc-application", + "memory", + format = "GBi", + factor = 4, + ).patch(listOf(initialDeployment), "2") + assertEquals(1, patchedDeployments.size) + val patchedDeployment = patchedDeployments[0] as Deployment + + val containers = patchedDeployment.spec.template.spec.containers.filter { it.name == "uc-application" } + assertEquals(1, containers.size) + containers.forEach { + assertEquals("8GBi", it.resources.requests["memory"].toString()) + } + } + + private fun getDeployment(fileName: String): Deployment { + return server.client.apps().deployments().load(javaClass.getResourceAsStream(fileName)).get() + } + } \ No newline at end of file diff --git a/theodolite/src/test/resources/k8s-resource-files/test-execution-1.yaml b/theodolite/src/test/resources/k8s-resource-files/test-execution-1.yaml index b758e9b9b1e39a9722ad9c0d6513f36b9a4961da..49fdd14fddf3ddc89100d5165e737d4b0aae0257 100644 --- a/theodolite/src/test/resources/k8s-resource-files/test-execution-1.yaml +++ b/theodolite/src/test/resources/k8s-resource-files/test-execution-1.yaml @@ -5,7 +5,7 @@ metadata: spec: name: test benchmark: "uc1-kstreams" - loads: + load: loadType: "NumSensors" loadValues: [25000, 50000, 75000, 100000, 125000, 150000] resources: @@ -13,11 +13,8 @@ spec: resourceValues: [1, 2, 3, 4, 5] slos: - name: "lag trend" - threshold: 2000 - prometheusUrl: "http://prometheus-operated:9090" - externalSloUrl: "http://localhost:80/evaluate-slope" - offset: 0 - warmup: 60 # in seconds + properties: + threshold: 2000 execution: strategy: name: "RestrictionSearch" diff --git a/theodolite/src/test/resources/k8s-resource-files/test-execution-update.yaml b/theodolite/src/test/resources/k8s-resource-files/test-execution-update.yaml index ccfd0160fc407863544e5b735e595b86a2669eea..2f58db5b45d139131204937d68fee4e141ad1883 100644 --- a/theodolite/src/test/resources/k8s-resource-files/test-execution-update.yaml +++ b/theodolite/src/test/resources/k8s-resource-files/test-execution-update.yaml @@ -5,7 +5,7 @@ metadata: spec: name: test benchmark: "uc1-kstreams-update" - loads: + load: loadType: "NumSensors" loadValues: [25000, 50000, 75000, 100000, 125000, 150000] resources: diff --git a/theodolite/src/test/resources/k8s-resource-files/test-execution.yaml b/theodolite/src/test/resources/k8s-resource-files/test-execution.yaml index 78c01c9e8e241c92a897dc00d024d92ecd9e82d6..8d274275ac6617ff6b2e3b0c448487e7a845e42b 100644 --- a/theodolite/src/test/resources/k8s-resource-files/test-execution.yaml +++ b/theodolite/src/test/resources/k8s-resource-files/test-execution.yaml @@ -5,7 +5,7 @@ metadata: spec: name: test benchmark: "uc1-kstreams" - loads: + load: loadType: "NumSensors" loadValues: [25000, 50000, 75000, 100000, 125000, 150000] resources: