diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4a412569fbd477f02a0b67b83e8814ab98b34031..01630912c52ccc728870d72c33a8f02cdf7c2c7f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -242,6 +242,7 @@ build-theodolite-native: script: - gu install native-image # TODO move to image - ./gradlew --build-cache assemble -Dquarkus.package.type=native + when: manual artifacts: paths: - "theodolite-quarkus/build/*-runner" @@ -252,7 +253,7 @@ test-theodolite: extends: .theodolite needs: - build-theodolite-jvm - - build-theodolite-native + #- build-theodolite-native script: ./gradlew test --stacktrace # Disabled for now @@ -279,12 +280,13 @@ deploy-theodolite: - .theodolite - .dind needs: - - build-theodolite-native + #- build-theodolite-native + - build-theodolite-jvm - test-theodolite script: - DOCKER_TAG_NAME=$(echo $CI_COMMIT_REF_SLUG- | sed 's/^master-$//') - - docker build -f src/main/docker/Dockerfile.native -t theodolite . - #- docker build -f src/main/docker/Dockerfile.jvm -t theodolite . + #- docker build -f src/main/docker/Dockerfile.native -t theodolite . + - docker build -f src/main/docker/Dockerfile.jvm -t theodolite . - "[ ! $CI_COMMIT_TAG ] && docker tag theodolite $CR_HOST/$CR_ORG/theodolite:${DOCKER_TAG_NAME}latest" - "[ ! $CI_COMMIT_TAG ] && docker tag theodolite $CR_HOST/$CR_ORG/theodolite:$DOCKER_TAG_NAME$CI_COMMIT_SHORT_SHA" - "[ $CI_COMMIT_TAG ] && docker tag theodolite $CR_HOST/$CR_ORG/theodolite:$CI_COMMIT_TAG" @@ -309,6 +311,7 @@ deploy-slo-checker-lag-trend: stage: deploy extends: - .dind + needs: [] script: - DOCKER_TAG_NAME=$(echo $CI_COMMIT_REF_SLUG- | sed 's/^master-$//') - docker build --pull -t theodolite-slo-checker-lag-trend slope-evaluator @@ -335,6 +338,7 @@ deploy-random-scheduler: stage: deploy extends: - .dind + needs: [] script: - DOCKER_TAG_NAME=$(echo $CI_COMMIT_REF_SLUG- | sed 's/^master-$//') - docker build --pull -t theodolite-random-scheduler execution/infrastructure/random-scheduler diff --git a/slope-evaluator/README.md b/slope-evaluator/README.md index 69831cd5f83665735c586ab25493ed257b93c2ad..5929fb157a7c783bd37497885a5e3bc373b84aa0 100644 --- a/slope-evaluator/README.md +++ b/slope-evaluator/README.md @@ -17,13 +17,13 @@ docker build . -t theodolite-evaluator Run the Docker image: ```sh - docker run -p 80:80 theodolite-evaluator +docker run -p 80:80 theodolite-evaluator ``` ## Configuration You can set the `HOST` and the `PORT` (and a lot of more parameters) via environment variables. Default is `0.0.0.0:80`. -For more information see [here](https://github.com/tiangolo/uvicorn-gunicorn-fastapi-docker#advanced-usage). +For more information see the [Gunicorn/FastAPI Docker docs](https://github.com/tiangolo/uvicorn-gunicorn-fastapi-docker#advanced-usage). ## API Documentation diff --git a/slope-evaluator/app/main.py b/slope-evaluator/app/main.py index 83709c0f71563d9bd1c29c5f064645144163ea72..5995a1175f781c4eb51ab3d3083a665fbc02d6fd 100644 --- a/slope-evaluator/app/main.py +++ b/slope-evaluator/app/main.py @@ -18,7 +18,7 @@ if os.getenv('LOG_LEVEL') == 'INFO': elif os.getenv('LOG_LEVEL') == 'WARNING': logger.setLevel(logging.WARNING) elif os.getenv('LOG_LEVEL') == 'DEBUG': - logger.setLevel((logging.DEBUG)) + logger.setLevel(logging.DEBUG) def execute(results, threshold, warmup): d = [] @@ -30,18 +30,18 @@ def execute(results, threshold, warmup): df = pd.DataFrame(d) - logger.info(df) + logger.info("Calculating trend slope with warmup of %s seconds for data frame:\n %s", warmup, df) try: trend_slope = trend_slope_computer.compute(df, warmup) except Exception as e: - err_msg = 'Computing trend slope failed' + err_msg = 'Computing trend slope failed.' logger.exception(err_msg) - logger.error('Mark this subexperiment as not successful and continue benchmark') + logger.error('Mark this subexperiment as not successful and continue benchmark.') return False - logger.info("Trend Slope: %s", trend_slope) - - return trend_slope < threshold + result = trend_slope < threshold + logger.info("Computed lag trend slope is '%s'. Result is: %s", trend_slope, result) + return result @app.post("/evaluate-slope",response_model=bool) async def evaluate_slope(request: Request): diff --git a/slope-evaluator/app/trend_slope_computer.py b/slope-evaluator/app/trend_slope_computer.py index c128d9f48c1e7ba20e43dfbfd6a0391eeec2b60b..51b28f2baa5110a6d64f3adc1ac9a94c6b6f3ce9 100644 --- a/slope-evaluator/app/trend_slope_computer.py +++ b/slope-evaluator/app/trend_slope_computer.py @@ -2,13 +2,12 @@ from sklearn.linear_model import LinearRegression import pandas as pd import os -def compute(x, warmup_sec): - input = x - input['sec_start'] = input.loc[0:, 'timestamp'] - input.iloc[0]['timestamp'] - regress = input.loc[input['sec_start'] >= warmup_sec] # Warm-Up +def compute(data, warmup_sec): + data['sec_start'] = data.loc[0:, 'timestamp'] - data.iloc[0]['timestamp'] + regress = data.loc[data['sec_start'] >= warmup_sec] # Warm-Up - X = regress.iloc[:, 2].values.reshape(-1, 1) # values converts it into a numpy array - Y = regress.iloc[:, 3].values.reshape(-1, 1) # -1 means that calculate the dimension of rows, but have 1 column + X = regress.iloc[:, 1].values.reshape(-1, 1) # values converts it into a numpy array + Y = regress.iloc[:, 2].values.reshape(-1, 1) # -1 means that calculate the dimension of rows, but have 1 column linear_regressor = LinearRegression() # create object for the class linear_regressor.fit(X, Y) # perform linear regression Y_pred = linear_regressor.predict(X) # make predictions diff --git a/theodolite-quarkus/build.gradle b/theodolite-quarkus/build.gradle index 2f58f4c8a587a4a17cf8586e507c0079d8cb56e6..8b61bf506dd241c5b0d4e75b1357ef3fd5966135 100644 --- a/theodolite-quarkus/build.gradle +++ b/theodolite-quarkus/build.gradle @@ -18,16 +18,16 @@ dependencies { implementation 'org.jetbrains.kotlin:kotlin-stdlib-jdk8' implementation 'io.quarkus:quarkus-arc' implementation 'io.quarkus:quarkus-resteasy' - testImplementation 'io.quarkus:quarkus-junit5' - testImplementation 'io.rest-assured:rest-assured' implementation 'com.google.code.gson:gson:2.8.5' - implementation 'org.slf4j:slf4j-simple:1.7.29' implementation 'io.github.microutils:kotlin-logging:1.12.0' implementation 'io.fabric8:kubernetes-client:5.0.0-alpha-2' implementation 'io.quarkus:quarkus-kubernetes-client' implementation 'org.apache.kafka:kafka-clients:2.7.0' implementation 'khttp:khttp:1.0.0' + + testImplementation 'io.quarkus:quarkus-junit5' + testImplementation 'io.rest-assured:rest-assured' } group 'theodolite' diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt index e76f6cf34fdec447fa4d581b94bbb51b972d888a..f15ec808ae8fdcd6ba31f500f819a08718db774b 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt @@ -4,8 +4,11 @@ import mu.KotlinLogging import theodolite.benchmark.BenchmarkExecution import theodolite.util.LoadDimension import theodolite.util.Resource +import java.text.Normalizer import java.time.Duration import java.time.Instant +import java.util.* +import java.util.regex.Pattern private val logger = KotlinLogging.logger {} @@ -41,7 +44,10 @@ class AnalysisExecutor( query = "sum by(group)(kafka_consumergroup_group_lag >= 0)" ) - CsvExporter().toCsv(name = "$executionId-${load.get()}-${res.get()}-${slo.sloType}", prom = prometheusData) + CsvExporter().toCsv( + name = "exp${executionId}_${load.get()}_${res.get()}_${slo.sloType.toSlug()}", + prom = prometheusData + ) val sloChecker = SloCheckerFactory().create( sloType = slo.sloType, externalSlopeURL = slo.externalSloUrl, @@ -55,8 +61,18 @@ class AnalysisExecutor( ) } catch (e: Exception) { - logger.error { "Evaluation failed for resource: ${res.get()} and load: ${load.get()} error: $e" } + logger.error { "Evaluation failed for resource '${res.get()}' and load '${load.get()}'. Error: $e" } } return result } + + private val NONLATIN: Pattern = Pattern.compile("[^\\w-]") + private val WHITESPACE: Pattern = Pattern.compile("[\\s]") + + fun String.toSlug(): String { + val noWhitespace: String = WHITESPACE.matcher(this).replaceAll("-") + val normalized: String = Normalizer.normalize(noWhitespace, Normalizer.Form.NFD) + val slug: String = NONLATIN.matcher(normalized).replaceAll("") + return slug.toLowerCase(Locale.ENGLISH) + } } diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/CsvExporter.kt b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/CsvExporter.kt index 68862851523934c533cf3af41f0a786ba2b5a73f..4ef78cd58085c4ff5fed1477fa08ae2e6342aa66 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/CsvExporter.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/CsvExporter.kt @@ -24,12 +24,12 @@ class CsvExporter { val csvOutputFile = File("$name.csv") PrintWriter(csvOutputFile).use { pw -> - pw.println(listOf("name", "time", "value").joinToString()) + pw.println(listOf("group", "timestamp", "value").joinToString()) responseArray.forEach { pw.println(it.joinToString()) } } - logger.info { "Wrote csv file: $name to ${csvOutputFile.absolutePath}" } + logger.info { "Wrote CSV file: $name to ${csvOutputFile.absolutePath}." } } /** @@ -41,9 +41,11 @@ class CsvExporter { val dataList = mutableListOf<List<String>>() if (values != null) { - for (x in values) { - val y = x as List<*> - dataList.add(listOf(name, "${y[0]}", "${y[1]}")) + for (maybeValuePair in values) { + val valuePair = maybeValuePair as List<*> + val timestamp = (valuePair[0] as Double).toLong().toString() + val value = valuePair[1].toString() + dataList.add(listOf(name, timestamp, value)) } } return Collections.unmodifiableList(dataList) diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt index fd901abc470a1f4b739b26e30276366e6bc69739..8eac05a815e0bc45e1abc38bcdf0352e61bd7730 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt @@ -17,8 +17,7 @@ class ExternalSloChecker( private val externalSlopeURL: String, private val threshold: Int, private val warmup: Int -) : - SloChecker { +) : SloChecker { private val RETRIES = 2 private val TIMEOUT = 60.0 @@ -38,19 +37,23 @@ class ExternalSloChecker( */ override fun evaluate(start: Instant, end: Instant, fetchedData: PrometheusResponse): Boolean { var counter = 0 - val data = - Gson().toJson(mapOf("total_lag" to fetchedData.data?.result, "threshold" to threshold, "warmup" to warmup)) + val data = Gson().toJson(mapOf( + "total_lag" to fetchedData.data?.result, + "threshold" to threshold, + "warmup" to warmup)) while (counter < RETRIES) { val result = post(externalSlopeURL, data = data, timeout = TIMEOUT) if (result.statusCode != 200) { counter++ - logger.error { "Could not reach external slope analysis" } + logger.error { "Could not reach external SLO checker" } } else { - return result.text.toBoolean() + val booleanResult = result.text.toBoolean() + logger.info { "SLO checker result is: $booleanResult" } + return booleanResult } } - throw ConnectException("Could not reach slope evaluation") + throw ConnectException("Could not reach external SLO checker") } } diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/MetricFetcher.kt b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/MetricFetcher.kt index bbfbf8c3269e442188f92a9b057fcc264acbbe78..833d7d1e16c2fbc91b58817b319a7d02af7f5b2b 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/MetricFetcher.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/MetricFetcher.kt @@ -48,18 +48,18 @@ class MetricFetcher(private val prometheusURL: String, private val offset: Durat val response = get("$prometheusURL/api/v1/query_range", params = parameter, timeout = TIMEOUT) if (response.statusCode != 200) { val message = response.jsonObject.toString() - logger.warn { "Could not connect to Prometheus: $message, retrying now" } + logger.warn { "Could not connect to Prometheus: $message. Retrying now." } counter++ } else { val values = parseValues(response) if (values.data?.result.isNullOrEmpty()) { - logger.error { "Empty query result: $values between $start and $end for querry $query" } + logger.error { "Empty query result: $values between $start and $end for query $query." } throw NoSuchFieldException() } return parseValues(response) } } - throw ConnectException("No answer from Prometheus received") + throw ConnectException("No answer from Prometheus received.") } /** diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt index e179f7fa9492fc4fbe069330046dfd5d83ff8374..6d4cd9ea9b5d03dda360b2ddcefcfb9682fd8383 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt @@ -47,7 +47,7 @@ abstract class BenchmarkExecutor( * */ fun waitAndLog() { - logger.info { "Execution of a new benchmark started." } + logger.info { "Execution of a new experiment started." } var secondsRunning = 0L diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt index 1ab8e215216cd6f2d3640ca113d438e4ed821042..cd85c143e3c416f115a4d301629caf4d46b7459f 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt @@ -30,7 +30,7 @@ class BenchmarkExecutorImpl( benchmarkDeployment.setup() this.waitAndLog() } catch (e: Exception) { - logger.error { "Error while setup experiment." } + logger.error { "Error while setting up experiment with id ${this.executionId}." } logger.error { "Error is: $e" } this.run.set(false) } @@ -39,20 +39,19 @@ class BenchmarkExecutorImpl( * Analyse the experiment, if [run] is true, otherwise the experiment was canceled by the user. */ if (this.run.get()) { - result = - AnalysisExecutor(slo = slo, executionId = executionId).analyze( - load = load, - res = res, - executionDuration = executionDuration - ) + result = AnalysisExecutor(slo = slo, executionId = executionId).analyze( + load = load, + res = res, + executionDuration = executionDuration + ) this.results.setResult(Pair(load, res), result) } try { benchmarkDeployment.teardown() } catch (e: Exception) { - logger.warn { "Error while teardown of deplyoment" } - logger.debug { "The Teardowm failed cause of: $e " } + logger.warn { "Error while tearing down the benchmark deployment." } + logger.debug { "Teardown failed, caused by: $e" } } return result diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/Main.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/Main.kt index 4518ef7957104819b26eae95cf4e6e9b35c4e995..64a40c0b11854d61900ab1fde3797e17427cac15 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/Main.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/Main.kt @@ -13,13 +13,17 @@ object Main { @JvmStatic fun main(args: Array<String>) { - val mode = System.getenv("MODE") ?: "yaml-executor" + val mode = System.getenv("MODE") ?: "standalone" logger.info { "Start Theodolite with mode $mode" } when(mode) { - "yaml-executor" -> TheodoliteYamlExecutor().start() + "standalone" -> TheodoliteYamlExecutor().start() + "yaml-executor" -> TheodoliteYamlExecutor().start() // TODO remove (#209) "operator" -> TheodoliteOperator().start() - else -> {logger.error { "MODE $mode not found" }; exitProcess(1)} + else -> { + logger.error { "MODE $mode not found" } + exitProcess(1) + } } } } \ No newline at end of file diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/k8s/K8sManager.kt b/theodolite-quarkus/src/main/kotlin/theodolite/k8s/K8sManager.kt index f8f7f9800ecb2b19f56d3dfe85c8f9cfc153b9f5..c58225bf855c70a5a7057132617418a89b0816a8 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/k8s/K8sManager.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/k8s/K8sManager.kt @@ -6,6 +6,9 @@ import io.fabric8.kubernetes.api.model.Service import io.fabric8.kubernetes.api.model.apps.Deployment import io.fabric8.kubernetes.api.model.apps.StatefulSet import io.fabric8.kubernetes.client.NamespacedKubernetesClient +import mu.KotlinLogging + +private val logger = KotlinLogging.logger {} /** * This class is used to deploy or remove different Kubernetes resources. @@ -42,7 +45,8 @@ class K8sManager(private val client: NamespacedKubernetesClient) { is Deployment -> { val label = resource.spec.selector.matchLabels["app"]!! this.client.apps().deployments().delete(resource) - blockUntilDeleted(label) + blockUntilPodsDeleted(label) + logger.info { "Deployment '${resource.metadata.name}' deleted." } } is Service -> this.client.services().delete(resource) @@ -51,23 +55,19 @@ class K8sManager(private val client: NamespacedKubernetesClient) { is StatefulSet -> { val label = resource.spec.selector.matchLabels["app"]!! this.client.apps().statefulSets().delete(resource) - blockUntilDeleted(label) + blockUntilPodsDeleted(label) + logger.info { "StatefulSet '$resource.metadata.name' deleted." } } is ServiceMonitorWrapper -> resource.delete(client) else -> throw IllegalArgumentException("Unknown Kubernetes resource.") } } - - private fun blockUntilDeleted(label: String) { - var deleted = false - do { - val pods = this.client.pods().withLabel(label).list().items - if (pods.isNullOrEmpty()) { - deleted = true - } + private fun blockUntilPodsDeleted(podLabel: String) { + while (!this.client.pods().withLabel(podLabel).list().items.isNullOrEmpty()) { + logger.info { "Wait for pods with label '$podLabel' to be deleted." } Thread.sleep(1000) - } while (!deleted) + } } } diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/k8s/TopicManager.kt b/theodolite-quarkus/src/main/kotlin/theodolite/k8s/TopicManager.kt index e82a133b3e5439e72987f3db107f4e81a1d01cd5..90c2cb8a9c810aa524b6608a558d31e15587719e 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/k8s/TopicManager.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/k8s/TopicManager.kt @@ -11,13 +11,14 @@ private const val RETRY_TIME = 2000L /** * Manages the topics related tasks - * @param kafkaConfig Kafka Configuration as HashMap + * @param kafkaConfig Kafka configuration as a Map * @constructor Creates a KafkaAdminClient */ -class TopicManager(private val kafkaConfig: HashMap<String, Any>) { +class TopicManager(private val kafkaConfig: Map<String, Any>) { + /** - * Creates topics. - * @param newTopics List of all Topic that should be created + * Create topics. + * @param newTopics Collection of all topic that should be created */ fun createTopics(newTopics: Collection<NewTopic>) { val kafkaAdmin: AdminClient = AdminClient.create(this.kafkaConfig) @@ -27,13 +28,14 @@ class TopicManager(private val kafkaConfig: HashMap<String, Any>) { var retryCreation = false try { result = kafkaAdmin.createTopics(newTopics) - result.all().get()// wait for the future object + result.all().get() // wait for the future to be completed } catch (e: Exception) { + logger.warn(e) { "Error during topic creation." } + logger.debug { e } // TODO remove? + logger.info { "Remove existing topics." } delete(newTopics.map { topic -> topic.name() }, kafkaAdmin) - logger.warn { "Error during topic creation." } - logger.debug { e } - logger.warn { "Will retry the topic creation after 2 seconds" } + logger.info { "Will retry the topic creation in $RETRY_TIME seconds." } sleep(RETRY_TIME) retryCreation = true } @@ -49,8 +51,8 @@ class TopicManager(private val kafkaConfig: HashMap<String, Any>) { } /** - * Removes topics. - * @param topics List of names with the topics to remove. + * Remove topics. + * @param topics Collection of names for the topics to remove. */ fun removeTopics(topics: List<String>) { val kafkaAdmin: AdminClient = AdminClient.create(this.kafkaConfig) @@ -64,7 +66,7 @@ class TopicManager(private val kafkaConfig: HashMap<String, Any>) { while (!deleted) { try { val result = kafkaAdmin.deleteTopics(topics) - result.all().get() // wait for the future object + result.all().get() // wait for the future to be completed logger.info { "Topics deletion finished with result: ${ result.values().map { it -> it.key + ": " + it.value.isDone } @@ -72,8 +74,8 @@ class TopicManager(private val kafkaConfig: HashMap<String, Any>) { }" } } catch (e: Exception) { - logger.error { "Error while removing topics: $e" } - logger.debug { "Existing topics are: ${kafkaAdmin.listTopics()}." } + logger.error(e) { "Error while removing topics: $e" } + logger.info { "Existing topics are: ${kafkaAdmin.listTopics()}." } } val toDelete = topics.filter { topic -> @@ -83,7 +85,7 @@ class TopicManager(private val kafkaConfig: HashMap<String, Any>) { if (toDelete.isNullOrEmpty()) { deleted = true } else { - logger.info { "Deletion of kafka topics failed retrying in 2 seconds" } + logger.info { "Deletion of kafka topics failed, will retry in $RETRY_TIME seconds." } sleep(RETRY_TIME) } } diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/patcher/AbstractPatcher.kt b/theodolite-quarkus/src/main/kotlin/theodolite/patcher/AbstractPatcher.kt index a1a4501c919748389089b9d81e3cf927b0ea2e2a..c0d17244b6a7a3f37b8d8a57713659b85b9b65b1 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/patcher/AbstractPatcher.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/patcher/AbstractPatcher.kt @@ -12,7 +12,7 @@ import io.fabric8.kubernetes.api.model.KubernetesResource * @param variableName *(optional)* The variable name to be patched * * - * **For example** to patch the load dimension of a workload generator, the Patcher should be created as follow: + * **For example** to patch the load dimension of a load generator, the patcher should be created as follow: * * k8sResource: `uc-1-workload-generator.yaml` * container: `workload` diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/strategies/StrategyFactory.kt b/theodolite-quarkus/src/main/kotlin/theodolite/strategies/StrategyFactory.kt index 9bef5587ac9c26d2323af41c5119ac36b95cf807..829370e8ce1c181c1a4cb9fdd8ccf0ecefd48d3d 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/strategies/StrategyFactory.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/strategies/StrategyFactory.kt @@ -4,12 +4,13 @@ import theodolite.execution.BenchmarkExecutor import theodolite.strategies.restriction.LowerBoundRestriction import theodolite.strategies.restriction.RestrictionStrategy import theodolite.strategies.searchstrategy.BinarySearch +import theodolite.strategies.searchstrategy.FullSearch import theodolite.strategies.searchstrategy.LinearSearch import theodolite.strategies.searchstrategy.SearchStrategy import theodolite.util.Results /** - * Factory for creating [SearchStrategy] and [RestrictionStrategy] Strategies. + * Factory for creating [SearchStrategy] and [RestrictionStrategy] strategies. */ class StrategyFactory { @@ -24,6 +25,7 @@ class StrategyFactory { */ fun createSearchStrategy(executor: BenchmarkExecutor, searchStrategyString: String): SearchStrategy { return when (searchStrategyString) { + "FullSearch" -> FullSearch(executor) "LinearSearch" -> LinearSearch(executor) "BinarySearch" -> BinarySearch(executor) else -> throw IllegalArgumentException("Search Strategy $searchStrategyString not found") diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/strategies/restriction/LowerBoundRestriction.kt b/theodolite-quarkus/src/main/kotlin/theodolite/strategies/restriction/LowerBoundRestriction.kt index 2911b6ac949a9d523e464c0ea2942063e996d767..13bfedfe055f2bd428137f89b2986f3967ec797c 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/strategies/restriction/LowerBoundRestriction.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/strategies/restriction/LowerBoundRestriction.kt @@ -5,12 +5,13 @@ import theodolite.util.Resource import theodolite.util.Results /** - * The Lower Bound Restriction sets the lower bound of the resources to be examined to the value + * The [LowerBoundRestriction] sets the lower bound of the resources to be examined to the value * needed to successfully execute the next smaller load. * * @param results [Result] object used as a basis to restrict the resources. */ class LowerBoundRestriction(results: Results) : RestrictionStrategy(results) { + override fun apply(load: LoadDimension, resources: List<Resource>): List<Resource> { val maxLoad: LoadDimension? = this.results.getMaxBenchmarkedLoad(load) var lowerBound: Resource? = this.results.getMinRequiredInstances(maxLoad) @@ -19,4 +20,5 @@ class LowerBoundRestriction(results: Results) : RestrictionStrategy(results) { } return resources.filter { x -> x.get() >= lowerBound.get() } } + } diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/strategies/searchstrategy/BinarySearch.kt b/theodolite-quarkus/src/main/kotlin/theodolite/strategies/searchstrategy/BinarySearch.kt index 027444fe36a47878af998abdf18dc7a7562d7afd..7f3311182e324f1ebe10bb664ea7766aca1aa783 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/strategies/searchstrategy/BinarySearch.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/strategies/searchstrategy/BinarySearch.kt @@ -1,9 +1,12 @@ package theodolite.strategies.searchstrategy +import mu.KotlinLogging import theodolite.execution.BenchmarkExecutor import theodolite.util.LoadDimension import theodolite.util.Resource +private val logger = KotlinLogging.logger {} + /** * Binary-search-like implementation for determining the smallest suitable number of instances. * @@ -32,6 +35,8 @@ class BinarySearch(benchmarkExecutor: BenchmarkExecutor) : SearchStrategy(benchm } // special case: length == 1 or 2 if (lower == upper) { + val res = resources[lower] + logger.info { "Running experiment with load '$load' and resources '$res'" } if (this.benchmarkExecutor.runExperiment(load, resources[lower])) return lower else { if (lower + 1 == resources.size) return -1 @@ -41,6 +46,8 @@ class BinarySearch(benchmarkExecutor: BenchmarkExecutor) : SearchStrategy(benchm // apply binary search for a list with // length > 2 and adjust upper and lower depending on the result for `resources[mid]` val mid = (upper + lower) / 2 + val res = resources[mid] + logger.info { "Running experiment with load '$load' and resources '$res'" } if (this.benchmarkExecutor.runExperiment(load, resources[mid])) { if (mid == lower) { return lower diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/strategies/searchstrategy/FullSearch.kt b/theodolite-quarkus/src/main/kotlin/theodolite/strategies/searchstrategy/FullSearch.kt new file mode 100644 index 0000000000000000000000000000000000000000..9698dab18c5eb804fc9a60ef23fa734a9917b338 --- /dev/null +++ b/theodolite-quarkus/src/main/kotlin/theodolite/strategies/searchstrategy/FullSearch.kt @@ -0,0 +1,31 @@ +package theodolite.strategies.searchstrategy + +import mu.KotlinLogging +import theodolite.execution.BenchmarkExecutor +import theodolite.util.LoadDimension +import theodolite.util.Resource + +private val logger = KotlinLogging.logger {} + +/** + * [SearchStrategy] that executes experiment for provides resources in a linear-search-like fashion, but **without + * stopping** once a suitable resource amount is found. + * + * @see LinearSearch for a SearchStrategy that stops once a suitable resource amount is found. + * + * @param benchmarkExecutor Benchmark executor which runs the individual benchmarks. + */ +class FullSearch(benchmarkExecutor: BenchmarkExecutor) : SearchStrategy(benchmarkExecutor) { + + override fun findSuitableResource(load: LoadDimension, resources: List<Resource>): Resource? { + var minimalSuitableResources: Resource? = null; + for (res in resources) { + logger.info { "Running experiment with load '$load' and resources '$res'" } + val result = this.benchmarkExecutor.runExperiment(load, res) + if (result && minimalSuitableResources != null) { + minimalSuitableResources = res + } + } + return minimalSuitableResources + } +} diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/strategies/searchstrategy/LinearSearch.kt b/theodolite-quarkus/src/main/kotlin/theodolite/strategies/searchstrategy/LinearSearch.kt index 08daa082d0eb8f2ecbb71193111a0263ae275fbc..3936d1982cc2e0d4701d3ff643199cfdc0d35ff4 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/strategies/searchstrategy/LinearSearch.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/strategies/searchstrategy/LinearSearch.kt @@ -1,9 +1,12 @@ package theodolite.strategies.searchstrategy +import mu.KotlinLogging import theodolite.execution.BenchmarkExecutor import theodolite.util.LoadDimension import theodolite.util.Resource +private val logger = KotlinLogging.logger {} + /** * Linear-search-like implementation for determining the smallest suitable number of instances. * @@ -13,6 +16,8 @@ class LinearSearch(benchmarkExecutor: BenchmarkExecutor) : SearchStrategy(benchm override fun findSuitableResource(load: LoadDimension, resources: List<Resource>): Resource? { for (res in resources) { + + logger.info { "Running experiment with load '$load' and resources '$res'" } if (this.benchmarkExecutor.runExperiment(load, res)) return res } return null diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/util/Results.kt b/theodolite-quarkus/src/main/kotlin/theodolite/util/Results.kt index 7116d73cf5b54325c8cfa41b1186d58695628874..ab40f3d1f722bab39c29e81621da86e8920bbf72 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/util/Results.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/util/Results.kt @@ -44,19 +44,21 @@ class Results { * yet, a Resource with the constant value Int.MIN_VALUE is returned. */ fun getMinRequiredInstances(load: LoadDimension?): Resource? { - if (this.results.isEmpty()) return Resource(Int.MIN_VALUE, emptyList()) + if (this.results.isEmpty()) { + return Resource(Int.MIN_VALUE, emptyList()) + } - var requiredInstances: Resource? = Resource(Int.MAX_VALUE, emptyList()) + var minRequiredInstances: Resource? = Resource(Int.MAX_VALUE, emptyList()) for (experiment in results) { + // Get all successful experiments for requested load if (experiment.key.first == load && experiment.value) { - if (requiredInstances == null) { - requiredInstances = experiment.key.second - } else if (experiment.key.second.get() < requiredInstances.get()) { - requiredInstances = experiment.key.second + if (minRequiredInstances == null || experiment.key.second.get() < minRequiredInstances.get()) { + // Found new smallest resources + minRequiredInstances = experiment.key.second } } } - return requiredInstances + return minRequiredInstances } /** diff --git a/theodolite-quarkus/src/test/kotlin/theodolite/strategies/restriction/LowerBoundRestrictionTest.kt b/theodolite-quarkus/src/test/kotlin/theodolite/strategies/restriction/LowerBoundRestrictionTest.kt new file mode 100644 index 0000000000000000000000000000000000000000..b6b9eaecd9fc35b25b9447611835c3d8469cea0e --- /dev/null +++ b/theodolite-quarkus/src/test/kotlin/theodolite/strategies/restriction/LowerBoundRestrictionTest.kt @@ -0,0 +1,116 @@ +package theodolite.strategies.restriction + +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.Disabled +import org.junit.jupiter.api.Test +import theodolite.util.LoadDimension +import theodolite.util.Resource +import theodolite.util.Results + +internal class LowerBoundRestrictionTest { + + @Test + fun testNoPreviousResults() { + val results = Results() + val strategy = LowerBoundRestriction(results) + val load = buildLoadDimension(10000) + val resources = listOf( + buildResourcesDimension(1), + buildResourcesDimension(2), + buildResourcesDimension(3) + ) + val restriction = strategy.apply(load, resources) + + assertEquals(3, restriction.size) + assertEquals(resources, restriction) + } + + @Test + fun testWithSuccessfulPreviousResults() { + val results = Results() + results.setResult(10000, 1, true) + results.setResult(20000, 1, false) + results.setResult(20000, 2, true) + val strategy = LowerBoundRestriction(results) + val load = buildLoadDimension(30000) + val resources = listOf( + buildResourcesDimension(1), + buildResourcesDimension(2), + buildResourcesDimension(3) + ) + val restriction = strategy.apply(load, resources) + + assertEquals(2, restriction.size) + assertEquals(resources.subList(1,3), restriction) + } + + @Test + @Disabled + fun testWithNoSuccessfulPreviousResults() { + // This test is currently not implemented this way, but might later be the desired behavior. + val results = Results() + results.setResult(10000, 1, true) + results.setResult(20000, 1, false) + results.setResult(20000, 2, false) + results.setResult(20000, 3, false) + val strategy = LowerBoundRestriction(results) + val load = buildLoadDimension(30000) + val resources = listOf( + buildResourcesDimension(1), + buildResourcesDimension(2), + buildResourcesDimension(3) + ) + val restriction = strategy.apply(load, resources) + + assertEquals(0, restriction.size) + assertEquals(emptyList<Resource>(), restriction) + } + + + @Test + fun testNoPreviousResults2() { + val results = Results() + results.setResult(10000, 1, true) + results.setResult(20000, 2, true) + results.setResult(10000, 1, false) + results.setResult(20000, 2, true) + + val minRequiredInstances = results.getMinRequiredInstances(LoadDimension(20000, emptyList())) + + assertNotNull(minRequiredInstances) + assertEquals(2, minRequiredInstances!!.get()) + } + + @Test + @Disabled + fun testMinRequiredInstancesWhenNotSuccessful() { + // This test is currently not implemented this way, but might later be the desired behavior. + val results = Results() + results.setResult(10000, 1, true) + results.setResult(20000, 2, true) + results.setResult(10000, 1, false) + results.setResult(20000, 2, false) + + val minRequiredInstances = results.getMinRequiredInstances(LoadDimension(20000, emptyList())) + + assertNotNull(minRequiredInstances) + assertEquals(2, minRequiredInstances!!.get()) + } + + private fun buildLoadDimension(load: Int): LoadDimension { + return LoadDimension(load, emptyList()) + } + + private fun buildResourcesDimension(resources: Int): Resource { + return Resource(resources, emptyList()) + } + + private fun Results.setResult(load: Int, resources: Int, successful: Boolean) { + this.setResult( + Pair( + buildLoadDimension(load), + buildResourcesDimension(resources) + ), + successful) + } +} \ No newline at end of file diff --git a/theodolite-quarkus/src/test/kotlin/theodolite/util/ResultsTest.kt b/theodolite-quarkus/src/test/kotlin/theodolite/util/ResultsTest.kt new file mode 100644 index 0000000000000000000000000000000000000000..6a69040a47abaf2e39225c563dfbad1594b1f261 --- /dev/null +++ b/theodolite-quarkus/src/test/kotlin/theodolite/util/ResultsTest.kt @@ -0,0 +1,50 @@ +package theodolite.util + +import io.quarkus.test.junit.QuarkusTest +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.Disabled +import org.junit.jupiter.api.Test + +@QuarkusTest +internal class ResultsTest { + + @Test + fun testMinRequiredInstancesWhenSuccessful() { + val results = Results() + results.setResult(10000, 1, true) + results.setResult(10000, 2, true) + results.setResult(20000, 1, false) + results.setResult(20000, 2, true) + + val minRequiredInstances = results.getMinRequiredInstances(LoadDimension(20000, emptyList())) + + assertNotNull(minRequiredInstances) + assertEquals(2, minRequiredInstances!!.get()) + } + + @Test + @Disabled + fun testMinRequiredInstancesWhenNotSuccessful() { + // This test is currently not implemented this way, but might later be the desired behavior. + val results = Results() + results.setResult(10000, 1, true) + results.setResult(10000, 2, true) + results.setResult(20000, 1, false) + results.setResult(20000, 2, false) + + val minRequiredInstances = results.getMinRequiredInstances(LoadDimension(20000, emptyList())) + + assertNotNull(minRequiredInstances) + assertEquals(2, minRequiredInstances!!.get()) + } + + private fun Results.setResult(load: Int, resources: Int, successful: Boolean) { + this.setResult( + Pair( + LoadDimension(load, emptyList()), + Resource(resources, emptyList()) + ), + successful) + } + +} \ No newline at end of file