diff --git a/helm/templates/theodolite/role.yaml b/helm/templates/theodolite/role.yaml index b9924ea4908718c361851ff6137b44a19589a0be..0b496f3ef506e56b74acf5b7c8d0f4edc4f2cd96 100644 --- a/helm/templates/theodolite/role.yaml +++ b/helm/templates/theodolite/role.yaml @@ -68,5 +68,11 @@ rules: - get - create - update + - apiGroups: + - "" + resources: + - events + verbs: + - create {{- end }} {{- end }} \ No newline at end of file diff --git a/helm/values.yaml b/helm/values.yaml index c45e62aed8126a8b9f20a6dd0ca6083cd162bc2b..6c20f4bab35825e66412381b454df5c9dedec0fc 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -155,7 +155,10 @@ cp-helm-charts: ### kafka-lag-exporter: enabled: true + image: + pullPolicy: IfNotPresent nodeSelector: {} + clusters: - name: "theodolite-cp-kafka" bootstrapBrokers: "theodolite-cp-kafka:9092" diff --git a/theodolite/examples/operator/example-benchmark.yaml b/theodolite/examples/operator/example-benchmark.yaml index fb2c5749dc469e30baabdbad910c302bef0b26bc..407d37d3637b4bbe6005b4c7bf1918814b3b7907 100644 --- a/theodolite/examples/operator/example-benchmark.yaml +++ b/theodolite/examples/operator/example-benchmark.yaml @@ -50,4 +50,4 @@ spec: name: "example-configmap" files: - uc1-load-generator-service.yaml - - uc1-load-generator-deployment.yaml \ No newline at end of file + - uc1-load-generator-deployment.yaml diff --git a/theodolite/examples/operator/example-configmap.yaml b/theodolite/examples/operator/example-configmap.yaml index 23bb3e021c49c4192bf42f065365c8f21dfc3f59..210ce32d3fc0f75b9ffce874d1fa0a1ea9bdc3cd 100644 --- a/theodolite/examples/operator/example-configmap.yaml +++ b/theodolite/examples/operator/example-configmap.yaml @@ -1,35 +1,8 @@ apiVersion: v1 +kind: ConfigMap +metadata: + name: example-configmap data: - aggregation-service.yaml: "apiVersion: v1\nkind: Service\nmetadata: \n name: titan-ccp-aggregation\n - \ labels:\n app: titan-ccp-aggregation\nspec:\n #type: NodePort\n selector: - \ \n app: titan-ccp-aggregation\n ports: \n - name: http\n port: 80\n - \ targetPort: 80\n protocol: TCP\n - name: metrics\n port: 5556\n" - jmx-configmap.yaml: | - apiVersion: v1 - kind: ConfigMap - metadata: - name: aggregation-jmx-configmap - data: - jmx-kafka-prometheus.yml: |+ - jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi - lowercaseOutputName: true - lowercaseOutputLabelNames: true - ssl: false - service-monitor.yaml: | - apiVersion: monitoring.coreos.com/v1 - kind: ServiceMonitor - metadata: - labels: - app: titan-ccp-aggregation - appScope: titan-ccp - name: titan-ccp-aggregation - spec: - selector: - matchLabels: - app: titan-ccp-aggregation - endpoints: - - port: metrics - interval: 10s uc1-kstreams-deployment.yaml: |- apiVersion: apps/v1 kind: Deployment @@ -49,9 +22,6 @@ data: containers: - name: uc-application image: ghcr.io/cau-se/theodolite-uc1-kstreams-app:latest - ports: - - containerPort: 5555 - name: jmx env: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-cp-kafka:9092" @@ -65,27 +35,6 @@ data: limits: memory: 4Gi cpu: 1000m - - name: prometheus-jmx-exporter - image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143" - command: - - java - - -XX:+UnlockExperimentalVMOptions - - -XX:+UseCGroupMemoryLimitForHeap - - -XX:MaxRAMFraction=1 - - -XshowSettings:vm - - -jar - - jmx_prometheus_httpserver.jar - - "5556" - - /etc/jmx-aggregation/jmx-kafka-prometheus.yml - ports: - - containerPort: 5556 - volumeMounts: - - name: jmx-config - mountPath: /etc/jmx-aggregation - volumes: - - name: jmx-config - configMap: - name: aggregation-jmx-configmap uc1-load-generator-deployment.yaml: | apiVersion: apps/v1 kind: Deployment @@ -109,10 +58,6 @@ data: - containerPort: 5701 name: coordination env: - - name: NUM_SENSORS - value: "25000" - - name: NUM_NESTED_GROUPS - value: "5" - name: KUBERNETES_NAMESPACE valueFrom: fieldRef: @@ -139,22 +84,4 @@ data: - name: coordination port: 5701 targetPort: 5701 - protocol: TCP - uc1-service-monitor.yaml: | - apiVersion: monitoring.coreos.com/v1 - kind: ServiceMonitor - metadata: - labels: - app: titan-ccp-aggregation - appScope: titan-ccp - name: titan-ccp-aggregation - spec: - selector: - matchLabels: - app: titan-ccp-aggregation - endpoints: - - port: metrics - interval: 10s -kind: ConfigMap -metadata: - name: example-configmap + protocol: TCP \ No newline at end of file diff --git a/theodolite/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt b/theodolite/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt index 777ea1c0ed43ac3af244dc0aaf770c69c11718cf..281c68e318784ee8206473cd014f814b3f5152a9 100644 --- a/theodolite/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt +++ b/theodolite/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt @@ -2,6 +2,7 @@ package theodolite.evaluation import mu.KotlinLogging import theodolite.benchmark.BenchmarkExecution +import theodolite.util.EvaluationFailedException import theodolite.util.IOHandler import theodolite.util.LoadDimension import theodolite.util.Resource @@ -12,7 +13,6 @@ import java.util.* import java.util.regex.Pattern private val logger = KotlinLogging.logger {} -private val RECORD_LAG_QUERY = "sum by(group)(kafka_consumergroup_group_lag >= 0)" /** * Contains the analysis. Fetches a metric from Prometheus, documents it, and evaluates it. @@ -37,7 +37,7 @@ class AnalysisExecutor( * @return true if the experiment succeeded. */ fun analyze(load: LoadDimension, res: Resource, executionIntervals: List<Pair<Instant, Instant>>): Boolean { - var result = false + var result: Boolean var repetitionCounter = 1 try { @@ -50,7 +50,7 @@ class AnalysisExecutor( fetcher.fetchMetric( start = interval.first, end = interval.second, - query = RECORD_LAG_QUERY + query = SloConfigHandler.getQueryString(sloType = slo.sloType) ) } @@ -58,7 +58,7 @@ class AnalysisExecutor( ioHandler.writeToCSVFile( fileURL = "${fileURL}_${repetitionCounter++}", data = data.getResultAsList(), - columns = listOf("group", "timestamp", "value") + columns = listOf("labels", "timestamp", "value") ) } @@ -71,8 +71,7 @@ class AnalysisExecutor( result = sloChecker.evaluate(prometheusData) } catch (e: Exception) { - // TODO(throw exception in order to make it possible to mark an experiment as unsuccessfully) - logger.error { "Evaluation failed for resource '${res.get()}' and load '${load.get()}'. Error: $e" } + throw EvaluationFailedException("Evaluation failed for resource '${res.get()}' and load '${load.get()} ", e) } return result } diff --git a/theodolite/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt b/theodolite/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt index 448a2a05f8dbeb1aef153895360bfb40e7275224..d646286b70bc5880df1f603afdc2bda22bcc3259 100644 --- a/theodolite/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt +++ b/theodolite/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt @@ -36,13 +36,12 @@ class ExternalSloChecker( */ override fun evaluate(fetchedData: List<PrometheusResponse>): Boolean { var counter = 0 - val data = Gson().toJson( - mapOf( - "total_lags" to fetchedData.map { it.data?.result }, - "threshold" to threshold, - "warmup" to warmup - ) - ) + val data = SloJson.Builder() + .results(fetchedData.map { it.data?.result }) + .addMetadata("threshold", threshold) + .addMetadata( "warmup", warmup) + .build() + .toJson() while (counter < RETRIES) { val result = post(externalSlopeURL, data = data, timeout = TIMEOUT) diff --git a/theodolite/src/main/kotlin/theodolite/evaluation/MetricFetcher.kt b/theodolite/src/main/kotlin/theodolite/evaluation/MetricFetcher.kt index 833d7d1e16c2fbc91b58817b319a7d02af7f5b2b..e54d79fe0f95b9f6079bd4295a74e81250b73a90 100644 --- a/theodolite/src/main/kotlin/theodolite/evaluation/MetricFetcher.kt +++ b/theodolite/src/main/kotlin/theodolite/evaluation/MetricFetcher.kt @@ -53,8 +53,7 @@ class MetricFetcher(private val prometheusURL: String, private val offset: Durat } else { val values = parseValues(response) if (values.data?.result.isNullOrEmpty()) { - logger.error { "Empty query result: $values between $start and $end for query $query." } - throw NoSuchFieldException() + throw NoSuchFieldException("Empty query result: $values between $start and $end for query $query.") } return parseValues(response) } diff --git a/theodolite/src/main/kotlin/theodolite/evaluation/SloCheckerFactory.kt b/theodolite/src/main/kotlin/theodolite/evaluation/SloCheckerFactory.kt index 93e8e6180f5a99486e500af022869d896067d128..64f9110cd931feef41dc65f88d6623e82f4e03a2 100644 --- a/theodolite/src/main/kotlin/theodolite/evaluation/SloCheckerFactory.kt +++ b/theodolite/src/main/kotlin/theodolite/evaluation/SloCheckerFactory.kt @@ -43,22 +43,23 @@ class SloCheckerFactory { properties: MutableMap<String, String>, load: LoadDimension ): SloChecker { - return when (sloType) { - "lag trend" -> ExternalSloChecker( + return when (sloType.toLowerCase()) { + SloTypes.LAG_TREND.value, SloTypes.DROPPED_RECORDS.value -> ExternalSloChecker( externalSlopeURL = properties["externalSloUrl"] ?: throw IllegalArgumentException("externalSloUrl expected"), threshold = properties["threshold"]?.toInt() ?: throw IllegalArgumentException("threshold expected"), warmup = properties["warmup"]?.toInt() ?: throw IllegalArgumentException("warmup expected") ) - "lag trend ratio" -> { - var thresholdRatio = + + SloTypes.LAG_TREND_RATIO.value, SloTypes.DROPPED_RECORDS_RATIO.value -> { + val thresholdRatio = properties["ratio"]?.toDouble() ?: throw IllegalArgumentException("ratio for threshold expected") if (thresholdRatio < 0.0) { throw IllegalArgumentException("Threshold ratio needs to be an Double greater or equal 0.0") } // cast to int, as rounding is not really necessary - var threshold = (load.get() * thresholdRatio).toInt() + val threshold = (load.get() * thresholdRatio).toInt() ExternalSloChecker( externalSlopeURL = properties["externalSloUrl"] diff --git a/theodolite/src/main/kotlin/theodolite/evaluation/SloConfigHandler.kt b/theodolite/src/main/kotlin/theodolite/evaluation/SloConfigHandler.kt new file mode 100644 index 0000000000000000000000000000000000000000..93929218c822030ff065dafb19cce1fbaa69a179 --- /dev/null +++ b/theodolite/src/main/kotlin/theodolite/evaluation/SloConfigHandler.kt @@ -0,0 +1,20 @@ +package theodolite.evaluation + +import theodolite.util.InvalidPatcherConfigurationException +import javax.enterprise.context.ApplicationScoped + +private const val CONSUMER_LAG_QUERY = "sum by(group)(kafka_consumergroup_group_lag >= 0)" +private const val DROPPED_RECORDS_QUERY = "sum by(job) (kafka_streams_stream_task_metrics_dropped_records_total>=0)" + +@ApplicationScoped +class SloConfigHandler() { + companion object { + fun getQueryString(sloType: String): String { + return when (sloType.toLowerCase()) { + SloTypes.LAG_TREND.value, SloTypes.LAG_TREND_RATIO.value -> CONSUMER_LAG_QUERY + SloTypes.DROPPED_RECORDS.value, SloTypes.DROPPED_RECORDS_RATIO.value -> DROPPED_RECORDS_QUERY + else -> throw InvalidPatcherConfigurationException("Could not find Prometheus query string for slo type $sloType") + } + } + } +} \ No newline at end of file diff --git a/theodolite/src/main/kotlin/theodolite/evaluation/SloJson.kt b/theodolite/src/main/kotlin/theodolite/evaluation/SloJson.kt new file mode 100644 index 0000000000000000000000000000000000000000..fc9fe17b255dbb5ae68881538d8d2a50a191edb1 --- /dev/null +++ b/theodolite/src/main/kotlin/theodolite/evaluation/SloJson.kt @@ -0,0 +1,63 @@ +package theodolite.evaluation + +import com.google.gson.Gson +import theodolite.util.PromResult + +class SloJson private constructor( + val results: List<List<PromResult>?>? = null, + var metadata: MutableMap<String, Any>? = null +) { + + data class Builder( + var results:List<List<PromResult>?>? = null, + var metadata: MutableMap<String, Any>? = null + ) { + + /** + * Set the results + * + * @param results list of prometheus results + */ + fun results(results: List<List<PromResult>?>) = apply { this.results = results } + + /** + * Add metadata as key value pairs + * + * @param key key of the metadata to be added + * @param value value of the metadata to be added + */ + fun addMetadata(key: String, value: String) = apply { + if (this.metadata.isNullOrEmpty()) { + this.metadata = mutableMapOf(key to value) + } else { + this.metadata!![key] = value + } + } + + /** + * Add metadata as key value pairs + * + * @param key key of the metadata to be added + * @param value value of the metadata to be added + */ + fun addMetadata(key: String, value: Int) = apply { + if (this.metadata.isNullOrEmpty()) { + this.metadata = mutableMapOf(key to value) + } else { + this.metadata!![key] = value + } + } + + fun build() = SloJson( + results = results, + metadata = metadata + ) + } + + fun toJson(): String { + return Gson().toJson(mapOf( + "results" to this.results, + "metadata" to this.metadata + )) + } +} \ No newline at end of file diff --git a/theodolite/src/main/kotlin/theodolite/evaluation/SloTypes.kt b/theodolite/src/main/kotlin/theodolite/evaluation/SloTypes.kt new file mode 100644 index 0000000000000000000000000000000000000000..ac9de35861b0bd9c012bfb0b8cfcb2e1aa5aed68 --- /dev/null +++ b/theodolite/src/main/kotlin/theodolite/evaluation/SloTypes.kt @@ -0,0 +1,10 @@ +package theodolite.evaluation + +enum class SloTypes(val value: String) { + LAG_TREND("lag trend"), + LAG_TREND_RATIO("lag trend ratio"), + DROPPED_RECORDS("dropped records"), + DROPPED_RECORDS_RATIO("dropped records ratio") + + +} \ No newline at end of file diff --git a/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt b/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt index e7b511d8c83b5abccece1204aad2a4a9ecfdfd26..3238f447be06ce6486bb7f6ca1758700f36ba558 100644 --- a/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt +++ b/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt @@ -25,11 +25,12 @@ abstract class BenchmarkExecutor( val results: Results, val executionDuration: Duration, val configurationOverrides: List<ConfigurationOverride?>, - val slo: BenchmarkExecution.Slo, + val slos: List<BenchmarkExecution.Slo>, val repetitions: Int, val executionId: Int, val loadGenerationDelay: Long, - val afterTeardownDelay: Long + val afterTeardownDelay: Long, + val executionName: String ) { var run: AtomicBoolean = AtomicBoolean(true) diff --git a/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt b/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt index c54d1878d4957a0b8ec6a8fdfb18ec6342d7bfc1..2e938be3a6e503a5e7e3f94c18a9454e173db5b0 100644 --- a/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt +++ b/theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt @@ -5,10 +5,8 @@ import mu.KotlinLogging import theodolite.benchmark.Benchmark import theodolite.benchmark.BenchmarkExecution import theodolite.evaluation.AnalysisExecutor -import theodolite.util.ConfigurationOverride -import theodolite.util.LoadDimension -import theodolite.util.Resource -import theodolite.util.Results +import theodolite.execution.operator.EventCreator +import theodolite.util.* import java.time.Duration import java.time.Instant @@ -20,29 +18,34 @@ class BenchmarkExecutorImpl( results: Results, executionDuration: Duration, configurationOverrides: List<ConfigurationOverride?>, - slo: BenchmarkExecution.Slo, + slos: List<BenchmarkExecution.Slo>, repetitions: Int, executionId: Int, loadGenerationDelay: Long, - afterTeardownDelay: Long + afterTeardownDelay: Long, + executionName: String ) : BenchmarkExecutor( benchmark, results, executionDuration, configurationOverrides, - slo, + slos, repetitions, executionId, loadGenerationDelay, - afterTeardownDelay + afterTeardownDelay, + executionName ) { + private val eventCreator = EventCreator() + private val mode = Configuration.EXECUTION_MODE + override fun runExperiment(load: LoadDimension, res: Resource): Boolean { var result = false val executionIntervals: MutableList<Pair<Instant, Instant>> = ArrayList() for (i in 1.rangeTo(repetitions)) { - logger.info { "Run repetition $i/$repetitions" } if (this.run.get()) { + logger.info { "Run repetition $i/$repetitions" } executionIntervals.add(runSingleExperiment(load, res)) } else { break @@ -53,14 +56,23 @@ class BenchmarkExecutorImpl( * Analyse the experiment, if [run] is true, otherwise the experiment was canceled by the user. */ if (this.run.get()) { - result = AnalysisExecutor(slo = slo, executionId = executionId) - .analyze( - load = load, - res = res, - executionIntervals = executionIntervals - ) + val experimentResults = slos.map { + AnalysisExecutor(slo = it, executionId = executionId) + .analyze( + load = load, + res = res, + executionIntervals = executionIntervals + ) + } + + result = (false !in experimentResults) this.results.setResult(Pair(load, res), result) } + + if(!this.run.get()) { + throw ExecutionFailedException("The execution was interrupted") + } + return result } @@ -73,22 +85,49 @@ class BenchmarkExecutorImpl( this.afterTeardownDelay ) val from = Instant.now() - // TODO(restructure try catch in order to throw exceptions if there are significant problems by running a experiment) + try { benchmarkDeployment.setup() this.waitAndLog() + if (mode == ExecutionModes.OPERATOR.value) { + eventCreator.createEvent( + executionName = executionName, + type = "NORMAL", + reason = "Start experiment", + message = "load: ${load.get()}, resources: ${res.get()}") + } } catch (e: Exception) { - logger.error { "Error while setup experiment." } - logger.error { "Error is: $e" } this.run.set(false) + + if (mode == ExecutionModes.OPERATOR.value) { + eventCreator.createEvent( + executionName = executionName, + type = "WARNING", + reason = "Start experiment failed", + message = "load: ${load.get()}, resources: ${res.get()}") + } + throw ExecutionFailedException("Error during setup the experiment", e) } val to = Instant.now() try { benchmarkDeployment.teardown() + if (mode == ExecutionModes.OPERATOR.value) { + eventCreator.createEvent( + executionName = executionName, + type = "NORMAL", + reason = "Stop experiment", + message = "Teardown complete") + } } catch (e: Exception) { - logger.warn { "Error while tearing down the benchmark deployment." } - logger.debug { "Teardown failed, caused by: $e" } + if (mode == ExecutionModes.OPERATOR.value) { + eventCreator.createEvent( + executionName = executionName, + type = "WARNING", + reason = "Stop experiment failed", + message = "Teardown failed: ${e.message}") + } + throw ExecutionFailedException("Error during teardown the experiment", e) } return Pair(from, to) } -} +} \ No newline at end of file diff --git a/theodolite/src/main/kotlin/theodolite/execution/ExecutionModes.kt b/theodolite/src/main/kotlin/theodolite/execution/ExecutionModes.kt new file mode 100644 index 0000000000000000000000000000000000000000..bf947be01b534fd000d3967f0b72ef25978d4110 --- /dev/null +++ b/theodolite/src/main/kotlin/theodolite/execution/ExecutionModes.kt @@ -0,0 +1,7 @@ +package theodolite.execution + +enum class ExecutionModes(val value: String) { + OPERATOR("operator"), + YAML_EXECUTOR("yaml-executor"), + STANDALONE("standalone") +} \ No newline at end of file diff --git a/theodolite/src/main/kotlin/theodolite/execution/Main.kt b/theodolite/src/main/kotlin/theodolite/execution/Main.kt index 7d5fca859422a194e81468d9766a9e7ba29fb998..11f696ddd739e987e92ecec724390948714d898b 100644 --- a/theodolite/src/main/kotlin/theodolite/execution/Main.kt +++ b/theodolite/src/main/kotlin/theodolite/execution/Main.kt @@ -3,6 +3,7 @@ package theodolite.execution import io.quarkus.runtime.annotations.QuarkusMain import mu.KotlinLogging import theodolite.execution.operator.TheodoliteOperator +import theodolite.util.Configuration import kotlin.system.exitProcess private val logger = KotlinLogging.logger {} @@ -13,13 +14,12 @@ object Main { @JvmStatic fun main(args: Array<String>) { - val mode = System.getenv("MODE") ?: "standalone" + val mode = Configuration.EXECUTION_MODE logger.info { "Start Theodolite with mode $mode" } - when (mode) { - "standalone" -> TheodoliteStandalone().start() - "yaml-executor" -> TheodoliteStandalone().start() // TODO remove (#209) - "operator" -> TheodoliteOperator().start() + when (mode.toLowerCase()) { + ExecutionModes.STANDALONE.value, ExecutionModes.YAML_EXECUTOR.value -> TheodoliteStandalone().start() // TODO remove standalone (#209) + ExecutionModes.OPERATOR.value -> TheodoliteOperator().start() else -> { logger.error { "MODE $mode not found" } exitProcess(1) diff --git a/theodolite/src/main/kotlin/theodolite/execution/Shutdown.kt b/theodolite/src/main/kotlin/theodolite/execution/Shutdown.kt index e795ada3e3bcb2dba19f1e088f426f38a824f4a7..6dedc94af864269d7d15929c69ec54aa384fc8e3 100644 --- a/theodolite/src/main/kotlin/theodolite/execution/Shutdown.kt +++ b/theodolite/src/main/kotlin/theodolite/execution/Shutdown.kt @@ -34,16 +34,15 @@ class Shutdown(private val benchmarkExecution: BenchmarkExecution, private val b afterTeardownDelay = 5L ) deployment.teardown() + logger.info { + "Finished teardown of all benchmark resources." + } } catch (e: Exception) { - // TODO(throw exception in order to make it possible to mark an experiment as unsuccessfully) logger.warn { "Could not delete all specified resources from Kubernetes. " + "This could be the case, if not all resources are deployed and running." } } - logger.info { - "Finished teardown of all benchmark resources." - } } } diff --git a/theodolite/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt b/theodolite/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt index addf30acde31ee8e3e53c20a5e2b57a03587d08e..a5a4904f8ea8de152932333a1b8302f9539e260b 100644 --- a/theodolite/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt +++ b/theodolite/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt @@ -61,11 +61,12 @@ class TheodoliteExecutor( results = results, executionDuration = executionDuration, configurationOverrides = config.configOverrides, - slo = config.slos[0], + slos = config.slos, repetitions = config.execution.repetitions, executionId = config.executionId, loadGenerationDelay = config.execution.loadGenerationDelay, - afterTeardownDelay = config.execution.afterTeardownDelay + afterTeardownDelay = config.execution.afterTeardownDelay, + executionName = config.name ) if (config.load.loadValues != config.load.loadValues.sorted()) { @@ -123,15 +124,18 @@ class TheodoliteExecutor( val config = buildConfig() // execute benchmarks for each load - for (load in config.loads) { - if (executor.run.get()) { - config.compositeStrategy.findSuitableResource(load, config.resources) + try { + for (load in config.loads) { + if (executor.run.get()) { + config.compositeStrategy.findSuitableResource(load, config.resources) + } } + } finally { + ioHandler.writeToJSONFile( + config.compositeStrategy.benchmarkExecutor.results, + "${resultsFolder}exp${this.config.executionId}-result" + ) } - ioHandler.writeToJSONFile( - config.compositeStrategy.benchmarkExecutor.results, - "${resultsFolder}exp${this.config.executionId}-result" - ) } private fun getAndIncrementExecutionID(fileURL: String): Int { diff --git a/theodolite/src/main/kotlin/theodolite/execution/TheodoliteStandalone.kt b/theodolite/src/main/kotlin/theodolite/execution/TheodoliteStandalone.kt index abb6e7f6ed454f748e0fe50d6a24214b06649d47..1bbf3e01f461a19dbe588aedd41be63b84c86162 100644 --- a/theodolite/src/main/kotlin/theodolite/execution/TheodoliteStandalone.kt +++ b/theodolite/src/main/kotlin/theodolite/execution/TheodoliteStandalone.kt @@ -4,6 +4,8 @@ import mu.KotlinLogging import theodolite.benchmark.BenchmarkExecution import theodolite.benchmark.KubernetesBenchmark import theodolite.util.YamlParserFromFile +import theodolite.util.EvaluationFailedException +import theodolite.util.ExecutionFailedException import kotlin.concurrent.thread import kotlin.system.exitProcess @@ -49,8 +51,14 @@ class TheodoliteStandalone { val shutdown = thread(start = false) { Shutdown(benchmarkExecution, benchmark).run() } Runtime.getRuntime().addShutdownHook(shutdown) - val executor = TheodoliteExecutor(benchmarkExecution, benchmark) - executor.run() + try { + TheodoliteExecutor(benchmarkExecution, benchmark).run() + } catch (e: EvaluationFailedException) { + logger.error { "Evaluation failed with error: ${e.message}" } + }catch (e: ExecutionFailedException) { + logger.error { "Execution failed with error: ${e.message}" } + } + logger.info { "Theodolite finished" } Runtime.getRuntime().removeShutdownHook(shutdown) exitProcess(0) diff --git a/theodolite/src/main/kotlin/theodolite/execution/operator/ClusterSetup.kt b/theodolite/src/main/kotlin/theodolite/execution/operator/ClusterSetup.kt index c3a2b7b25ed71e797c45d8b497bad6cad15e21e8..6987372f96a6d956378a928011be9b5406590a16 100644 --- a/theodolite/src/main/kotlin/theodolite/execution/operator/ClusterSetup.kt +++ b/theodolite/src/main/kotlin/theodolite/execution/operator/ClusterSetup.kt @@ -54,11 +54,8 @@ class ClusterSetup( benchmark.spec.name = benchmark.metadata.name Shutdown(execution.spec, benchmark.spec).start() } else { - logger.error { - "Execution with state ${States.RUNNING.value} was found, but no corresponding benchmark. " + - "Could not initialize cluster." - } - throw IllegalStateException("Cluster state is invalid, required Benchmark for running execution not found.") + throw IllegalStateException("Execution with state ${States.RUNNING.value} was found, but no corresponding benchmark. " + + "Could not initialize cluster.") } } } diff --git a/theodolite/src/main/kotlin/theodolite/execution/operator/EventCreator.kt b/theodolite/src/main/kotlin/theodolite/execution/operator/EventCreator.kt new file mode 100644 index 0000000000000000000000000000000000000000..fab098ebd5fe765a455d787ddb7fcbfbb6c9ffc7 --- /dev/null +++ b/theodolite/src/main/kotlin/theodolite/execution/operator/EventCreator.kt @@ -0,0 +1,60 @@ +package theodolite.execution.operator + +import io.fabric8.kubernetes.api.model.EventBuilder +import io.fabric8.kubernetes.api.model.EventSource +import io.fabric8.kubernetes.api.model.ObjectReference +import io.fabric8.kubernetes.client.DefaultKubernetesClient +import io.fabric8.kubernetes.client.NamespacedKubernetesClient +import mu.KotlinLogging +import theodolite.util.Configuration +import java.time.Instant +import java.util.* +import kotlin.NoSuchElementException +private val logger = KotlinLogging.logger {} + +class EventCreator { + val client: NamespacedKubernetesClient = DefaultKubernetesClient().inNamespace(Configuration.NAMESPACE) + + fun createEvent(executionName: String, type: String, message: String, reason: String) { + val uuid = UUID.randomUUID().toString() + try { + val objectRef = buildObjectReference(executionName) + val event = EventBuilder() + .withNewMetadata() + .withName(uuid) + .endMetadata() + .withMessage(message) + .withReason(reason) + .withType(type) + .withFirstTimestamp(Instant.now().toString()) // TODO change datetime format + .build() + + val source = EventSource() + source.component = Configuration.COMPONENT_NAME + event.source = source + + event.involvedObject = objectRef + client.v1().events().inNamespace(Configuration.NAMESPACE).createOrReplace(event) + } catch (e: NoSuchElementException) { + logger.warn {"Could not create event: type: $type, message: $message, reason: $reason, no corresponding execution found."} + } + } + + private fun buildObjectReference(executionName: String): ObjectReference { + val exec = TheodoliteOperator() + .getExecutionClient(client = client) + .list() + .items + .first{it.metadata.name == executionName} + + val objectRef = ObjectReference() + objectRef.apiVersion = exec.apiVersion + objectRef.kind = exec.kind + objectRef.uid = exec.metadata.uid + objectRef.name = exec.metadata.name + objectRef.namespace = exec.metadata.namespace + objectRef.resourceVersion = exec.metadata.resourceVersion + + return objectRef + } +} \ No newline at end of file diff --git a/theodolite/src/main/kotlin/theodolite/execution/operator/TheodoliteController.kt b/theodolite/src/main/kotlin/theodolite/execution/operator/TheodoliteController.kt index 8ead498a727a9ac6d88dab22cf99b0f2dd51e74a..9058f1f314be9e71e882dff789e914bc4085c6f2 100644 --- a/theodolite/src/main/kotlin/theodolite/execution/operator/TheodoliteController.kt +++ b/theodolite/src/main/kotlin/theodolite/execution/operator/TheodoliteController.kt @@ -5,9 +5,11 @@ import io.fabric8.kubernetes.client.dsl.Resource import mu.KotlinLogging import theodolite.benchmark.BenchmarkExecution import theodolite.benchmark.KubernetesBenchmark +import theodolite.execution.ExecutionModes import theodolite.execution.TheodoliteExecutor import theodolite.model.crd.* import theodolite.patcher.ConfigOverrideModifier +import theodolite.util.ExecutionFailedException import theodolite.util.ExecutionStateComparator import java.lang.Thread.sleep @@ -94,9 +96,18 @@ class TheodoliteController( States.RUNNING -> { executionStateHandler.setExecutionState(execution.name, States.FINISHED) logger.info { "Execution of ${execution.name} is finally stopped." } + } + else -> { + executionStateHandler.setExecutionState(execution.name, States.FAILURE) + logger.warn { "Unexpected execution state, set state to ${States.FAILURE.value}" } } } } catch (e: Exception) { + EventCreator().createEvent( + executionName = execution.name, + type = "WARNING", + reason = "Execution failed", + message = "An error occurs while executing: ${e.message}") logger.error { "Failure while executing execution ${execution.name} with benchmark ${benchmark.name}." } logger.error { "Problem is: $e" } executionStateHandler.setExecutionState(execution.name, States.FAILURE) diff --git a/theodolite/src/main/kotlin/theodolite/execution/operator/TheodoliteOperator.kt b/theodolite/src/main/kotlin/theodolite/execution/operator/TheodoliteOperator.kt index 1cb1d8e4bc5ffee89e47aa23546b6e6100d33e9b..d078b52c4c72d71d4f9f773831ea1a0736be6c99 100644 --- a/theodolite/src/main/kotlin/theodolite/execution/operator/TheodoliteOperator.kt +++ b/theodolite/src/main/kotlin/theodolite/execution/operator/TheodoliteOperator.kt @@ -11,6 +11,7 @@ import theodolite.model.crd.BenchmarkCRD import theodolite.model.crd.BenchmarkExecutionList import theodolite.model.crd.ExecutionCRD import theodolite.model.crd.KubernetesBenchmarkList +import theodolite.util.Configuration private const val DEFAULT_NAMESPACE = "default" @@ -27,7 +28,7 @@ private val logger = KotlinLogging.logger {} * **See Also:** [Kubernetes Operator Pattern](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) */ class TheodoliteOperator { - private val namespace = System.getenv("NAMESPACE") ?: DEFAULT_NAMESPACE + private val namespace = Configuration.NAMESPACE private val client: NamespacedKubernetesClient = DefaultKubernetesClient().inNamespace(namespace) private lateinit var controller: TheodoliteController @@ -37,7 +38,7 @@ class TheodoliteOperator { fun start() { LeaderElector( client = client, - name = "theodolite-operator" // TODO(make leaslock name configurable via env var) + name = Configuration.COMPONENT_NAME ) .getLeadership(::startOperator) } @@ -115,7 +116,7 @@ class TheodoliteOperator { return this.controller } - private fun getExecutionClient(client: NamespacedKubernetesClient): MixedOperation< + fun getExecutionClient(client: NamespacedKubernetesClient): MixedOperation< ExecutionCRD, BenchmarkExecutionList, Resource<ExecutionCRD>> { diff --git a/theodolite/src/main/kotlin/theodolite/k8s/CustomResourceWrapper.kt b/theodolite/src/main/kotlin/theodolite/k8s/CustomResourceWrapper.kt index a7f18ce6f0f860865a18e0ac324ea5819c61c918..797ed88389947d66aa626ba2ef3fdf6732f8369d 100644 --- a/theodolite/src/main/kotlin/theodolite/k8s/CustomResourceWrapper.kt +++ b/theodolite/src/main/kotlin/theodolite/k8s/CustomResourceWrapper.kt @@ -33,7 +33,7 @@ class CustomResourceWrapper( client.customResource(this.context) .delete(client.configuration.namespace, this.getName()) } catch (e: Exception) { - logger.warn { "Could not delete service monitor" } + logger.warn { "Could not delete custom resource" } } } diff --git a/theodolite/src/main/kotlin/theodolite/k8s/TopicManager.kt b/theodolite/src/main/kotlin/theodolite/k8s/TopicManager.kt index 8e83883fc881db0f7e2b1b75b2fb7c7322a11a00..f2afd71f6e4b4cf8e7106a8fc8a9bd113d9f36e6 100644 --- a/theodolite/src/main/kotlin/theodolite/k8s/TopicManager.kt +++ b/theodolite/src/main/kotlin/theodolite/k8s/TopicManager.kt @@ -30,8 +30,7 @@ class TopicManager(private val kafkaConfig: Map<String, Any>) { result = kafkaAdmin.createTopics(newTopics) result.all().get() // wait for the future to be completed } catch (e: Exception) { // TopicExistsException - logger.warn(e) { "Error during topic creation." } - logger.debug { e } // TODO remove due to attached exception to warn log? + logger.warn { "Error during topic creation. Error is: ${e.message}" } logger.info { "Remove existing topics." } delete(newTopics.map { topic -> topic.name() }, kafkaAdmin) logger.info { "Will retry the topic creation in ${RETRY_TIME / 1000} seconds." } @@ -94,7 +93,7 @@ class TopicManager(private val kafkaConfig: Map<String, Any>) { }" } } catch (e: Exception) { - logger.error(e) { "Error while removing topics: $e" } + logger.error { "Error while removing topics: ${e.message}" } logger.info { "Existing topics are: ${kafkaAdmin.listTopics().names().get()}." } } diff --git a/theodolite/src/main/kotlin/theodolite/util/Configuration.kt b/theodolite/src/main/kotlin/theodolite/util/Configuration.kt new file mode 100644 index 0000000000000000000000000000000000000000..dac3b943e69bd7e208d318f2a788275f19db11e4 --- /dev/null +++ b/theodolite/src/main/kotlin/theodolite/util/Configuration.kt @@ -0,0 +1,18 @@ +package theodolite.util + +import theodolite.execution.ExecutionModes + +// Defaults +private const val DEFAULT_NAMESPACE = "default" +private const val DEFAULT_COMPONENT_NAME = "theodolite-operator" + + +class Configuration( +) { + companion object { + val NAMESPACE = System.getenv("NAMESPACE") ?: DEFAULT_NAMESPACE + val COMPONENT_NAME = System.getenv("COMPONENT_NAME") ?: DEFAULT_COMPONENT_NAME + val EXECUTION_MODE = System.getenv("MODE") ?: ExecutionModes.STANDALONE.value + } + +} diff --git a/theodolite/src/main/kotlin/theodolite/util/DeploymentFailedException.kt b/theodolite/src/main/kotlin/theodolite/util/DeploymentFailedException.kt index 9439024eede8fd66d04a85af16501fc136dc3d8f..9f4caedf3db1e09dca7924bf0035c6ace0b835d7 100644 --- a/theodolite/src/main/kotlin/theodolite/util/DeploymentFailedException.kt +++ b/theodolite/src/main/kotlin/theodolite/util/DeploymentFailedException.kt @@ -1,4 +1,4 @@ package theodolite.util -class DeploymentFailedException(message: String, e: Exception? = null): Exception(message, e) \ No newline at end of file +open class DeploymentFailedException(message: String, e: Exception? = null) : TheodoliteException(message,e) diff --git a/theodolite/src/main/kotlin/theodolite/util/EvaluationFailedException.kt b/theodolite/src/main/kotlin/theodolite/util/EvaluationFailedException.kt new file mode 100644 index 0000000000000000000000000000000000000000..c67ed7ffd79afc733a97dae05c3203f8e78722ea --- /dev/null +++ b/theodolite/src/main/kotlin/theodolite/util/EvaluationFailedException.kt @@ -0,0 +1,4 @@ +package theodolite.util + +class EvaluationFailedException(message: String, e: Exception? = null) : ExecutionFailedException(message,e) { +} diff --git a/theodolite/src/main/kotlin/theodolite/util/ExecutionFailedException.kt b/theodolite/src/main/kotlin/theodolite/util/ExecutionFailedException.kt new file mode 100644 index 0000000000000000000000000000000000000000..6566a451a3e273214f59962531b6bd17b33a850d --- /dev/null +++ b/theodolite/src/main/kotlin/theodolite/util/ExecutionFailedException.kt @@ -0,0 +1,4 @@ +package theodolite.util + +open class ExecutionFailedException(message: String, e: Exception? = null) : TheodoliteException(message,e) { +} \ No newline at end of file diff --git a/theodolite/src/main/kotlin/theodolite/util/InvalidPatcherConfigurationException.kt b/theodolite/src/main/kotlin/theodolite/util/InvalidPatcherConfigurationException.kt index 81ea227d0d9871c2420a414d81749a34b97676b8..d02948ad341207051c4653ba9400ac0ffe5b03aa 100644 --- a/theodolite/src/main/kotlin/theodolite/util/InvalidPatcherConfigurationException.kt +++ b/theodolite/src/main/kotlin/theodolite/util/InvalidPatcherConfigurationException.kt @@ -1,4 +1,3 @@ package theodolite.util -class InvalidPatcherConfigurationException(message: String, e: Exception? = null) : Exception(message, e) - +class InvalidPatcherConfigurationException(message: String, e: Exception? = null) : DeploymentFailedException(message,e) diff --git a/theodolite/src/main/kotlin/theodolite/util/PrometheusResponse.kt b/theodolite/src/main/kotlin/theodolite/util/PrometheusResponse.kt index bf33fcf6104645727a13b92cf3a13d36e04a10c6..9b0b0dd4e0a5a48072ca576e874cb850c5f8df3b 100644 --- a/theodolite/src/main/kotlin/theodolite/util/PrometheusResponse.kt +++ b/theodolite/src/main/kotlin/theodolite/util/PrometheusResponse.kt @@ -23,7 +23,7 @@ data class PrometheusResponse( * The format of the returned list is: `[[ group, timestamp, value ], [ group, timestamp, value ], ... ]` */ fun getResultAsList(): List<List<String>> { - val group = data?.result?.get(0)?.metric?.group.toString() + val group = data?.result?.get(0)?.metric?.toString()!! val values = data?.result?.get(0)?.values val result = mutableListOf<List<String>>() @@ -64,18 +64,9 @@ data class PromResult( /** * Label of the metric */ - var metric: PromMetric? = null, + var metric: Map<String, String>? = null, /** * Values of the metric (e.g. [ [ <unix_time>, "<sample_value>" ], ... ]) */ var values: List<Any>? = null -) - -/** - * Corresponds to the metric field in the range-vector result format of a Prometheus range-query response. - */ -@RegisterForReflection -data class PromMetric( - var group: String? = null -) - +) \ No newline at end of file diff --git a/theodolite/src/main/kotlin/theodolite/util/TheodoliteException.kt b/theodolite/src/main/kotlin/theodolite/util/TheodoliteException.kt new file mode 100644 index 0000000000000000000000000000000000000000..fc7453bae6aaa4c5c526eee72c006562ea887eb5 --- /dev/null +++ b/theodolite/src/main/kotlin/theodolite/util/TheodoliteException.kt @@ -0,0 +1,3 @@ +package theodolite.util + +open class TheodoliteException (message: String, e: Exception? = null) : Exception(message,e) \ No newline at end of file diff --git a/theodolite/src/test/kotlin/theodolite/CompositeStrategyTest.kt b/theodolite/src/test/kotlin/theodolite/CompositeStrategyTest.kt index 49131352cfe517a382ddd7aa1be09d3fbe317466..580d9e747bde687a91ffb1bce2e7c9dfb6f166a2 100644 --- a/theodolite/src/test/kotlin/theodolite/CompositeStrategyTest.kt +++ b/theodolite/src/test/kotlin/theodolite/CompositeStrategyTest.kt @@ -31,7 +31,7 @@ class CompositeStrategyTest { val results = Results() val benchmark = TestBenchmark() val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo() - val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, sloChecker, 0, 0, 5) + val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, listOf(sloChecker), 0, 0, 5) val linearSearch = LinearSearch(benchmarkExecutor) val lowerBoundRestriction = LowerBoundRestriction(results) val strategy = @@ -65,7 +65,7 @@ class CompositeStrategyTest { val benchmark = TestBenchmark() val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo() val benchmarkExecutorImpl = - TestBenchmarkExecutorImpl(mockResults, benchmark, results, sloChecker, 0, 0, 0) + TestBenchmarkExecutorImpl(mockResults, benchmark, results, listOf(sloChecker), 0, 0, 0) val binarySearch = BinarySearch(benchmarkExecutorImpl) val lowerBoundRestriction = LowerBoundRestriction(results) val strategy = @@ -98,7 +98,7 @@ class CompositeStrategyTest { val results = Results() val benchmark = TestBenchmark() val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo() - val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, sloChecker, 0, 0, 0) + val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, listOf(sloChecker), 0, 0, 0) val binarySearch = BinarySearch(benchmarkExecutor) val lowerBoundRestriction = LowerBoundRestriction(results) val strategy = diff --git a/theodolite/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt b/theodolite/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt index cbd2d5926d61b0bfd4de6fab0c14422ddf88f190..2efddc48cb93a0870d1716c58a7018145c16e2ff 100644 --- a/theodolite/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt +++ b/theodolite/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt @@ -12,7 +12,7 @@ class TestBenchmarkExecutorImpl( private val mockResults: Array<Array<Boolean>>, benchmark: Benchmark, results: Results, - slo: BenchmarkExecution.Slo, + slo: List<BenchmarkExecution.Slo>, executionId: Int, loadGenerationDelay: Long, afterTeardownDelay: Long @@ -22,11 +22,12 @@ class TestBenchmarkExecutorImpl( results, executionDuration = Duration.ofSeconds(1), configurationOverrides = emptyList(), - slo = slo, + slos = slo, repetitions = 1, executionId = executionId, loadGenerationDelay = loadGenerationDelay, - afterTeardownDelay = afterTeardownDelay + afterTeardownDelay = afterTeardownDelay, + executionName = "test-execution" ) { override fun runExperiment(load: LoadDimension, res: Resource): Boolean {