diff --git a/slope-evaluator/README.md b/slope-evaluator/README.md
index 25c02b42e6a6eb4611972febf935403b8b8703c8..69831cd5f83665735c586ab25493ed257b93c2ad 100644
--- a/slope-evaluator/README.md
+++ b/slope-evaluator/README.md
@@ -24,3 +24,36 @@ Run the Docker image:
You can set the `HOST` and the `PORT` (and a lot of more parameters) via environment variables. Default is `0.0.0.0:80`.
For more information see [here](https://github.com/tiangolo/uvicorn-gunicorn-fastapi-docker#advanced-usage).
+
+## API Documentation
+
+The running webserver provides a REST API with the following route:
+
+* /evaluate-slope
+ * Method: POST
+ * Body:
+ * total_lag
+ * threshold
+ * warmup
+
+The body of the request must be a JSON string that satisfies the following conditions:
+
+* **total_lag**: This property is based on the [Range Vector type](https://www.prometheus.io/docs/prometheus/latest/querying/api/#range-vectors) from Prometheus and must have the following JSON structure:
+ ```
+ {
+ "metric": {
+ "group": "<label_value>"
+ },
+ "values": [
+ [
+ <unix_timestamp>,
+ "<sample_value>"
+ ]
+ ]
+ }
+ ```
+ * The `<label_value>` provided in "metric.group" must be equal to the id of the Kafka consumer group.
+ * The `<unix_timestamp>` provided as the first element of each element in the "values" array must be the timestamp of the measurement value in seconds (with optional decimal precision)
+ * The `<sample_value>` must be the measurement value as string.
+* **threshold**: Must be an unsigned integer that specifies the threshold for the SLO evaluation. The SLO is considered fulfilled, if the result value is below the threshold. If the result value is equal or above the threshold, the SLO is considered not fulfilled.
+* **warmup**: Specifieds the warmup time in seconds that are ignored for evaluating the SLO.
\ No newline at end of file
diff --git a/theodolite-quarkus/build.gradle b/theodolite-quarkus/build.gradle
index 7f32c0c5bab25b1d62b6cd9aa6f5db5eb6d78e47..2f58f4c8a587a4a17cf8586e507c0079d8cb56e6 100644
--- a/theodolite-quarkus/build.gradle
+++ b/theodolite-quarkus/build.gradle
@@ -31,7 +31,7 @@ dependencies {
}
group 'theodolite'
-version '1.0.0-SNAPSHOT'
+version '0.5.0-SNAPSHOT'
java {
sourceCompatibility = JavaVersion.VERSION_11
diff --git a/theodolite-quarkus/config/example-benchmark-yaml-resource.yaml b/theodolite-quarkus/config/example-benchmark-yaml-resource.yaml
index 60eb3bb9c31e3eab3e70f916b450372d56db4968..d507934fb87127748a1f6e22177f9dd9430f5f0b 100644
--- a/theodolite-quarkus/config/example-benchmark-yaml-resource.yaml
+++ b/theodolite-quarkus/config/example-benchmark-yaml-resource.yaml
@@ -19,6 +19,8 @@ loadTypes:
resource: "uc1-load-generator-deployment.yaml"
container: "workload-generator"
variableName: "NUM_SENSORS"
+ - type: "NumSensorsLoadGeneratorReplicaPatcher"
+ resource: "uc1-load-generator-deployment.yaml"
kafkaConfig:
bootstrapServer: "theodolite-cp-kafka:9092"
topics:
diff --git a/theodolite-quarkus/config/example-operator-benchmark.yaml b/theodolite-quarkus/config/example-operator-benchmark.yaml
index 419042fdd0b1e58fed4d402b4bb329d54602d23f..93b42e8690c3b9f432aea1c6711ee0118d14adb3 100644
--- a/theodolite-quarkus/config/example-operator-benchmark.yaml
+++ b/theodolite-quarkus/config/example-operator-benchmark.yaml
@@ -23,6 +23,8 @@ loadTypes:
resource: "uc1-load-generator-deployment.yaml"
container: "workload-generator"
variableName: "NUM_SENSORS"
+ - type: "NumSensorsLoadGeneratorReplicaPatcher"
+ resource: "uc1-load-generator-deployment.yaml"
kafkaConfig:
bootstrapServer: "theodolite-cp-kafka:9092"
topics:
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmarkDeployment.kt b/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmarkDeployment.kt
index 3d73dd67ddcb5363e752d9a0a65d5a8bff98b4e9..0de00fb5007b646240661310d037e34ef1ea1ae2 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmarkDeployment.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmarkDeployment.kt
@@ -46,10 +46,10 @@ class KubernetesBenchmarkDeployment(
* - Remove the [KubernetesResource]s.
*/
override fun teardown() {
- KafkaLagExporterRemover(client).remove(LABEL)
- kafkaController.removeTopics(this.topics.map { topic -> topic.name() })
resources.forEach {
kubernetesManager.remove(it)
}
+ kafkaController.removeTopics(this.topics.map { topic -> topic.name() })
+ KafkaLagExporterRemover(client).remove(LABEL)
}
}
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt
index efbfe4df41d70b1b35ea91667c8e0c85d8b58953..1ab8e215216cd6f2d3640ca113d438e4ed821042 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt
@@ -29,7 +29,7 @@ class BenchmarkExecutorImpl(
try {
benchmarkDeployment.setup()
this.waitAndLog()
- } catch(e: Exception) {
+ } catch (e: Exception) {
logger.error { "Error while setup experiment." }
logger.error { "Error is: $e" }
this.run.set(false)
@@ -40,10 +40,20 @@ class BenchmarkExecutorImpl(
*/
if (this.run.get()) {
result =
- AnalysisExecutor(slo = slo, executionId = executionId).analyze(load = load, res = res, executionDuration = executionDuration)
+ AnalysisExecutor(slo = slo, executionId = executionId).analyze(
+ load = load,
+ res = res,
+ executionDuration = executionDuration
+ )
this.results.setResult(Pair(load, res), result)
}
- benchmarkDeployment.teardown()
+
+ try {
+ benchmarkDeployment.teardown()
+ } catch (e: Exception) {
+ logger.warn { "Error while teardown of deplyoment" }
+ logger.debug { "The Teardowm failed cause of: $e " }
+ }
return result
}
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/k8s/K8sManager.kt b/theodolite-quarkus/src/main/kotlin/theodolite/k8s/K8sManager.kt
index ac2165303f083be066c4398e294e456f1d268dad..f8f7f9800ecb2b19f56d3dfe85c8f9cfc153b9f5 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/k8s/K8sManager.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/k8s/K8sManager.kt
@@ -39,16 +39,35 @@ class K8sManager(private val client: NamespacedKubernetesClient) {
*/
fun remove(resource: KubernetesResource) {
when (resource) {
- is Deployment ->
+ is Deployment -> {
+ val label = resource.spec.selector.matchLabels["app"]!!
this.client.apps().deployments().delete(resource)
+ blockUntilDeleted(label)
+ }
is Service ->
this.client.services().delete(resource)
is ConfigMap ->
this.client.configMaps().delete(resource)
- is StatefulSet ->
+ is StatefulSet -> {
+ val label = resource.spec.selector.matchLabels["app"]!!
this.client.apps().statefulSets().delete(resource)
+ blockUntilDeleted(label)
+ }
is ServiceMonitorWrapper -> resource.delete(client)
else -> throw IllegalArgumentException("Unknown Kubernetes resource.")
}
}
+
+
+ private fun blockUntilDeleted(label: String) {
+ var deleted = false
+ do {
+ val pods = this.client.pods().withLabel(label).list().items
+ if (pods.isNullOrEmpty()) {
+ deleted = true
+ }
+ Thread.sleep(1000)
+ } while (!deleted)
+ }
+
}
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/k8s/TopicManager.kt b/theodolite-quarkus/src/main/kotlin/theodolite/k8s/TopicManager.kt
index 2cbe16b5a460f0caf55bf2c99bc84dc0b3b5ac69..e82a133b3e5439e72987f3db107f4e81a1d01cd5 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/k8s/TopicManager.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/k8s/TopicManager.kt
@@ -2,9 +2,12 @@ package theodolite.k8s
import mu.KotlinLogging
import org.apache.kafka.clients.admin.AdminClient
+import org.apache.kafka.clients.admin.CreateTopicsResult
import org.apache.kafka.clients.admin.NewTopic
+import java.lang.Thread.sleep
private val logger = KotlinLogging.logger {}
+private const val RETRY_TIME = 2000L
/**
* Manages the topics related tasks
@@ -12,17 +15,32 @@ private val logger = KotlinLogging.logger {}
* @constructor Creates a KafkaAdminClient
*/
class TopicManager(private val kafkaConfig: HashMap<String, Any>) {
-
/**
* Creates topics.
* @param newTopics List of all Topic that should be created
*/
fun createTopics(newTopics: Collection<NewTopic>) {
- var kafkaAdmin: AdminClient = AdminClient.create(this.kafkaConfig)
- val result = kafkaAdmin.createTopics(newTopics)
- result.all().get()// wait for the future object
+ val kafkaAdmin: AdminClient = AdminClient.create(this.kafkaConfig)
+ lateinit var result: CreateTopicsResult
+
+ do {
+ var retryCreation = false
+ try {
+ result = kafkaAdmin.createTopics(newTopics)
+ result.all().get()// wait for the future object
+
+ } catch (e: Exception) {
+ delete(newTopics.map { topic -> topic.name() }, kafkaAdmin)
+ logger.warn { "Error during topic creation." }
+ logger.debug { e }
+ logger.warn { "Will retry the topic creation after 2 seconds" }
+ sleep(RETRY_TIME)
+ retryCreation = true
+ }
+ } while (retryCreation)
+
logger.info {
- "Topics created finished with result: ${
+ "Topics creation finished with result: ${
result.values().map { it -> it.key + ": " + it.value.isDone }
.joinToString(separator = ",")
} "
@@ -35,20 +53,40 @@ class TopicManager(private val kafkaConfig: HashMap<String, Any>) {
* @param topics List of names with the topics to remove.
*/
fun removeTopics(topics: List<String>) {
- var kafkaAdmin: AdminClient = AdminClient.create(this.kafkaConfig)
- try {
- val result = kafkaAdmin.deleteTopics(topics)
- result.all().get() // wait for the future object
- logger.info {
- "\"Topics deletion finished with result: ${
- result.values().map { it -> it.key + ": " + it.value.isDone }
- .joinToString(separator = ",")
- } "
+ val kafkaAdmin: AdminClient = AdminClient.create(this.kafkaConfig)
+ delete(topics, kafkaAdmin)
+ kafkaAdmin.close()
+ }
+
+ private fun delete(topics: List<String>, kafkaAdmin: AdminClient) {
+ var deleted = false
+
+ while (!deleted) {
+ try {
+ val result = kafkaAdmin.deleteTopics(topics)
+ result.all().get() // wait for the future object
+ logger.info {
+ "Topics deletion finished with result: ${
+ result.values().map { it -> it.key + ": " + it.value.isDone }
+ .joinToString(separator = ",")
+ }"
+ }
+ } catch (e: Exception) {
+ logger.error { "Error while removing topics: $e" }
+ logger.debug { "Existing topics are: ${kafkaAdmin.listTopics()}." }
+ }
+
+ val toDelete = topics.filter { topic ->
+ kafkaAdmin.listTopics().names().get().contains(topic)
+ }
+
+ if (toDelete.isNullOrEmpty()) {
+ deleted = true
+ } else {
+ logger.info { "Deletion of kafka topics failed retrying in 2 seconds" }
+ sleep(RETRY_TIME)
}
- } catch (e: Exception) {
- logger.error { "Error while removing topics: $e" }
- logger.debug { "Existing topics are: ${kafkaAdmin.listTopics()}." }
}
- kafkaAdmin.close()
}
+
}
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/patcher/EnvVarPatcher.kt b/theodolite-quarkus/src/main/kotlin/theodolite/patcher/EnvVarPatcher.kt
index 16bd9aa34127b79c97e8f9d195d4757145a3fa93..b640df1da2ca1c139bb5b02e9e42bad9e7d08d74 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/patcher/EnvVarPatcher.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/patcher/EnvVarPatcher.kt
@@ -2,7 +2,6 @@ package theodolite.patcher
import io.fabric8.kubernetes.api.model.Container
import io.fabric8.kubernetes.api.model.EnvVar
-import io.fabric8.kubernetes.api.model.EnvVarSource
import io.fabric8.kubernetes.api.model.KubernetesResource
import io.fabric8.kubernetes.api.model.apps.Deployment
@@ -39,7 +38,9 @@ class EnvVarPatcher(
val x = container.env.filter { envVar -> envVar.name == k }
if (x.isEmpty()) {
- val newVar = EnvVar(k, v, EnvVarSource())
+ val newVar = EnvVar()
+ newVar.name = k
+ newVar.value = v
container.env.add(newVar)
} else {
x.forEach {
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/patcher/NumNestedGroupsLoadGeneratorReplicaPatcher.kt b/theodolite-quarkus/src/main/kotlin/theodolite/patcher/NumNestedGroupsLoadGeneratorReplicaPatcher.kt
new file mode 100644
index 0000000000000000000000000000000000000000..7cf56f8452949e387a186aa8f8c962e1ee1aad15
--- /dev/null
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/patcher/NumNestedGroupsLoadGeneratorReplicaPatcher.kt
@@ -0,0 +1,20 @@
+package theodolite.patcher
+
+import io.fabric8.kubernetes.api.model.KubernetesResource
+import io.fabric8.kubernetes.api.model.apps.Deployment
+import kotlin.math.pow
+
+private const val NUM_SENSORS = 4.0
+private const val LOAD_GEN_MAX_RECORDS = 150000
+
+class NumNestedGroupsLoadGeneratorReplicaPatcher(private val k8sResource: KubernetesResource) : AbstractPatcher(k8sResource) {
+ override fun <String> patch(value: String) {
+ if (k8sResource is Deployment) {
+ if (value is kotlin.String) {
+ val approxNumSensors = NUM_SENSORS.pow(Integer.parseInt(value).toDouble())
+ val loadGenInstances = (approxNumSensors + LOAD_GEN_MAX_RECORDS -1) / LOAD_GEN_MAX_RECORDS
+ this.k8sResource.spec.replicas = loadGenInstances.toInt()
+ }
+ }
+ }
+}
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/patcher/NumSensorsLoadGeneratorReplicaPatcher.kt b/theodolite-quarkus/src/main/kotlin/theodolite/patcher/NumSensorsLoadGeneratorReplicaPatcher.kt
new file mode 100644
index 0000000000000000000000000000000000000000..6f2ebcb8b1eb37801c7f6bb2f28c251a07ae44e8
--- /dev/null
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/patcher/NumSensorsLoadGeneratorReplicaPatcher.kt
@@ -0,0 +1,17 @@
+package theodolite.patcher
+
+import io.fabric8.kubernetes.api.model.KubernetesResource
+import io.fabric8.kubernetes.api.model.apps.Deployment
+
+private const val LOAD_GEN_MAX_RECORDS = 150000
+
+class NumSensorsLoadGeneratorReplicaPatcher(private val k8sResource: KubernetesResource) : AbstractPatcher(k8sResource) {
+ override fun <String> patch(value: String) {
+ if (k8sResource is Deployment) {
+ if (value is kotlin.String) {
+ val loadGenInstances = (Integer.parseInt(value) + LOAD_GEN_MAX_RECORDS - 1) / LOAD_GEN_MAX_RECORDS
+ this.k8sResource.spec.replicas = loadGenInstances
+ }
+ }
+ }
+}
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/patcher/PatcherFactory.kt b/theodolite-quarkus/src/main/kotlin/theodolite/patcher/PatcherFactory.kt
index 5b1274ea6f57b8bd0594ddf8b6c1f3410b3fa107..2ee1f6c7b46322cb0f8de03c37aabe64ccf0ba5a 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/patcher/PatcherFactory.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/patcher/PatcherFactory.kt
@@ -30,6 +30,8 @@ class PatcherFactory {
k8sResources.filter { it.first == patcherDefinition.resource }.map { resource -> resource.second }[0]
return when (patcherDefinition.type) {
"ReplicaPatcher" -> ReplicaPatcher(resource)
+ "NumNestedGroupsLoadGeneratorReplicaPatcher" -> NumNestedGroupsLoadGeneratorReplicaPatcher(resource)
+ "NumSensorsLoadGeneratorReplicaPatcher" -> NumSensorsLoadGeneratorReplicaPatcher(resource)
"EnvVarPatcher" -> EnvVarPatcher(resource, patcherDefinition.container, patcherDefinition.variableName)
"NodeSelectorPatcher" -> NodeSelectorPatcher(resource, patcherDefinition.variableName)
"ResourceLimitPatcher" -> ResourceLimitPatcher(