Merge branch 'theodolite-kotlin' of git.se.informatik.uni-kiel.de:she/spesb into theodolite-kotlin

.gitlab-ci.yml

@@ -51,7 +51,7 @@ build-benchmarks:
       - "theodolite-benchmarks/build/libs/*.jar"
       - "theodolite-benchmarks/*/build/libs/*.jar"
       - "theodolite-benchmarks/*/build/distributions/*.tar"
-    expire_in: 1 day
+    expire_in: 6 hours
 
 test-benchmarks:
   stage: test
@@ -234,7 +234,7 @@ build-theodolite-jvm:
     paths:
       - "theodolite-quarkus/build/lib/*"
       - "theodolite-quarkus/build/*-runner.jar"
-    expire_in: 1 day
+    expire_in: 6 hours
 
 build-theodolite-native:
   stage: build
@@ -245,7 +245,7 @@ build-theodolite-native:
   artifacts:
     paths:
       - "theodolite-quarkus/build/*-runner"
-    expire_in: 1 day
+    expire_in: 6 hours
 
 test-theodolite:
   stage: test

execution/helm/README.md

@@ -6,11 +6,9 @@ Install the chart via:
 
 ```sh
 helm dependencies update .
-helm install my-confluent .
+helm install theodolite .
 ```
 
-**Please note: Theodolite currently uses hard-coded URLs, to connect to Kafka and Zookeeper. For that reason, the name of this chart must be `my-confluent`.** We will change this behavior soon.
-
 This chart installs requirements to execute benchmarks with Theodolite.
 
 Dependencies and subcharts:
@@ -27,7 +25,7 @@ Dependencies and subcharts:
 Test the installation:
 
 ```sh
-helm test <release-name>
+helm test theodolite
 ```
 
 Our test files are located [here](templates/../../theodolite-chart/templates/tests). Many subcharts have their own tests, these are also executed and are placed in the respective /templates folders. 
@@ -46,18 +44,18 @@ helm install theodolite . -f preconfigs/one-broker-values.yaml
 
 ## Uninstall this Chart
 
-To uninstall/delete the `my-release` deployment:
+To uninstall/delete the `theodolite` deployment:
 
 ```sh
-helm delete my-release
+helm delete theodolite
 ```
 
 This command does not remove the CRDs which are created by this chart. Remove them manually with:
 
 ```sh
 # CRDs from Theodolite
-kubectl delete crd execution
-kubectl delete crd benchmark
+kubectl delete crd executions.theodolite.com
+kubectl delete crd benchmarks.theodolite.com
 # CRDs from Prometheus operator (see https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack#uninstall-chart)
 kubectl delete crd alertmanagerconfigs.monitoring.coreos.com
 kubectl delete crd alertmanagers.monitoring.coreos.com

execution/helm/templates/theodolite/role.yaml

@@ -42,7 +42,7 @@ rules:
     - delete
     - list
     - create
-  {{- if .Values.operator.enabled -}}
+  {{- if .Values.operator.enabled }}
   - apiGroups:
     - theodolite.com
     resources: 

execution/helm/templates/theodolite/thedolite-operator.yaml

@@ -23,9 +23,19 @@ spec:
               value: {{ .Release.Namespace }}
             - name: MODE
               value: operator
+            - name: THEODOLITE_APP_RESOURCES
+              value: "./benchmark-resources"
+          volumeMounts:
+            - name: benchmark-resources
+              mountPath: /work/benchmark-resources
         - name: lag-analysis
           image: ghcr.io/cau-se/theodolite-slo-checker-lag-trend:theodolite-kotlin-latest
           ports:
           - containerPort: 80
             name: analysis
+      volumes:
+      - name: benchmark-resources
+        configMap:
+          name: benchmark-resources
+          optional: true
 {{- end }}

execution/helm/values.yaml

@@ -24,6 +24,9 @@ grafana:
   # Administrator credentials when not using an existing secret (see below)
   adminUser: admin
   adminPassword: admin
+  grafana.ini:
+    users:
+      default_theme: light
   ## Sidecars that collect the configmaps with specified label and stores the included files them into the respective folders
   ## Requires at least Grafana 5 to work and can't be used together with parameters dashboardProviders, datasources and dashboards
   sidecar:
@@ -145,8 +148,8 @@ kafka-lag-exporter:
   enabled: true
   nodeSelector: {}
   clusters:
-    - name: "my-confluent-cp-kafka"
-      bootstrapBrokers: "my-confluent-cp-kafka:9092"
+    - name: "theodolite-cp-kafka"
+      bootstrapBrokers: "theodolite-cp-kafka:9092"
 
   ## The interval between refreshing metrics
   pollIntervalSeconds: 15

execution/uc-application/aggregation-deployment.yaml

@@ -21,9 +21,9 @@ spec:
           name: jmx
         env:
         - name: KAFKA_BOOTSTRAP_SERVERS
-          value: "my-confluent-cp-kafka:9092"
+          value: "theodolite-cp-kafka:9092"
         - name: SCHEMA_REGISTRY_URL
-          value: "http://my-confluent-cp-schema-registry:8081"
+          value: "http://theodolite-cp-schema-registry:8081"
         - name: JAVA_OPTS
           value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555"
         - name: COMMIT_INTERVAL_MS # Set as default for the applications

execution/uc-workload-generator/workloadGenerator.yaml

@@ -32,6 +32,6 @@ spec:
         - name: KUBERNETES_DNS_NAME
           value: "titan-ccp-load-generator.$(KUBERNETES_NAMESPACE).svc.cluster.local"
         - name: KAFKA_BOOTSTRAP_SERVERS
-          value: "my-confluent-cp-kafka:9092"
+          value: "theodolite-cp-kafka:9092"
         - name: SCHEMA_REGISTRY_URL
-          value: "http://my-confluent-cp-schema-registry:8081"
+          value: "http://theodolite-cp-schema-registry:8081"

theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.flink.gradle

@@ -6,6 +6,23 @@ plugins {
 
 applicationDefaultJvmArgs = ["-Dlog4j.configuration=log4j.properties"]
 
+
+run.classpath = sourceSets.main.runtimeClasspath
+
+jar {
+    manifest {
+        attributes 'Built-By': System.getProperty('user.name'),
+                   'Build-Jdk': System.getProperty('java.version')
+    }
+}
+
+shadowJar {
+    configurations = [project.configurations.compile]
+    zip64 true
+}
+
+tasks.distZip.enabled = false
+
 ext {
   flinkVersion = '1.12.2'
   scalaBinaryVersion = '2.12'
@@ -48,17 +65,3 @@ dependencies {
     // Use JUnit test framework
     testImplementation 'junit:junit:4.12'
 }
-
-run.classpath = sourceSets.main.runtimeClasspath
-
-jar {
-    manifest {
-        attributes 'Built-By': System.getProperty('user.name'),
-                   'Build-Jdk': System.getProperty('java.version')
-    }
-}
-
-shadowJar {
-    configurations = [project.configurations.compile]
-    zip64 true
-}

theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.kstreams.gradle

@@ -6,6 +6,8 @@ plugins {
   id 'application'
 }
 
+tasks.distZip.enabled = false
+
 repositories {
   jcenter()
   maven {

theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.load-generator.gradle

@@ -6,6 +6,8 @@ plugins {
   id 'application'
 }
 
+tasks.distZip.enabled = false
+
 repositories {
   jcenter()
   maven {

theodolite-quarkus/config/aggregation-service.yaml

@@ -14,4 +14,4 @@ spec:
     targetPort: 80
     protocol: TCP
   - name: metrics
-    port: 9980
+    port: 5556

theodolite-quarkus/config/example-benchmark-yaml-resource.yaml

@@ -20,7 +20,7 @@ loadTypes:
         container: "workload-generator"
         variableName: "NUM_SENSORS"
 kafkaConfig:
-  bootstrapServer: "my-confluent-cp-kafka:9092"
+  bootstrapServer: "theodolite-cp-kafka:9092"
   topics:
     - name: "input"
       numPartitions: 40

theodolite-quarkus/config/example-execution-yaml-resource.yaml

@@ -2,36 +2,34 @@ name: example-execution
 benchmark: "uc1-kstreams"
 load:
   loadType: "NumSensors"
-  loadValues:
-    - 50000
+  loadValues: [25000, 50000, 75000, 100000, 125000, 150000]
 resources:
   resourceType: "Instances"
-  resourceValues:
-    - 1
+  resourceValues: [1, 2, 3, 4, 5]
 slos:
   - sloType: "lag trend"
-    threshold: 1000
+    threshold: 2000
     prometheusUrl: "http://prometheus-operated:9090"
     externalSloUrl: "http://localhost:80/evaluate-slope"
     offset: 0
-    warmup: 0
+    warmup: 60 # in seconds
 execution:
   strategy: "LinearSearch"
-  duration: 60
+  duration: 300 # in seconds
   repetitions: 1
   restrictions:
     - "LowerBound"
-configOverrides:
-  - patcher:
-      type: "NodeSelectorPatcher"
-      resource: "uc1-load-generator-deployment.yaml"
-      variableName: "env"
-    value: "prod"
-  - patcher:
-      type: "NodeSelectorPatcher"
-      resource: "uc1-kstreams-deployment.yaml"
-      variableName: "env"
-    value: "prod"
+configOverrides: []
+#  - patcher:
+#      type: "NodeSelectorPatcher"
+#      resource: "uc1-load-generator-deployment.yaml"
+#      variableName: "env"
+#    value: "prod"
+#  - patcher:
+#      type: "NodeSelectorPatcher"
+#      resource: "uc1-kstreams-deployment.yaml"
+#      variableName: "env"
+#    value: "prod"
 #  - patcher:
 #      type: "ResourceLimitPatcher"
 #      resource: "uc1-kstreams-deployment.yaml"

theodolite-quarkus/config/example-operator-benchmark.yaml

+apiVersion: theodolite.com/v1alpha1
+kind: benchmark
+metadata:
+  name: uc1-kstreams
+#name: "uc1-kstreams"
+appResource:
+  - "uc1-kstreams-deployment.yaml"
+  - "aggregation-service.yaml"
+  - "jmx-configmap.yaml"
+  - "uc1-service-monitor.yaml"
+loadGenResource:
+  - "uc1-load-generator-deployment.yaml"
+  - "uc1-load-generator-service.yaml"
+resourceTypes:
+  - typeName: "Instances"
+    patchers:
+      - type: "ReplicaPatcher"
+        resource: "uc1-kstreams-deployment.yaml"
+loadTypes:
+  - typeName: "NumSensors"
+    patchers:
+      - type: "EnvVarPatcher"
+        resource: "uc1-load-generator-deployment.yaml"
+        container: "workload-generator"
+        variableName: "NUM_SENSORS"
+kafkaConfig:
+  bootstrapServer: "theodolite-cp-kafka:9092"
+  topics:
+    - name: "input"
+      numPartitions: 40
+      replicationFactor: 1
\ No newline at end of file

theodolite-quarkus/config/example-operator-execution.yaml

+apiVersion: theodolite.com/v1alpha1
+kind: execution
+metadata:
+  name: example-execution
+#name: example-execution
+benchmark: "uc1-kstreams"
+load:
+  loadType: "NumSensors"
+  loadValues: [25000, 50000, 75000, 100000, 125000, 150000]
+resources:
+  resourceType: "Instances"
+  resourceValues: [1, 2, 3, 4, 5]
+slos:
+  - sloType: "lag trend"
+    threshold: 2000
+    prometheusUrl: "http://prometheus-operated:9090"
+    externalSloUrl: "http://localhost:80/evaluate-slope"
+    offset: 0
+    warmup: 60 # in seconds
+execution:
+  strategy: "LinearSearch"
+  duration: 300 # in seconds
+  repetitions: 1
+  restrictions:
+    - "LowerBound"
+configOverrides: []
+#  - patcher:
+#      type: "NodeSelectorPatcher"
+#      resource: "uc1-load-generator-deployment.yaml"
+#      variableName: "env"
+#    value: "prod"
+#  - patcher:
+#      type: "NodeSelectorPatcher"
+#      resource: "uc1-kstreams-deployment.yaml"
+#      variableName: "env"
+#    value: "prod"
+#  - patcher:
+#      type: "ResourceLimitPatcher"
+#      resource: "uc1-kstreams-deployment.yaml"
+#      container: "uc-application"
+#      variableName: "cpu"
+#    value: "1000m"
+#  - patcher:
+#      type: "ResourceLimitPatcher"
+#      resource: "uc1-kstreams-deployment.yaml"
+#      container: "uc-application"
+#      variableName: "memory"
+#    value: "2Gi"
+#  - patcher:
+#      type: "SchedulerNamePatcher"
+#      resource: "uc1-kstreams-deployment.yaml"
+#    value: "random-scheduler"

theodolite-quarkus/config/uc1-kstreams-deployment.yaml

@@ -21,9 +21,9 @@ spec:
               name: jmx
           env:
             - name: KAFKA_BOOTSTRAP_SERVERS
-              value: "my-confluent-cp-kafka:9092"
+              value: "theodolite-cp-kafka:9092"
             - name: SCHEMA_REGISTRY_URL
-              value: "http://my-confluent-cp-schema-registry:8081"
+              value: "http://theodolite-cp-schema-registry:8081"
             - name: JAVA_OPTS
               value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555"
             - name: COMMIT_INTERVAL_MS # Set as default for the applications

theodolite-quarkus/config/uc1-load-generator-deployment.yaml

@@ -31,6 +31,6 @@ spec:
             - name: KUBERNETES_DNS_NAME
               value: "titan-ccp-load-generator.$(KUBERNETES_NAMESPACE).svc.cluster.local"
             - name: KAFKA_BOOTSTRAP_SERVERS
-              value: "my-confluent-cp-kafka:9092"
+              value: "theodolite-cp-kafka:9092"
             - name: SCHEMA_REGISTRY_URL
-              value: "http://my-confluent-cp-schema-registry:8081"
+              value: "http://theodolite-cp-schema-registry:8081"

theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/ExecutionEventHandler.kt

@@ -3,6 +3,7 @@ package theodolite.execution.operator
 import io.fabric8.kubernetes.client.informers.ResourceEventHandler
 import mu.KotlinLogging
 import theodolite.benchmark.BenchmarkExecution
+import java.lang.NullPointerException
 
 private val logger = KotlinLogging.logger {}
 
@@ -16,7 +17,11 @@ class ExecutionHandler(private val controller: TheodoliteController): ResourceEv
     override fun onUpdate(oldExecution: BenchmarkExecution, newExecution: BenchmarkExecution) {
         logger.info { "Add updated execution to queue." }
         newExecution.name = newExecution.metadata.name
+        try {
             this.controller.executionsQueue.removeIf { e -> e.name == newExecution.metadata.name }
+        } catch(e: NullPointerException) {
+            logger.warn { "No execution found for deletion" }
+        }
         this.controller.executionsQueue.addFirst(newExecution)
         if (this.controller.isInitialized() && this.controller.executor.getExecution().name == newExecution.metadata.name) {
             this.controller.isUpdated.set(true)
@@ -25,8 +30,12 @@ class ExecutionHandler(private val controller: TheodoliteController): ResourceEv
     }
 
     override fun onDelete(execution: BenchmarkExecution, b: Boolean) {
-        logger.info { "Delete execution ${execution.metadata.name} from queue." }
+        try {
             this.controller.executionsQueue.removeIf { e -> e.name == execution.metadata.name }
+            logger.info { "Delete execution ${execution.metadata.name} from queue." }
+        } catch(e: NullPointerException) {
+            logger.warn { "No execution found for deletion" }
+        }
         if (this.controller.isInitialized() && this.controller.executor.getExecution().name == execution.metadata.name) {
             this.controller.isUpdated.set(true)
             this.controller.executor.executor.run.compareAndSet(true, false)

theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/TheodoliteController.kt

@@ -70,9 +70,15 @@ class TheodoliteController(
         executor = TheodoliteExecutor(config = execution, kubernetesBenchmark = benchmark)
         executor.run()
 
+        try {
             if (!isUpdated.get()) {
+                this.executionsQueue.removeIf { e -> e.name == execution.name }
                 client.customResource(executionContext).delete(client.namespace, execution.metadata.name)
             }
+        } catch (e: Exception) {
+            logger.warn { "Deletion skipped." }
+        }
+
         logger.info { "Execution of ${execution.name} is finally stopped." }
     }
 

theodolite-quarkus/src/main/kotlin/theodolite/k8s/TopicManager.kt

@@ -2,7 +2,6 @@ package theodolite.k8s
 
 import mu.KotlinLogging
 import org.apache.kafka.clients.admin.AdminClient
-import org.apache.kafka.clients.admin.ListTopicsResult
 import org.apache.kafka.clients.admin.NewTopic
 import java.util.*
 
@@ -20,9 +19,15 @@ class TopicManager(private val kafkaConfig: HashMap<String, Any>) {
      */
     fun createTopics(newTopics: Collection<NewTopic>) {
         var kafkaAdmin: AdminClient = AdminClient.create(this.kafkaConfig)
-        kafkaAdmin.createTopics(newTopics)
+        val result = kafkaAdmin.createTopics(newTopics)
+        result.all().get()// wait for the future object
+        logger.info {
+            "Topics created finished with result: ${
+                result.values().map { it -> it.key + ": " + it.value.isDone }
+                    .joinToString(separator = ",")
+            } "
+        }
         kafkaAdmin.close()
-        logger.info { "Topics created" }
     }
 
 
@@ -32,15 +37,19 @@ class TopicManager(private val kafkaConfig: HashMap<String, Any>) {
      */
     fun removeTopics(topics: List<String>) {
         var kafkaAdmin: AdminClient = AdminClient.create(this.kafkaConfig)
-        val result = kafkaAdmin.deleteTopics(topics)
-
         try {
-            result.all().get()
+            val result = kafkaAdmin.deleteTopics(topics)
+            result.all().get() // wait for the future object
+            logger.info {
+                "\"Topics deletion finished with result: ${
+                    result.values().map { it -> it.key + ": " + it.value.isDone }
+                        .joinToString(separator = ",")
+                } "
+            }
         } catch (e: Exception) {
             logger.error { "Error while removing topics: $e" }
             logger.debug { "Existing topics are: ${kafkaAdmin.listTopics()}." }
         }
         kafkaAdmin.close()
-        logger.info { "Topics removed" }
     }
 }