diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b7d75d5470d8073a2021e06428e9170f605ec7b3..4a412569fbd477f02a0b67b83e8814ab98b34031 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -51,7 +51,7 @@ build-benchmarks: - "theodolite-benchmarks/build/libs/*.jar" - "theodolite-benchmarks/*/build/libs/*.jar" - "theodolite-benchmarks/*/build/distributions/*.tar" - expire_in: 1 day + expire_in: 6 hours test-benchmarks: stage: test @@ -234,7 +234,7 @@ build-theodolite-jvm: paths: - "theodolite-quarkus/build/lib/*" - "theodolite-quarkus/build/*-runner.jar" - expire_in: 1 day + expire_in: 6 hours build-theodolite-native: stage: build @@ -245,7 +245,7 @@ build-theodolite-native: artifacts: paths: - "theodolite-quarkus/build/*-runner" - expire_in: 1 day + expire_in: 6 hours test-theodolite: stage: test diff --git a/execution/helm/README.md b/execution/helm/README.md index 6a2f083559fd0fe7b5674834e99a075e9bea1851..c545804aaec8eb8ed91054f1f7ee97dd293816a4 100644 --- a/execution/helm/README.md +++ b/execution/helm/README.md @@ -6,11 +6,9 @@ Install the chart via: ```sh helm dependencies update . -helm install my-confluent . +helm install theodolite . ``` -**Please note: Theodolite currently uses hard-coded URLs, to connect to Kafka and Zookeeper. For that reason, the name of this chart must be `my-confluent`.** We will change this behavior soon. - This chart installs requirements to execute benchmarks with Theodolite. Dependencies and subcharts: @@ -27,7 +25,7 @@ Dependencies and subcharts: Test the installation: ```sh -helm test <release-name> +helm test theodolite ``` Our test files are located [here](templates/../../theodolite-chart/templates/tests). Many subcharts have their own tests, these are also executed and are placed in the respective /templates folders. @@ -46,18 +44,18 @@ helm install theodolite . -f preconfigs/one-broker-values.yaml ## Uninstall this Chart -To uninstall/delete the `my-release` deployment: +To uninstall/delete the `theodolite` deployment: ```sh -helm delete my-release +helm delete theodolite ``` This command does not remove the CRDs which are created by this chart. Remove them manually with: ```sh # CRDs from Theodolite -kubectl delete crd execution -kubectl delete crd benchmark +kubectl delete crd executions.theodolite.com +kubectl delete crd benchmarks.theodolite.com # CRDs from Prometheus operator (see https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack#uninstall-chart) kubectl delete crd alertmanagerconfigs.monitoring.coreos.com kubectl delete crd alertmanagers.monitoring.coreos.com diff --git a/execution/helm/templates/theodolite/role.yaml b/execution/helm/templates/theodolite/role.yaml index 73e3e5098448c88e6b030f1dc2056caf999da3ce..ee699a5b861c64e355ca2ba44330a7d600756b77 100644 --- a/execution/helm/templates/theodolite/role.yaml +++ b/execution/helm/templates/theodolite/role.yaml @@ -42,7 +42,7 @@ rules: - delete - list - create - {{- if .Values.operator.enabled -}} + {{- if .Values.operator.enabled }} - apiGroups: - theodolite.com resources: diff --git a/execution/helm/templates/theodolite/thedolite-operator.yaml b/execution/helm/templates/theodolite/thedolite-operator.yaml index cb3f193b89c3a198bab853fb85a0db1184394dce..3af9cfffd71059993688d5be77ae110a2e3cdc2d 100644 --- a/execution/helm/templates/theodolite/thedolite-operator.yaml +++ b/execution/helm/templates/theodolite/thedolite-operator.yaml @@ -23,9 +23,19 @@ spec: value: {{ .Release.Namespace }} - name: MODE value: operator + - name: THEODOLITE_APP_RESOURCES + value: "./benchmark-resources" + volumeMounts: + - name: benchmark-resources + mountPath: /work/benchmark-resources - name: lag-analysis image: ghcr.io/cau-se/theodolite-slo-checker-lag-trend:theodolite-kotlin-latest ports: - containerPort: 80 - name: analysis + name: analysis + volumes: + - name: benchmark-resources + configMap: + name: benchmark-resources + optional: true {{- end }} diff --git a/execution/helm/values.yaml b/execution/helm/values.yaml index 5e48ad46b01689c974189f471b0016e45d71f958..67dab74c3931ce13a1ab0f7504a946a208b4dfb8 100644 --- a/execution/helm/values.yaml +++ b/execution/helm/values.yaml @@ -24,6 +24,9 @@ grafana: # Administrator credentials when not using an existing secret (see below) adminUser: admin adminPassword: admin + grafana.ini: + users: + default_theme: light ## Sidecars that collect the configmaps with specified label and stores the included files them into the respective folders ## Requires at least Grafana 5 to work and can't be used together with parameters dashboardProviders, datasources and dashboards sidecar: @@ -145,8 +148,8 @@ kafka-lag-exporter: enabled: true nodeSelector: {} clusters: - - name: "my-confluent-cp-kafka" - bootstrapBrokers: "my-confluent-cp-kafka:9092" + - name: "theodolite-cp-kafka" + bootstrapBrokers: "theodolite-cp-kafka:9092" ## The interval between refreshing metrics pollIntervalSeconds: 15 diff --git a/execution/uc-application/aggregation-deployment.yaml b/execution/uc-application/aggregation-deployment.yaml index 07732ca1dd1e6b2b06f098dfb10a53d38e8d5cae..1d3ebdb20dd06433e97e112edef76d7deac17395 100644 --- a/execution/uc-application/aggregation-deployment.yaml +++ b/execution/uc-application/aggregation-deployment.yaml @@ -21,9 +21,9 @@ spec: name: jmx env: - name: KAFKA_BOOTSTRAP_SERVERS - value: "my-confluent-cp-kafka:9092" + value: "theodolite-cp-kafka:9092" - name: SCHEMA_REGISTRY_URL - value: "http://my-confluent-cp-schema-registry:8081" + value: "http://theodolite-cp-schema-registry:8081" - name: JAVA_OPTS value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555" - name: COMMIT_INTERVAL_MS # Set as default for the applications diff --git a/execution/uc-workload-generator/workloadGenerator.yaml b/execution/uc-workload-generator/workloadGenerator.yaml index 146e285f66d4c0e1a88d613e4ac2d5571234fad6..8f7cc3a8df20752eccb321242bb774c18f4e1d0a 100644 --- a/execution/uc-workload-generator/workloadGenerator.yaml +++ b/execution/uc-workload-generator/workloadGenerator.yaml @@ -32,6 +32,6 @@ spec: - name: KUBERNETES_DNS_NAME value: "titan-ccp-load-generator.$(KUBERNETES_NAMESPACE).svc.cluster.local" - name: KAFKA_BOOTSTRAP_SERVERS - value: "my-confluent-cp-kafka:9092" + value: "theodolite-cp-kafka:9092" - name: SCHEMA_REGISTRY_URL - value: "http://my-confluent-cp-schema-registry:8081" + value: "http://theodolite-cp-schema-registry:8081" diff --git a/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.flink.gradle b/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.flink.gradle index f9c1d98d9f88a95bdc3fa25e7c1bec2f3c9bddb4..d6d5217667a73a1529d73ac59260bcf47d8cf2e1 100644 --- a/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.flink.gradle +++ b/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.flink.gradle @@ -6,6 +6,23 @@ plugins { applicationDefaultJvmArgs = ["-Dlog4j.configuration=log4j.properties"] + +run.classpath = sourceSets.main.runtimeClasspath + +jar { + manifest { + attributes 'Built-By': System.getProperty('user.name'), + 'Build-Jdk': System.getProperty('java.version') + } +} + +shadowJar { + configurations = [project.configurations.compile] + zip64 true +} + +tasks.distZip.enabled = false + ext { flinkVersion = '1.12.2' scalaBinaryVersion = '2.12' @@ -48,17 +65,3 @@ dependencies { // Use JUnit test framework testImplementation 'junit:junit:4.12' } - -run.classpath = sourceSets.main.runtimeClasspath - -jar { - manifest { - attributes 'Built-By': System.getProperty('user.name'), - 'Build-Jdk': System.getProperty('java.version') - } -} - -shadowJar { - configurations = [project.configurations.compile] - zip64 true -} diff --git a/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.kstreams.gradle b/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.kstreams.gradle index c6779fbc4348a8d665776e68688858ab3d2f4146..eece7b835ae9d6f39283ea371ce8b0b8194cdaa0 100644 --- a/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.kstreams.gradle +++ b/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.kstreams.gradle @@ -6,6 +6,8 @@ plugins { id 'application' } +tasks.distZip.enabled = false + repositories { jcenter() maven { diff --git a/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.load-generator.gradle b/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.load-generator.gradle index 13b7ea191d11c942cd0ca58b882ffda7bc7912be..c6c2b6057cf35c32faa4d67b6ea6dba9e5c13beb 100644 --- a/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.load-generator.gradle +++ b/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.load-generator.gradle @@ -6,6 +6,8 @@ plugins { id 'application' } +tasks.distZip.enabled = false + repositories { jcenter() maven { diff --git a/theodolite-quarkus/config/aggregation-service.yaml b/theodolite-quarkus/config/aggregation-service.yaml index 916dd6677d60370b1d62e5d7e708c3ee966bda23..85432d04f225c30469f3232153ef6bd72bd02bdf 100644 --- a/theodolite-quarkus/config/aggregation-service.yaml +++ b/theodolite-quarkus/config/aggregation-service.yaml @@ -14,4 +14,4 @@ spec: targetPort: 80 protocol: TCP - name: metrics - port: 9980 + port: 5556 diff --git a/theodolite-quarkus/config/example-benchmark-yaml-resource.yaml b/theodolite-quarkus/config/example-benchmark-yaml-resource.yaml index 42b1cd08dc1949355c97edebc92ea7b5ab8799b2..60eb3bb9c31e3eab3e70f916b450372d56db4968 100644 --- a/theodolite-quarkus/config/example-benchmark-yaml-resource.yaml +++ b/theodolite-quarkus/config/example-benchmark-yaml-resource.yaml @@ -20,7 +20,7 @@ loadTypes: container: "workload-generator" variableName: "NUM_SENSORS" kafkaConfig: - bootstrapServer: "my-confluent-cp-kafka:9092" + bootstrapServer: "theodolite-cp-kafka:9092" topics: - name: "input" numPartitions: 40 diff --git a/theodolite-quarkus/config/example-execution-yaml-resource.yaml b/theodolite-quarkus/config/example-execution-yaml-resource.yaml index 80e4575d653bff8d67d8f66cf4ae9806a1fe102e..23c1587ec1e5c2a88fcf69d7127edbcc1ffdb00f 100644 --- a/theodolite-quarkus/config/example-execution-yaml-resource.yaml +++ b/theodolite-quarkus/config/example-execution-yaml-resource.yaml @@ -2,36 +2,34 @@ name: example-execution benchmark: "uc1-kstreams" load: loadType: "NumSensors" - loadValues: - - 50000 + loadValues: [25000, 50000, 75000, 100000, 125000, 150000] resources: resourceType: "Instances" - resourceValues: - - 1 + resourceValues: [1, 2, 3, 4, 5] slos: - sloType: "lag trend" - threshold: 1000 + threshold: 2000 prometheusUrl: "http://prometheus-operated:9090" externalSloUrl: "http://localhost:80/evaluate-slope" offset: 0 - warmup: 0 + warmup: 60 # in seconds execution: strategy: "LinearSearch" - duration: 60 + duration: 300 # in seconds repetitions: 1 restrictions: - "LowerBound" -configOverrides: - - patcher: - type: "NodeSelectorPatcher" - resource: "uc1-load-generator-deployment.yaml" - variableName: "env" - value: "prod" - - patcher: - type: "NodeSelectorPatcher" - resource: "uc1-kstreams-deployment.yaml" - variableName: "env" - value: "prod" +configOverrides: [] +# - patcher: +# type: "NodeSelectorPatcher" +# resource: "uc1-load-generator-deployment.yaml" +# variableName: "env" +# value: "prod" +# - patcher: +# type: "NodeSelectorPatcher" +# resource: "uc1-kstreams-deployment.yaml" +# variableName: "env" +# value: "prod" # - patcher: # type: "ResourceLimitPatcher" # resource: "uc1-kstreams-deployment.yaml" diff --git a/theodolite-quarkus/config/example-operator-benchmark.yaml b/theodolite-quarkus/config/example-operator-benchmark.yaml new file mode 100644 index 0000000000000000000000000000000000000000..419042fdd0b1e58fed4d402b4bb329d54602d23f --- /dev/null +++ b/theodolite-quarkus/config/example-operator-benchmark.yaml @@ -0,0 +1,31 @@ +apiVersion: theodolite.com/v1alpha1 +kind: benchmark +metadata: + name: uc1-kstreams +#name: "uc1-kstreams" +appResource: + - "uc1-kstreams-deployment.yaml" + - "aggregation-service.yaml" + - "jmx-configmap.yaml" + - "uc1-service-monitor.yaml" +loadGenResource: + - "uc1-load-generator-deployment.yaml" + - "uc1-load-generator-service.yaml" +resourceTypes: + - typeName: "Instances" + patchers: + - type: "ReplicaPatcher" + resource: "uc1-kstreams-deployment.yaml" +loadTypes: + - typeName: "NumSensors" + patchers: + - type: "EnvVarPatcher" + resource: "uc1-load-generator-deployment.yaml" + container: "workload-generator" + variableName: "NUM_SENSORS" +kafkaConfig: + bootstrapServer: "theodolite-cp-kafka:9092" + topics: + - name: "input" + numPartitions: 40 + replicationFactor: 1 \ No newline at end of file diff --git a/theodolite-quarkus/config/example-operator-execution.yaml b/theodolite-quarkus/config/example-operator-execution.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3df1a723dd771453ab1b267335176e4ae74c3ed5 --- /dev/null +++ b/theodolite-quarkus/config/example-operator-execution.yaml @@ -0,0 +1,52 @@ +apiVersion: theodolite.com/v1alpha1 +kind: execution +metadata: + name: example-execution +#name: example-execution +benchmark: "uc1-kstreams" +load: + loadType: "NumSensors" + loadValues: [25000, 50000, 75000, 100000, 125000, 150000] +resources: + resourceType: "Instances" + resourceValues: [1, 2, 3, 4, 5] +slos: + - sloType: "lag trend" + threshold: 2000 + prometheusUrl: "http://prometheus-operated:9090" + externalSloUrl: "http://localhost:80/evaluate-slope" + offset: 0 + warmup: 60 # in seconds +execution: + strategy: "LinearSearch" + duration: 300 # in seconds + repetitions: 1 + restrictions: + - "LowerBound" +configOverrides: [] +# - patcher: +# type: "NodeSelectorPatcher" +# resource: "uc1-load-generator-deployment.yaml" +# variableName: "env" +# value: "prod" +# - patcher: +# type: "NodeSelectorPatcher" +# resource: "uc1-kstreams-deployment.yaml" +# variableName: "env" +# value: "prod" +# - patcher: +# type: "ResourceLimitPatcher" +# resource: "uc1-kstreams-deployment.yaml" +# container: "uc-application" +# variableName: "cpu" +# value: "1000m" +# - patcher: +# type: "ResourceLimitPatcher" +# resource: "uc1-kstreams-deployment.yaml" +# container: "uc-application" +# variableName: "memory" +# value: "2Gi" +# - patcher: +# type: "SchedulerNamePatcher" +# resource: "uc1-kstreams-deployment.yaml" +# value: "random-scheduler" diff --git a/theodolite-quarkus/config/uc1-kstreams-deployment.yaml b/theodolite-quarkus/config/uc1-kstreams-deployment.yaml index 85fe243714f9523e760ad827ef697b3dcefdc8e3..171c3446db2719ee91bd8954233015316851fcf9 100644 --- a/theodolite-quarkus/config/uc1-kstreams-deployment.yaml +++ b/theodolite-quarkus/config/uc1-kstreams-deployment.yaml @@ -21,9 +21,9 @@ spec: name: jmx env: - name: KAFKA_BOOTSTRAP_SERVERS - value: "my-confluent-cp-kafka:9092" + value: "theodolite-cp-kafka:9092" - name: SCHEMA_REGISTRY_URL - value: "http://my-confluent-cp-schema-registry:8081" + value: "http://theodolite-cp-schema-registry:8081" - name: JAVA_OPTS value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555" - name: COMMIT_INTERVAL_MS # Set as default for the applications diff --git a/theodolite-quarkus/config/uc1-load-generator-deployment.yaml b/theodolite-quarkus/config/uc1-load-generator-deployment.yaml index e49c22d6a478fbd7f3f5987222c76ca73a661cb0..374dd60113e133ef0a793149e3786efb38973287 100644 --- a/theodolite-quarkus/config/uc1-load-generator-deployment.yaml +++ b/theodolite-quarkus/config/uc1-load-generator-deployment.yaml @@ -31,6 +31,6 @@ spec: - name: KUBERNETES_DNS_NAME value: "titan-ccp-load-generator.$(KUBERNETES_NAMESPACE).svc.cluster.local" - name: KAFKA_BOOTSTRAP_SERVERS - value: "my-confluent-cp-kafka:9092" + value: "theodolite-cp-kafka:9092" - name: SCHEMA_REGISTRY_URL - value: "http://my-confluent-cp-schema-registry:8081" + value: "http://theodolite-cp-schema-registry:8081" diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/ExecutionEventHandler.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/ExecutionEventHandler.kt index 1752ac112ea84ea179e238f7ab8d808779014d1b..0152cd7bef808d4652cd893fb282e0cb8b18dd5a 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/ExecutionEventHandler.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/ExecutionEventHandler.kt @@ -3,6 +3,7 @@ package theodolite.execution.operator import io.fabric8.kubernetes.client.informers.ResourceEventHandler import mu.KotlinLogging import theodolite.benchmark.BenchmarkExecution +import java.lang.NullPointerException private val logger = KotlinLogging.logger {} @@ -16,17 +17,25 @@ class ExecutionHandler(private val controller: TheodoliteController): ResourceEv override fun onUpdate(oldExecution: BenchmarkExecution, newExecution: BenchmarkExecution) { logger.info { "Add updated execution to queue." } newExecution.name = newExecution.metadata.name - this.controller.executionsQueue.removeIf { e -> e.name == newExecution.metadata.name } + try { + this.controller.executionsQueue.removeIf { e -> e.name == newExecution.metadata.name } + } catch(e: NullPointerException) { + logger.warn { "No execution found for deletion" } + } this.controller.executionsQueue.addFirst(newExecution) - if (this.controller.isInitialized() && this.controller.executor.getExecution().name == newExecution.metadata.name) { + if (this.controller.isInitialized() && this.controller.executor.getExecution().name == newExecution.metadata.name) { this.controller.isUpdated.set(true) this.controller.executor.executor.run.compareAndSet(true, false) } } override fun onDelete(execution: BenchmarkExecution, b: Boolean) { - logger.info { "Delete execution ${execution.metadata.name} from queue." } - this.controller.executionsQueue.removeIf { e -> e.name == execution.metadata.name } + try { + this.controller.executionsQueue.removeIf { e -> e.name == execution.metadata.name } + logger.info { "Delete execution ${execution.metadata.name} from queue." } + } catch(e: NullPointerException) { + logger.warn { "No execution found for deletion" } + } if (this.controller.isInitialized() && this.controller.executor.getExecution().name == execution.metadata.name) { this.controller.isUpdated.set(true) this.controller.executor.executor.run.compareAndSet(true, false) diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/TheodoliteController.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/TheodoliteController.kt index 9e6280cf3c8160ef686fd6dcee45276de7e67fa7..9f6cd64528874a1dc5f20c6d6c1563b1aa9f003d 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/TheodoliteController.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/TheodoliteController.kt @@ -70,9 +70,15 @@ class TheodoliteController( executor = TheodoliteExecutor(config = execution, kubernetesBenchmark = benchmark) executor.run() - if (!isUpdated.get()) { - client.customResource(executionContext).delete(client.namespace, execution.metadata.name) + try { + if (!isUpdated.get()) { + this.executionsQueue.removeIf { e -> e.name == execution.name } + client.customResource(executionContext).delete(client.namespace, execution.metadata.name) + } + } catch (e: Exception) { + logger.warn { "Deletion skipped." } } + logger.info { "Execution of ${execution.name} is finally stopped." } } diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/k8s/TopicManager.kt b/theodolite-quarkus/src/main/kotlin/theodolite/k8s/TopicManager.kt index 390974cd247645197ebe6044bf785710164155aa..7529a45310f219fc0c5248b4031a020227a86049 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/k8s/TopicManager.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/k8s/TopicManager.kt @@ -2,7 +2,6 @@ package theodolite.k8s import mu.KotlinLogging import org.apache.kafka.clients.admin.AdminClient -import org.apache.kafka.clients.admin.ListTopicsResult import org.apache.kafka.clients.admin.NewTopic import java.util.* @@ -20,9 +19,15 @@ class TopicManager(private val kafkaConfig: HashMap<String, Any>) { */ fun createTopics(newTopics: Collection<NewTopic>) { var kafkaAdmin: AdminClient = AdminClient.create(this.kafkaConfig) - kafkaAdmin.createTopics(newTopics) + val result = kafkaAdmin.createTopics(newTopics) + result.all().get()// wait for the future object + logger.info { + "Topics created finished with result: ${ + result.values().map { it -> it.key + ": " + it.value.isDone } + .joinToString(separator = ",") + } " + } kafkaAdmin.close() - logger.info { "Topics created" } } @@ -32,15 +37,19 @@ class TopicManager(private val kafkaConfig: HashMap<String, Any>) { */ fun removeTopics(topics: List<String>) { var kafkaAdmin: AdminClient = AdminClient.create(this.kafkaConfig) - val result = kafkaAdmin.deleteTopics(topics) - try { - result.all().get() + val result = kafkaAdmin.deleteTopics(topics) + result.all().get() // wait for the future object + logger.info { + "\"Topics deletion finished with result: ${ + result.values().map { it -> it.key + ": " + it.value.isDone } + .joinToString(separator = ",") + } " + } } catch (e: Exception) { - logger.error { "Error while removing topics: $e" } - logger.debug { "Existing topics are: ${kafkaAdmin.listTopics()}." } + logger.error { "Error while removing topics: $e" } + logger.debug { "Existing topics are: ${kafkaAdmin.listTopics()}." } } kafkaAdmin.close() - logger.info { "Topics removed" } } }