diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9b10ffeabbc08a1f25a88d2b351f3e8dd6309443..b7d75d5470d8073a2021e06428e9170f605ec7b3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -303,6 +303,32 @@ deploy-theodolite: allow_failure: true +# Theodolite SLO Checker: Lag Trend + +deploy-slo-checker-lag-trend: + stage: deploy + extends: + - .dind + script: + - DOCKER_TAG_NAME=$(echo $CI_COMMIT_REF_SLUG- | sed 's/^master-$//') + - docker build --pull -t theodolite-slo-checker-lag-trend slope-evaluator + - "[ ! $CI_COMMIT_TAG ] && docker tag theodolite-slo-checker-lag-trend $CR_HOST/$CR_ORG/theodolite-slo-checker-lag-trend:${DOCKER_TAG_NAME}latest" + - "[ $CI_COMMIT_TAG ] && docker tag theodolite-slo-checker-lag-trend $CR_HOST/$CR_ORG/theodolite-slo-checker-lag-trend:$CI_COMMIT_TAG" + - echo $CR_PW | docker login $CR_HOST -u $CR_USER --password-stdin + - docker push $CR_HOST/$CR_ORG/theodolite-slo-checker-lag-trend + - docker logout + rules: + - if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW && $CI_COMMIT_TAG" + when: always + - changes: + - slope-evaluator/**/* + if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW" + when: always + - if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW" + when: manual + allow_failure: true + + # Theodolite Random Scheduler deploy-random-scheduler: diff --git a/execution/helm/README.md b/execution/helm/README.md index 4cacd06c8181970e78cb4f62e93b77fa169fcdfa..6a2f083559fd0fe7b5674834e99a075e9bea1851 100644 --- a/execution/helm/README.md +++ b/execution/helm/README.md @@ -44,6 +44,31 @@ In development environments Kubernetes resources are often low. To reduce resour helm install theodolite . -f preconfigs/one-broker-values.yaml ``` +## Uninstall this Chart + +To uninstall/delete the `my-release` deployment: + +```sh +helm delete my-release +``` + +This command does not remove the CRDs which are created by this chart. Remove them manually with: + +```sh +# CRDs from Theodolite +kubectl delete crd execution +kubectl delete crd benchmark +# CRDs from Prometheus operator (see https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack#uninstall-chart) +kubectl delete crd alertmanagerconfigs.monitoring.coreos.com +kubectl delete crd alertmanagers.monitoring.coreos.com +kubectl delete crd podmonitors.monitoring.coreos.com +kubectl delete crd probes.monitoring.coreos.com +kubectl delete crd prometheuses.monitoring.coreos.com +kubectl delete crd prometheusrules.monitoring.coreos.com +kubectl delete crd servicemonitors.monitoring.coreos.com +kubectl delete crd thanosrulers.monitoring.coreos.com +``` + ## Development **Hints**: diff --git a/execution/helm/preconfigs/one-broker-values.yaml b/execution/helm/preconfigs/one-broker-values.yaml index fdbc3207ee37f49cf176645851d91e62ba354d28..c53c1f1eb8bc7a17f192d70a6f10f8cacc09c98f 100644 --- a/execution/helm/preconfigs/one-broker-values.yaml +++ b/execution/helm/preconfigs/one-broker-values.yaml @@ -9,7 +9,7 @@ cp-helm-charts: ## Kafka ## ------------------------------------------------------ cp-kafka: - brokers: 1 # deauflt: 10 + brokers: 1 # default: 10 configurationOverrides: offsets.topic.replication.factor: "1" \ No newline at end of file diff --git a/execution/helm/templates/theodolite/crd-benchmark.yaml b/execution/helm/templates/theodolite/crd-benchmark.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9d7468b490fa2f2a6cf829bdcafab8c4bd6fc5bf --- /dev/null +++ b/execution/helm/templates/theodolite/crd-benchmark.yaml @@ -0,0 +1,15 @@ +{{- if .Values.benchmarkCRD.create -}} +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: benchmarks.theodolite.com +spec: + group: theodolite.com + version: v1alpha1 + names: + kind: benchmark + plural: benchmarks + scope: Namespaced + subresources: + status: {} +{{- end }} \ No newline at end of file diff --git a/execution/helm/templates/theodolite/crd-execution.yaml b/execution/helm/templates/theodolite/crd-execution.yaml new file mode 100644 index 0000000000000000000000000000000000000000..73b58397b8c1fc15ffef5da74e8f1dbdabaa3a30 --- /dev/null +++ b/execution/helm/templates/theodolite/crd-execution.yaml @@ -0,0 +1,15 @@ +{{- if .Values.executionCRD.create -}} +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: executions.theodolite.com +spec: + group: theodolite.com + version: v1alpha1 + names: + kind: execution + plural: executions + scope: Namespaced + subresources: + status: {} +{{- end }} \ No newline at end of file diff --git a/execution/helm/templates/theodolite/role-binding.yaml b/execution/helm/templates/theodolite/role-binding.yaml new file mode 100644 index 0000000000000000000000000000000000000000..93d8c34e7bc544c3b0c231e986bc58c792cce38e --- /dev/null +++ b/execution/helm/templates/theodolite/role-binding.yaml @@ -0,0 +1,15 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: {{ include "theodolite.fullname" . }} + labels: + app: {{ include "theodolite.name" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "theodolite.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ include "theodolite.serviceAccountName" . }} +{{- end }} \ No newline at end of file diff --git a/execution/helm/templates/theodolite/role.yaml b/execution/helm/templates/theodolite/role.yaml new file mode 100644 index 0000000000000000000000000000000000000000..73e3e5098448c88e6b030f1dc2056caf999da3ce --- /dev/null +++ b/execution/helm/templates/theodolite/role.yaml @@ -0,0 +1,60 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "theodolite.fullname" . }} +rules: + - apiGroups: + - apps + resources: + - deployments + verbs: + - delete + - list + - get + - create + - update + - apiGroups: + - "" + resources: + - services + - pods + - configmaps + verbs: + - update + - delete + - list + - get + - create + - apiGroups: + - "" + resources: + - pods/exec + verbs: + - create + - get + - apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + verbs: + - update + - delete + - list + - create + {{- if .Values.operator.enabled -}} + - apiGroups: + - theodolite.com + resources: + - executions + - benchmarks + verbs: + - delete + - list + - get + - create + - watch + - update + - patch + {{- end }} +{{- end }} \ No newline at end of file diff --git a/execution/helm/templates/theodolite/serviceaccount.yaml b/execution/helm/templates/theodolite/serviceaccount.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4585b8ce413bf3d36cb986163788c353f2a4a2de --- /dev/null +++ b/execution/helm/templates/theodolite/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "theodolite.serviceAccountName" . }} + labels: + {{- include "theodolite.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/execution/helm/templates/theodolite/thedolite-operator.yaml b/execution/helm/templates/theodolite/thedolite-operator.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb3f193b89c3a198bab853fb85a0db1184394dce --- /dev/null +++ b/execution/helm/templates/theodolite/thedolite-operator.yaml @@ -0,0 +1,31 @@ +{{- if .Values.operator.enabled -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "theodolite.fullname" . }}-operator +spec: + selector: + matchLabels: + app: {{ include "theodolite.fullname" . }} + replicas: 1 + template: + metadata: + labels: + app: {{ include "theodolite.fullname" . }} + spec: + terminationGracePeriodSeconds: 0 + serviceAccountName: {{ include "theodolite.serviceAccountName" . }} + containers: + - name: theodolite + image: ghcr.io/cau-se/theodolite:theodolite-kotlin-latest + env: + - name: NAMESPACE + value: {{ .Release.Namespace }} + - name: MODE + value: operator + - name: lag-analysis + image: ghcr.io/cau-se/theodolite-slo-checker-lag-trend:theodolite-kotlin-latest + ports: + - containerPort: 80 + name: analysis +{{- end }} diff --git a/execution/helm/values.yaml b/execution/helm/values.yaml index d35912de6ffc72fff27f5d389c94761bc80eecd1..5e48ad46b01689c974189f471b0016e45d71f958 100644 --- a/execution/helm/values.yaml +++ b/execution/helm/values.yaml @@ -226,4 +226,22 @@ prometheus: clusterRole: enabled: true clusterRoleBinding: - enabled: true \ No newline at end of file + enabled: true + +### +# Theodolite Operator +### +operator: + enabled: true + +executionCRD: + create: true + +benchmarkCRD: + create: true + +serviceAccount: + create: true + +rbac: + create: true diff --git a/execution/infrastructure/kubernetes/rbac/role.yaml b/execution/infrastructure/kubernetes/rbac/role.yaml index 84ba14a8bc7a6eceb8a20596ede057ca2271b967..a21fd554f0f56b3955e9a9b6cf4bf95442b5d7af 100644 --- a/execution/infrastructure/kubernetes/rbac/role.yaml +++ b/execution/infrastructure/kubernetes/rbac/role.yaml @@ -38,4 +38,18 @@ rules: verbs: - delete - list - - create \ No newline at end of file + - create + - apiGroups: + - theodolite.com + resources: + - executions + - benchmarks + verbs: + - delete + - list + - get + - create + - watch + - update + - patch + diff --git a/theodolite-quarkus/config/example-benchmark-yaml-resource.yaml b/theodolite-quarkus/config/example-benchmark-yaml-resource.yaml index 60eb3bb9c31e3eab3e70f916b450372d56db4968..42b1cd08dc1949355c97edebc92ea7b5ab8799b2 100644 --- a/theodolite-quarkus/config/example-benchmark-yaml-resource.yaml +++ b/theodolite-quarkus/config/example-benchmark-yaml-resource.yaml @@ -20,7 +20,7 @@ loadTypes: container: "workload-generator" variableName: "NUM_SENSORS" kafkaConfig: - bootstrapServer: "theodolite-cp-kafka:9092" + bootstrapServer: "my-confluent-cp-kafka:9092" topics: - name: "input" numPartitions: 40 diff --git a/theodolite-quarkus/config/thedolite-operator.yaml b/theodolite-quarkus/config/thedolite-operator.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e0e60248c2474cc8493179c003b806030f79f8c --- /dev/null +++ b/theodolite-quarkus/config/thedolite-operator.yaml @@ -0,0 +1,24 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: theodolite +spec: + selector: + matchLabels: + app: theodolite + replicas: 1 + template: + metadata: + labels: + app: theodolite + spec: + terminationGracePeriodSeconds: 0 + serviceAccountName: theodolite + containers: + - name: thedolite + image: ghcr.io/cau-se/theodolite:theodolite-kotlin-latest + env: + - name: KUBECONFIG + value: "~/.kube/config" + - name: NAMESPACE + value: "default" \ No newline at end of file diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt b/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt index f129b6bf557e5484c375c0d61c02e93caf1602fd..25ab9aba92a4b77bc444209a0046561e25bc62fc 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt @@ -38,8 +38,8 @@ class KubernetesBenchmark : Benchmark, CustomResource(), Namespaced { lateinit var resourceTypes: List<TypeName> lateinit var loadTypes: List<TypeName> lateinit var kafkaConfig: KafkaConfig - lateinit var namespace: String - lateinit var path: String + private val namespace = System.getenv("NAMESPACE") ?: DEFAULT_NAMESPACE + var path = System.getenv("THEODOLITE_APP_RESOURCES") ?: "./config" /** * Loads [KubernetesResource]s. @@ -49,13 +49,6 @@ class KubernetesBenchmark : Benchmark, CustomResource(), Namespaced { private fun loadKubernetesResources(resources: List<String>): List<Pair<String, KubernetesResource>> { val parser = YamlParser() - namespace = System.getenv("NAMESPACE") ?: DEFAULT_NAMESPACE - logger.info { "Using $namespace as namespace." } - - path = System.getenv("THEODOLITE_APP_RESOURCES") ?: "./config" - logger.info { "Using $path as path for resources." } - - val loader = K8sResourceLoader(DefaultKubernetesClient().inNamespace(namespace)) return resources .map { resource -> @@ -80,6 +73,9 @@ class KubernetesBenchmark : Benchmark, CustomResource(), Namespaced { res: Resource, configurationOverrides: List<ConfigurationOverride?> ): BenchmarkDeployment { + logger.info { "Using $namespace as namespace." } + logger.info { "Using $path as resource path." } + val resources = loadKubernetesResources(this.appResource + this.loadGenResource) val patcherFactory = PatcherFactory() @@ -97,12 +93,12 @@ class KubernetesBenchmark : Benchmark, CustomResource(), Namespaced { patcherFactory.createPatcher(it.patcher, resources).patch(override.value) } } - return KubernetesBenchmarkDeployment( namespace = namespace, resources = resources.map { r -> r.second }, kafkaConfig = hashMapOf("bootstrap.servers" to kafkaConfig.bootstrapServer), - topics = kafkaConfig.getKafkaTopics() + topics = kafkaConfig.getKafkaTopics(), + client = DefaultKubernetesClient().inNamespace(namespace) ) } } diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmarkDeployment.kt b/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmarkDeployment.kt index c26555e6fb9dd80c773af434f0793d874f6d1108..76ba76e57f0e52d21bf5d62ac003d6b64e781ab0 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmarkDeployment.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmarkDeployment.kt @@ -1,7 +1,7 @@ package theodolite.benchmark import io.fabric8.kubernetes.api.model.KubernetesResource -import io.fabric8.kubernetes.client.DefaultKubernetesClient +import io.fabric8.kubernetes.client.NamespacedKubernetesClient import io.quarkus.runtime.annotations.RegisterForReflection import org.apache.kafka.clients.admin.NewTopic import theodolite.k8s.K8sManager @@ -20,14 +20,12 @@ class KubernetesBenchmarkDeployment( val namespace: String, val resources: List<KubernetesResource>, private val kafkaConfig: HashMap<String, Any>, - private val topics: Collection<NewTopic> + private val topics: Collection<NewTopic>, + private val client: NamespacedKubernetesClient ) : BenchmarkDeployment { private val kafkaController = TopicManager(this.kafkaConfig) - private val kubernetesManager = K8sManager(DefaultKubernetesClient().inNamespace(namespace)) - - //label of the KafkaLagExporter. + private val kubernetesManager = K8sManager(client) private val LABEL = "app.kubernetes.io/name=kafka-lag-exporter" - private val client = DefaultKubernetesClient().inNamespace(namespace) /** * Setup a [KubernetesBenchmark] using the [TopicManager] and the [K8sManager]: diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/Main.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/Main.kt index 3246d0a7930aab22d16ed94a6c0f10a9d3fde10e..4518ef7957104819b26eae95cf4e6e9b35c4e995 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/Main.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/Main.kt @@ -13,7 +13,7 @@ object Main { @JvmStatic fun main(args: Array<String>) { - val mode = System.getenv("MODE") ?: "operator" + val mode = System.getenv("MODE") ?: "yaml-executor" logger.info { "Start Theodolite with mode $mode" } when(mode) { diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteYamlExecutor.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteYamlExecutor.kt index 3a75b891965ea05345afbd7bb681254181da6e47..f455f486edb97a12cc82629711e747a8fd2dd7db 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteYamlExecutor.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteYamlExecutor.kt @@ -33,11 +33,9 @@ class TheodoliteYamlExecutor { val executionPath = System.getenv("THEODOLITE_EXECUTION") ?: "./config/BenchmarkExecution.yaml" val benchmarkPath = System.getenv("THEODOLITE_BENCHMARK") ?: "./config/BenchmarkType.yaml" - val appResource = System.getenv("THEODOLITE_APP_RESOURCES") ?: "./config" logger.info { "Using $executionPath for BenchmarkExecution" } logger.info { "Using $benchmarkPath for BenchmarkType" } - logger.info { "Using $appResource for Resources" } // load the BenchmarkExecution and the BenchmarkType @@ -45,7 +43,6 @@ class TheodoliteYamlExecutor { parser.parse(path = executionPath, E = BenchmarkExecution::class.java)!! val benchmark = parser.parse(path = benchmarkPath, E = KubernetesBenchmark::class.java)!! - benchmark.path = appResource val shutdown = Shutdown(benchmarkExecution, benchmark) Runtime.getRuntime().addShutdownHook(thread { shutdown.run()}) diff --git a/theodolite-quarkus/src/main/resources/application.properties b/theodolite-quarkus/src/main/resources/application.properties index d5ff26fd074ec74c02a25bc4dd0dd3734433401b..42647e2391706286602945cf2be7baa96857ba19 100644 --- a/theodolite-quarkus/src/main/resources/application.properties +++ b/theodolite-quarkus/src/main/resources/application.properties @@ -1,3 +1,6 @@ quarkus.native.additional-build-args=\ --initialize-at-run-time=io.fabric8.kubernetes.client.internal.CertUtils,\ - --report-unsupported-elements-at-runtime \ No newline at end of file + --initialize-at-run-time=io.fabric8.kubernetes.client.dsl.internal.uploadable.PodUpload,\ + --initialize-at-run-time=io.fabric8.kubernetes.client.dsl.internal.core.v1.PodOperationsImpl$1,\ + --initialize-at-run-time=io.fabric8.kubernetes.client.dsl.internal.core.v1.PodOperationsImpl$3,\ + --report-unsupported-elements-at-runtime