Merge branch 'feature/346-migrate-strimzi-kafka' into 'master'

Migrate to Strimzi Kafka

Closes #346 and #345

See merge request !243

docs/creating-a-benchmark.md

@@ -42,7 +42,7 @@ spec:
           properties:
             loadGenMaxRecords: "150000"
   kafkaConfig:
-    bootstrapServer: "theodolite-cp-kafka:9092"
+    bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092"
     topics:
       - name: "input"
         numPartitions: 40
@@ -54,7 +54,7 @@ spec:
 
 ## System under Test (SUT), Load Generator and Infrastructure
 
-In Thedolite, the system under test (SUT), the load generator as well as additional infrastructure (e.g., a middleware) are described by Kubernetes resources files.
+In Theodolite, the system under test (SUT), the load generator as well as additional infrastructure (e.g., a middleware) are described by Kubernetes resources files.
 All resources defined for the SUT and the load generator are started and stopped for each SLO experiment, with SUT resources being started before the load generator.
 Infrastructure resources live over the entire duration of a benchmark run. They avoid time-consuming recreation of software components like middlewares, but should be used with caution to not let previous SLO experiments influence latte ones.
 

docs/installation.md

@@ -58,13 +58,13 @@ In cases, where you need to install multiple Theodolite instances, it's best to
 
 ### Installation with a release name other than `theodolite`
 
-When using another release name than `theodolite`, make sure to adjust the Kafka Lag Exporter configuration of you `values.yaml` accordingly:
+When using another release name than `theodolite`, make sure to adjust the Confluent Schema Registry configuration of you `values.yaml` accordingly:
 
 ```yaml
-kafka-lag-exporter:
-  clusters:
-  - name: "<your-release-name>-cp-kafka"
-    bootstrapBrokers: "<your-release-name>-cp-kafka:9092"
+cp-helm-charts:
+  cp-schema-registry:
+    kafka:
+      bootstrapServers: <your-release-name>-kafka-kafka-bootstrap:9092
 ```
 
 This seems unfortunately to be necessary as Helm does not let us inject values into dependency charts.

helm/Chart.yaml

@@ -24,10 +24,11 @@ dependencies:
     version: 0.6.0
     repository: https://soerenhenning.github.io/cp-helm-charts
     condition: cp-helm-charts.enabled
-  - name: kafka-lag-exporter
-    version: 0.6.7
-    repository: https://seanglover.com/kafka-lag-exporter/repo
-    condition: kafka-lag-exporter.enabled
+  - name: strimzi-kafka-operator
+    version: 0.28.0
+    repository: https://strimzi.io/charts/
+    condition: strimzi.enabled
+
 
 version: 0.7.0-SNAPSHOT
 

helm/templates/grafana/dashboard-config-map.yaml

@@ -71,10 +71,10 @@ data:
         "steppedLine": false,
         "targets": [
           {
-            "expr": "sum(cp_kafka_server_brokertopicmetrics_messagesinpersec_topic_input)",
+            "expr": "sum by (topic) (rate(kafka_server_brokertopicmetrics_messagesin_total{topic='input'}[30s]))",
             "format": "time_series",
             "intervalFactor": 1,
-            "legendFormat": "{{Messages In Per Second}}",
+            "legendFormat": "{{topic}}",
             "refId": "D"
           }
         ],
@@ -162,10 +162,10 @@ data:
         "steppedLine": false,
         "targets": [
           {
-            "expr": "sum(cp_kafka_server_brokertopicmetrics_messagesinpersec_topic_output)",
+            "expr": "sum by (topic) (rate(kafka_server_brokertopicmetrics_messagesin_total{topic='output'}[30s]))",
             "format": "time_series",
             "intervalFactor": 1,
-            "legendFormat": "{{Messages Out Per Second}}",
+            "legendFormat": "{{topic}}",
             "refId": "D"
           }
         ],
@@ -253,7 +253,7 @@ data:
         "steppedLine": false,
         "targets": [
           {
-            "expr": "sum by(group, topic) (kafka_consumergroup_group_lag >= 0)",
+            "expr": "sum by(consumergroup, topic) (kafka_consumergroup_lag >= 0)",
             "format": "time_series",
             "intervalFactor": 1,
             "legendFormat": "{{topic}}",
@@ -344,10 +344,10 @@ data:
         "steppedLine": false,
         "targets": [
           {
-            "expr": "count(count (kafka_consumer_consumer_fetch_manager_metrics_records_lag) by(pod))",
+            "expr": "sum by(consumergroup) (kafka_consumergroup_members >= 0)",
             "format": "time_series",
             "intervalFactor": 1,
-            "legendFormat": "instances",
+            "legendFormat": "{{consumergroup}}",
             "refId": "D"
           }
         ],
@@ -436,7 +436,7 @@ data:
         "steppedLine": false,
         "targets": [
           {
-            "expr": "sum by(group,topic) (kafka_consumergroup_group_offset >= 0)",
+            "expr": "sum by(consumergroup,topic) (kafka_consumergroup_current_offset{topic='input'} >= 0)",
             "format": "time_series",
             "intervalFactor": 1,
             "legendFormat": "{{topic}}",
@@ -527,7 +527,7 @@ data:
         "steppedLine": false,
         "targets": [
           {
-            "expr": "count by(group,topic) (kafka_consumergroup_group_offset >= 0)",
+            "expr": "count by(topic) (kafka_topic_partitions >= 0)",
             "format": "time_series",
             "intervalFactor": 1,
             "legendFormat": "{{topic}}",
@@ -618,7 +618,7 @@ data:
         "steppedLine": false,
         "targets": [
           {
-            "expr": "sum by(group,topic) (kafka_partition_latest_offset)",
+            "expr": "sum by(topic) (kafka_topic_partition_current_offset)",
             "format": "time_series",
             "intervalFactor": 1,
             "legendFormat": "{{topic}}",

helm/templates/prometheus/prometheus.yaml

@@ -5,11 +5,12 @@ metadata:
   name: {{ template "theodolite.fullname" . }}-prometheus
 spec:
   serviceAccountName: {{ template "theodolite.fullname" . }}-prometheus
+  podMonitorSelector: {}
   serviceMonitorSelector: {}
   resources:
     requests:
       memory: 400Mi
-  #scrapeInterval: 1s
+  scrapeInterval: 15s
   enableAdminAPI: true
   {{- with .Values.prometheus.nodeSelector }}
   nodeSelector:

helm/templates/strimzi/kafka-cluster.yaml

+{{- if .Values.strimzi.enabled -}}
+apiVersion: kafka.strimzi.io/v1beta2
+kind: Kafka
+metadata:
+  name: {{ template "theodolite.fullname" . }}-kafka
+spec:
+  kafka:
+    jmxOptions: {}
+    {{- with .Values.strimzi.kafka.listeners }}
+    listeners:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+    {{- with .Values.strimzi.kafka.replicas }}
+    replicas:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+    {{- with .Values.strimzi.kafka.config }}
+    config:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+    {{- with .Values.strimzi.kafka.jvmOptions }}
+    jvmOptions:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+    storage:
+      type: ephemeral
+    metricsConfig:
+      type: jmxPrometheusExporter
+      valueFrom:
+        configMapKeyRef:
+          name: {{ template "theodolite.fullname" . }}-kafka-metrics
+          key: kafka-metrics-config.yml
+  
+  kafkaExporter: {}
+
+  zookeeper:
+    {{- with .Values.strimzi.zookeeper.replicas }}
+    replicas:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+    storage:
+      type: ephemeral
+{{- end }}
\ No newline at end of file

helm/templates/strimzi/kafka-exporter-podmonitor.yaml

+{{- if .Values.strimzi.enabled -}}
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: {{ template "theodolite.fullname" . }}-kafka-exporter-podmonitor
+  labels:
+    app: theodolite
+spec:
+  selector:
+    selector:
+    matchLabels:
+      strimzi.io/name: {{ template "theodolite.fullname" . }}-strimzi-kafka-exporter
+  podMetricsEndpoints:
+  - path: /metrics
+    port: tcp-prometheus
+{{- end }}
\ No newline at end of file

helm/templates/strimzi/kafka-metrics-configmap.yaml

+{{- if .Values.strimzi.enabled -}}
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: {{ template "theodolite.fullname" . }}-kafka-metrics
+  labels:
+    app: strimzi
+data:
+  kafka-metrics-config.yml: |
+    # See https://github.com/prometheus/jmx_exporter for more info about JMX Prometheus Exporter metrics
+    lowercaseOutputName: true
+    rules:
+    # Special cases and very specific rules
+    - pattern: kafka.server<type=(.+), name=(.+), clientId=(.+), topic=(.+), partition=(.*)><>Value
+      name: kafka_server_$1_$2
+      type: GAUGE
+      labels:
+       clientId: "$3"
+       topic: "$4"
+       partition: "$5"
+    - pattern: kafka.server<type=(.+), name=(.+), clientId=(.+), brokerHost=(.+), brokerPort=(.+)><>Value
+      name: kafka_server_$1_$2
+      type: GAUGE
+      labels:
+       clientId: "$3"
+       broker: "$4:$5"
+    - pattern: kafka.server<type=(.+), cipher=(.+), protocol=(.+), listener=(.+), networkProcessor=(.+)><>connections
+      name: kafka_server_$1_connections_tls_info
+      type: GAUGE
+      labels:
+        cipher: "$2"
+        protocol: "$3"
+        listener: "$4"
+        networkProcessor: "$5"
+    - pattern: kafka.server<type=(.+), clientSoftwareName=(.+), clientSoftwareVersion=(.+), listener=(.+), networkProcessor=(.+)><>connections
+      name: kafka_server_$1_connections_software
+      type: GAUGE
+      labels:
+        clientSoftwareName: "$2"
+        clientSoftwareVersion: "$3"
+        listener: "$4"
+        networkProcessor: "$5"
+    - pattern: "kafka.server<type=(.+), listener=(.+), networkProcessor=(.+)><>(.+):"
+      name: kafka_server_$1_$4
+      type: GAUGE
+      labels:
+       listener: "$2"
+       networkProcessor: "$3"
+    - pattern: kafka.server<type=(.+), listener=(.+), networkProcessor=(.+)><>(.+)
+      name: kafka_server_$1_$4
+      type: GAUGE
+      labels:
+       listener: "$2"
+       networkProcessor: "$3"
+    # Some percent metrics use MeanRate attribute
+    # Ex) kafka.server<type=(KafkaRequestHandlerPool), name=(RequestHandlerAvgIdlePercent)><>MeanRate
+    - pattern: kafka.(\w+)<type=(.+), name=(.+)Percent\w*><>MeanRate
+      name: kafka_$1_$2_$3_percent
+      type: GAUGE
+    # Generic gauges for percents
+    - pattern: kafka.(\w+)<type=(.+), name=(.+)Percent\w*><>Value
+      name: kafka_$1_$2_$3_percent
+      type: GAUGE
+    - pattern: kafka.(\w+)<type=(.+), name=(.+)Percent\w*, (.+)=(.+)><>Value
+      name: kafka_$1_$2_$3_percent
+      type: GAUGE
+      labels:
+        "$4": "$5"
+    # Generic per-second counters with 0-2 key/value pairs
+    - pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*, (.+)=(.+), (.+)=(.+)><>Count
+      name: kafka_$1_$2_$3_total
+      type: COUNTER
+      labels:
+        "$4": "$5"
+        "$6": "$7"
+    - pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*, (.+)=(.+)><>Count
+      name: kafka_$1_$2_$3_total
+      type: COUNTER
+      labels:
+        "$4": "$5"
+    - pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*><>Count
+      name: kafka_$1_$2_$3_total
+      type: COUNTER
+    # Generic gauges with 0-2 key/value pairs
+    - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>Value
+      name: kafka_$1_$2_$3
+      type: GAUGE
+      labels:
+        "$4": "$5"
+        "$6": "$7"
+    - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+)><>Value
+      name: kafka_$1_$2_$3
+      type: GAUGE
+      labels:
+        "$4": "$5"
+    - pattern: kafka.(\w+)<type=(.+), name=(.+)><>Value
+      name: kafka_$1_$2_$3
+      type: GAUGE
+    # Emulate Prometheus 'Summary' metrics for the exported 'Histogram's.
+    # Note that these are missing the '_sum' metric!
+    - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>Count
+      name: kafka_$1_$2_$3_count
+      type: COUNTER
+      labels:
+        "$4": "$5"
+        "$6": "$7"
+    - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.*), (.+)=(.+)><>(\d+)thPercentile
+      name: kafka_$1_$2_$3
+      type: GAUGE
+      labels:
+        "$4": "$5"
+        "$6": "$7"
+        quantile: "0.$8"
+    - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+)><>Count
+      name: kafka_$1_$2_$3_count
+      type: COUNTER
+      labels:
+        "$4": "$5"
+    - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.*)><>(\d+)thPercentile
+      name: kafka_$1_$2_$3
+      type: GAUGE
+      labels:
+        "$4": "$5"
+        quantile: "0.$6"
+    - pattern: kafka.(\w+)<type=(.+), name=(.+)><>Count
+      name: kafka_$1_$2_$3_count
+      type: COUNTER
+    - pattern: kafka.(\w+)<type=(.+), name=(.+)><>(\d+)thPercentile
+      name: kafka_$1_$2_$3
+      type: GAUGE
+      labels:
+        quantile: "0.$4"
+{{- end }}
\ No newline at end of file

helm/templates/strimzi/kafka-podmonitor.yaml

+{{- if .Values.strimzi.enabled -}}
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: {{ template "theodolite.fullname" . }}-kafka-resources-metrics
+  labels:
+    app: theodolite
+spec:
+  selector:
+    selector:
+    matchLabels:
+      strimzi.io/name: {{ template "theodolite.fullname" . }}-kafka-kafka
+  podMetricsEndpoints:
+  - path: /metrics
+    port: tcp-prometheus
+{{- end }}
\ No newline at end of file

helm/templates/theodolite/role.yaml

@@ -38,6 +38,7 @@ rules:
     - monitoring.coreos.com
     resources:
     - servicemonitors
+    - podmonitors
     verbs:
     - update
     - delete

helm/values.yaml

@@ -64,7 +64,7 @@ cp-helm-charts:
   ## Zookeeper
   ## ------------------------------------------------------
   cp-zookeeper:
-    enabled: true
+    enabled: false
     nodeSelector: {}
     servers: 3
     image: confluentinc/cp-zookeeper
@@ -81,7 +81,7 @@ cp-helm-charts:
   ## Kafka
   ## ------------------------------------------------------
   cp-kafka:
-    enabled: true
+    enabled: false
     nodeSelector: {}
     brokers: 3
     image: confluentinc/cp-enterprise-kafka
@@ -137,6 +137,9 @@ cp-helm-charts:
       nodePort: 30099
       annotations: {}
 
+    kafka:
+      bootstrapServers: theodolite-kafka-kafka-bootstrap:9092
+
   cp-kafka-rest:
     enabled: false
 
@@ -149,29 +152,30 @@ cp-helm-charts:
   cp-control-center:
     enabled: false
 
-
-###
-# Kafka Lag Exporter
-###
-kafka-lag-exporter:
-  enabled: true
-  image:
-    pullPolicy: IfNotPresent
-  nodeSelector: {}
-  
-  clusters:
-    - name: "theodolite-cp-kafka"
-      bootstrapBrokers: "theodolite-cp-kafka:9092"
-
   ## The interval between refreshing metrics
   pollIntervalSeconds: 15
 
-  prometheus:
-    serviceMonitor:
-      enabled: true
-      interval: "5s"
-      additionalLabels:
-        appScope: titan-ccp
+strimzi:
+  enabled: true
+  kafka:
+    listeners:
+      - name: plain
+        port: 9092
+        type: internal
+        tls: false
+    replicas: 3
+    config:
+      "message.max.bytes": "134217728" # 128 MB
+      "replica.fetch.max.bytes": "134217728" #128 MB
+      "auto.create.topics.enable": false
+      "log.retention.ms": "7200000" # 2h
+      "metrics.sample.window.ms": "5000" #5s
+    jvmOptions:
+      "-Xmx": "512M"
+      "-Xms": "512M"
+
+  zookeeper:
+    replicas: 3
 
 
 ###

slo-checker/record-lag/app/main.py

@@ -24,7 +24,7 @@ elif os.getenv('LOG_LEVEL') == 'DEBUG':
 def calculate_slope_trend(results, warmup):
     d = []
     for result in results:
-        group = result['metric']['group']
+        group = result['metric']['consumergroup']
         for value in result['values']:
             d.append({'group': group, 'timestamp': int(
                 value[0]), 'value': int(value[1]) if value[1] != 'NaN' else 0})

slo-checker/record-lag/resources/test-1-rep-success.json

@@ -3,7 +3,7 @@
         [
             {
                 "metric": {
-                    "group": "theodolite-uc1-application-0.0.1"
+                    "consumergroup": "theodolite-uc1-application-0.0.1"
                 },
                 "values": [
                     [

slo-checker/record-lag/resources/test-3-rep-success.json

@@ -3,7 +3,7 @@
         [
             {
                 "metric": {
-                    "group": "theodolite-uc1-application-0.0.1"
+                    "consumergroup": "theodolite-uc1-application-0.0.1"
                 },
                 "values": [
                     [
@@ -100,7 +100,7 @@
         [
             {
                 "metric": {
-                    "group": "theodolite-uc1-application-0.0.1"
+                    "consumergroup": "theodolite-uc1-application-0.0.1"
                 },
                 "values": [
                     [
@@ -193,7 +193,7 @@
         [
             {
                 "metric": {
-                    "group": "theodolite-uc1-application-0.0.1"
+                    "consumergroup": "theodolite-uc1-application-0.0.1"
                 },
                 "values": [
                     [

theodolite-benchmarks/definitions/uc1-flink/resources/jobmanager-deployment.yaml

@@ -20,7 +20,7 @@ spec:
           image: ghcr.io/cau-se/theodolite-uc1-flink:latest
           env:
             - name: KAFKA_BOOTSTRAP_SERVERS
-              value: "theodolite-cp-kafka:9092"
+              value: "theodolite-kafka-kafka-bootstrap:9092"
             - name: SCHEMA_REGISTRY_URL
               value: "http://theodolite-cp-schema-registry:8081"
             - name: COMMIT_INTERVAL_MS

theodolite-benchmarks/definitions/uc1-flink/resources/taskmanager-deployment.yaml

@@ -20,7 +20,7 @@ spec:
           image: ghcr.io/cau-se/theodolite-uc1-flink:latest
           env:
             - name: KAFKA_BOOTSTRAP_SERVERS
-              value: "theodolite-cp-kafka:9092"
+              value: "theodolitee-kafka-kafka-bootstrap:9092"
             - name: SCHEMA_REGISTRY_URL
               value: "http://theodolite-cp-schema-registry:8081"
             - name: COMMIT_INTERVAL_MS

theodolite-benchmarks/definitions/uc1-flink/uc1-flink-benchmark-operator.yaml

@@ -50,7 +50,7 @@ spec:
           properties:
             loadGenMaxRecords: "150000"
   kafkaConfig:
-    bootstrapServer: "theodolite-cp-kafka:9092"
+    bootstrapServer: "theodolitee-kafka-kafka-bootstrap:9092"
     topics:
       - name: "input"
         numPartitions: 40

theodolite-benchmarks/definitions/uc1-kstreams/resources/uc1-kstreams-deployment.yaml

@@ -21,7 +21,7 @@ spec:
               name: jmx
           env:
             - name: KAFKA_BOOTSTRAP_SERVERS
-              value: "theodolite-cp-kafka:9092"
+              value: "theodolite-kafka-kafka-bootstrap:9092"
             - name: SCHEMA_REGISTRY_URL
               value: "http://theodolite-cp-schema-registry:8081"
             - name: JAVA_OPTS

theodolite-benchmarks/definitions/uc1-kstreams/uc1-kstreams-benchmark-operator.yaml

@@ -37,7 +37,7 @@ spec:
           properties:
             loadGenMaxRecords: "150000"
   kafkaConfig:
-    bootstrapServer: "theodolite-cp-kafka:9092"
+    bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092"
     topics:
       - name: "input"
         numPartitions: 40

theodolite-benchmarks/definitions/uc1-kstreams/uc1-kstreams-benchmark-standalone.yaml

@@ -32,7 +32,7 @@ loadTypes:
         properties:
           loadGenMaxRecords: "150000"
 kafkaConfig:
-  bootstrapServer: "theodolite-cp-kafka:9092"
+  bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092"
   topics:
     - name: "input"
       numPartitions: 40