From f798df08d350feb65a05a5c823a05703b2a3cfbc Mon Sep 17 00:00:00 2001
From: Simon Ehrenstein <simon.ehrenstein@gmail.com>
Date: Tue, 15 Feb 2022 16:41:49 +0100
Subject: [PATCH] Add strimzi kafka
---
helm/Chart.yaml | 6 +-
helm/templates/prometheus/prometheus.yaml | 5 +-
helm/templates/strimzi/kafka-cluster.yaml | 27 ++++
helm/templates/strimzi/kafka-config.yaml | 131 ++++++++++++++++++
.../strimzi/kafka-exporter-podmonitor.yaml | 17 +++
helm/templates/strimzi/kafka-podmonitor.yaml | 46 ++++++
helm/values.yaml | 2 +-
7 files changed, 229 insertions(+), 5 deletions(-)
create mode 100644 helm/templates/strimzi/kafka-cluster.yaml
create mode 100644 helm/templates/strimzi/kafka-config.yaml
create mode 100644 helm/templates/strimzi/kafka-exporter-podmonitor.yaml
create mode 100644 helm/templates/strimzi/kafka-podmonitor.yaml
diff --git a/helm/Chart.yaml b/helm/Chart.yaml
index 27451ad55..243a49648 100644
--- a/helm/Chart.yaml
+++ b/helm/Chart.yaml
@@ -24,10 +24,10 @@ dependencies:
version: 0.6.0
repository: https://soerenhenning.github.io/cp-helm-charts
condition: cp-helm-charts.enabled
- - name: kafka-lag-exporter
- version: 0.6.7
- repository: https://lightbend.github.io/kafka-lag-exporter/repo/
condition: kafka-lag-exporter.enabled
+ - name: strimzi-kafka-operator
+ version: 0.27.1
+ repository: https://strimzi.io/charts/
version: 0.7.0-SNAPSHOT
diff --git a/helm/templates/prometheus/prometheus.yaml b/helm/templates/prometheus/prometheus.yaml
index 4e297b202..2b655dd70 100644
--- a/helm/templates/prometheus/prometheus.yaml
+++ b/helm/templates/prometheus/prometheus.yaml
@@ -5,10 +5,13 @@ metadata:
name: {{ template "theodolite.fullname" . }}-prometheus
spec:
serviceAccountName: {{ template "theodolite.fullname" . }}-prometheus
+ podMonitorSelector:
+ matchLabels:
+ app: theodolite
serviceMonitorSelector:
matchLabels:
#app: cp-kafka
- appScope: titan-ccp
+ appScope: theodolite
resources:
requests:
memory: 400Mi
diff --git a/helm/templates/strimzi/kafka-cluster.yaml b/helm/templates/strimzi/kafka-cluster.yaml
new file mode 100644
index 000000000..b56033cc8
--- /dev/null
+++ b/helm/templates/strimzi/kafka-cluster.yaml
@@ -0,0 +1,27 @@
+apiVersion: kafka.strimzi.io/v1beta2
+kind: Kafka
+metadata:
+ name: theodolite-strimzi
+spec:
+ kafka:
+ listeners:
+ - name: plain
+ port: 9092
+ type: internal
+ tls: false
+ storage:
+ type: ephemeral
+ replicas: 3
+ metricsConfig:
+ type: jmxPrometheusExporter
+ valueFrom:
+ configMapKeyRef:
+ name: kafka-metrics
+ key: kafka-metrics-config.yml
+
+ kafkaExporter: {}
+
+ zookeeper:
+ replicas: 3
+ storage:
+ type: ephemeral
diff --git a/helm/templates/strimzi/kafka-config.yaml b/helm/templates/strimzi/kafka-config.yaml
new file mode 100644
index 000000000..1ce8c3d08
--- /dev/null
+++ b/helm/templates/strimzi/kafka-config.yaml
@@ -0,0 +1,131 @@
+kind: ConfigMap
+apiVersion: v1
+metadata:
+ name: kafka-metrics
+ labels:
+ app: strimzi
+data:
+ kafka-metrics-config.yml: |
+ # See https://github.com/prometheus/jmx_exporter for more info about JMX Prometheus Exporter metrics
+ lowercaseOutputName: true
+ rules:
+ # Special cases and very specific rules
+ - pattern: kafka.server<type=(.+), name=(.+), clientId=(.+), topic=(.+), partition=(.*)><>Value
+ name: kafka_server_$1_$2
+ type: GAUGE
+ labels:
+ clientId: "$3"
+ topic: "$4"
+ partition: "$5"
+ - pattern: kafka.server<type=(.+), name=(.+), clientId=(.+), brokerHost=(.+), brokerPort=(.+)><>Value
+ name: kafka_server_$1_$2
+ type: GAUGE
+ labels:
+ clientId: "$3"
+ broker: "$4:$5"
+ - pattern: kafka.server<type=(.+), cipher=(.+), protocol=(.+), listener=(.+), networkProcessor=(.+)><>connections
+ name: kafka_server_$1_connections_tls_info
+ type: GAUGE
+ labels:
+ cipher: "$2"
+ protocol: "$3"
+ listener: "$4"
+ networkProcessor: "$5"
+ - pattern: kafka.server<type=(.+), clientSoftwareName=(.+), clientSoftwareVersion=(.+), listener=(.+), networkProcessor=(.+)><>connections
+ name: kafka_server_$1_connections_software
+ type: GAUGE
+ labels:
+ clientSoftwareName: "$2"
+ clientSoftwareVersion: "$3"
+ listener: "$4"
+ networkProcessor: "$5"
+ - pattern: "kafka.server<type=(.+), listener=(.+), networkProcessor=(.+)><>(.+):"
+ name: kafka_server_$1_$4
+ type: GAUGE
+ labels:
+ listener: "$2"
+ networkProcessor: "$3"
+ - pattern: kafka.server<type=(.+), listener=(.+), networkProcessor=(.+)><>(.+)
+ name: kafka_server_$1_$4
+ type: GAUGE
+ labels:
+ listener: "$2"
+ networkProcessor: "$3"
+ # Some percent metrics use MeanRate attribute
+ # Ex) kafka.server<type=(KafkaRequestHandlerPool), name=(RequestHandlerAvgIdlePercent)><>MeanRate
+ - pattern: kafka.(\w+)<type=(.+), name=(.+)Percent\w*><>MeanRate
+ name: kafka_$1_$2_$3_percent
+ type: GAUGE
+ # Generic gauges for percents
+ - pattern: kafka.(\w+)<type=(.+), name=(.+)Percent\w*><>Value
+ name: kafka_$1_$2_$3_percent
+ type: GAUGE
+ - pattern: kafka.(\w+)<type=(.+), name=(.+)Percent\w*, (.+)=(.+)><>Value
+ name: kafka_$1_$2_$3_percent
+ type: GAUGE
+ labels:
+ "$4": "$5"
+ # Generic per-second counters with 0-2 key/value pairs
+ - pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*, (.+)=(.+), (.+)=(.+)><>Count
+ name: kafka_$1_$2_$3_total
+ type: COUNTER
+ labels:
+ "$4": "$5"
+ "$6": "$7"
+ - pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*, (.+)=(.+)><>Count
+ name: kafka_$1_$2_$3_total
+ type: COUNTER
+ labels:
+ "$4": "$5"
+ - pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*><>Count
+ name: kafka_$1_$2_$3_total
+ type: COUNTER
+ # Generic gauges with 0-2 key/value pairs
+ - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>Value
+ name: kafka_$1_$2_$3
+ type: GAUGE
+ labels:
+ "$4": "$5"
+ "$6": "$7"
+ - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+)><>Value
+ name: kafka_$1_$2_$3
+ type: GAUGE
+ labels:
+ "$4": "$5"
+ - pattern: kafka.(\w+)<type=(.+), name=(.+)><>Value
+ name: kafka_$1_$2_$3
+ type: GAUGE
+ # Emulate Prometheus 'Summary' metrics for the exported 'Histogram's.
+ # Note that these are missing the '_sum' metric!
+ - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>Count
+ name: kafka_$1_$2_$3_count
+ type: COUNTER
+ labels:
+ "$4": "$5"
+ "$6": "$7"
+ - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.*), (.+)=(.+)><>(\d+)thPercentile
+ name: kafka_$1_$2_$3
+ type: GAUGE
+ labels:
+ "$4": "$5"
+ "$6": "$7"
+ quantile: "0.$8"
+ - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+)><>Count
+ name: kafka_$1_$2_$3_count
+ type: COUNTER
+ labels:
+ "$4": "$5"
+ - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.*)><>(\d+)thPercentile
+ name: kafka_$1_$2_$3
+ type: GAUGE
+ labels:
+ "$4": "$5"
+ quantile: "0.$6"
+ - pattern: kafka.(\w+)<type=(.+), name=(.+)><>Count
+ name: kafka_$1_$2_$3_count
+ type: COUNTER
+ - pattern: kafka.(\w+)<type=(.+), name=(.+)><>(\d+)thPercentile
+ name: kafka_$1_$2_$3
+ type: GAUGE
+ labels:
+ quantile: "0.$4"
\ No newline at end of file
diff --git a/helm/templates/strimzi/kafka-exporter-podmonitor.yaml b/helm/templates/strimzi/kafka-exporter-podmonitor.yaml
new file mode 100644
index 000000000..679b9dbdb
--- /dev/null
+++ b/helm/templates/strimzi/kafka-exporter-podmonitor.yaml
@@ -0,0 +1,17 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+ name: kafka-exporter-podmonitor
+ labels:
+ app: theodolite
+spec:
+ selector:
+ selector:
+ matchLabels:
+ strimzi.io/name: theodolite-strimzi-kafka-exporter
+ namespaceSelector:
+ matchNames:
+ - default
+ podMetricsEndpoints:
+ - path: /metrics
+ port: tcp-prometheus
\ No newline at end of file
diff --git a/helm/templates/strimzi/kafka-podmonitor.yaml b/helm/templates/strimzi/kafka-podmonitor.yaml
new file mode 100644
index 000000000..d232d3d18
--- /dev/null
+++ b/helm/templates/strimzi/kafka-podmonitor.yaml
@@ -0,0 +1,46 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+ name: kafka-resources-metrics
+ labels:
+ app: theodolite
+spec:
+ selector:
+ selector:
+ matchLabels:
+ strimzi.io/name: theodolite-strimzi-kafka
+ namespaceSelector:
+ matchNames:
+ - default
+ podMetricsEndpoints:
+ - path: /metrics
+ port: tcp-prometheus
+ relabelings:
+ - separator: ;
+ regex: __meta_kubernetes_pod_label_(strimzi_io_.+)
+ replacement: $1
+ action: labelmap
+ - sourceLabels: [__meta_kubernetes_namespace]
+ separator: ;
+ regex: (.*)
+ targetLabel: namespace
+ replacement: $1
+ action: replace
+ - sourceLabels: [__meta_kubernetes_pod_name]
+ separator: ;
+ regex: (.*)
+ targetLabel: kubernetes_pod_name
+ replacement: $1
+ action: replace
+ - sourceLabels: [__meta_kubernetes_pod_node_name]
+ separator: ;
+ regex: (.*)
+ targetLabel: node_name
+ replacement: $1
+ action: replace
+ - sourceLabels: [__meta_kubernetes_pod_host_ip]
+ separator: ;
+ regex: (.*)
+ targetLabel: node_ip
+ replacement: $1
+ action: replace
\ No newline at end of file
diff --git a/helm/values.yaml b/helm/values.yaml
index ba58b0409..82f3c6970 100644
--- a/helm/values.yaml
+++ b/helm/values.yaml
@@ -154,7 +154,7 @@ cp-helm-charts:
# Kafka Lag Exporter
###
kafka-lag-exporter:
- enabled: true
+ enabled: false
image:
pullPolicy: IfNotPresent
nodeSelector: {}
--
GitLab