diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 27451ad55ce75592db9dc7550b1f81dced3951bc..e3307341c3f455215c74a8449ac7160606524b63 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -24,10 +24,10 @@ dependencies: version: 0.6.0 repository: https://soerenhenning.github.io/cp-helm-charts condition: cp-helm-charts.enabled - - name: kafka-lag-exporter - version: 0.6.7 - repository: https://lightbend.github.io/kafka-lag-exporter/repo/ - condition: kafka-lag-exporter.enabled + - name: strimzi-kafka-operator + version: 0.27.1 + repository: https://strimzi.io/charts/ + version: 0.7.0-SNAPSHOT diff --git a/helm/templates/strimzi/kafka-cluster.yaml b/helm/templates/strimzi/kafka-cluster.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b56033cc8a03bb546b6ce48870ca2fc2a44a5fbf --- /dev/null +++ b/helm/templates/strimzi/kafka-cluster.yaml @@ -0,0 +1,27 @@ +apiVersion: kafka.strimzi.io/v1beta2 +kind: Kafka +metadata: + name: theodolite-strimzi +spec: + kafka: + listeners: + - name: plain + port: 9092 + type: internal + tls: false + storage: + type: ephemeral + replicas: 3 + metricsConfig: + type: jmxPrometheusExporter + valueFrom: + configMapKeyRef: + name: kafka-metrics + key: kafka-metrics-config.yml + + kafkaExporter: {} + + zookeeper: + replicas: 3 + storage: + type: ephemeral diff --git a/helm/templates/strimzi/kafka-config.yaml b/helm/templates/strimzi/kafka-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1ce8c3d08c11b3591dfe36557afeed6204f4fbb7 --- /dev/null +++ b/helm/templates/strimzi/kafka-config.yaml @@ -0,0 +1,131 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: kafka-metrics + labels: + app: strimzi +data: + kafka-metrics-config.yml: | + # See https://github.com/prometheus/jmx_exporter for more info about JMX Prometheus Exporter metrics + lowercaseOutputName: true + rules: + # Special cases and very specific rules + - pattern: kafka.server<type=(.+), name=(.+), clientId=(.+), topic=(.+), partition=(.*)><>Value + name: kafka_server_$1_$2 + type: GAUGE + labels: + clientId: "$3" + topic: "$4" + partition: "$5" + - pattern: kafka.server<type=(.+), name=(.+), clientId=(.+), brokerHost=(.+), brokerPort=(.+)><>Value + name: kafka_server_$1_$2 + type: GAUGE + labels: + clientId: "$3" + broker: "$4:$5" + - pattern: kafka.server<type=(.+), cipher=(.+), protocol=(.+), listener=(.+), networkProcessor=(.+)><>connections + name: kafka_server_$1_connections_tls_info + type: GAUGE + labels: + cipher: "$2" + protocol: "$3" + listener: "$4" + networkProcessor: "$5" + - pattern: kafka.server<type=(.+), clientSoftwareName=(.+), clientSoftwareVersion=(.+), listener=(.+), networkProcessor=(.+)><>connections + name: kafka_server_$1_connections_software + type: GAUGE + labels: + clientSoftwareName: "$2" + clientSoftwareVersion: "$3" + listener: "$4" + networkProcessor: "$5" + - pattern: "kafka.server<type=(.+), listener=(.+), networkProcessor=(.+)><>(.+):" + name: kafka_server_$1_$4 + type: GAUGE + labels: + listener: "$2" + networkProcessor: "$3" + - pattern: kafka.server<type=(.+), listener=(.+), networkProcessor=(.+)><>(.+) + name: kafka_server_$1_$4 + type: GAUGE + labels: + listener: "$2" + networkProcessor: "$3" + # Some percent metrics use MeanRate attribute + # Ex) kafka.server<type=(KafkaRequestHandlerPool), name=(RequestHandlerAvgIdlePercent)><>MeanRate + - pattern: kafka.(\w+)<type=(.+), name=(.+)Percent\w*><>MeanRate + name: kafka_$1_$2_$3_percent + type: GAUGE + # Generic gauges for percents + - pattern: kafka.(\w+)<type=(.+), name=(.+)Percent\w*><>Value + name: kafka_$1_$2_$3_percent + type: GAUGE + - pattern: kafka.(\w+)<type=(.+), name=(.+)Percent\w*, (.+)=(.+)><>Value + name: kafka_$1_$2_$3_percent + type: GAUGE + labels: + "$4": "$5" + # Generic per-second counters with 0-2 key/value pairs + - pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*, (.+)=(.+), (.+)=(.+)><>Count + name: kafka_$1_$2_$3_total + type: COUNTER + labels: + "$4": "$5" + "$6": "$7" + - pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*, (.+)=(.+)><>Count + name: kafka_$1_$2_$3_total + type: COUNTER + labels: + "$4": "$5" + - pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*><>Count + name: kafka_$1_$2_$3_total + type: COUNTER + # Generic gauges with 0-2 key/value pairs + - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>Value + name: kafka_$1_$2_$3 + type: GAUGE + labels: + "$4": "$5" + "$6": "$7" + - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+)><>Value + name: kafka_$1_$2_$3 + type: GAUGE + labels: + "$4": "$5" + - pattern: kafka.(\w+)<type=(.+), name=(.+)><>Value + name: kafka_$1_$2_$3 + type: GAUGE + # Emulate Prometheus 'Summary' metrics for the exported 'Histogram's. + # Note that these are missing the '_sum' metric! + - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>Count + name: kafka_$1_$2_$3_count + type: COUNTER + labels: + "$4": "$5" + "$6": "$7" + - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.*), (.+)=(.+)><>(\d+)thPercentile + name: kafka_$1_$2_$3 + type: GAUGE + labels: + "$4": "$5" + "$6": "$7" + quantile: "0.$8" + - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+)><>Count + name: kafka_$1_$2_$3_count + type: COUNTER + labels: + "$4": "$5" + - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.*)><>(\d+)thPercentile + name: kafka_$1_$2_$3 + type: GAUGE + labels: + "$4": "$5" + quantile: "0.$6" + - pattern: kafka.(\w+)<type=(.+), name=(.+)><>Count + name: kafka_$1_$2_$3_count + type: COUNTER + - pattern: kafka.(\w+)<type=(.+), name=(.+)><>(\d+)thPercentile + name: kafka_$1_$2_$3 + type: GAUGE + labels: + quantile: "0.$4" \ No newline at end of file diff --git a/helm/templates/strimzi/kafka-exporter-podmonitor.yaml b/helm/templates/strimzi/kafka-exporter-podmonitor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..679b9dbdb8bd7440f890172867adafffbd5a4aa9 --- /dev/null +++ b/helm/templates/strimzi/kafka-exporter-podmonitor.yaml @@ -0,0 +1,17 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: kafka-exporter-podmonitor + labels: + app: theodolite +spec: + selector: + selector: + matchLabels: + strimzi.io/name: theodolite-strimzi-kafka-exporter + namespaceSelector: + matchNames: + - default + podMetricsEndpoints: + - path: /metrics + port: tcp-prometheus \ No newline at end of file diff --git a/helm/templates/strimzi/kafka-podmonitor.yaml b/helm/templates/strimzi/kafka-podmonitor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d232d3d187e25a3ffbb07b60fe6486dc1c4c228c --- /dev/null +++ b/helm/templates/strimzi/kafka-podmonitor.yaml @@ -0,0 +1,46 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: kafka-resources-metrics + labels: + app: theodolite +spec: + selector: + selector: + matchLabels: + strimzi.io/name: theodolite-strimzi-kafka + namespaceSelector: + matchNames: + - default + podMetricsEndpoints: + - path: /metrics + port: tcp-prometheus + relabelings: + - separator: ; + regex: __meta_kubernetes_pod_label_(strimzi_io_.+) + replacement: $1 + action: labelmap + - sourceLabels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + targetLabel: namespace + replacement: $1 + action: replace + - sourceLabels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + targetLabel: kubernetes_pod_name + replacement: $1 + action: replace + - sourceLabels: [__meta_kubernetes_pod_node_name] + separator: ; + regex: (.*) + targetLabel: node_name + replacement: $1 + action: replace + - sourceLabels: [__meta_kubernetes_pod_host_ip] + separator: ; + regex: (.*) + targetLabel: node_ip + replacement: $1 + action: replace \ No newline at end of file diff --git a/helm/values.yaml b/helm/values.yaml index ba58b040974886518ab111d668cb0db1140b2eb8..6724094907b89e600df45df0bf00346ad5d30a23 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -64,7 +64,7 @@ cp-helm-charts: ## Zookeeper ## ------------------------------------------------------ cp-zookeeper: - enabled: true + enabled: false nodeSelector: {} servers: 3 image: confluentinc/cp-zookeeper @@ -81,7 +81,7 @@ cp-helm-charts: ## Kafka ## ------------------------------------------------------ cp-kafka: - enabled: true + enabled: false nodeSelector: {} brokers: 3 image: confluentinc/cp-enterprise-kafka @@ -137,6 +137,10 @@ cp-helm-charts: nodePort: 30099 annotations: {} + + kafka: + bootstrapServers: theodolite-strimzi-kafka-bootstrap:9092 + cp-kafka-rest: enabled: false @@ -149,20 +153,6 @@ cp-helm-charts: cp-control-center: enabled: false - -### -# Kafka Lag Exporter -### -kafka-lag-exporter: - enabled: true - image: - pullPolicy: IfNotPresent - nodeSelector: {} - - clusters: - - name: "theodolite-cp-kafka" - bootstrapBrokers: "theodolite-cp-kafka:9092" - ## The interval between refreshing metrics pollIntervalSeconds: 15