Skip to content
Snippets Groups Projects
Commit 5d188eb0 authored by Simon Ehrenstein's avatar Simon Ehrenstein
Browse files

Closes 346

parent 7cd0201c
No related branches found
No related tags found
2 merge requests!243Migrate to Strimzi Kafka,!240Draft: Integrate strimzi
Hide whitespace changes
Inline Side-by-side
helm/Chart.yaml
+ 4
− 4
View file @ 5d188eb0
......@@ -24,10 +24,10 @@ dependencies:
version: 0.6.0
repository: https://soerenhenning.github.io/cp-helm-charts
condition: cp-helm-charts.enabled
- name: kafka-lag-exporter
version: 0.6.7
repository: https://lightbend.github.io/kafka-lag-exporter/repo/
condition: kafka-lag-exporter.enabled
- name: strimzi-kafka-operator
version: 0.27.1
repository: https://strimzi.io/charts/
version: 0.7.0-SNAPSHOT
......
helm/templates/strimzi/kafka-cluster.yaml 0 → 100644
+ 27
− 0
View file @ 5d188eb0
apiVersion: kafka.strimzi.io/v1beta2
kind: Kafka
metadata:
name: theodolite-strimzi
spec:
kafka:
listeners:
- name: plain
port: 9092
type: internal
tls: false
storage:
type: ephemeral
replicas: 3
metricsConfig:
type: jmxPrometheusExporter
valueFrom:
configMapKeyRef:
name: kafka-metrics
key: kafka-metrics-config.yml
kafkaExporter: {}
zookeeper:
replicas: 3
storage:
type: ephemeral
helm/templates/strimzi/kafka-config.yaml 0 → 100644
+ 131
− 0
View file @ 5d188eb0
kind: ConfigMap
apiVersion: v1
metadata:
name: kafka-metrics
labels:
app: strimzi
data:
kafka-metrics-config.yml: |
# See https://github.com/prometheus/jmx_exporter for more info about JMX Prometheus Exporter metrics
lowercaseOutputName: true
rules:
# Special cases and very specific rules
- pattern: kafka.server<type=(.+), name=(.+), clientId=(.+), topic=(.+), partition=(.*)><>Value
name: kafka_server_$1_$2
type: GAUGE
labels:
clientId: "$3"
topic: "$4"
partition: "$5"
- pattern: kafka.server<type=(.+), name=(.+), clientId=(.+), brokerHost=(.+), brokerPort=(.+)><>Value
name: kafka_server_$1_$2
type: GAUGE
labels:
clientId: "$3"
broker: "$4:$5"
- pattern: kafka.server<type=(.+), cipher=(.+), protocol=(.+), listener=(.+), networkProcessor=(.+)><>connections
name: kafka_server_$1_connections_tls_info
type: GAUGE
labels:
cipher: "$2"
protocol: "$3"
listener: "$4"
networkProcessor: "$5"
- pattern: kafka.server<type=(.+), clientSoftwareName=(.+), clientSoftwareVersion=(.+), listener=(.+), networkProcessor=(.+)><>connections
name: kafka_server_$1_connections_software
type: GAUGE
labels:
clientSoftwareName: "$2"
clientSoftwareVersion: "$3"
listener: "$4"
networkProcessor: "$5"
- pattern: "kafka.server<type=(.+), listener=(.+), networkProcessor=(.+)><>(.+):"
name: kafka_server_$1_$4
type: GAUGE
labels:
listener: "$2"
networkProcessor: "$3"
- pattern: kafka.server<type=(.+), listener=(.+), networkProcessor=(.+)><>(.+)
name: kafka_server_$1_$4
type: GAUGE
labels:
listener: "$2"
networkProcessor: "$3"
# Some percent metrics use MeanRate attribute
# Ex) kafka.server<type=(KafkaRequestHandlerPool), name=(RequestHandlerAvgIdlePercent)><>MeanRate
- pattern: kafka.(\w+)<type=(.+), name=(.+)Percent\w*><>MeanRate
name: kafka_$1_$2_$3_percent
type: GAUGE
# Generic gauges for percents
- pattern: kafka.(\w+)<type=(.+), name=(.+)Percent\w*><>Value
name: kafka_$1_$2_$3_percent
type: GAUGE
- pattern: kafka.(\w+)<type=(.+), name=(.+)Percent\w*, (.+)=(.+)><>Value
name: kafka_$1_$2_$3_percent
type: GAUGE
labels:
"$4": "$5"
# Generic per-second counters with 0-2 key/value pairs
- pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*, (.+)=(.+), (.+)=(.+)><>Count
name: kafka_$1_$2_$3_total
type: COUNTER
labels:
"$4": "$5"
"$6": "$7"
- pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*, (.+)=(.+)><>Count
name: kafka_$1_$2_$3_total
type: COUNTER
labels:
"$4": "$5"
- pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*><>Count
name: kafka_$1_$2_$3_total
type: COUNTER
# Generic gauges with 0-2 key/value pairs
- pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>Value
name: kafka_$1_$2_$3
type: GAUGE
labels:
"$4": "$5"
"$6": "$7"
- pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+)><>Value
name: kafka_$1_$2_$3
type: GAUGE
labels:
"$4": "$5"
- pattern: kafka.(\w+)<type=(.+), name=(.+)><>Value
name: kafka_$1_$2_$3
type: GAUGE
# Emulate Prometheus 'Summary' metrics for the exported 'Histogram's.
# Note that these are missing the '_sum' metric!
- pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>Count
name: kafka_$1_$2_$3_count
type: COUNTER
labels:
"$4": "$5"
"$6": "$7"
- pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.*), (.+)=(.+)><>(\d+)thPercentile
name: kafka_$1_$2_$3
type: GAUGE
labels:
"$4": "$5"
"$6": "$7"
quantile: "0.$8"
- pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+)><>Count
name: kafka_$1_$2_$3_count
type: COUNTER
labels:
"$4": "$5"
- pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.*)><>(\d+)thPercentile
name: kafka_$1_$2_$3
type: GAUGE
labels:
"$4": "$5"
quantile: "0.$6"
- pattern: kafka.(\w+)<type=(.+), name=(.+)><>Count
name: kafka_$1_$2_$3_count
type: COUNTER
- pattern: kafka.(\w+)<type=(.+), name=(.+)><>(\d+)thPercentile
name: kafka_$1_$2_$3
type: GAUGE
labels:
quantile: "0.$4"
\ No newline at end of file
helm/templates/strimzi/kafka-exporter-podmonitor.yaml 0 → 100644
+ 17
− 0
View file @ 5d188eb0
apiVersion: monitoring.coreos.com/v1
kind: PodMonitor
metadata:
name: kafka-exporter-podmonitor
labels:
app: theodolite
spec:
selector:
selector:
matchLabels:
strimzi.io/name: theodolite-strimzi-kafka-exporter
namespaceSelector:
matchNames:
- default
podMetricsEndpoints:
- path: /metrics
port: tcp-prometheus
\ No newline at end of file
helm/templates/strimzi/kafka-podmonitor.yaml 0 → 100644
+ 46
− 0
View file @ 5d188eb0
apiVersion: monitoring.coreos.com/v1
kind: PodMonitor
metadata:
name: kafka-resources-metrics
labels:
app: theodolite
spec:
selector:
selector:
matchLabels:
strimzi.io/name: theodolite-strimzi-kafka
namespaceSelector:
matchNames:
- default
podMetricsEndpoints:
- path: /metrics
port: tcp-prometheus
relabelings:
- separator: ;
regex: __meta_kubernetes_pod_label_(strimzi_io_.+)
replacement: $1
action: labelmap
- sourceLabels: [__meta_kubernetes_namespace]
separator: ;
regex: (.*)
targetLabel: namespace
replacement: $1
action: replace
- sourceLabels: [__meta_kubernetes_pod_name]
separator: ;
regex: (.*)
targetLabel: kubernetes_pod_name
replacement: $1
action: replace
- sourceLabels: [__meta_kubernetes_pod_node_name]
separator: ;
regex: (.*)
targetLabel: node_name
replacement: $1
action: replace
- sourceLabels: [__meta_kubernetes_pod_host_ip]
separator: ;
regex: (.*)
targetLabel: node_ip
replacement: $1
action: replace
\ No newline at end of file
helm/values.yaml
+ 6
− 16
View file @ 5d188eb0
......@@ -64,7 +64,7 @@ cp-helm-charts:
## Zookeeper
## ------------------------------------------------------
cp-zookeeper:
enabled: true
enabled: false
nodeSelector: {}
servers: 3
image: confluentinc/cp-zookeeper
......@@ -81,7 +81,7 @@ cp-helm-charts:
## Kafka
## ------------------------------------------------------
cp-kafka:
enabled: true
enabled: false
nodeSelector: {}
brokers: 3
image: confluentinc/cp-enterprise-kafka
......@@ -137,6 +137,10 @@ cp-helm-charts:
nodePort: 30099
annotations: {}
kafka:
bootstrapServers: theodolite-strimzi-kafka-bootstrap:9092
cp-kafka-rest:
enabled: false
......@@ -149,20 +153,6 @@ cp-helm-charts:
cp-control-center:
enabled: false
###
# Kafka Lag Exporter
###
kafka-lag-exporter:
enabled: true
image:
pullPolicy: IfNotPresent
nodeSelector: {}
clusters:
- name: "theodolite-cp-kafka"
bootstrapBrokers: "theodolite-cp-kafka:9092"
## The interval between refreshing metrics
pollIntervalSeconds: 15
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment