From f3ebed03e83be50825ff91c87aec2c3a9e5f08e3 Mon Sep 17 00:00:00 2001
From: Simon Ehrenstein <simon.ehrenstein@gmail.com>
Date: Fri, 18 Feb 2022 18:15:07 +0100
Subject: [PATCH] Adjust grafana dashboard metrics, prometheus scrape interval,
and slo checker
---
helm/templates/grafana/dashboard-config-map.yaml | 10 +++++-----
helm/templates/prometheus/prometheus.yaml | 2 +-
slo-checker/record-lag/app/main.py | 2 +-
.../kotlin/theodolite/evaluation/SloConfigHandler.kt | 2 +-
4 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/helm/templates/grafana/dashboard-config-map.yaml b/helm/templates/grafana/dashboard-config-map.yaml
index 6c2d5a2f2..c1d4471aa 100644
--- a/helm/templates/grafana/dashboard-config-map.yaml
+++ b/helm/templates/grafana/dashboard-config-map.yaml
@@ -71,10 +71,10 @@ data:
"steppedLine": false,
"targets": [
{
- "expr": "sum by (topic) (rate(kafka_server_brokertopicmetrics_messagesin_total{topic='input'}[60s]))",
+ "expr": "sum by (topic) (rate(kafka_server_brokertopicmetrics_messagesin_total{topic='input'}[30s]))",
"format": "time_series",
"intervalFactor": 1,
- "legendFormat": "{{Messages In Per Second}}",
+ "legendFormat": "{{topic}}",
"refId": "D"
}
],
@@ -162,10 +162,10 @@ data:
"steppedLine": false,
"targets": [
{
- "expr": "sum by (topic) (rate(kafka_server_brokertopicmetrics_messagesin_total{topic='output'}[60s]))",
+ "expr": "sum by (topic) (rate(kafka_server_brokertopicmetrics_messagesin_total{topic='output'}[30s]))",
"format": "time_series",
"intervalFactor": 1,
- "legendFormat": "{{Messages Out Per Second}}",
+ "legendFormat": "{{topic}}",
"refId": "D"
}
],
@@ -347,7 +347,7 @@ data:
"expr": "sum by(consumergroup) (kafka_consumergroup_members >= 0)",
"format": "time_series",
"intervalFactor": 1,
- "legendFormat": "instances",
+ "legendFormat": "{{consumergroup}}",
"refId": "D"
}
],
diff --git a/helm/templates/prometheus/prometheus.yaml b/helm/templates/prometheus/prometheus.yaml
index e1faa92c4..196d68487 100644
--- a/helm/templates/prometheus/prometheus.yaml
+++ b/helm/templates/prometheus/prometheus.yaml
@@ -10,7 +10,7 @@ spec:
resources:
requests:
memory: 400Mi
- #scrapeInterval: 1s
+ scrapeInterval: 15s
enableAdminAPI: true
{{- with .Values.prometheus.nodeSelector }}
nodeSelector:
diff --git a/slo-checker/record-lag/app/main.py b/slo-checker/record-lag/app/main.py
index 621fa0cfc..2e38354d4 100644
--- a/slo-checker/record-lag/app/main.py
+++ b/slo-checker/record-lag/app/main.py
@@ -24,7 +24,7 @@ elif os.getenv('LOG_LEVEL') == 'DEBUG':
def calculate_slope_trend(results, warmup):
d = []
for result in results:
- group = result['metric']['group']
+ group = result['metric']['consumergroup']
for value in result['values']:
d.append({'group': group, 'timestamp': int(
value[0]), 'value': int(value[1]) if value[1] != 'NaN' else 0})
diff --git a/theodolite/src/main/kotlin/theodolite/evaluation/SloConfigHandler.kt b/theodolite/src/main/kotlin/theodolite/evaluation/SloConfigHandler.kt
index 10d8df74c..924305660 100644
--- a/theodolite/src/main/kotlin/theodolite/evaluation/SloConfigHandler.kt
+++ b/theodolite/src/main/kotlin/theodolite/evaluation/SloConfigHandler.kt
@@ -4,7 +4,7 @@ import theodolite.benchmark.BenchmarkExecution
import theodolite.util.InvalidPatcherConfigurationException
import javax.enterprise.context.ApplicationScoped
-private const val CONSUMER_LAG_QUERY = "sum by(group) (kafka_consumergroup_lag >= 0)"
+private const val CONSUMER_LAG_QUERY = "sum by(consumergroup) (kafka_consumergroup_lag >= 0)"
private const val DROPPED_RECORDS_QUERY = "sum by(job) (kafka_streams_stream_task_metrics_dropped_records_total>=0)"
@ApplicationScoped
--
GitLab