Adjust grafana dashboard metrics, prometheus scrape interval, and slo checker

helm/templates/grafana/dashboard-config-map.yaml

@@ -71,10 +71,10 @@ data:
         "steppedLine": false,
         "targets": [
           {
-            "expr": "sum by (topic) (rate(kafka_server_brokertopicmetrics_messagesin_total{topic='input'}[60s]))",
+            "expr": "sum by (topic) (rate(kafka_server_brokertopicmetrics_messagesin_total{topic='input'}[30s]))",
             "format": "time_series",
             "intervalFactor": 1,
-            "legendFormat": "{{Messages In Per Second}}",
+            "legendFormat": "{{topic}}",
             "refId": "D"
           }
         ],
@@ -162,10 +162,10 @@ data:
         "steppedLine": false,
         "targets": [
           {
-            "expr": "sum by (topic) (rate(kafka_server_brokertopicmetrics_messagesin_total{topic='output'}[60s]))",
+            "expr": "sum by (topic) (rate(kafka_server_brokertopicmetrics_messagesin_total{topic='output'}[30s]))",
             "format": "time_series",
             "intervalFactor": 1,
-            "legendFormat": "{{Messages Out Per Second}}",
+            "legendFormat": "{{topic}}",
             "refId": "D"
           }
         ],
@@ -347,7 +347,7 @@ data:
             "expr": "sum by(consumergroup) (kafka_consumergroup_members >= 0)",
             "format": "time_series",
             "intervalFactor": 1,
-            "legendFormat": "instances",
+            "legendFormat": "{{consumergroup}}",
             "refId": "D"
           }
         ],

helm/templates/prometheus/prometheus.yaml

@@ -10,7 +10,7 @@ spec:
   resources:
     requests:
       memory: 400Mi
-  #scrapeInterval: 1s
+  scrapeInterval: 15s
   enableAdminAPI: true
   {{- with .Values.prometheus.nodeSelector }}
   nodeSelector:

slo-checker/record-lag/app/main.py

@@ -24,7 +24,7 @@ elif os.getenv('LOG_LEVEL') == 'DEBUG':
 def calculate_slope_trend(results, warmup):
     d = []
     for result in results:
-        group = result['metric']['group']
+        group = result['metric']['consumergroup']
         for value in result['values']:
             d.append({'group': group, 'timestamp': int(
                 value[0]), 'value': int(value[1]) if value[1] != 'NaN' else 0})

theodolite/src/main/kotlin/theodolite/evaluation/SloConfigHandler.kt

@@ -4,7 +4,7 @@ import theodolite.benchmark.BenchmarkExecution
 import theodolite.util.InvalidPatcherConfigurationException
 import javax.enterprise.context.ApplicationScoped
 
-private const val CONSUMER_LAG_QUERY = "sum by(group) (kafka_consumergroup_lag >= 0)"
+private const val CONSUMER_LAG_QUERY = "sum by(consumergroup) (kafka_consumergroup_lag >= 0)"
 private const val DROPPED_RECORDS_QUERY = "sum by(job) (kafka_streams_stream_task_metrics_dropped_records_total>=0)"
 
 @ApplicationScoped