From 495be913f7aecc74dca199554936bc361ddb4daa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=B6ren=20Henning?= <post@soeren-henning.de>
Date: Fri, 17 Apr 2020 17:09:32 +0200
Subject: [PATCH] Use kafka-lag-exporter instead of Kafka Stream metrics
---
execution/lag_analysis.py | 5 ++---
execution/run_uc1-new.sh | 3 +++
execution/run_uc2-new.sh | 3 +++
execution/run_uc3-new.sh | 2 ++
execution/run_uc4-new.sh | 3 +++
5 files changed, 13 insertions(+), 3 deletions(-)
diff --git a/execution/lag_analysis.py b/execution/lag_analysis.py
index c05c0501b..59475620b 100644
--- a/execution/lag_analysis.py
+++ b/execution/lag_analysis.py
@@ -26,13 +26,12 @@ start = now - timedelta(minutes=5)
#print(start.isoformat().replace('+00:00', 'Z'))
#print(end.isoformat().replace('+00:00', 'Z'))
-#http://localhost:9090
response = requests.get('http://kube1.se.internal:32529/api/v1/query_range', params={
- 'query': "sum by(job,topic)(kafka_consumer_consumer_fetch_manager_metrics_records_lag)",
+ #'query': "sum by(job,topic)(kafka_consumer_consumer_fetch_manager_metrics_records_lag)",
+ 'query': "sum by(group, topic)(kafka_consumergroup_group_lag)",
'start': start.isoformat(),
'end': end.isoformat(),
'step': '5s'})
-# TODO Check if start=now()-5m and end=now() also works
#response
#print(response.request.path_url)
diff --git a/execution/run_uc1-new.sh b/execution/run_uc1-new.sh
index 1e41219bc..1029dee49 100755
--- a/execution/run_uc1-new.sh
+++ b/execution/run_uc1-new.sh
@@ -61,3 +61,6 @@ done
echo "Finish topic deletion, print topics:"
#kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --list" | sed -n -r '/^(titan-.*|input|output|configuration)( - marked for deletion)?$/p'
echo "Exiting script"
+
+KAFKA_LAG_EXPORTER_POD=$(kubectl get pod -l app.kubernetes.io/name=kafka-lag-exporter -o jsonpath="{.items[0].metadata.name}")
+kubectl delete pod $KAFKA_LAG_EXPORTER_POD
diff --git a/execution/run_uc2-new.sh b/execution/run_uc2-new.sh
index d06002846..8383e602e 100755
--- a/execution/run_uc2-new.sh
+++ b/execution/run_uc2-new.sh
@@ -58,3 +58,6 @@ done
echo "Finish topic deletion, print topics:"
#kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --list" | sed -n -r '/^(titan-.*|input|output|configuration)( - marked for deletion)?$/p'
echo "Exiting script"
+
+KAFKA_LAG_EXPORTER_POD=$(kubectl get pod -l app.kubernetes.io/name=kafka-lag-exporter -o jsonpath="{.items[0].metadata.name}")
+kubectl delete pod $KAFKA_LAG_EXPORTER_POD
diff --git a/execution/run_uc3-new.sh b/execution/run_uc3-new.sh
index d0ec923a1..24353fdf2 100755
--- a/execution/run_uc3-new.sh
+++ b/execution/run_uc3-new.sh
@@ -59,3 +59,5 @@ echo "Finish topic deletion, print topics:"
#kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --list" | sed -n -r '/^(titan-.*|input|output|configuration)( - marked for deletion)?$/p'
echo "Exiting script"
+KAFKA_LAG_EXPORTER_POD=$(kubectl get pod -l app.kubernetes.io/name=kafka-lag-exporter -o jsonpath="{.items[0].metadata.name}")
+kubectl delete pod $KAFKA_LAG_EXPORTER_POD
diff --git a/execution/run_uc4-new.sh b/execution/run_uc4-new.sh
index b40d2a423..c3774bb9e 100755
--- a/execution/run_uc4-new.sh
+++ b/execution/run_uc4-new.sh
@@ -63,3 +63,6 @@ echo "Exiting script"
#TODO maybe delete schemas
#https://docs.confluent.io/current/schema-registry/schema-deletion-guidelines.html
#curl -X DELETE http://localhost:8081/subjects/Kafka-value
+
+KAFKA_LAG_EXPORTER_POD=$(kubectl get pod -l app.kubernetes.io/name=kafka-lag-exporter -o jsonpath="{.items[0].metadata.name}")
+kubectl delete pod $KAFKA_LAG_EXPORTER_POD
--
GitLab