Use kafka-lag-exporter instead of Kafka Stream metrics

execution/lag_analysis.py

@@ -26,13 +26,12 @@ start = now - timedelta(minutes=5)
 #print(start.isoformat().replace('+00:00', 'Z'))
 #print(end.isoformat().replace('+00:00', 'Z'))
 
-#http://localhost:9090
 response = requests.get('http://kube1.se.internal:32529/api/v1/query_range', params={
-    'query': "sum by(job,topic)(kafka_consumer_consumer_fetch_manager_metrics_records_lag)",
+    #'query': "sum by(job,topic)(kafka_consumer_consumer_fetch_manager_metrics_records_lag)",
+    'query': "sum by(group, topic)(kafka_consumergroup_group_lag)",
     'start': start.isoformat(),
     'end': end.isoformat(),
     'step': '5s'})
-# TODO Check if start=now()-5m and end=now() also works
 
 #response
 #print(response.request.path_url)

execution/run_uc1-new.sh

@@ -61,3 +61,6 @@ done
 echo "Finish topic deletion, print topics:"
 #kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --list" | sed -n -r '/^(titan-.*|input|output|configuration)( - marked for deletion)?$/p'
 echo "Exiting script"
+
+KAFKA_LAG_EXPORTER_POD=$(kubectl get pod -l app.kubernetes.io/name=kafka-lag-exporter -o jsonpath="{.items[0].metadata.name}")
+kubectl delete pod $KAFKA_LAG_EXPORTER_POD

execution/run_uc2-new.sh

@@ -58,3 +58,6 @@ done
 echo "Finish topic deletion, print topics:"
 #kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --list" | sed -n -r '/^(titan-.*|input|output|configuration)( - marked for deletion)?$/p'
 echo "Exiting script"
+
+KAFKA_LAG_EXPORTER_POD=$(kubectl get pod -l app.kubernetes.io/name=kafka-lag-exporter -o jsonpath="{.items[0].metadata.name}")
+kubectl delete pod $KAFKA_LAG_EXPORTER_POD

execution/run_uc3-new.sh

@@ -59,3 +59,5 @@ echo "Finish topic deletion, print topics:"
 #kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --list" | sed -n -r '/^(titan-.*|input|output|configuration)( - marked for deletion)?$/p'
 echo "Exiting script"
 
+KAFKA_LAG_EXPORTER_POD=$(kubectl get pod -l app.kubernetes.io/name=kafka-lag-exporter -o jsonpath="{.items[0].metadata.name}")
+kubectl delete pod $KAFKA_LAG_EXPORTER_POD

execution/run_uc4-new.sh

@@ -63,3 +63,6 @@ echo "Exiting script"
 #TODO maybe delete schemas
 #https://docs.confluent.io/current/schema-registry/schema-deletion-guidelines.html
 #curl -X DELETE http://localhost:8081/subjects/Kafka-value
+
+KAFKA_LAG_EXPORTER_POD=$(kubectl get pod -l app.kubernetes.io/name=kafka-lag-exporter -o jsonpath="{.items[0].metadata.name}")
+kubectl delete pod $KAFKA_LAG_EXPORTER_POD