From d38abce5ff62f6b5ce0b6044a5fb76b51bc438f3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=B6ren=20Henning?= <soeren.henning@email.uni-kiel.de>
Date: Sat, 3 Dec 2022 15:38:23 +0100
Subject: [PATCH] Add metrics to Samza (part of #408)
---
.../resources/beam-samza-service.yaml | 5 --
.../resources/uc1-beam-samza-deployment.yaml | 46 +++++++++----------
.../resources/uc1-jmx-configmap.yaml | 10 ++++
.../uc1-beam-samza-benchmark-operator.yaml | 1 +
.../resources/beam-samza-service.yaml | 5 --
.../resources/uc2-beam-samza-deployment.yaml | 25 +++++++++-
.../resources/uc2-jmx-configmap.yaml | 10 ++++
.../uc2-beam-samza-benchmark-operator.yaml | 1 +
.../resources/beam-samza-service.yaml | 5 --
.../resources/uc3-beam-samza-deployment.yaml | 25 +++++++++-
.../resources/uc3-jmx-configmap.yaml | 10 ++++
.../uc3-beam-samza-benchmark-operator.yaml | 1 +
.../resources/beam-samza-service.yaml | 5 --
.../resources/uc4-beam-samza-deployment.yaml | 25 +++++++++-
.../resources/uc4-jmx-configmap.yaml | 10 ++++
.../uc4-beam-samza-benchmark-operator.yaml | 1 +
16 files changed, 139 insertions(+), 46 deletions(-)
create mode 100644 theodolite-benchmarks/definitions/uc1-beam-samza/resources/uc1-jmx-configmap.yaml
create mode 100644 theodolite-benchmarks/definitions/uc2-beam-samza/resources/uc2-jmx-configmap.yaml
create mode 100644 theodolite-benchmarks/definitions/uc3-beam-samza/resources/uc3-jmx-configmap.yaml
create mode 100644 theodolite-benchmarks/definitions/uc4-beam-samza/resources/uc4-jmx-configmap.yaml
diff --git a/theodolite-benchmarks/definitions/uc1-beam-samza/resources/beam-samza-service.yaml b/theodolite-benchmarks/definitions/uc1-beam-samza/resources/beam-samza-service.yaml
index 6317caf9f..ef285310c 100644
--- a/theodolite-benchmarks/definitions/uc1-beam-samza/resources/beam-samza-service.yaml
+++ b/theodolite-benchmarks/definitions/uc1-beam-samza/resources/beam-samza-service.yaml
@@ -5,13 +5,8 @@ metadata:
labels:
app: titan-ccp-aggregation
spec:
- #type: NodePort
selector:
app: titan-ccp-aggregation
ports:
- - name: http
- port: 80
- targetPort: 80
- protocol: TCP
- name: metrics
port: 5556
diff --git a/theodolite-benchmarks/definitions/uc1-beam-samza/resources/uc1-beam-samza-deployment.yaml b/theodolite-benchmarks/definitions/uc1-beam-samza/resources/uc1-beam-samza-deployment.yaml
index d10840efe..05eb8a777 100644
--- a/theodolite-benchmarks/definitions/uc1-beam-samza/resources/uc1-beam-samza-deployment.yaml
+++ b/theodolite-benchmarks/definitions/uc1-beam-samza/resources/uc1-beam-samza-deployment.yaml
@@ -31,30 +31,30 @@ spec:
value: "theodolite-kafka-kafka-bootstrap:9092"
- name: SCHEMA_REGISTRY_URL
value: "http://theodolite-kafka-schema-registry:8081"
- # - name: JAVA_OPTS
- # value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555"
+ - name: JAVA_OPTS
+ value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555"
resources:
limits:
memory: 4Gi
cpu: 1000m
- # - name: prometheus-jmx-exporter
- # image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143"
- # command:
- # - java
- # - -XX:+UnlockExperimentalVMOptions
- # - -XX:+UseCGroupMemoryLimitForHeap
- # - -XX:MaxRAMFraction=1
- # - -XshowSettings:vm
- # - -jar
- # - jmx_prometheus_httpserver.jar
- # - "5556"
- # - /etc/jmx-aggregation/jmx-kafka-prometheus.yml
- # ports:
- # - containerPort: 5556
- # volumeMounts:
- # - name: jmx-config
- # mountPath: /etc/jmx-aggregation
- # volumes:
- # - name: jmx-config
- # configMap:
- # name: uc1-jmx-configmap
\ No newline at end of file
+ - name: prometheus-jmx-exporter
+ image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143"
+ command:
+ - java
+ - -XX:+UnlockExperimentalVMOptions
+ - -XX:+UseCGroupMemoryLimitForHeap
+ - -XX:MaxRAMFraction=1
+ - -XshowSettings:vm
+ - -jar
+ - jmx_prometheus_httpserver.jar
+ - "5556"
+ - /etc/jmx-aggregation/jmx-prometheus.yml
+ ports:
+ - containerPort: 5556
+ volumeMounts:
+ - name: jmx-config
+ mountPath: /etc/jmx-aggregation
+ volumes:
+ - name: jmx-config
+ configMap:
+ name: aggregation-jmx-configmap
diff --git a/theodolite-benchmarks/definitions/uc1-beam-samza/resources/uc1-jmx-configmap.yaml b/theodolite-benchmarks/definitions/uc1-beam-samza/resources/uc1-jmx-configmap.yaml
new file mode 100644
index 000000000..714e11593
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc1-beam-samza/resources/uc1-jmx-configmap.yaml
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: aggregation-jmx-configmap
+data:
+ jmx-prometheus.yml: |+
+ jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi
+ lowercaseOutputName: true
+ lowercaseOutputLabelNames: true
+ ssl: false
diff --git a/theodolite-benchmarks/definitions/uc1-beam-samza/uc1-beam-samza-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc1-beam-samza/uc1-beam-samza-benchmark-operator.yaml
index 615511caf..a4d307fdb 100644
--- a/theodolite-benchmarks/definitions/uc1-beam-samza/uc1-beam-samza-benchmark-operator.yaml
+++ b/theodolite-benchmarks/definitions/uc1-beam-samza/uc1-beam-samza-benchmark-operator.yaml
@@ -15,6 +15,7 @@ spec:
- "uc1-beam-samza-deployment.yaml"
- "beam-samza-service.yaml"
- "service-monitor.yaml"
+ - "uc1-jmx-configmap.yaml"
afterActions:
- exec:
selector: # delete zookeeper nodes to reset zookeeper
diff --git a/theodolite-benchmarks/definitions/uc2-beam-samza/resources/beam-samza-service.yaml b/theodolite-benchmarks/definitions/uc2-beam-samza/resources/beam-samza-service.yaml
index 6317caf9f..ef285310c 100644
--- a/theodolite-benchmarks/definitions/uc2-beam-samza/resources/beam-samza-service.yaml
+++ b/theodolite-benchmarks/definitions/uc2-beam-samza/resources/beam-samza-service.yaml
@@ -5,13 +5,8 @@ metadata:
labels:
app: titan-ccp-aggregation
spec:
- #type: NodePort
selector:
app: titan-ccp-aggregation
ports:
- - name: http
- port: 80
- targetPort: 80
- protocol: TCP
- name: metrics
port: 5556
diff --git a/theodolite-benchmarks/definitions/uc2-beam-samza/resources/uc2-beam-samza-deployment.yaml b/theodolite-benchmarks/definitions/uc2-beam-samza/resources/uc2-beam-samza-deployment.yaml
index 986740cbf..8f078fc0a 100644
--- a/theodolite-benchmarks/definitions/uc2-beam-samza/resources/uc2-beam-samza-deployment.yaml
+++ b/theodolite-benchmarks/definitions/uc2-beam-samza/resources/uc2-beam-samza-deployment.yaml
@@ -31,7 +31,30 @@ spec:
value: "theodolite-kafka-kafka-bootstrap:9092"
- name: SCHEMA_REGISTRY_URL
value: "http://theodolite-kafka-schema-registry:8081"
+ - name: JAVA_OPTS
+ value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555"
resources:
limits:
memory: 4Gi
- cpu: 1000m
\ No newline at end of file
+ cpu: 1000m
+ - name: prometheus-jmx-exporter
+ image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143"
+ command:
+ - java
+ - -XX:+UnlockExperimentalVMOptions
+ - -XX:+UseCGroupMemoryLimitForHeap
+ - -XX:MaxRAMFraction=1
+ - -XshowSettings:vm
+ - -jar
+ - jmx_prometheus_httpserver.jar
+ - "5556"
+ - /etc/jmx-aggregation/jmx-prometheus.yml
+ ports:
+ - containerPort: 5556
+ volumeMounts:
+ - name: jmx-config
+ mountPath: /etc/jmx-aggregation
+ volumes:
+ - name: jmx-config
+ configMap:
+ name: aggregation-jmx-configmap
diff --git a/theodolite-benchmarks/definitions/uc2-beam-samza/resources/uc2-jmx-configmap.yaml b/theodolite-benchmarks/definitions/uc2-beam-samza/resources/uc2-jmx-configmap.yaml
new file mode 100644
index 000000000..714e11593
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc2-beam-samza/resources/uc2-jmx-configmap.yaml
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: aggregation-jmx-configmap
+data:
+ jmx-prometheus.yml: |+
+ jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi
+ lowercaseOutputName: true
+ lowercaseOutputLabelNames: true
+ ssl: false
diff --git a/theodolite-benchmarks/definitions/uc2-beam-samza/uc2-beam-samza-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc2-beam-samza/uc2-beam-samza-benchmark-operator.yaml
index c385df649..14c8ed5a0 100644
--- a/theodolite-benchmarks/definitions/uc2-beam-samza/uc2-beam-samza-benchmark-operator.yaml
+++ b/theodolite-benchmarks/definitions/uc2-beam-samza/uc2-beam-samza-benchmark-operator.yaml
@@ -15,6 +15,7 @@ spec:
- "uc2-beam-samza-deployment.yaml"
- "beam-samza-service.yaml"
- "service-monitor.yaml"
+ - "uc2-jmx-configmap.yaml"
afterActions:
- exec:
selector: # delete zookeeper nodes to reset zookeeper
diff --git a/theodolite-benchmarks/definitions/uc3-beam-samza/resources/beam-samza-service.yaml b/theodolite-benchmarks/definitions/uc3-beam-samza/resources/beam-samza-service.yaml
index 6317caf9f..ef285310c 100644
--- a/theodolite-benchmarks/definitions/uc3-beam-samza/resources/beam-samza-service.yaml
+++ b/theodolite-benchmarks/definitions/uc3-beam-samza/resources/beam-samza-service.yaml
@@ -5,13 +5,8 @@ metadata:
labels:
app: titan-ccp-aggregation
spec:
- #type: NodePort
selector:
app: titan-ccp-aggregation
ports:
- - name: http
- port: 80
- targetPort: 80
- protocol: TCP
- name: metrics
port: 5556
diff --git a/theodolite-benchmarks/definitions/uc3-beam-samza/resources/uc3-beam-samza-deployment.yaml b/theodolite-benchmarks/definitions/uc3-beam-samza/resources/uc3-beam-samza-deployment.yaml
index 8ccbf16df..0b87a4da7 100644
--- a/theodolite-benchmarks/definitions/uc3-beam-samza/resources/uc3-beam-samza-deployment.yaml
+++ b/theodolite-benchmarks/definitions/uc3-beam-samza/resources/uc3-beam-samza-deployment.yaml
@@ -31,7 +31,30 @@ spec:
value: "theodolite-kafka-kafka-bootstrap:9092"
- name: SCHEMA_REGISTRY_URL
value: "http://theodolite-kafka-schema-registry:8081"
+ - name: JAVA_OPTS
+ value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555"
resources:
limits:
memory: 4Gi
- cpu: 1000m
\ No newline at end of file
+ cpu: 1000m
+ - name: prometheus-jmx-exporter
+ image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143"
+ command:
+ - java
+ - -XX:+UnlockExperimentalVMOptions
+ - -XX:+UseCGroupMemoryLimitForHeap
+ - -XX:MaxRAMFraction=1
+ - -XshowSettings:vm
+ - -jar
+ - jmx_prometheus_httpserver.jar
+ - "5556"
+ - /etc/jmx-aggregation/jmx-prometheus.yml
+ ports:
+ - containerPort: 5556
+ volumeMounts:
+ - name: jmx-config
+ mountPath: /etc/jmx-aggregation
+ volumes:
+ - name: jmx-config
+ configMap:
+ name: aggregation-jmx-configmap
diff --git a/theodolite-benchmarks/definitions/uc3-beam-samza/resources/uc3-jmx-configmap.yaml b/theodolite-benchmarks/definitions/uc3-beam-samza/resources/uc3-jmx-configmap.yaml
new file mode 100644
index 000000000..714e11593
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc3-beam-samza/resources/uc3-jmx-configmap.yaml
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: aggregation-jmx-configmap
+data:
+ jmx-prometheus.yml: |+
+ jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi
+ lowercaseOutputName: true
+ lowercaseOutputLabelNames: true
+ ssl: false
diff --git a/theodolite-benchmarks/definitions/uc3-beam-samza/uc3-beam-samza-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc3-beam-samza/uc3-beam-samza-benchmark-operator.yaml
index 6fde25874..c613fc8fd 100644
--- a/theodolite-benchmarks/definitions/uc3-beam-samza/uc3-beam-samza-benchmark-operator.yaml
+++ b/theodolite-benchmarks/definitions/uc3-beam-samza/uc3-beam-samza-benchmark-operator.yaml
@@ -15,6 +15,7 @@ spec:
- "uc3-beam-samza-deployment.yaml"
- "beam-samza-service.yaml"
- "service-monitor.yaml"
+ - "uc3-jmx-configmap.yaml"
afterActions:
- exec:
selector: # delete zookeeper nodes to reset zookeeper
diff --git a/theodolite-benchmarks/definitions/uc4-beam-samza/resources/beam-samza-service.yaml b/theodolite-benchmarks/definitions/uc4-beam-samza/resources/beam-samza-service.yaml
index 6317caf9f..ef285310c 100644
--- a/theodolite-benchmarks/definitions/uc4-beam-samza/resources/beam-samza-service.yaml
+++ b/theodolite-benchmarks/definitions/uc4-beam-samza/resources/beam-samza-service.yaml
@@ -5,13 +5,8 @@ metadata:
labels:
app: titan-ccp-aggregation
spec:
- #type: NodePort
selector:
app: titan-ccp-aggregation
ports:
- - name: http
- port: 80
- targetPort: 80
- protocol: TCP
- name: metrics
port: 5556
diff --git a/theodolite-benchmarks/definitions/uc4-beam-samza/resources/uc4-beam-samza-deployment.yaml b/theodolite-benchmarks/definitions/uc4-beam-samza/resources/uc4-beam-samza-deployment.yaml
index 506802d09..fcd93394d 100644
--- a/theodolite-benchmarks/definitions/uc4-beam-samza/resources/uc4-beam-samza-deployment.yaml
+++ b/theodolite-benchmarks/definitions/uc4-beam-samza/resources/uc4-beam-samza-deployment.yaml
@@ -31,7 +31,30 @@ spec:
value: "theodolite-kafka-kafka-bootstrap:9092"
- name: SCHEMA_REGISTRY_URL
value: "http://theodolite-kafka-schema-registry:8081"
+ - name: JAVA_OPTS
+ value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555"
resources:
limits:
memory: 4Gi
- cpu: 1000m
\ No newline at end of file
+ cpu: 1000m
+ - name: prometheus-jmx-exporter
+ image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143"
+ command:
+ - java
+ - -XX:+UnlockExperimentalVMOptions
+ - -XX:+UseCGroupMemoryLimitForHeap
+ - -XX:MaxRAMFraction=1
+ - -XshowSettings:vm
+ - -jar
+ - jmx_prometheus_httpserver.jar
+ - "5556"
+ - /etc/jmx-aggregation/jmx-prometheus.yml
+ ports:
+ - containerPort: 5556
+ volumeMounts:
+ - name: jmx-config
+ mountPath: /etc/jmx-aggregation
+ volumes:
+ - name: jmx-config
+ configMap:
+ name: aggregation-jmx-configmap
diff --git a/theodolite-benchmarks/definitions/uc4-beam-samza/resources/uc4-jmx-configmap.yaml b/theodolite-benchmarks/definitions/uc4-beam-samza/resources/uc4-jmx-configmap.yaml
new file mode 100644
index 000000000..714e11593
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc4-beam-samza/resources/uc4-jmx-configmap.yaml
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: aggregation-jmx-configmap
+data:
+ jmx-prometheus.yml: |+
+ jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi
+ lowercaseOutputName: true
+ lowercaseOutputLabelNames: true
+ ssl: false
diff --git a/theodolite-benchmarks/definitions/uc4-beam-samza/uc4-beam-samza-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc4-beam-samza/uc4-beam-samza-benchmark-operator.yaml
index b07976d82..1955f6e1a 100644
--- a/theodolite-benchmarks/definitions/uc4-beam-samza/uc4-beam-samza-benchmark-operator.yaml
+++ b/theodolite-benchmarks/definitions/uc4-beam-samza/uc4-beam-samza-benchmark-operator.yaml
@@ -15,6 +15,7 @@ spec:
- "uc4-beam-samza-deployment.yaml"
- "beam-samza-service.yaml"
- "service-monitor.yaml"
+ - "uc4-jmx-configmap.yaml"
afterActions:
- exec:
selector: # delete zookeeper nodes to reset zookeeper
--
GitLab