diff --git a/docs/creating-a-benchmark.md b/docs/creating-a-benchmark.md index fde8ba0759407ddea8befc18e244784a9ba34c1f..b09c989e59d910fc352af9d1c5690b224e3346e6 100644 --- a/docs/creating-a-benchmark.md +++ b/docs/creating-a-benchmark.md @@ -42,7 +42,7 @@ spec: properties: loadGenMaxRecords: "150000" kafkaConfig: - bootstrapServer: "theodolite-cp-kafka:9092" + bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092" topics: - name: "input" numPartitions: 40 @@ -54,7 +54,7 @@ spec: ## System under Test (SUT), Load Generator and Infrastructure -In Thedolite, the system under test (SUT), the load generator as well as additional infrastructure (e.g., a middleware) are described by Kubernetes resources files. +In Theodolite, the system under test (SUT), the load generator as well as additional infrastructure (e.g., a middleware) are described by Kubernetes resources files. All resources defined for the SUT and the load generator are started and stopped for each SLO experiment, with SUT resources being started before the load generator. Infrastructure resources live over the entire duration of a benchmark run. They avoid time-consuming recreation of software components like middlewares, but should be used with caution to not let previous SLO experiments influence latte ones. diff --git a/docs/installation.md b/docs/installation.md index a97e5ea499657cdc3c40f3c03a13c974b5a39bab..d1c7ac3d1dd68d244c556e1ade53b50330aec6ed 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -58,13 +58,13 @@ In cases, where you need to install multiple Theodolite instances, it's best to ### Installation with a release name other than `theodolite` -When using another release name than `theodolite`, make sure to adjust the Kafka Lag Exporter configuration of you `values.yaml` accordingly: +When using another release name than `theodolite`, make sure to adjust the Confluent Schema Registry configuration of you `values.yaml` accordingly: ```yaml -kafka-lag-exporter: - clusters: - - name: "<your-release-name>-cp-kafka" - bootstrapBrokers: "<your-release-name>-cp-kafka:9092" +cp-helm-charts: + cp-schema-registry: + kafka: + bootstrapServers: <your-release-name>-kafka-kafka-bootstrap:9092 ``` This seems unfortunately to be necessary as Helm does not let us inject values into dependency charts. diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 52fda2bbc83c722d70f6179c4ec53fb3775bd436..973c985b5bdaa4d53390954017ed9176bb396f55 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -24,10 +24,11 @@ dependencies: version: 0.6.0 repository: https://soerenhenning.github.io/cp-helm-charts condition: cp-helm-charts.enabled - - name: kafka-lag-exporter - version: 0.6.7 - repository: https://seanglover.com/kafka-lag-exporter/repo - condition: kafka-lag-exporter.enabled + - name: strimzi-kafka-operator + version: 0.28.0 + repository: https://strimzi.io/charts/ + condition: strimzi.enabled + version: 0.7.0-SNAPSHOT diff --git a/helm/templates/grafana/dashboard-config-map.yaml b/helm/templates/grafana/dashboard-config-map.yaml index 0df01b20efa0fb1100fe4b7289b00b3058eb032f..cc9ec623f3ef71459ccad65128bf8cd65f0d6eb6 100644 --- a/helm/templates/grafana/dashboard-config-map.yaml +++ b/helm/templates/grafana/dashboard-config-map.yaml @@ -71,10 +71,10 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(cp_kafka_server_brokertopicmetrics_messagesinpersec_topic_input)", + "expr": "sum by (topic) (rate(kafka_server_brokertopicmetrics_messagesin_total{topic='input'}[30s]))", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{Messages In Per Second}}", + "legendFormat": "{{topic}}", "refId": "D" } ], @@ -162,10 +162,10 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(cp_kafka_server_brokertopicmetrics_messagesinpersec_topic_output)", + "expr": "sum by (topic) (rate(kafka_server_brokertopicmetrics_messagesin_total{topic='output'}[30s]))", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{Messages Out Per Second}}", + "legendFormat": "{{topic}}", "refId": "D" } ], @@ -253,7 +253,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum by(group, topic) (kafka_consumergroup_group_lag >= 0)", + "expr": "sum by(consumergroup, topic) (kafka_consumergroup_lag >= 0)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{topic}}", @@ -344,10 +344,10 @@ data: "steppedLine": false, "targets": [ { - "expr": "count(count (kafka_consumer_consumer_fetch_manager_metrics_records_lag) by(pod))", + "expr": "sum by(consumergroup) (kafka_consumergroup_members >= 0)", "format": "time_series", "intervalFactor": 1, - "legendFormat": "instances", + "legendFormat": "{{consumergroup}}", "refId": "D" } ], @@ -436,7 +436,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum by(group,topic) (kafka_consumergroup_group_offset >= 0)", + "expr": "sum by(consumergroup,topic) (kafka_consumergroup_current_offset{topic='input'} >= 0)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{topic}}", @@ -527,7 +527,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "count by(group,topic) (kafka_consumergroup_group_offset >= 0)", + "expr": "sum by(topic) (kafka_topic_partitions >= 0)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{topic}}", @@ -618,7 +618,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum by(group,topic) (kafka_partition_latest_offset)", + "expr": "sum by(topic) (kafka_topic_partition_current_offset)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{topic}}", diff --git a/helm/templates/kafka/kafka-client.yaml b/helm/templates/kafka/kafka-client.yaml index 02e16d33dfc9595dd16c41fa6bfe1404fd7889ab..1edf1b96c7326d6a3cfdb1a45640ec9d8d720fba 100644 --- a/helm/templates/kafka/kafka-client.yaml +++ b/helm/templates/kafka/kafka-client.yaml @@ -2,7 +2,6 @@ apiVersion: v1 kind: Pod metadata: - # name: {{ template "theodolite.fullname" . }}-kafka-client name: {{ template "theodolite.fullname" . }}-kafka-client spec: containers: @@ -12,6 +11,9 @@ spec: - sh - -c - "exec tail -f /dev/null" + env: + - name: BOOTSTRAP_SERVER + value: {{ template "theodolite.fullname" . }}-kafka-kafka-bootstrap:9092 {{- with .Values.kafkaClient.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/helm/templates/prometheus/prometheus.yaml b/helm/templates/prometheus/prometheus.yaml index 23a015250e19cc14550ce73e8162ba27f65be774..196d68487824d7d8e130c56d11cec2687304d7e6 100644 --- a/helm/templates/prometheus/prometheus.yaml +++ b/helm/templates/prometheus/prometheus.yaml @@ -5,11 +5,12 @@ metadata: name: {{ template "theodolite.fullname" . }}-prometheus spec: serviceAccountName: {{ template "theodolite.fullname" . }}-prometheus + podMonitorSelector: {} serviceMonitorSelector: {} resources: requests: memory: 400Mi - #scrapeInterval: 1s + scrapeInterval: 15s enableAdminAPI: true {{- with .Values.prometheus.nodeSelector }} nodeSelector: diff --git a/helm/templates/strimzi/kafka-cluster.yaml b/helm/templates/strimzi/kafka-cluster.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0d7eccfd279c62f7d996a8e3e41a55a5ebdd4e96 --- /dev/null +++ b/helm/templates/strimzi/kafka-cluster.yaml @@ -0,0 +1,43 @@ +{{- if .Values.strimzi.enabled -}} +apiVersion: kafka.strimzi.io/v1beta2 +kind: Kafka +metadata: + name: {{ template "theodolite.fullname" . }}-kafka +spec: + kafka: + jmxOptions: {} + {{- with .Values.strimzi.kafka.listeners }} + listeners: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.strimzi.kafka.replicas }} + replicas: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.strimzi.kafka.config }} + config: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.strimzi.kafka.jvmOptions }} + jvmOptions: + {{- toYaml . | nindent 6 }} + {{- end }} + storage: + type: ephemeral + metricsConfig: + type: jmxPrometheusExporter + valueFrom: + configMapKeyRef: + name: {{ template "theodolite.fullname" . }}-kafka-metrics + key: kafka-metrics-config.yml + + kafkaExporter: {} + + zookeeper: + {{- with .Values.strimzi.zookeeper.replicas }} + replicas: + {{- toYaml . | nindent 6 }} + {{- end }} + storage: + type: ephemeral +{{- end }} \ No newline at end of file diff --git a/helm/templates/strimzi/kafka-exporter-podmonitor.yaml b/helm/templates/strimzi/kafka-exporter-podmonitor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..41395dfd6a9ea9f74598ff89c4d9e2e7f8757ff0 --- /dev/null +++ b/helm/templates/strimzi/kafka-exporter-podmonitor.yaml @@ -0,0 +1,16 @@ +{{- if .Values.strimzi.enabled -}} +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: {{ template "theodolite.fullname" . }}-kafka-exporter-podmonitor + labels: + app: theodolite +spec: + selector: + selector: + matchLabels: + strimzi.io/name: {{ template "theodolite.fullname" . }}-kafka-kafka-exporter + podMetricsEndpoints: + - path: /metrics + port: tcp-prometheus +{{- end }} \ No newline at end of file diff --git a/helm/templates/strimzi/kafka-metrics-configmap.yaml b/helm/templates/strimzi/kafka-metrics-configmap.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ad75bfa6ab9ec89a229125d167b897cb58744c42 --- /dev/null +++ b/helm/templates/strimzi/kafka-metrics-configmap.yaml @@ -0,0 +1,133 @@ +{{- if .Values.strimzi.enabled -}} +kind: ConfigMap +apiVersion: v1 +metadata: + name: {{ template "theodolite.fullname" . }}-kafka-metrics + labels: + app: strimzi +data: + kafka-metrics-config.yml: | + # See https://github.com/prometheus/jmx_exporter for more info about JMX Prometheus Exporter metrics + lowercaseOutputName: true + rules: + # Special cases and very specific rules + - pattern: kafka.server<type=(.+), name=(.+), clientId=(.+), topic=(.+), partition=(.*)><>Value + name: kafka_server_$1_$2 + type: GAUGE + labels: + clientId: "$3" + topic: "$4" + partition: "$5" + - pattern: kafka.server<type=(.+), name=(.+), clientId=(.+), brokerHost=(.+), brokerPort=(.+)><>Value + name: kafka_server_$1_$2 + type: GAUGE + labels: + clientId: "$3" + broker: "$4:$5" + - pattern: kafka.server<type=(.+), cipher=(.+), protocol=(.+), listener=(.+), networkProcessor=(.+)><>connections + name: kafka_server_$1_connections_tls_info + type: GAUGE + labels: + cipher: "$2" + protocol: "$3" + listener: "$4" + networkProcessor: "$5" + - pattern: kafka.server<type=(.+), clientSoftwareName=(.+), clientSoftwareVersion=(.+), listener=(.+), networkProcessor=(.+)><>connections + name: kafka_server_$1_connections_software + type: GAUGE + labels: + clientSoftwareName: "$2" + clientSoftwareVersion: "$3" + listener: "$4" + networkProcessor: "$5" + - pattern: "kafka.server<type=(.+), listener=(.+), networkProcessor=(.+)><>(.+):" + name: kafka_server_$1_$4 + type: GAUGE + labels: + listener: "$2" + networkProcessor: "$3" + - pattern: kafka.server<type=(.+), listener=(.+), networkProcessor=(.+)><>(.+) + name: kafka_server_$1_$4 + type: GAUGE + labels: + listener: "$2" + networkProcessor: "$3" + # Some percent metrics use MeanRate attribute + # Ex) kafka.server<type=(KafkaRequestHandlerPool), name=(RequestHandlerAvgIdlePercent)><>MeanRate + - pattern: kafka.(\w+)<type=(.+), name=(.+)Percent\w*><>MeanRate + name: kafka_$1_$2_$3_percent + type: GAUGE + # Generic gauges for percents + - pattern: kafka.(\w+)<type=(.+), name=(.+)Percent\w*><>Value + name: kafka_$1_$2_$3_percent + type: GAUGE + - pattern: kafka.(\w+)<type=(.+), name=(.+)Percent\w*, (.+)=(.+)><>Value + name: kafka_$1_$2_$3_percent + type: GAUGE + labels: + "$4": "$5" + # Generic per-second counters with 0-2 key/value pairs + - pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*, (.+)=(.+), (.+)=(.+)><>Count + name: kafka_$1_$2_$3_total + type: COUNTER + labels: + "$4": "$5" + "$6": "$7" + - pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*, (.+)=(.+)><>Count + name: kafka_$1_$2_$3_total + type: COUNTER + labels: + "$4": "$5" + - pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*><>Count + name: kafka_$1_$2_$3_total + type: COUNTER + # Generic gauges with 0-2 key/value pairs + - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>Value + name: kafka_$1_$2_$3 + type: GAUGE + labels: + "$4": "$5" + "$6": "$7" + - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+)><>Value + name: kafka_$1_$2_$3 + type: GAUGE + labels: + "$4": "$5" + - pattern: kafka.(\w+)<type=(.+), name=(.+)><>Value + name: kafka_$1_$2_$3 + type: GAUGE + # Emulate Prometheus 'Summary' metrics for the exported 'Histogram's. + # Note that these are missing the '_sum' metric! + - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>Count + name: kafka_$1_$2_$3_count + type: COUNTER + labels: + "$4": "$5" + "$6": "$7" + - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.*), (.+)=(.+)><>(\d+)thPercentile + name: kafka_$1_$2_$3 + type: GAUGE + labels: + "$4": "$5" + "$6": "$7" + quantile: "0.$8" + - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+)><>Count + name: kafka_$1_$2_$3_count + type: COUNTER + labels: + "$4": "$5" + - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.*)><>(\d+)thPercentile + name: kafka_$1_$2_$3 + type: GAUGE + labels: + "$4": "$5" + quantile: "0.$6" + - pattern: kafka.(\w+)<type=(.+), name=(.+)><>Count + name: kafka_$1_$2_$3_count + type: COUNTER + - pattern: kafka.(\w+)<type=(.+), name=(.+)><>(\d+)thPercentile + name: kafka_$1_$2_$3 + type: GAUGE + labels: + quantile: "0.$4" +{{- end }} \ No newline at end of file diff --git a/helm/templates/strimzi/kafka-podmonitor.yaml b/helm/templates/strimzi/kafka-podmonitor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..522f23f305b53615251d80e8cb13eac50e5530ad --- /dev/null +++ b/helm/templates/strimzi/kafka-podmonitor.yaml @@ -0,0 +1,16 @@ +{{- if .Values.strimzi.enabled -}} +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: {{ template "theodolite.fullname" . }}-kafka-resources-metrics + labels: + app: theodolite +spec: + selector: + selector: + matchLabels: + strimzi.io/name: {{ template "theodolite.fullname" . }}-kafka-kafka + podMetricsEndpoints: + - path: /metrics + port: tcp-prometheus +{{- end }} \ No newline at end of file diff --git a/helm/templates/theodolite/role.yaml b/helm/templates/theodolite/role.yaml index 43ee0e43d6974cd95548df32d6c4b1df8f3e497e..ba5a223b6527df94b64fac3574ee5f90fdb3903b 100644 --- a/helm/templates/theodolite/role.yaml +++ b/helm/templates/theodolite/role.yaml @@ -38,6 +38,7 @@ rules: - monitoring.coreos.com resources: - servicemonitors + - podmonitors verbs: - update - delete diff --git a/helm/values.yaml b/helm/values.yaml index ba58b040974886518ab111d668cb0db1140b2eb8..797098b1a2316389134827cfd6be37c0aaf3c4e6 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -64,7 +64,7 @@ cp-helm-charts: ## Zookeeper ## ------------------------------------------------------ cp-zookeeper: - enabled: true + enabled: false nodeSelector: {} servers: 3 image: confluentinc/cp-zookeeper @@ -81,7 +81,7 @@ cp-helm-charts: ## Kafka ## ------------------------------------------------------ cp-kafka: - enabled: true + enabled: false nodeSelector: {} brokers: 3 image: confluentinc/cp-enterprise-kafka @@ -137,6 +137,9 @@ cp-helm-charts: nodePort: 30099 annotations: {} + kafka: + bootstrapServers: theodolite-kafka-kafka-bootstrap:9092 + cp-kafka-rest: enabled: false @@ -149,29 +152,30 @@ cp-helm-charts: cp-control-center: enabled: false - -### -# Kafka Lag Exporter -### -kafka-lag-exporter: - enabled: true - image: - pullPolicy: IfNotPresent - nodeSelector: {} - - clusters: - - name: "theodolite-cp-kafka" - bootstrapBrokers: "theodolite-cp-kafka:9092" - ## The interval between refreshing metrics pollIntervalSeconds: 15 - prometheus: - serviceMonitor: - enabled: true - interval: "5s" - additionalLabels: - appScope: titan-ccp +strimzi: + enabled: true + kafka: + listeners: + - name: plain + port: 9092 + type: internal + tls: false + replicas: 3 + config: + "message.max.bytes": "134217728" # 128 MB + "replica.fetch.max.bytes": "134217728" #128 MB + "auto.create.topics.enable": false + "log.retention.ms": "7200000" # 2h + "metrics.sample.window.ms": "5000" #5s + jvmOptions: + "-Xmx": "512M" + "-Xms": "512M" + + zookeeper: + replicas: 3 ### diff --git a/slo-checker/record-lag/app/main.py b/slo-checker/record-lag/app/main.py index 621fa0cfc9c27e809fd92752de93f2795fa32c05..2e38354d45df57087a94e57d5c9ca412ed5534d3 100644 --- a/slo-checker/record-lag/app/main.py +++ b/slo-checker/record-lag/app/main.py @@ -24,7 +24,7 @@ elif os.getenv('LOG_LEVEL') == 'DEBUG': def calculate_slope_trend(results, warmup): d = [] for result in results: - group = result['metric']['group'] + group = result['metric']['consumergroup'] for value in result['values']: d.append({'group': group, 'timestamp': int( value[0]), 'value': int(value[1]) if value[1] != 'NaN' else 0}) diff --git a/slo-checker/record-lag/resources/test-1-rep-success.json b/slo-checker/record-lag/resources/test-1-rep-success.json index dfe11282720ebfcdd60582b7717da892bc85a923..09320c5f1eb0f96d847ddaa17cbd48c47a989155 100644 --- a/slo-checker/record-lag/resources/test-1-rep-success.json +++ b/slo-checker/record-lag/resources/test-1-rep-success.json @@ -3,7 +3,7 @@ [ { "metric": { - "group": "theodolite-uc1-application-0.0.1" + "consumergroup": "theodolite-uc1-application-0.0.1" }, "values": [ [ diff --git a/slo-checker/record-lag/resources/test-3-rep-success.json b/slo-checker/record-lag/resources/test-3-rep-success.json index cf483f42f3783aecd1f428ac7bbbe2090c4cade0..e752430e09e00e6ea12128df44f2d687ba037a7a 100644 --- a/slo-checker/record-lag/resources/test-3-rep-success.json +++ b/slo-checker/record-lag/resources/test-3-rep-success.json @@ -3,7 +3,7 @@ [ { "metric": { - "group": "theodolite-uc1-application-0.0.1" + "consumergroup": "theodolite-uc1-application-0.0.1" }, "values": [ [ @@ -100,7 +100,7 @@ [ { "metric": { - "group": "theodolite-uc1-application-0.0.1" + "consumergroup": "theodolite-uc1-application-0.0.1" }, "values": [ [ @@ -193,7 +193,7 @@ [ { "metric": { - "group": "theodolite-uc1-application-0.0.1" + "consumergroup": "theodolite-uc1-application-0.0.1" }, "values": [ [ diff --git a/theodolite-benchmarks/definitions/uc1-flink/resources/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc1-flink/resources/jobmanager-deployment.yaml index 1f328b1cd553c8036e570d28b97795fb2b00ec81..f81b851aa935408112e2a2fadbb72046720068c6 100644 --- a/theodolite-benchmarks/definitions/uc1-flink/resources/jobmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc1-flink/resources/jobmanager-deployment.yaml @@ -20,7 +20,7 @@ spec: image: ghcr.io/cau-se/theodolite-uc1-flink:latest env: - name: KAFKA_BOOTSTRAP_SERVERS - value: "theodolite-cp-kafka:9092" + value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL value: "http://theodolite-cp-schema-registry:8081" - name: COMMIT_INTERVAL_MS diff --git a/theodolite-benchmarks/definitions/uc1-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc1-flink/resources/taskmanager-deployment.yaml index c2266a4aeb21302262279f147e6512d5264e1dc1..9710d44cbb1ffb75e4d1ebafe1ffe60042587adc 100644 --- a/theodolite-benchmarks/definitions/uc1-flink/resources/taskmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc1-flink/resources/taskmanager-deployment.yaml @@ -20,7 +20,7 @@ spec: image: ghcr.io/cau-se/theodolite-uc1-flink:latest env: - name: KAFKA_BOOTSTRAP_SERVERS - value: "theodolite-cp-kafka:9092" + value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL value: "http://theodolite-cp-schema-registry:8081" - name: COMMIT_INTERVAL_MS diff --git a/theodolite-benchmarks/definitions/uc1-flink/uc1-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc1-flink/uc1-flink-benchmark-operator.yaml index 89bac41ee5c8dcefa628b3cb01052df5a1df9292..20953c2d1e64895417f4f5339a0f3820d78735ac 100644 --- a/theodolite-benchmarks/definitions/uc1-flink/uc1-flink-benchmark-operator.yaml +++ b/theodolite-benchmarks/definitions/uc1-flink/uc1-flink-benchmark-operator.yaml @@ -50,7 +50,7 @@ spec: properties: loadGenMaxRecords: "150000" kafkaConfig: - bootstrapServer: "theodolite-cp-kafka:9092" + bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092" topics: - name: "input" numPartitions: 40 diff --git a/theodolite-benchmarks/definitions/uc1-kstreams/resources/uc1-kstreams-deployment.yaml b/theodolite-benchmarks/definitions/uc1-kstreams/resources/uc1-kstreams-deployment.yaml index 171c3446db2719ee91bd8954233015316851fcf9..b4d2bfe738fd9c96d0219a825d5c4ef2cf5385f3 100644 --- a/theodolite-benchmarks/definitions/uc1-kstreams/resources/uc1-kstreams-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc1-kstreams/resources/uc1-kstreams-deployment.yaml @@ -21,7 +21,7 @@ spec: name: jmx env: - name: KAFKA_BOOTSTRAP_SERVERS - value: "theodolite-cp-kafka:9092" + value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL value: "http://theodolite-cp-schema-registry:8081" - name: JAVA_OPTS diff --git a/theodolite-benchmarks/definitions/uc1-kstreams/uc1-kstreams-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-kstreams-benchmark-operator.yaml index fb5557c2df8b483164d3c1000717db4c7cface81..c340547c703c03a2e91738d4f53537938da97e0e 100644 --- a/theodolite-benchmarks/definitions/uc1-kstreams/uc1-kstreams-benchmark-operator.yaml +++ b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-kstreams-benchmark-operator.yaml @@ -37,7 +37,7 @@ spec: properties: loadGenMaxRecords: "150000" kafkaConfig: - bootstrapServer: "theodolite-cp-kafka:9092" + bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092" topics: - name: "input" numPartitions: 40 diff --git a/theodolite-benchmarks/definitions/uc1-kstreams/uc1-kstreams-benchmark-standalone.yaml b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-kstreams-benchmark-standalone.yaml index 5aaf87e724a4e8c728c3c15b998cb927ff57f3d5..545474b1c556a8185d3725c51908f996b89e91e2 100644 --- a/theodolite-benchmarks/definitions/uc1-kstreams/uc1-kstreams-benchmark-standalone.yaml +++ b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-kstreams-benchmark-standalone.yaml @@ -32,7 +32,7 @@ loadTypes: properties: loadGenMaxRecords: "150000" kafkaConfig: - bootstrapServer: "theodolite-cp-kafka:9092" + bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092" topics: - name: "input" numPartitions: 40 diff --git a/theodolite-benchmarks/definitions/uc1-load-generator/resources/uc1-load-generator-deployment.yaml b/theodolite-benchmarks/definitions/uc1-load-generator/resources/uc1-load-generator-deployment.yaml index 9f9ccc6ae39407bb1f027e1e23cb152944b869e0..65048a97d5de3d831f782db329e295a5e5ceb727 100644 --- a/theodolite-benchmarks/definitions/uc1-load-generator/resources/uc1-load-generator-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc1-load-generator/resources/uc1-load-generator-deployment.yaml @@ -27,6 +27,6 @@ spec: - name: KUBERNETES_DNS_NAME value: "titan-ccp-load-generator.$(KUBERNETES_NAMESPACE).svc.cluster.local" - name: KAFKA_BOOTSTRAP_SERVERS - value: "theodolite-cp-kafka:9092" + value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL value: "http://theodolite-cp-schema-registry:8081" diff --git a/theodolite-benchmarks/definitions/uc2-flink/resources/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc2-flink/resources/jobmanager-deployment.yaml index 87ea174f71c592bbffab4e5fc9ce6e3963596b9c..ab8816fbf82dde2d71705f5d6977d045fe20f4f5 100644 --- a/theodolite-benchmarks/definitions/uc2-flink/resources/jobmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc2-flink/resources/jobmanager-deployment.yaml @@ -20,7 +20,7 @@ spec: image: ghcr.io/cau-se/theodolite-uc2-flink:latest env: - name: KAFKA_BOOTSTRAP_SERVERS - value: "theodolite-cp-kafka:9092" + value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL value: "http://theodolite-cp-schema-registry:8081" - name: COMMIT_INTERVAL_MS diff --git a/theodolite-benchmarks/definitions/uc2-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc2-flink/resources/taskmanager-deployment.yaml index c37df972a334a4a0e27f0420030f99f1dff15b53..b9a917ee825637988cf6902186d99421aac3d671 100644 --- a/theodolite-benchmarks/definitions/uc2-flink/resources/taskmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc2-flink/resources/taskmanager-deployment.yaml @@ -20,7 +20,7 @@ spec: image: ghcr.io/cau-se/theodolite-uc2-flink:latest env: - name: KAFKA_BOOTSTRAP_SERVERS - value: "theodolite-cp-kafka:9092" + value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL value: "http://theodolite-cp-schema-registry:8081" - name: COMMIT_INTERVAL_MS diff --git a/theodolite-benchmarks/definitions/uc2-flink/uc2-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc2-flink/uc2-flink-benchmark-operator.yaml index 206fbf9683659fcc074341d7077da04c36909b75..3020bb317c8b500562f1edcf2dc770f1288a8788 100644 --- a/theodolite-benchmarks/definitions/uc2-flink/uc2-flink-benchmark-operator.yaml +++ b/theodolite-benchmarks/definitions/uc2-flink/uc2-flink-benchmark-operator.yaml @@ -50,7 +50,7 @@ spec: properties: loadGenMaxRecords: "150000" kafkaConfig: - bootstrapServer: "theodolite-cp-kafka:9092" + bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092" topics: - name: "input" numPartitions: 40 diff --git a/theodolite-benchmarks/definitions/uc2-kstreams/resources/uc2-kstreams-deployment.yaml b/theodolite-benchmarks/definitions/uc2-kstreams/resources/uc2-kstreams-deployment.yaml index e07bb3f9e536655712c06a004c5d1fb60ffa67e0..86932cafb26248736fbe060ba7f23ee5dded412d 100644 --- a/theodolite-benchmarks/definitions/uc2-kstreams/resources/uc2-kstreams-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc2-kstreams/resources/uc2-kstreams-deployment.yaml @@ -21,7 +21,7 @@ spec: name: jmx env: - name: KAFKA_BOOTSTRAP_SERVERS - value: "theodolite-cp-kafka:9092" + value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL value: "http://theodolite-cp-schema-registry:8081" - name: JAVA_OPTS diff --git a/theodolite-benchmarks/definitions/uc2-kstreams/uc2-kstreams-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-kstreams-benchmark-operator.yaml index 0db22fa95f46d1cb484fa1a7730b8b6801dac67c..b9f2b14e369b3c8e241be62c04bd480f38d847dc 100644 --- a/theodolite-benchmarks/definitions/uc2-kstreams/uc2-kstreams-benchmark-operator.yaml +++ b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-kstreams-benchmark-operator.yaml @@ -37,7 +37,7 @@ spec: properties: loadGenMaxRecords: "150000" kafkaConfig: - bootstrapServer: "theodolite-cp-kafka:9092" + bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092" topics: - name: "input" numPartitions: 40 diff --git a/theodolite-benchmarks/definitions/uc2-kstreams/uc2-kstreams-benchmark-standalone.yaml b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-kstreams-benchmark-standalone.yaml index 67376d76bf0a7cc4cd47563a1d8da8dc0aa3b944..572d2b3ceac5fa43a324d4c687f72ddd1e1cdb78 100644 --- a/theodolite-benchmarks/definitions/uc2-kstreams/uc2-kstreams-benchmark-standalone.yaml +++ b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-kstreams-benchmark-standalone.yaml @@ -33,7 +33,7 @@ loadTypes: properties: loadGenMaxRecords: "150000" kafkaConfig: - bootstrapServer: "theodolite-cp-kafka:9092" + bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092" topics: - name: "input" numPartitions: 40 diff --git a/theodolite-benchmarks/definitions/uc2-load-generator/resources/uc2-load-generator-deployment.yaml b/theodolite-benchmarks/definitions/uc2-load-generator/resources/uc2-load-generator-deployment.yaml index dfc0af71543c15b12b5c850919feb0e0a4f52f28..d758c66f88fa93c98258febf6c5e6a35f7171820 100644 --- a/theodolite-benchmarks/definitions/uc2-load-generator/resources/uc2-load-generator-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc2-load-generator/resources/uc2-load-generator-deployment.yaml @@ -27,6 +27,6 @@ spec: - name: KUBERNETES_DNS_NAME value: "titan-ccp-load-generator.$(KUBERNETES_NAMESPACE).svc.cluster.local" - name: KAFKA_BOOTSTRAP_SERVERS - value: "theodolite-cp-kafka:9092" + value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL value: "http://theodolite-cp-schema-registry:8081" diff --git a/theodolite-benchmarks/definitions/uc3-flink/resources/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc3-flink/resources/jobmanager-deployment.yaml index d01123b13fe2d63637ee4000051091a99bad0546..2b3f33e3748dab0fd62747ff1f0caeb768dcd4e2 100644 --- a/theodolite-benchmarks/definitions/uc3-flink/resources/jobmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc3-flink/resources/jobmanager-deployment.yaml @@ -20,7 +20,7 @@ spec: image: ghcr.io/cau-se/theodolite-uc3-flink:latest env: - name: KAFKA_BOOTSTRAP_SERVERS - value: "theodolite-cp-kafka:9092" + value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL value: "http://theodolite-cp-schema-registry:8081" - name: COMMIT_INTERVAL_MS diff --git a/theodolite-benchmarks/definitions/uc3-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc3-flink/resources/taskmanager-deployment.yaml index 495f97817e43d692c30fe898c4ef3118cae682d7..cc1efa23c32220c7c664d8aaa4669f3af6492d15 100644 --- a/theodolite-benchmarks/definitions/uc3-flink/resources/taskmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc3-flink/resources/taskmanager-deployment.yaml @@ -20,7 +20,7 @@ spec: image: ghcr.io/cau-se/theodolite-uc3-flink:latest env: - name: KAFKA_BOOTSTRAP_SERVERS - value: "theodolite-cp-kafka:9092" + value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL value: "http://theodolite-cp-schema-registry:8081" - name: COMMIT_INTERVAL_MS diff --git a/theodolite-benchmarks/definitions/uc3-flink/uc3-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc3-flink/uc3-flink-benchmark-operator.yaml index 47b64d9890fc0f300ee1bd8e67acbdf7c8c4e4f9..0b6e2490f3b58e5c843f2719b24378b46406c6a9 100644 --- a/theodolite-benchmarks/definitions/uc3-flink/uc3-flink-benchmark-operator.yaml +++ b/theodolite-benchmarks/definitions/uc3-flink/uc3-flink-benchmark-operator.yaml @@ -50,7 +50,7 @@ spec: properties: loadGenMaxRecords: "150000" kafkaConfig: - bootstrapServer: "theodolite-cp-kafka:9092" + bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092" topics: - name: "input" numPartitions: 40 diff --git a/theodolite-benchmarks/definitions/uc3-kstreams/resources/uc3-kstreams-deployment.yaml b/theodolite-benchmarks/definitions/uc3-kstreams/resources/uc3-kstreams-deployment.yaml index e3f63fae9e245e6116e0fe451480d9bc74b36433..0b58c15fd123c9d06e441084063eae40d20cc48a 100644 --- a/theodolite-benchmarks/definitions/uc3-kstreams/resources/uc3-kstreams-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc3-kstreams/resources/uc3-kstreams-deployment.yaml @@ -21,7 +21,7 @@ spec: name: jmx env: - name: KAFKA_BOOTSTRAP_SERVERS - value: "theodolite-cp-kafka:9092" + value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL value: "http://theodolite-cp-schema-registry:8081" - name: JAVA_OPTS diff --git a/theodolite-benchmarks/definitions/uc3-kstreams/uc3-kstreams-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-kstreams-benchmark-operator.yaml index 25374ad92a32782857cea5924ea6482060832eac..1db619303fe1bb108205654c2245b8032b723c15 100644 --- a/theodolite-benchmarks/definitions/uc3-kstreams/uc3-kstreams-benchmark-operator.yaml +++ b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-kstreams-benchmark-operator.yaml @@ -37,7 +37,7 @@ spec: properties: loadGenMaxRecords: "150000" kafkaConfig: - bootstrapServer: "theodolite-cp-kafka:9092" + bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092" topics: - name: "input" numPartitions: 40 diff --git a/theodolite-benchmarks/definitions/uc3-kstreams/uc3-kstreams-benchmark-standalone.yaml b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-kstreams-benchmark-standalone.yaml index aa92913d2c992835078174747ea849ce296c3eb1..f879e0949e325e2e2cd830231170775935bda65d 100644 --- a/theodolite-benchmarks/definitions/uc3-kstreams/uc3-kstreams-benchmark-standalone.yaml +++ b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-kstreams-benchmark-standalone.yaml @@ -33,7 +33,7 @@ loadTypes: properties: loadGenMaxRecords: "150000" kafkaConfig: - bootstrapServer: "theodolite-cp-kafka:9092" + bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092" topics: - name: "input" numPartitions: 40 diff --git a/theodolite-benchmarks/definitions/uc3-load-generator/resources/uc3-load-generator-deployment.yaml b/theodolite-benchmarks/definitions/uc3-load-generator/resources/uc3-load-generator-deployment.yaml index c1cad0b70fd82a5bbb43792ee79f9cf5cc71d95f..928e0a91b210e786b3d6b156e964a7fb9a7cc184 100644 --- a/theodolite-benchmarks/definitions/uc3-load-generator/resources/uc3-load-generator-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc3-load-generator/resources/uc3-load-generator-deployment.yaml @@ -27,6 +27,6 @@ spec: - name: KUBERNETES_DNS_NAME value: "titan-ccp-load-generator.$(KUBERNETES_NAMESPACE).svc.cluster.local" - name: KAFKA_BOOTSTRAP_SERVERS - value: "theodolite-cp-kafka:9092" + value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL value: "http://theodolite-cp-schema-registry:8081" diff --git a/theodolite-benchmarks/definitions/uc4-flink/resources/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc4-flink/resources/jobmanager-deployment.yaml index 032499ea498f8155fd80e42ec4cbdd850498b217..2f6eaecc4fd1aede1b9a5ff2341149d1aa8c5ccd 100644 --- a/theodolite-benchmarks/definitions/uc4-flink/resources/jobmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc4-flink/resources/jobmanager-deployment.yaml @@ -20,7 +20,7 @@ spec: image: ghcr.io/cau-se/theodolite-uc4-flink:latest env: - name: KAFKA_BOOTSTRAP_SERVERS - value: "theodolite-cp-kafka:9092" + value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL value: "http://theodolite-cp-schema-registry:8081" - name: COMMIT_INTERVAL_MS diff --git a/theodolite-benchmarks/definitions/uc4-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc4-flink/resources/taskmanager-deployment.yaml index 7af13f20b6b2edf3c8878adf4f381dc1c1add115..7db560db53bac827dd92386e0de5621a2b911e35 100644 --- a/theodolite-benchmarks/definitions/uc4-flink/resources/taskmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc4-flink/resources/taskmanager-deployment.yaml @@ -20,7 +20,7 @@ spec: image: ghcr.io/cau-se/theodolite-uc4-flink:latest env: - name: KAFKA_BOOTSTRAP_SERVERS - value: "theodolite-cp-kafka:9092" + value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL value: "http://theodolite-cp-schema-registry:8081" - name: COMMIT_INTERVAL_MS diff --git a/theodolite-benchmarks/definitions/uc4-flink/uc4-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc4-flink/uc4-flink-benchmark-operator.yaml index 8a73f5b0f87198def7b152ea52008e3d4a1aa4ee..28ae937e964127ded0e34d637ab307fa08db8ec3 100644 --- a/theodolite-benchmarks/definitions/uc4-flink/uc4-flink-benchmark-operator.yaml +++ b/theodolite-benchmarks/definitions/uc4-flink/uc4-flink-benchmark-operator.yaml @@ -51,7 +51,7 @@ spec: loadGenMaxRecords: "150000" numSensors: "4.0" kafkaConfig: - bootstrapServer: "theodolite-cp-kafka:9092" + bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092" topics: - name: "input" numPartitions: 40 diff --git a/theodolite-benchmarks/definitions/uc4-kstreams/resources/uc4-kstreams-deployment.yaml b/theodolite-benchmarks/definitions/uc4-kstreams/resources/uc4-kstreams-deployment.yaml index 20e0872d262df46b5c213d9d529983f5f4155735..3c9a96e36ebf31397f91930426fc028be9d5a2c6 100644 --- a/theodolite-benchmarks/definitions/uc4-kstreams/resources/uc4-kstreams-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc4-kstreams/resources/uc4-kstreams-deployment.yaml @@ -21,7 +21,7 @@ spec: name: jmx env: - name: KAFKA_BOOTSTRAP_SERVERS - value: "theodolite-cp-kafka:9092" + value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL value: "http://theodolite-cp-schema-registry:8081" - name: JAVA_OPTS diff --git a/theodolite-benchmarks/definitions/uc4-kstreams/uc4-kstreams-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-kstreams-benchmark-operator.yaml index 655db2fd4122c9e0e844eed3bfe7c0a878c6d7ec..9ce6daa2dc14e8beecba1c43381defea6bba0d37 100644 --- a/theodolite-benchmarks/definitions/uc4-kstreams/uc4-kstreams-benchmark-operator.yaml +++ b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-kstreams-benchmark-operator.yaml @@ -38,7 +38,7 @@ spec: loadGenMaxRecords: "150000" numSensors: "4.0" kafkaConfig: - bootstrapServer: "theodolite-cp-kafka:9092" + bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092" topics: - name: "input" numPartitions: 40 diff --git a/theodolite-benchmarks/definitions/uc4-kstreams/uc4-kstreams-benchmark-standalone.yaml b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-kstreams-benchmark-standalone.yaml index 5c50b6f95d796941c0b2830549ef825f4a4ff6fb..afed6115102499f225b11c25633e07168a903ea8 100644 --- a/theodolite-benchmarks/definitions/uc4-kstreams/uc4-kstreams-benchmark-standalone.yaml +++ b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-kstreams-benchmark-standalone.yaml @@ -34,7 +34,7 @@ loadTypes: loadGenMaxRecords: "150000" numSensors: "4.0" kafkaConfig: - bootstrapServer: "theodolite-cp-kafka:9092" + bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092" topics: - name: "input" numPartitions: 40 diff --git a/theodolite-benchmarks/definitions/uc4-load-generator/resources/uc4-load-generator-deployment.yaml b/theodolite-benchmarks/definitions/uc4-load-generator/resources/uc4-load-generator-deployment.yaml index 7a69d13daae57b06c77f316da9aa953b21ac096b..0f64fc881b5beb31a6dad4ff2a755413049b96fa 100644 --- a/theodolite-benchmarks/definitions/uc4-load-generator/resources/uc4-load-generator-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc4-load-generator/resources/uc4-load-generator-deployment.yaml @@ -27,7 +27,7 @@ spec: - name: KUBERNETES_DNS_NAME value: "titan-ccp-load-generator.$(KUBERNETES_NAMESPACE).svc.cluster.local" - name: KAFKA_BOOTSTRAP_SERVERS - value: "theodolite-cp-kafka:9092" + value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL value: "http://theodolite-cp-schema-registry:8081" - name: NUM_NESTED_GROUPS diff --git a/theodolite-benchmarks/http-bridge/build.gradle b/theodolite-benchmarks/http-bridge/build.gradle index 0377eefc76b456d8e0f94087b06d0c2689f977cf..6bc7c721894e36799015a0f2fb155c9b838238b8 100644 --- a/theodolite-benchmarks/http-bridge/build.gradle +++ b/theodolite-benchmarks/http-bridge/build.gradle @@ -24,6 +24,8 @@ dependencies { implementation project(':load-generator-commons') implementation 'io.javalin:javalin:4.3.0' + implementation 'io.micrometer:micrometer-core:1.8.3' + implementation 'io.micrometer:micrometer-registry-prometheus:1.8.3' implementation 'com.google.code.gson:gson:2.8.2' runtimeOnly 'org.slf4j:slf4j-simple:1.7.25' diff --git a/theodolite-benchmarks/http-bridge/src/main/java/theodolite/commons/httpbridge/JavalinWebServer.java b/theodolite-benchmarks/http-bridge/src/main/java/theodolite/commons/httpbridge/JavalinWebServer.java index c23a17588d661fc5d1c6e9eb294d2d29fc165675..e7073a421582d74009f4a89acfcfb816a689acf3 100644 --- a/theodolite-benchmarks/http-bridge/src/main/java/theodolite/commons/httpbridge/JavalinWebServer.java +++ b/theodolite-benchmarks/http-bridge/src/main/java/theodolite/commons/httpbridge/JavalinWebServer.java @@ -1,6 +1,10 @@ package theodolite.commons.httpbridge; import io.javalin.Javalin; +import io.javalin.plugin.metrics.MicrometerPlugin; +import io.micrometer.prometheus.PrometheusConfig; +import io.micrometer.prometheus.PrometheusMeterRegistry; +import io.prometheus.client.exporter.common.TextFormat; import java.util.Collection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -14,7 +18,9 @@ public class JavalinWebServer { private static final int HTTP_SUCCESS = 200; - private final Javalin app = Javalin.create(); + private final Javalin app; + + private final PrometheusMeterRegistry registry; private final String host; private final int port; @@ -28,6 +34,10 @@ public class JavalinWebServer { final int port) { this.host = host; this.port = port; + this.registry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT); + this.app = Javalin.create(config -> { + config.registerPlugin(new MicrometerPlugin(this.registry)); + }); this.configureRoutes(converters); } @@ -40,6 +50,9 @@ public class JavalinWebServer { ctx.status(HTTP_SUCCESS); }); } + this.app.get("/metrics", ctx -> ctx + .contentType(TextFormat.CONTENT_TYPE_004) + .result(this.registry.scrape())); } public void start() { diff --git a/theodolite/examples/operator/example-benchmark.yaml b/theodolite/examples/operator/example-benchmark.yaml index 5f68af04750bcd779c9682ede69d6c68b9fb3e92..62920091e831ff914fb67e85a67cd3f1d98995ab 100644 --- a/theodolite/examples/operator/example-benchmark.yaml +++ b/theodolite/examples/operator/example-benchmark.yaml @@ -34,7 +34,7 @@ spec: properties: loadGenMaxRecords: "150000" kafkaConfig: - bootstrapServer: "theodolite-cp-kafka:9092" + bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092" topics: - name: "input" numPartitions: 40 diff --git a/theodolite/examples/operator/example-configmap.yaml b/theodolite/examples/operator/example-configmap.yaml index 210ce32d3fc0f75b9ffce874d1fa0a1ea9bdc3cd..db511a94cc903869677f2e447d45baf0d983ac6c 100644 --- a/theodolite/examples/operator/example-configmap.yaml +++ b/theodolite/examples/operator/example-configmap.yaml @@ -24,7 +24,7 @@ data: image: ghcr.io/cau-se/theodolite-uc1-kstreams-app:latest env: - name: KAFKA_BOOTSTRAP_SERVERS - value: "theodolite-cp-kafka:9092" + value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL value: "http://theodolite-cp-schema-registry:8081" - name: JAVA_OPTS @@ -65,7 +65,7 @@ data: - name: KUBERNETES_DNS_NAME value: "titan-ccp-load-generator.$(KUBERNETES_NAMESPACE).svc.cluster.local" - name: KAFKA_BOOTSTRAP_SERVERS - value: "theodolite-cp-kafka:9092" + value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL value: "http://theodolite-cp-schema-registry:8081" uc1-load-generator-service.yaml: | diff --git a/theodolite/examples/resources/uc1-kstreams-deployment.yaml b/theodolite/examples/resources/uc1-kstreams-deployment.yaml index fdd1ff867ac83beb10856baec53569c88169232e..1951b1177572dbd1276afcbc6770f91f9f5ff168 100644 --- a/theodolite/examples/resources/uc1-kstreams-deployment.yaml +++ b/theodolite/examples/resources/uc1-kstreams-deployment.yaml @@ -21,7 +21,7 @@ spec: name: jmx env: - name: KAFKA_BOOTSTRAP_SERVERS - value: "theodolite-cp-kafka:9092" + value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL value: "http://theodolite-cp-schema-registry:8081" - name: JAVA_OPTS diff --git a/theodolite/examples/resources/uc1-load-generator-deployment.yaml b/theodolite/examples/resources/uc1-load-generator-deployment.yaml index 9f9ccc6ae39407bb1f027e1e23cb152944b869e0..65048a97d5de3d831f782db329e295a5e5ceb727 100644 --- a/theodolite/examples/resources/uc1-load-generator-deployment.yaml +++ b/theodolite/examples/resources/uc1-load-generator-deployment.yaml @@ -27,6 +27,6 @@ spec: - name: KUBERNETES_DNS_NAME value: "titan-ccp-load-generator.$(KUBERNETES_NAMESPACE).svc.cluster.local" - name: KAFKA_BOOTSTRAP_SERVERS - value: "theodolite-cp-kafka:9092" + value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL value: "http://theodolite-cp-schema-registry:8081" diff --git a/theodolite/examples/standalone/example-benchmark.yaml b/theodolite/examples/standalone/example-benchmark.yaml index 4d67399231778c91cebb3ffe088e2d26ef388008..254fb4628a595b627f9f4260e3d5478984cec1c6 100644 --- a/theodolite/examples/standalone/example-benchmark.yaml +++ b/theodolite/examples/standalone/example-benchmark.yaml @@ -31,7 +31,7 @@ loadTypes: properties: loadGenMaxRecords: "150000" kafkaConfig: - bootstrapServer: "theodolite-cp-kafka:9092" + bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092" topics: - name: "input" numPartitions: 40 diff --git a/theodolite/src/main/kotlin/theodolite/evaluation/SloConfigHandler.kt b/theodolite/src/main/kotlin/theodolite/evaluation/SloConfigHandler.kt index 425a4f3b0634d53f8b1d5c4b8abdba9ca81c3f2b..924305660798e6dbed06662ef4e393c63f5f2bfa 100644 --- a/theodolite/src/main/kotlin/theodolite/evaluation/SloConfigHandler.kt +++ b/theodolite/src/main/kotlin/theodolite/evaluation/SloConfigHandler.kt @@ -4,7 +4,7 @@ import theodolite.benchmark.BenchmarkExecution import theodolite.util.InvalidPatcherConfigurationException import javax.enterprise.context.ApplicationScoped -private const val CONSUMER_LAG_QUERY = "sum by(group)(kafka_consumergroup_group_lag >= 0)" +private const val CONSUMER_LAG_QUERY = "sum by(consumergroup) (kafka_consumergroup_lag >= 0)" private const val DROPPED_RECORDS_QUERY = "sum by(job) (kafka_streams_stream_task_metrics_dropped_records_total>=0)" @ApplicationScoped diff --git a/theodolite/src/test/resources/k8s-resource-files/test-benchmark.yaml b/theodolite/src/test/resources/k8s-resource-files/test-benchmark.yaml index e690aa56d74d695b0b81469023ccf82d0046cf45..1ba204bb2821f9b734706d322322b28220ef19d5 100644 --- a/theodolite/src/test/resources/k8s-resource-files/test-benchmark.yaml +++ b/theodolite/src/test/resources/k8s-resource-files/test-benchmark.yaml @@ -29,7 +29,7 @@ spec: properties: loadGenMaxRecords: "15000" kafkaConfig: - bootstrapServer: "theodolite-cp-kafka:9092" + bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092" topics: - name: "input" numPartitions: 40