diff --git a/execution/infrastructure/zookeeper-client.yaml b/execution/infrastructure/zookeeper-client.yaml
index 9d6d00275c46a888ed4e8ff08533ca245c4a684c..85739bba751be62773f511aa95030e6d71a4a869 100644
--- a/execution/infrastructure/zookeeper-client.yaml
+++ b/execution/infrastructure/zookeeper-client.yaml
@@ -2,10 +2,12 @@ apiVersion: v1
kind: Pod
metadata:
name: zookeeper-client
+ labels:
+ app: zookeeper-client
spec:
containers:
- name: zookeeper-client
- image: confluentinc/cp-zookeeper:5.4.0
+ image: zookeeper:3.7.0
command:
- sh
- -c
diff --git a/helm/templates/strimzi/kafka-cluster.yaml b/helm/templates/kafka/kafka-cluster.yaml
similarity index 100%
rename from helm/templates/strimzi/kafka-cluster.yaml
rename to helm/templates/kafka/kafka-cluster.yaml
diff --git a/helm/templates/strimzi/kafka-exporter-podmonitor.yaml b/helm/templates/kafka/kafka-exporter-podmonitor.yaml
similarity index 100%
rename from helm/templates/strimzi/kafka-exporter-podmonitor.yaml
rename to helm/templates/kafka/kafka-exporter-podmonitor.yaml
diff --git a/helm/templates/strimzi/kafka-metrics-configmap.yaml b/helm/templates/kafka/kafka-metrics-configmap.yaml
similarity index 100%
rename from helm/templates/strimzi/kafka-metrics-configmap.yaml
rename to helm/templates/kafka/kafka-metrics-configmap.yaml
diff --git a/helm/templates/strimzi/kafka-podmonitor.yaml b/helm/templates/kafka/kafka-podmonitor.yaml
similarity index 100%
rename from helm/templates/strimzi/kafka-podmonitor.yaml
rename to helm/templates/kafka/kafka-podmonitor.yaml
diff --git a/helm/templates/kafka/zookeeper-client.yaml b/helm/templates/kafka/zookeeper-client.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..2154ec116b71bbb33808d3c32d42170a8d9589da
--- /dev/null
+++ b/helm/templates/kafka/zookeeper-client.yaml
@@ -0,0 +1,23 @@
+{{- if .Values.strimzi.zookeeper.zooEntrance.zookeeperClient.enabled -}}
+apiVersion: v1
+kind: Pod
+metadata:
+ name: {{ template "theodolite.fullname" . }}-kafka-zookeeper-client
+ labels:
+ app: zookeeper-client
+spec:
+ containers:
+ - name: zookeeper-client
+ image: zookeeper:3.7.0
+ command:
+ - sh
+ - -c
+ - "exec tail -f /dev/null"
+ env:
+ - name: ZOOKEEPER_SERVER
+ value: {{ template "theodolite.fullname" . }}-kafka-zoo-entrance:2181
+ {{- with .Values.strimzi.zookeeper.zooEntrance.zookeeperClient.nodeSelector }}
+ nodeSelector:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+{{- end }}
\ No newline at end of file
diff --git a/helm/templates/kafka/zookeeper-entrance-deployment.yaml b/helm/templates/kafka/zookeeper-entrance-deployment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a0c2841760aa641b1f1105863465a08069d164bd
--- /dev/null
+++ b/helm/templates/kafka/zookeeper-entrance-deployment.yaml
@@ -0,0 +1,71 @@
+{{- if .Values.strimzi.zookeeper.zooEntrance.enabled -}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: {{ template "theodolite.fullname" . }}-kafka-zoo-entrance
+ labels:
+ app: zoo-entrance
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: zoo-entrance
+ strategy:
+ type: Recreate
+ template:
+ metadata:
+ labels:
+ app: zoo-entrance
+ spec:
+ containers:
+ - name: zoo-entrance
+ image: 'ghcr.io/scholzj/zoo-entrance:latest'
+ command:
+ - /opt/stunnel/stunnel_run.sh
+ ports:
+ - containerPort: 2181
+ name: zoo
+ protocol: TCP
+ env:
+ - name: LOG_LEVEL
+ value: notice
+ - name: STRIMZI_ZOOKEEPER_CONNECT
+ value: {{ template "theodolite.fullname" . }}-kafka-zookeeper-client:2181
+ imagePullPolicy: Always
+ livenessProbe:
+ exec:
+ command:
+ - /opt/stunnel/stunnel_healthcheck.sh
+ - '2181'
+ failureThreshold: 3
+ initialDelaySeconds: 15
+ periodSeconds: 10
+ successThreshold: 1
+ timeoutSeconds: 5
+ readinessProbe:
+ exec:
+ command:
+ - /opt/stunnel/stunnel_healthcheck.sh
+ - '2181'
+ failureThreshold: 3
+ initialDelaySeconds: 15
+ periodSeconds: 10
+ successThreshold: 1
+ timeoutSeconds: 5
+ volumeMounts:
+ - mountPath: /etc/cluster-operator-certs/
+ name: cluster-operator-certs
+ - mountPath: /etc/cluster-ca-certs/
+ name: cluster-ca-certs
+ restartPolicy: Always
+ terminationGracePeriodSeconds: 30
+ volumes:
+ - name: cluster-operator-certs
+ secret:
+ defaultMode: 288
+ secretName: {{ template "theodolite.fullname" . }}-kafka-cluster-operator-certs
+ - name: cluster-ca-certs
+ secret:
+ defaultMode: 288
+ secretName: {{ template "theodolite.fullname" . }}-kafka-cluster-ca-cert
+{{- end }}
diff --git a/helm/templates/kafka/zookeeper-entrance-network.yaml b/helm/templates/kafka/zookeeper-entrance-network.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..604095c1ab3f5e207666383e23dfb8f006b196ae
--- /dev/null
+++ b/helm/templates/kafka/zookeeper-entrance-network.yaml
@@ -0,0 +1,22 @@
+{{- if .Values.strimzi.zookeeper.zooEntrance.enabled -}}
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+ labels:
+ app: zoo-entrance
+ name: {{ template "theodolite.fullname" . }}-kafka-zoo-entrance
+spec:
+ ingress:
+ - from:
+ - podSelector:
+ matchLabels:
+ app: zoo-entrance
+ ports:
+ - port: 2181
+ protocol: TCP
+ podSelector:
+ matchLabels:
+ strimzi.io/name: {{ template "theodolite.fullname" . }}-kafka-zookeeper
+ policyTypes:
+ - Ingress
+{{- end }}
diff --git a/helm/templates/kafka/zookeeper-entrance-service.yaml b/helm/templates/kafka/zookeeper-entrance-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..bbb466c3d9ad4e7ff9cb012aaaa95ad9a5380a6e
--- /dev/null
+++ b/helm/templates/kafka/zookeeper-entrance-service.yaml
@@ -0,0 +1,17 @@
+{{- if .Values.strimzi.zookeeper.zooEntrance.enabled -}}
+apiVersion: v1
+kind: Service
+metadata:
+ labels:
+ app: zoo-entrance
+ name: {{ template "theodolite.fullname" . }}-kafka-zoo-entrance
+spec:
+ ports:
+ - name: zoo
+ port: 2181
+ protocol: TCP
+ targetPort: 2181
+ selector:
+ app: zoo-entrance
+ type: ClusterIP
+{{- end }}
diff --git a/helm/values.yaml b/helm/values.yaml
index 797098b1a2316389134827cfd6be37c0aaf3c4e6..f3f6093d1e66432495f8acfc6db1e7e269330c4e 100644
--- a/helm/values.yaml
+++ b/helm/values.yaml
@@ -176,6 +176,11 @@ strimzi:
zookeeper:
replicas: 3
+ zooEntrance:
+ enabled: true
+ zookeeperClient:
+ enabled: true
+ nodeSelector: {}
###
@@ -295,24 +300,40 @@ operator:
uc1LoadGenerator: true
uc1Kstreams: true
uc1Flink: true
+ uc1BeamFlink: true
+ uc1BeamSamza: true
uc2LoadGenerator: true
uc2Kstreams: true
uc2Flink: true
+ uc2BeamFlink: true
+ uc2BeamSamza: true
uc3LoadGenerator: true
uc3Kstreams: true
uc3Flink: true
+ uc3BeamFlink: true
+ uc3BeamSamza: true
uc4LoadGenerator: true
uc4Kstreams: true
uc4Flink: true
+ uc4BeamFlink: true
+ uc4BeamSamza: true
benchmarks:
uc1Kstreams: true
uc1Flink: true
+ uc1BeamFlink: true
+ uc1BeamSamza: true
uc2Kstreams: true
uc2Flink: true
+ uc2BeamFlink: true
+ uc2BeamSamza: true
uc3Kstreams: true
uc3Flink: true
+ uc3BeamFlink: true
+ uc3BeamSamza: true
uc4Kstreams: true
uc4Flink: true
+ uc4BeamFlink: true
+ uc4BeamSamza: true
serviceAccount:
create: true
diff --git a/theodolite-benchmarks/definitions/install-configmaps.sh b/theodolite-benchmarks/definitions/install-configmaps.sh
index 841a293bbb77c4960a2532a13a009a42227223d3..03d87bf971e98693be7936368421c91a83fd16bb 100755
--- a/theodolite-benchmarks/definitions/install-configmaps.sh
+++ b/theodolite-benchmarks/definitions/install-configmaps.sh
@@ -10,6 +10,18 @@ kubectl create configmap benchmark-resources-uc2-kstreams --from-file uc2-kstrea
kubectl create configmap benchmark-resources-uc3-kstreams --from-file uc3-kstreams/resources
kubectl create configmap benchmark-resources-uc4-kstreams --from-file uc4-kstreams/resources
+# Beam Flink
+kubectl create configmap benchmark-resources-uc1-beam-flink --from-file uc1-beam-flink/resources
+kubectl create configmap benchmark-resources-uc2-beam-flink --from-file uc2-beam-flink/resources
+kubectl create configmap benchmark-resources-uc3-beam-flink --from-file uc3-beam-flink/resources
+kubectl create configmap benchmark-resources-uc4-beam-flink --from-file uc4-beam-flink/resources
+
+# Beam Samza
+kubectl create configmap benchmark-resources-uc1-beam-samza --from-file uc1-beam-samza/resources
+kubectl create configmap benchmark-resources-uc2-beam-samza --from-file uc2-beam-samza/resources
+kubectl create configmap benchmark-resources-uc3-beam-samza --from-file uc3-beam-samza/resources
+kubectl create configmap benchmark-resources-uc4-beam-samza --from-file uc4-beam-samza/resources
+
# Load Generator
kubectl create configmap benchmark-resources-uc1-load-generator --from-file uc1-load-generator/resources
kubectl create configmap benchmark-resources-uc2-load-generator --from-file uc2-load-generator/resources
diff --git a/theodolite-benchmarks/definitions/uc1-beam-flink/resources/flink-configuration-configmap.yaml b/theodolite-benchmarks/definitions/uc1-beam-flink/resources/flink-configuration-configmap.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..36178e2bebdac96b8648bd6c299009aa49d3fff6
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc1-beam-flink/resources/flink-configuration-configmap.yaml
@@ -0,0 +1,66 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: flink-config
+ labels:
+ app: flink
+data:
+ flink-conf.yaml: |+
+ jobmanager.rpc.address: flink-jobmanager
+ taskmanager.numberOfTaskSlots: 1 #TODO
+ #blob.server.port: 6124
+ #jobmanager.rpc.port: 6123
+ #taskmanager.rpc.port: 6122
+ #queryable-state.proxy.ports: 6125
+ #jobmanager.memory.process.size: 4Gb
+ #taskmanager.memory.process.size: 4Gb
+ #parallelism.default: 1 #TODO
+ metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
+ metrics.reporter.prom.interval: 10 SECONDS
+ taskmanager.network.detailed-metrics: true
+ # -> gives metrics about inbound/outbound network queue lengths
+ log4j-console.properties: |+
+ # This affects logging for both user code and Flink
+ rootLogger.level = INFO
+ rootLogger.appenderRef.console.ref = ConsoleAppender
+ rootLogger.appenderRef.rolling.ref = RollingFileAppender
+
+ # Uncomment this if you want to _only_ change Flink's logging
+ #logger.flink.name = org.apache.flink
+ #logger.flink.level = INFO
+
+ # The following lines keep the log level of common libraries/connectors on
+ # log level INFO. The root logger does not override this. You have to manually
+ # change the log levels here.
+ logger.akka.name = akka
+ logger.akka.level = INFO
+ logger.kafka.name= org.apache.kafka
+ logger.kafka.level = INFO
+ logger.hadoop.name = org.apache.hadoop
+ logger.hadoop.level = INFO
+ logger.zookeeper.name = org.apache.zookeeper
+ logger.zookeeper.level = INFO
+
+ # Log all infos to the console
+ appender.console.name = ConsoleAppender
+ appender.console.type = CONSOLE
+ appender.console.layout.type = PatternLayout
+ appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+
+ # Log all infos in the given rolling file
+ appender.rolling.name = RollingFileAppender
+ appender.rolling.type = RollingFile
+ appender.rolling.append = false
+ appender.rolling.fileName = ${sys:log.file}
+ appender.rolling.filePattern = ${sys:log.file}.%i
+ appender.rolling.layout.type = PatternLayout
+ appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+ appender.rolling.policies.type = Policies
+ appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
+ appender.rolling.policies.size.size=100MB
+ appender.rolling.strategy.type = DefaultRolloverStrategy
+ appender.rolling.strategy.max = 10
+
+ # Suppress the irrelevant (wrong) warnings from the Netty channel handler
+ logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
+ logger.netty.level = OFF
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc1-beam-flink/resources/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc1-beam-flink/resources/jobmanager-deployment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5108ff272f93d83fddaddfe1ebe821073f75ade7
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc1-beam-flink/resources/jobmanager-deployment.yaml
@@ -0,0 +1,97 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: flink-jobmanager
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: flink
+ component: jobmanager
+ template:
+ metadata:
+ labels:
+ app: flink
+ component: jobmanager
+ spec:
+ terminationGracePeriodSeconds: 0
+ containers:
+ - name: jobmanager
+ image: ghcr.io/cau-se/theodolite-uc1-beam-flink:latest
+ args: ["standalone-job", "--job-classname", "application.Uc1BeamFlink",
+ "--parallelism=$(PARALLELISM)",
+ "--disableMetrics=true",
+ "--fasterCopy"]
+ # optional arguments: ["--job-id", "<job id>", "--fromSavepoint", "/path/to/savepoint", "--allowNonRestoredState"]
+ env:
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-kafka-kafka-bootstrap:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
+ - name: COMMIT_INTERVAL_MS
+ value: "100"
+ - name: CHECKPOINTING
+ value: "false"
+ - name: PARALLELISM
+ value: "1"
+ - name: "FLINK_STATE_BACKEND"
+ value: "rocksdb"
+ - name: JOB_MANAGER_RPC_ADDRESS
+ value: "flink-jobmanager"
+ - name: FLINK_PROPERTIES
+ value: |+
+ blob.server.port: 6124
+ jobmanager.rpc.port: 6123
+ taskmanager.rpc.port: 6122
+ queryable-state.proxy.ports: 6125
+ jobmanager.memory.process.size: 4Gb
+ taskmanager.memory.process.size: 4Gb
+ parallelism.default: 1 #TODO
+ resources:
+ limits:
+ memory: 4Gi
+ cpu: 1000m
+ ports:
+ - containerPort: 6123
+ name: rpc
+ - containerPort: 6124
+ name: blob-server
+ - containerPort: 8081
+ name: webui
+ - containerPort: 9249
+ name: metrics
+ livenessProbe:
+ tcpSocket:
+ port: 6123
+ initialDelaySeconds: 30
+ periodSeconds: 60
+ # volumeMounts:
+ # - name: flink-config-volume-rw
+ # mountPath: /opt/flink/conf
+# - name: job-artifacts-volume
+# mountPath: /opt/flink/usrlib
+ securityContext:
+ runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
+ # initContainers:
+ # - name: init-jobmanager
+ # image: busybox:1.28
+ # command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
+ # volumeMounts:
+ # - name: flink-config-volume
+ # mountPath: /flink-config/
+ # - name: flink-config-volume-rw
+ # mountPath: /flink-config-rw/
+ # volumes:
+ # - name: flink-config-volume
+ # configMap:
+ # name: flink-config
+ # items:
+ # - key: flink-conf.yaml
+ # path: flink-conf.yaml
+ # - key: log4j-console.properties
+ # path: log4j-console.properties
+ # - name: flink-config-volume-rw
+ # emptyDir: {}
+# - name: job-artifacts-volume
+# hostPath:
+# path: /host/path/to/job/artifacts
diff --git a/theodolite-benchmarks/definitions/uc1-beam-flink/resources/jobmanager-rest-service.yaml b/theodolite-benchmarks/definitions/uc1-beam-flink/resources/jobmanager-rest-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..3d74aaf7f625c6922e2e1b4f20c19e50a39b68ac
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc1-beam-flink/resources/jobmanager-rest-service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flink-jobmanager-rest
+spec:
+ type: NodePort
+ ports:
+ - name: rest
+ port: 8081
+ targetPort: 8081
+ nodePort: 30081
+ selector:
+ app: flink
+ component: jobmanager
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc1-beam-flink/resources/jobmanager-service.yaml b/theodolite-benchmarks/definitions/uc1-beam-flink/resources/jobmanager-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..e2ff5d9898eb1ebf5db9a827472a47514ab1473c
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc1-beam-flink/resources/jobmanager-service.yaml
@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flink-jobmanager
+ labels:
+ app: flink
+spec:
+ type: ClusterIP
+ ports:
+ - name: rpc
+ port: 6123
+ - name: blob-server
+ port: 6124
+ - name: webui
+ port: 8081
+ - name: metrics
+ port: 9249
+ selector:
+ app: flink
+ component: jobmanager
\ No newline at end of file
diff --git a/helm/templates/kafka/service-monitor.yaml b/theodolite-benchmarks/definitions/uc1-beam-flink/resources/service-monitor.yaml
similarity index 51%
rename from helm/templates/kafka/service-monitor.yaml
rename to theodolite-benchmarks/definitions/uc1-beam-flink/resources/service-monitor.yaml
index 68fd5f7599d36187fa7c4dee2fab211eb263c67d..02f78823c627e27ddfe1db5eac3f6a7f7a7f1bf8 100644
--- a/helm/templates/kafka/service-monitor.yaml
+++ b/theodolite-benchmarks/definitions/uc1-beam-flink/resources/service-monitor.yaml
@@ -1,16 +1,14 @@
-{{- if .Values.kafkaClient.enabled -}}
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
labels:
- app: cp-kafka
+ app: flink
appScope: titan-ccp
- name: {{ template "theodolite.fullname" . }}-cp-kafka
+ name: flink
spec:
selector:
matchLabels:
- app: cp-kafka
+ app: flink
endpoints:
- port: metrics
- interval: 7s
-{{- end}}
+ interval: 10s
diff --git a/theodolite-benchmarks/definitions/uc1-beam-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc1-beam-flink/resources/taskmanager-deployment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..7d70908589cfa2124af64cf3c671ed62c00b1be6
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc1-beam-flink/resources/taskmanager-deployment.yaml
@@ -0,0 +1,88 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: flink-taskmanager
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: flink
+ component: taskmanager
+ template:
+ metadata:
+ labels:
+ app: flink
+ component: taskmanager
+ spec:
+ terminationGracePeriodSeconds: 0
+ containers:
+ - name: taskmanager
+ image: ghcr.io/cau-se/theodolite-uc1-beam-flink:latest
+ args: ["taskmanager"]
+ env:
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-kafka-kafka-bootstrap:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
+ - name: COMMIT_INTERVAL_MS
+ value: "100"
+ - name: CHECKPOINTING
+ value: "false"
+ # - name: PARALLELISM
+ # value: "1"
+ - name: "FLINK_STATE_BACKEND"
+ value: "rocksdb"
+ - name: JOB_MANAGER_RPC_ADDRESS
+ value: "flink-jobmanager"
+ # - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS
+ # value: "1" #TODO
+ # - name: FLINK_PROPERTIES
+ # value: |+
+ # blob.server.port: 6124
+ # jobmanager.rpc.port: 6123
+ # taskmanager.rpc.port: 6122
+ # queryable-state.proxy.ports: 6125
+ # jobmanager.memory.process.size: 4Gb
+ # taskmanager.memory.process.size: 4Gb
+ # #parallelism.default: 1 #TODO
+ resources:
+ limits:
+ memory: 4Gi
+ cpu: 1000m
+ ports:
+ - containerPort: 6122
+ name: rpc
+ - containerPort: 6125
+ name: query-state
+ - containerPort: 9249
+ name: metrics
+ # livenessProbe:
+ # tcpSocket:
+ # port: 6122
+ # initialDelaySeconds: 30
+ # periodSeconds: 60
+ # volumeMounts:
+ # - name: flink-config-volume-rw
+ # mountPath: /opt/flink/conf/
+ securityContext:
+ runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
+ # initContainers:
+ # - name: init-taskmanager
+ # image: busybox:1.28
+ # command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
+ # volumeMounts:
+ # - name: flink-config-volume
+ # mountPath: /flink-config/
+ # - name: flink-config-volume-rw
+ # mountPath: /flink-config-rw/
+ # volumes:
+ # - name: flink-config-volume
+ # configMap:
+ # name: flink-config
+ # items:
+ # - key: flink-conf.yaml
+ # path: flink-conf.yaml
+ # - key: log4j-console.properties
+ # path: log4j-console.properties
+ # - name: flink-config-volume-rw
+ # emptyDir: {}
diff --git a/theodolite-benchmarks/definitions/uc1-beam-flink/resources/taskmanager-service.yaml b/theodolite-benchmarks/definitions/uc1-beam-flink/resources/taskmanager-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a2e27f64af1cfd1a26da142b8a50bb41c8ba5fcb
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc1-beam-flink/resources/taskmanager-service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flink-taskmanager
+ labels:
+ app: flink
+spec:
+ type: ClusterIP
+ ports:
+ - name: metrics
+ port: 9249
+ selector:
+ app: flink
+ component: taskmanager
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc1-beam-flink/uc1-beam-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc1-beam-flink/uc1-beam-flink-benchmark-operator.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..2e110abfc13e1537792da381de2260fb3adbd606
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc1-beam-flink/uc1-beam-flink-benchmark-operator.yaml
@@ -0,0 +1,57 @@
+apiVersion: theodolite.com/v1
+kind: benchmark
+metadata:
+ name: uc1-beam-flink
+spec:
+ sut:
+ resources:
+ - configMap:
+ name: "benchmark-resources-uc1-beam-flink"
+ files:
+ - "flink-configuration-configmap.yaml"
+ - "taskmanager-deployment.yaml"
+ - "taskmanager-service.yaml"
+ - "service-monitor.yaml"
+ - "jobmanager-service.yaml"
+ - "jobmanager-deployment.yaml"
+ #- "jobmanager-rest-service.yaml"
+ loadGenerator:
+ resources:
+ - configMap:
+ name: "benchmark-resources-uc1-load-generator"
+ files:
+ - "uc1-load-generator-deployment.yaml"
+ - "uc1-load-generator-service.yaml"
+ resourceTypes:
+ - typeName: "Instances"
+ patchers:
+ - type: "ReplicaPatcher"
+ resource: "taskmanager-deployment.yaml"
+ - type: "EnvVarPatcher"
+ resource: "jobmanager-deployment.yaml"
+ properties:
+ container: "jobmanager"
+ variableName: "PARALLELISM"
+ - type: "EnvVarPatcher" # required?
+ resource: "taskmanager-deployment.yaml"
+ properties:
+ container: "taskmanager"
+ variableName: "PARALLELISM"
+ loadTypes:
+ - typeName: "NumSensors"
+ patchers:
+ - type: "EnvVarPatcher"
+ resource: "uc1-load-generator-deployment.yaml"
+ properties:
+ container: "workload-generator"
+ variableName: "NUM_SENSORS"
+ - type: NumSensorsLoadGeneratorReplicaPatcher
+ resource: "uc1-load-generator-deployment.yaml"
+ properties:
+ loadGenMaxRecords: "150000"
+ kafkaConfig:
+ bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092"
+ topics:
+ - name: "input"
+ numPartitions: 40
+ replicationFactor: 1
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc1-beam-samza/resources/beam-samza-service.yaml b/theodolite-benchmarks/definitions/uc1-beam-samza/resources/beam-samza-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..6317caf9fe624e42449b8f630d040a068709cda3
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc1-beam-samza/resources/beam-samza-service.yaml
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: titan-ccp-aggregation
+ labels:
+ app: titan-ccp-aggregation
+spec:
+ #type: NodePort
+ selector:
+ app: titan-ccp-aggregation
+ ports:
+ - name: http
+ port: 80
+ targetPort: 80
+ protocol: TCP
+ - name: metrics
+ port: 5556
diff --git a/theodolite-benchmarks/definitions/uc1-beam-samza/resources/service-monitor.yaml b/theodolite-benchmarks/definitions/uc1-beam-samza/resources/service-monitor.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..4e7e758cacb5086305efa26292ddef2afc958096
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc1-beam-samza/resources/service-monitor.yaml
@@ -0,0 +1,14 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+ labels:
+ app: titan-ccp-aggregation
+ appScope: titan-ccp
+ name: titan-ccp-aggregation
+spec:
+ selector:
+ matchLabels:
+ app: titan-ccp-aggregation
+ endpoints:
+ - port: metrics
+ interval: 10s
diff --git a/theodolite-benchmarks/definitions/uc1-beam-samza/resources/uc1-beam-samza-deployment.yaml b/theodolite-benchmarks/definitions/uc1-beam-samza/resources/uc1-beam-samza-deployment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..607767382b4b2fce406fe3820c0f8c6ed1cef777
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc1-beam-samza/resources/uc1-beam-samza-deployment.yaml
@@ -0,0 +1,60 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: titan-ccp-aggregation
+spec:
+ selector:
+ matchLabels:
+ app: titan-ccp-aggregation
+ replicas: 1
+ template:
+ metadata:
+ labels:
+ app: titan-ccp-aggregation
+ spec:
+ terminationGracePeriodSeconds: 0
+ containers:
+ - name: uc1-beam-samza
+ image: ghcr.io/cau-se/theodolite-uc1-beam-samza:latest
+ env:
+ - name: SAMZA_JOB_COORDINATOR_ZK_CONNECT
+ value: "theodolite-kafka-zoo-entrance:2181"
+ - name: SAMZA_SYSTEMS_KAFKA_PRODUCER_BOOTSTRAP_SERVERS
+ value: "theodolite-kafka-kafka-bootstrap:9092"
+ - name: SAMZA_SYSTEMS_KAFKA_CONSUMER_BOOTSTRAP_SERVERS
+ value: "theodolite-kafka-kafka-bootstrap:9092"
+ - name: MAX_SOURCE_PARALLELISM
+ value: "1024"
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-kafka-kafka-bootstrap:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
+ # - name: JAVA_OPTS
+ # value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555"
+ - name: COMMIT_INTERVAL_MS # Set as default for the applications
+ value: "100"
+ resources:
+ limits:
+ memory: 4Gi
+ cpu: 1000m
+ # - name: prometheus-jmx-exporter
+ # image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143"
+ # command:
+ # - java
+ # - -XX:+UnlockExperimentalVMOptions
+ # - -XX:+UseCGroupMemoryLimitForHeap
+ # - -XX:MaxRAMFraction=1
+ # - -XshowSettings:vm
+ # - -jar
+ # - jmx_prometheus_httpserver.jar
+ # - "5556"
+ # - /etc/jmx-aggregation/jmx-kafka-prometheus.yml
+ # ports:
+ # - containerPort: 5556
+ # volumeMounts:
+ # - name: jmx-config
+ # mountPath: /etc/jmx-aggregation
+ # volumes:
+ # - name: jmx-config
+ # configMap:
+ # name: uc1-jmx-configmap
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc1-beam-samza/uc1-beam-samza-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc1-beam-samza/uc1-beam-samza-benchmark-operator.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..46030c67b135c97fcefe99482bd511f0c8138a0e
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc1-beam-samza/uc1-beam-samza-benchmark-operator.yaml
@@ -0,0 +1,56 @@
+apiVersion: theodolite.com/v1
+kind: benchmark
+metadata:
+ name: uc1-beam-samza
+spec:
+ sut:
+ resources:
+ - configMap:
+ name: "benchmark-resources-uc1-beam-samza"
+ files:
+ - "uc1-beam-samza-deployment.yaml"
+ - "beam-samza-service.yaml"
+ - "service-monitor.yaml"
+ afterActions:
+ - selector: # delete zookeeper nodes to reset zookeeper
+ pod:
+ matchLabels:
+ app: "zookeeper-client"
+ container: "zookeeper-client"
+ exec:
+ command: ["bash", "-c", "bin/zkCli.sh -server $ZOOKEEPER_SERVER deleteall /app-theodolite-uc1-application-1"]
+ timeoutSeconds: 60
+ loadGenerator:
+ resources:
+ - configMap:
+ name: "benchmark-resources-uc1-load-generator"
+ files:
+ - "uc1-load-generator-deployment.yaml"
+ - "uc1-load-generator-service.yaml"
+ resourceTypes:
+ - typeName: "Instances"
+ patchers:
+ - type: "ReplicaPatcher"
+ resource: "uc1-beam-samza-deployment.yaml"
+ loadTypes:
+ - typeName: "NumSensors"
+ patchers:
+ - type: "EnvVarPatcher"
+ resource: "uc1-load-generator-deployment.yaml"
+ properties:
+ container: "workload-generator"
+ variableName: "NUM_SENSORS"
+ - type: NumSensorsLoadGeneratorReplicaPatcher
+ resource: "uc1-load-generator-deployment.yaml"
+ properties:
+ loadGenMaxRecords: "150000"
+ kafkaConfig:
+ bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092"
+ topics:
+ - name: "input"
+ numPartitions: 40
+ replicationFactor: 1
+ - name: ".*samza.*"
+ removeOnly: true
+ - name: ".*uc1-application.*"
+ removeOnly: true
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc2-beam-flink/resources/flink-configuration-configmap.yaml b/theodolite-benchmarks/definitions/uc2-beam-flink/resources/flink-configuration-configmap.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..36178e2bebdac96b8648bd6c299009aa49d3fff6
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc2-beam-flink/resources/flink-configuration-configmap.yaml
@@ -0,0 +1,66 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: flink-config
+ labels:
+ app: flink
+data:
+ flink-conf.yaml: |+
+ jobmanager.rpc.address: flink-jobmanager
+ taskmanager.numberOfTaskSlots: 1 #TODO
+ #blob.server.port: 6124
+ #jobmanager.rpc.port: 6123
+ #taskmanager.rpc.port: 6122
+ #queryable-state.proxy.ports: 6125
+ #jobmanager.memory.process.size: 4Gb
+ #taskmanager.memory.process.size: 4Gb
+ #parallelism.default: 1 #TODO
+ metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
+ metrics.reporter.prom.interval: 10 SECONDS
+ taskmanager.network.detailed-metrics: true
+ # -> gives metrics about inbound/outbound network queue lengths
+ log4j-console.properties: |+
+ # This affects logging for both user code and Flink
+ rootLogger.level = INFO
+ rootLogger.appenderRef.console.ref = ConsoleAppender
+ rootLogger.appenderRef.rolling.ref = RollingFileAppender
+
+ # Uncomment this if you want to _only_ change Flink's logging
+ #logger.flink.name = org.apache.flink
+ #logger.flink.level = INFO
+
+ # The following lines keep the log level of common libraries/connectors on
+ # log level INFO. The root logger does not override this. You have to manually
+ # change the log levels here.
+ logger.akka.name = akka
+ logger.akka.level = INFO
+ logger.kafka.name= org.apache.kafka
+ logger.kafka.level = INFO
+ logger.hadoop.name = org.apache.hadoop
+ logger.hadoop.level = INFO
+ logger.zookeeper.name = org.apache.zookeeper
+ logger.zookeeper.level = INFO
+
+ # Log all infos to the console
+ appender.console.name = ConsoleAppender
+ appender.console.type = CONSOLE
+ appender.console.layout.type = PatternLayout
+ appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+
+ # Log all infos in the given rolling file
+ appender.rolling.name = RollingFileAppender
+ appender.rolling.type = RollingFile
+ appender.rolling.append = false
+ appender.rolling.fileName = ${sys:log.file}
+ appender.rolling.filePattern = ${sys:log.file}.%i
+ appender.rolling.layout.type = PatternLayout
+ appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+ appender.rolling.policies.type = Policies
+ appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
+ appender.rolling.policies.size.size=100MB
+ appender.rolling.strategy.type = DefaultRolloverStrategy
+ appender.rolling.strategy.max = 10
+
+ # Suppress the irrelevant (wrong) warnings from the Netty channel handler
+ logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
+ logger.netty.level = OFF
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc2-beam-flink/resources/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc2-beam-flink/resources/jobmanager-deployment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..2f6203c3357c24c39c06a233765ccf8e7bbf2712
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc2-beam-flink/resources/jobmanager-deployment.yaml
@@ -0,0 +1,69 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: flink-jobmanager
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: flink
+ component: jobmanager
+ template:
+ metadata:
+ labels:
+ app: flink
+ component: jobmanager
+ spec:
+ terminationGracePeriodSeconds: 0
+ containers:
+ - name: jobmanager
+ image: ghcr.io/cau-se/theodolite-uc2-beam-flink:latest
+ args: ["standalone-job", "--job-classname", "application.Uc2BeamFlink",
+ "--parallelism=$(PARALLELISM)",
+ "--disableMetrics=true",
+ "--fasterCopy"]
+ # optional arguments: ["--job-id", "<job id>", "--fromSavepoint", "/path/to/savepoint", "--allowNonRestoredState"]
+ env:
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-kafka-kafka-bootstrap:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
+ - name: COMMIT_INTERVAL_MS
+ value: "100"
+ - name: CHECKPOINTING
+ value: "false"
+ - name: PARALLELISM
+ value: "1"
+ - name: "FLINK_STATE_BACKEND"
+ value: "rocksdb"
+ - name: JOB_MANAGER_RPC_ADDRESS
+ value: "flink-jobmanager"
+ - name: FLINK_PROPERTIES
+ value: |+
+ blob.server.port: 6124
+ jobmanager.rpc.port: 6123
+ taskmanager.rpc.port: 6122
+ queryable-state.proxy.ports: 6125
+ jobmanager.memory.process.size: 4Gb
+ taskmanager.memory.process.size: 4Gb
+ parallelism.default: 1 #TODO
+ resources:
+ limits:
+ memory: 4Gi
+ cpu: 1000m
+ ports:
+ - containerPort: 6123
+ name: rpc
+ - containerPort: 6124
+ name: blob-server
+ - containerPort: 8081
+ name: webui
+ - containerPort: 9249
+ name: metrics
+ livenessProbe:
+ tcpSocket:
+ port: 6123
+ initialDelaySeconds: 30
+ periodSeconds: 60
+ securityContext:
+ runAsUser: 9999
diff --git a/theodolite-benchmarks/definitions/uc2-beam-flink/resources/jobmanager-rest-service.yaml b/theodolite-benchmarks/definitions/uc2-beam-flink/resources/jobmanager-rest-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..3d74aaf7f625c6922e2e1b4f20c19e50a39b68ac
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc2-beam-flink/resources/jobmanager-rest-service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flink-jobmanager-rest
+spec:
+ type: NodePort
+ ports:
+ - name: rest
+ port: 8081
+ targetPort: 8081
+ nodePort: 30081
+ selector:
+ app: flink
+ component: jobmanager
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc2-beam-flink/resources/jobmanager-service.yaml b/theodolite-benchmarks/definitions/uc2-beam-flink/resources/jobmanager-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..e2ff5d9898eb1ebf5db9a827472a47514ab1473c
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc2-beam-flink/resources/jobmanager-service.yaml
@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flink-jobmanager
+ labels:
+ app: flink
+spec:
+ type: ClusterIP
+ ports:
+ - name: rpc
+ port: 6123
+ - name: blob-server
+ port: 6124
+ - name: webui
+ port: 8081
+ - name: metrics
+ port: 9249
+ selector:
+ app: flink
+ component: jobmanager
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc2-beam-flink/resources/service-monitor.yaml b/theodolite-benchmarks/definitions/uc2-beam-flink/resources/service-monitor.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..02f78823c627e27ddfe1db5eac3f6a7f7a7f1bf8
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc2-beam-flink/resources/service-monitor.yaml
@@ -0,0 +1,14 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+ labels:
+ app: flink
+ appScope: titan-ccp
+ name: flink
+spec:
+ selector:
+ matchLabels:
+ app: flink
+ endpoints:
+ - port: metrics
+ interval: 10s
diff --git a/theodolite-benchmarks/definitions/uc2-beam-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc2-beam-flink/resources/taskmanager-deployment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..86e17e96e83f991bd0b6c7bd4576d100c9222cb1
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc2-beam-flink/resources/taskmanager-deployment.yaml
@@ -0,0 +1,64 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: flink-taskmanager
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: flink
+ component: taskmanager
+ template:
+ metadata:
+ labels:
+ app: flink
+ component: taskmanager
+ spec:
+ terminationGracePeriodSeconds: 0
+ containers:
+ - name: taskmanager
+ image: ghcr.io/cau-se/theodolite-uc2-beam-flink:latest
+ args: ["taskmanager"]
+ env:
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-kafka-kafka-bootstrap:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
+ - name: COMMIT_INTERVAL_MS
+ value: "100"
+ - name: CHECKPOINTING
+ value: "false"
+ - name: "FLINK_STATE_BACKEND"
+ value: "rocksdb"
+ - name: JOB_MANAGER_RPC_ADDRESS
+ value: "flink-jobmanager"
+ # - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS
+ # value: "1" #TODO
+ # - name: FLINK_PROPERTIES
+ # value: |+
+ # blob.server.port: 6124
+ # jobmanager.rpc.port: 6123
+ # taskmanager.rpc.port: 6122
+ # queryable-state.proxy.ports: 6125
+ # jobmanager.memory.process.size: 4Gb
+ # taskmanager.memory.process.size: 4Gb
+ # #parallelism.default: 1 #TODO
+ resources:
+ limits:
+ memory: 4Gi
+ cpu: 1000m
+ ports:
+ - containerPort: 6122
+ name: rpc
+ - containerPort: 6125
+ name: query-state
+ - containerPort: 9249
+ name: metrics
+ # livenessProbe:
+ # tcpSocket:
+ # port: 6122
+ # initialDelaySeconds: 30
+ # periodSeconds: 60
+ securityContext:
+ runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
+
diff --git a/theodolite-benchmarks/definitions/uc2-beam-flink/resources/taskmanager-service.yaml b/theodolite-benchmarks/definitions/uc2-beam-flink/resources/taskmanager-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a2e27f64af1cfd1a26da142b8a50bb41c8ba5fcb
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc2-beam-flink/resources/taskmanager-service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flink-taskmanager
+ labels:
+ app: flink
+spec:
+ type: ClusterIP
+ ports:
+ - name: metrics
+ port: 9249
+ selector:
+ app: flink
+ component: taskmanager
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc2-beam-flink/uc2-beam-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc2-beam-flink/uc2-beam-flink-benchmark-operator.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..ffe613b401301e00cb1d368ec44609210d299df1
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc2-beam-flink/uc2-beam-flink-benchmark-operator.yaml
@@ -0,0 +1,62 @@
+apiVersion: theodolite.com/v1
+kind: benchmark
+metadata:
+ name: uc2-beam-flink
+spec:
+ sut:
+ resources:
+ - configMap:
+ name: "benchmark-resources-uc2-beam-flink"
+ files:
+ - "flink-configuration-configmap.yaml"
+ - "taskmanager-deployment.yaml"
+ - "taskmanager-service.yaml"
+ - "service-monitor.yaml"
+ - "jobmanager-service.yaml"
+ - "jobmanager-deployment.yaml"
+ #- "jobmanager-rest-service.yaml"
+ loadGenerator:
+ resources:
+ - configMap:
+ name: "benchmark-resources-uc2-load-generator"
+ files:
+ - "uc2-load-generator-deployment.yaml"
+ - "uc2-load-generator-service.yaml"
+ resourceTypes:
+ - typeName: "Instances"
+ patchers:
+ - type: "ReplicaPatcher"
+ resource: "taskmanager-deployment.yaml"
+ - type: "EnvVarPatcher"
+ resource: "jobmanager-deployment.yaml"
+ properties:
+ container: "jobmanager"
+ variableName: "PARALLELISM"
+ - type: "EnvVarPatcher" # required?
+ resource: "taskmanager-deployment.yaml"
+ properties:
+ container: "taskmanager"
+ variableName: "PARALLELISM"
+ loadTypes:
+ - typeName: "NumSensors"
+ patchers:
+ - type: "EnvVarPatcher"
+ resource: "uc2-load-generator-deployment.yaml"
+ properties:
+ container: "workload-generator"
+ variableName: "NUM_SENSORS"
+ - type: NumSensorsLoadGeneratorReplicaPatcher
+ resource: "uc2-load-generator-deployment.yaml"
+ properties:
+ loadGenMaxRecords: "150000"
+ kafkaConfig:
+ bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092"
+ topics:
+ - name: "input"
+ numPartitions: 40
+ replicationFactor: 1
+ - name: "output"
+ numPartitions: 40
+ replicationFactor: 1
+ - name: "theodolite-.*"
+ removeOnly: True
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc2-beam-samza/resources/beam-samza-service.yaml b/theodolite-benchmarks/definitions/uc2-beam-samza/resources/beam-samza-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..6317caf9fe624e42449b8f630d040a068709cda3
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc2-beam-samza/resources/beam-samza-service.yaml
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: titan-ccp-aggregation
+ labels:
+ app: titan-ccp-aggregation
+spec:
+ #type: NodePort
+ selector:
+ app: titan-ccp-aggregation
+ ports:
+ - name: http
+ port: 80
+ targetPort: 80
+ protocol: TCP
+ - name: metrics
+ port: 5556
diff --git a/theodolite-benchmarks/definitions/uc2-beam-samza/resources/service-monitor.yaml b/theodolite-benchmarks/definitions/uc2-beam-samza/resources/service-monitor.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..4e7e758cacb5086305efa26292ddef2afc958096
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc2-beam-samza/resources/service-monitor.yaml
@@ -0,0 +1,14 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+ labels:
+ app: titan-ccp-aggregation
+ appScope: titan-ccp
+ name: titan-ccp-aggregation
+spec:
+ selector:
+ matchLabels:
+ app: titan-ccp-aggregation
+ endpoints:
+ - port: metrics
+ interval: 10s
diff --git a/theodolite-benchmarks/definitions/uc2-beam-samza/resources/uc2-beam-samza-deployment.yaml b/theodolite-benchmarks/definitions/uc2-beam-samza/resources/uc2-beam-samza-deployment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..bcb7d82ed463a194fb0851287487a6d52cfd0198
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc2-beam-samza/resources/uc2-beam-samza-deployment.yaml
@@ -0,0 +1,37 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: titan-ccp-aggregation
+spec:
+ selector:
+ matchLabels:
+ app: titan-ccp-aggregation
+ replicas: 1
+ template:
+ metadata:
+ labels:
+ app: titan-ccp-aggregation
+ spec:
+ terminationGracePeriodSeconds: 0
+ containers:
+ - name: uc2-beam-samza
+ image: ghcr.io/cau-se/theodolite-uc2-beam-samza:latest
+ env:
+ - name: SAMZA_JOB_COORDINATOR_ZK_CONNECT
+ value: "theodolite-kafka-zoo-entrance:2181"
+ - name: SAMZA_SYSTEMS_KAFKA_PRODUCER_BOOTSTRAP_SERVERS
+ value: "theodolite-kafka-kafka-bootstrap:9092"
+ - name: SAMZA_SYSTEMS_KAFKA_CONSUMER_BOOTSTRAP_SERVERS
+ value: "theodolite-kafka-kafka-bootstrap:9092"
+ - name: MAX_SOURCE_PARALLELISM
+ value: "1024"
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-kafka-kafka-bootstrap:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
+ - name: COMMIT_INTERVAL_MS # Set as default for the applications
+ value: "100"
+ resources:
+ limits:
+ memory: 4Gi
+ cpu: 1000m
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc2-beam-samza/uc2-beam-samza-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc2-beam-samza/uc2-beam-samza-benchmark-operator.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..46b61aaeaead664d6ed723278384c1bdf8b99ed1
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc2-beam-samza/uc2-beam-samza-benchmark-operator.yaml
@@ -0,0 +1,59 @@
+apiVersion: theodolite.com/v1
+kind: benchmark
+metadata:
+ name: uc2-beam-samza
+spec:
+ sut:
+ resources:
+ - configMap:
+ name: "benchmark-resources-uc2-beam-samza"
+ files:
+ - "uc2-beam-samza-deployment.yaml"
+ - "beam-samza-service.yaml"
+ - "service-monitor.yaml"
+ afterActions:
+ - selector: # delete zookeeper nodes to reset zookeeper
+ pod:
+ matchLabels:
+ app: "zookeeper-client"
+ container: "zookeeper-client"
+ exec:
+ command: ["bash", "-c", "bin/zkCli.sh -server $ZOOKEEPER_SERVER deleteall /app-theodolite-uc2-application-1"]
+ timeoutSeconds: 60
+ loadGenerator:
+ resources:
+ - configMap:
+ name: "benchmark-resources-uc2-load-generator"
+ files:
+ - "uc2-load-generator-deployment.yaml"
+ - "uc2-load-generator-service.yaml"
+ resourceTypes:
+ - typeName: "Instances"
+ patchers:
+ - type: "ReplicaPatcher"
+ resource: "uc2-beam-samza-deployment.yaml"
+ loadTypes:
+ - typeName: "NumSensors"
+ patchers:
+ - type: "EnvVarPatcher"
+ resource: "uc2-load-generator-deployment.yaml"
+ properties:
+ container: "workload-generator"
+ variableName: "NUM_SENSORS"
+ - type: NumSensorsLoadGeneratorReplicaPatcher
+ resource: "uc2-load-generator-deployment.yaml"
+ properties:
+ loadGenMaxRecords: "150000"
+ kafkaConfig:
+ bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092"
+ topics:
+ - name: "input"
+ numPartitions: 40
+ replicationFactor: 1
+ - name: "output"
+ numPartitions: 40
+ replicationFactor: 1
+ - name: ".*samza.*"
+ removeOnly: true
+ - name: ".*uc2-application.*"
+ removeOnly: true
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc3-beam-flink/resources/flink-configuration-configmap.yaml b/theodolite-benchmarks/definitions/uc3-beam-flink/resources/flink-configuration-configmap.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..36178e2bebdac96b8648bd6c299009aa49d3fff6
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc3-beam-flink/resources/flink-configuration-configmap.yaml
@@ -0,0 +1,66 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: flink-config
+ labels:
+ app: flink
+data:
+ flink-conf.yaml: |+
+ jobmanager.rpc.address: flink-jobmanager
+ taskmanager.numberOfTaskSlots: 1 #TODO
+ #blob.server.port: 6124
+ #jobmanager.rpc.port: 6123
+ #taskmanager.rpc.port: 6122
+ #queryable-state.proxy.ports: 6125
+ #jobmanager.memory.process.size: 4Gb
+ #taskmanager.memory.process.size: 4Gb
+ #parallelism.default: 1 #TODO
+ metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
+ metrics.reporter.prom.interval: 10 SECONDS
+ taskmanager.network.detailed-metrics: true
+ # -> gives metrics about inbound/outbound network queue lengths
+ log4j-console.properties: |+
+ # This affects logging for both user code and Flink
+ rootLogger.level = INFO
+ rootLogger.appenderRef.console.ref = ConsoleAppender
+ rootLogger.appenderRef.rolling.ref = RollingFileAppender
+
+ # Uncomment this if you want to _only_ change Flink's logging
+ #logger.flink.name = org.apache.flink
+ #logger.flink.level = INFO
+
+ # The following lines keep the log level of common libraries/connectors on
+ # log level INFO. The root logger does not override this. You have to manually
+ # change the log levels here.
+ logger.akka.name = akka
+ logger.akka.level = INFO
+ logger.kafka.name= org.apache.kafka
+ logger.kafka.level = INFO
+ logger.hadoop.name = org.apache.hadoop
+ logger.hadoop.level = INFO
+ logger.zookeeper.name = org.apache.zookeeper
+ logger.zookeeper.level = INFO
+
+ # Log all infos to the console
+ appender.console.name = ConsoleAppender
+ appender.console.type = CONSOLE
+ appender.console.layout.type = PatternLayout
+ appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+
+ # Log all infos in the given rolling file
+ appender.rolling.name = RollingFileAppender
+ appender.rolling.type = RollingFile
+ appender.rolling.append = false
+ appender.rolling.fileName = ${sys:log.file}
+ appender.rolling.filePattern = ${sys:log.file}.%i
+ appender.rolling.layout.type = PatternLayout
+ appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+ appender.rolling.policies.type = Policies
+ appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
+ appender.rolling.policies.size.size=100MB
+ appender.rolling.strategy.type = DefaultRolloverStrategy
+ appender.rolling.strategy.max = 10
+
+ # Suppress the irrelevant (wrong) warnings from the Netty channel handler
+ logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
+ logger.netty.level = OFF
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc3-beam-flink/resources/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc3-beam-flink/resources/jobmanager-deployment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..24572528bb7c5fb741db740919ee400cdcd2a8ad
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc3-beam-flink/resources/jobmanager-deployment.yaml
@@ -0,0 +1,69 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: flink-jobmanager
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: flink
+ component: jobmanager
+ template:
+ metadata:
+ labels:
+ app: flink
+ component: jobmanager
+ spec:
+ terminationGracePeriodSeconds: 0
+ containers:
+ - name: jobmanager
+ image: ghcr.io/cau-se/theodolite-uc3-beam-flink:latest
+ args: ["standalone-job", "--job-classname", "application.Uc3BeamFlink",
+ "--parallelism=$(PARALLELISM)",
+ "--disableMetrics=true",
+ "--fasterCopy"]
+ # optional arguments: ["--job-id", "<job id>", "--fromSavepoint", "/path/to/savepoint", "--allowNonRestoredState"]
+ env:
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-kafka-kafka-bootstrap:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
+ - name: COMMIT_INTERVAL_MS
+ value: "100"
+ - name: CHECKPOINTING
+ value: "false"
+ - name: PARALLELISM
+ value: "1"
+ - name: "FLINK_STATE_BACKEND"
+ value: "rocksdb"
+ - name: JOB_MANAGER_RPC_ADDRESS
+ value: "flink-jobmanager"
+ - name: FLINK_PROPERTIES
+ value: |+
+ blob.server.port: 6124
+ jobmanager.rpc.port: 6123
+ taskmanager.rpc.port: 6122
+ queryable-state.proxy.ports: 6125
+ jobmanager.memory.process.size: 4Gb
+ taskmanager.memory.process.size: 4Gb
+ parallelism.default: 1 #TODO
+ resources:
+ limits:
+ memory: 4Gi
+ cpu: 1000m
+ ports:
+ - containerPort: 6123
+ name: rpc
+ - containerPort: 6124
+ name: blob-server
+ - containerPort: 8081
+ name: webui
+ - containerPort: 9249
+ name: metrics
+ livenessProbe:
+ tcpSocket:
+ port: 6123
+ initialDelaySeconds: 30
+ periodSeconds: 60
+ securityContext:
+ runAsUser: 9999
diff --git a/theodolite-benchmarks/definitions/uc3-beam-flink/resources/jobmanager-rest-service.yaml b/theodolite-benchmarks/definitions/uc3-beam-flink/resources/jobmanager-rest-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..3d74aaf7f625c6922e2e1b4f20c19e50a39b68ac
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc3-beam-flink/resources/jobmanager-rest-service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flink-jobmanager-rest
+spec:
+ type: NodePort
+ ports:
+ - name: rest
+ port: 8081
+ targetPort: 8081
+ nodePort: 30081
+ selector:
+ app: flink
+ component: jobmanager
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc3-beam-flink/resources/jobmanager-service.yaml b/theodolite-benchmarks/definitions/uc3-beam-flink/resources/jobmanager-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..e2ff5d9898eb1ebf5db9a827472a47514ab1473c
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc3-beam-flink/resources/jobmanager-service.yaml
@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flink-jobmanager
+ labels:
+ app: flink
+spec:
+ type: ClusterIP
+ ports:
+ - name: rpc
+ port: 6123
+ - name: blob-server
+ port: 6124
+ - name: webui
+ port: 8081
+ - name: metrics
+ port: 9249
+ selector:
+ app: flink
+ component: jobmanager
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc3-beam-flink/resources/service-monitor.yaml b/theodolite-benchmarks/definitions/uc3-beam-flink/resources/service-monitor.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..02f78823c627e27ddfe1db5eac3f6a7f7a7f1bf8
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc3-beam-flink/resources/service-monitor.yaml
@@ -0,0 +1,14 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+ labels:
+ app: flink
+ appScope: titan-ccp
+ name: flink
+spec:
+ selector:
+ matchLabels:
+ app: flink
+ endpoints:
+ - port: metrics
+ interval: 10s
diff --git a/theodolite-benchmarks/definitions/uc3-beam-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc3-beam-flink/resources/taskmanager-deployment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a0a840fad96fa21212bb25c78a597ea5cb41558c
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc3-beam-flink/resources/taskmanager-deployment.yaml
@@ -0,0 +1,64 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: flink-taskmanager
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: flink
+ component: taskmanager
+ template:
+ metadata:
+ labels:
+ app: flink
+ component: taskmanager
+ spec:
+ terminationGracePeriodSeconds: 0
+ containers:
+ - name: taskmanager
+ image: ghcr.io/cau-se/theodolite-uc3-beam-flink:latest
+ args: ["taskmanager"]
+ env:
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-kafka-kafka-bootstrap:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
+ - name: COMMIT_INTERVAL_MS
+ value: "100"
+ - name: CHECKPOINTING
+ value: "false"
+ - name: "FLINK_STATE_BACKEND"
+ value: "rocksdb"
+ - name: JOB_MANAGER_RPC_ADDRESS
+ value: "flink-jobmanager"
+ # - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS
+ # value: "1" #TODO
+ # - name: FLINK_PROPERTIES
+ # value: |+
+ # blob.server.port: 6124
+ # jobmanager.rpc.port: 6123
+ # taskmanager.rpc.port: 6122
+ # queryable-state.proxy.ports: 6125
+ # jobmanager.memory.process.size: 4Gb
+ # taskmanager.memory.process.size: 4Gb
+ # #parallelism.default: 1 #TODO
+ resources:
+ limits:
+ memory: 4Gi
+ cpu: 1000m
+ ports:
+ - containerPort: 6122
+ name: rpc
+ - containerPort: 6125
+ name: query-state
+ - containerPort: 9249
+ name: metrics
+ # livenessProbe:
+ # tcpSocket:
+ # port: 6122
+ # initialDelaySeconds: 30
+ # periodSeconds: 60
+ securityContext:
+ runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
+
diff --git a/theodolite-benchmarks/definitions/uc3-beam-flink/resources/taskmanager-service.yaml b/theodolite-benchmarks/definitions/uc3-beam-flink/resources/taskmanager-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a2e27f64af1cfd1a26da142b8a50bb41c8ba5fcb
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc3-beam-flink/resources/taskmanager-service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flink-taskmanager
+ labels:
+ app: flink
+spec:
+ type: ClusterIP
+ ports:
+ - name: metrics
+ port: 9249
+ selector:
+ app: flink
+ component: taskmanager
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc3-beam-flink/uc3-beam-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc3-beam-flink/uc3-beam-flink-benchmark-operator.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..4449252c0de5aa81c3069b65c19325256bab57e9
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc3-beam-flink/uc3-beam-flink-benchmark-operator.yaml
@@ -0,0 +1,62 @@
+apiVersion: theodolite.com/v1
+kind: benchmark
+metadata:
+ name: uc3-beam-flink
+spec:
+ sut:
+ resources:
+ - configMap:
+ name: "benchmark-resources-uc3-beam-flink"
+ files:
+ - "flink-configuration-configmap.yaml"
+ - "taskmanager-deployment.yaml"
+ - "taskmanager-service.yaml"
+ - "service-monitor.yaml"
+ - "jobmanager-service.yaml"
+ - "jobmanager-deployment.yaml"
+ #- "jobmanager-rest-service.yaml"
+ loadGenerator:
+ resources:
+ - configMap:
+ name: "benchmark-resources-uc3-load-generator"
+ files:
+ - "uc3-load-generator-deployment.yaml"
+ - "uc3-load-generator-service.yaml"
+ resourceTypes:
+ - typeName: "Instances"
+ patchers:
+ - type: "ReplicaPatcher"
+ resource: "taskmanager-deployment.yaml"
+ - type: "EnvVarPatcher"
+ resource: "jobmanager-deployment.yaml"
+ properties:
+ container: "jobmanager"
+ variableName: "PARALLELISM"
+ - type: "EnvVarPatcher" # required?
+ resource: "taskmanager-deployment.yaml"
+ properties:
+ container: "taskmanager"
+ variableName: "PARALLELISM"
+ loadTypes:
+ - typeName: "NumSensors"
+ patchers:
+ - type: "EnvVarPatcher"
+ resource: "uc3-load-generator-deployment.yaml"
+ properties:
+ container: "workload-generator"
+ variableName: "NUM_SENSORS"
+ - type: NumSensorsLoadGeneratorReplicaPatcher
+ resource: "uc3-load-generator-deployment.yaml"
+ properties:
+ loadGenMaxRecords: "150000"
+ kafkaConfig:
+ bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092"
+ topics:
+ - name: "input"
+ numPartitions: 40
+ replicationFactor: 1
+ - name: "output"
+ numPartitions: 40
+ replicationFactor: 1
+ - name: "theodolite-.*"
+ removeOnly: True
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc3-beam-samza/resources/beam-samza-service.yaml b/theodolite-benchmarks/definitions/uc3-beam-samza/resources/beam-samza-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..6317caf9fe624e42449b8f630d040a068709cda3
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc3-beam-samza/resources/beam-samza-service.yaml
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: titan-ccp-aggregation
+ labels:
+ app: titan-ccp-aggregation
+spec:
+ #type: NodePort
+ selector:
+ app: titan-ccp-aggregation
+ ports:
+ - name: http
+ port: 80
+ targetPort: 80
+ protocol: TCP
+ - name: metrics
+ port: 5556
diff --git a/theodolite-benchmarks/definitions/uc3-beam-samza/resources/service-monitor.yaml b/theodolite-benchmarks/definitions/uc3-beam-samza/resources/service-monitor.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..4e7e758cacb5086305efa26292ddef2afc958096
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc3-beam-samza/resources/service-monitor.yaml
@@ -0,0 +1,14 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+ labels:
+ app: titan-ccp-aggregation
+ appScope: titan-ccp
+ name: titan-ccp-aggregation
+spec:
+ selector:
+ matchLabels:
+ app: titan-ccp-aggregation
+ endpoints:
+ - port: metrics
+ interval: 10s
diff --git a/theodolite-benchmarks/definitions/uc3-beam-samza/resources/uc3-beam-samza-deployment.yaml b/theodolite-benchmarks/definitions/uc3-beam-samza/resources/uc3-beam-samza-deployment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..aff9da6a0f7ee490a94a6f9954c2b3cc8a2b5ed5
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc3-beam-samza/resources/uc3-beam-samza-deployment.yaml
@@ -0,0 +1,37 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: titan-ccp-aggregation
+spec:
+ selector:
+ matchLabels:
+ app: titan-ccp-aggregation
+ replicas: 1
+ template:
+ metadata:
+ labels:
+ app: titan-ccp-aggregation
+ spec:
+ terminationGracePeriodSeconds: 0
+ containers:
+ - name: uc3-beam-samza
+ image: ghcr.io/cau-se/theodolite-uc3-beam-samza:latest
+ env:
+ - name: SAMZA_JOB_COORDINATOR_ZK_CONNECT
+ value: "theodolite-kafka-zoo-entrance:2181"
+ - name: SAMZA_SYSTEMS_KAFKA_PRODUCER_BOOTSTRAP_SERVERS
+ value: "theodolite-kafka-kafka-bootstrap:9092"
+ - name: SAMZA_SYSTEMS_KAFKA_CONSUMER_BOOTSTRAP_SERVERS
+ value: "theodolite-kafka-kafka-bootstrap:9092"
+ - name: MAX_SOURCE_PARALLELISM
+ value: "1024"
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-kafka-kafka-bootstrap:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
+ - name: COMMIT_INTERVAL_MS # Set as default for the applications
+ value: "100"
+ resources:
+ limits:
+ memory: 4Gi
+ cpu: 1000m
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc3-beam-samza/uc3-beam-samza-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc3-beam-samza/uc3-beam-samza-benchmark-operator.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..36d812d4ca1fd226c7edcd96472b5aefff26bfda
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc3-beam-samza/uc3-beam-samza-benchmark-operator.yaml
@@ -0,0 +1,59 @@
+apiVersion: theodolite.com/v1
+kind: benchmark
+metadata:
+ name: uc3-beam-samza
+spec:
+ sut:
+ resources:
+ - configMap:
+ name: "benchmark-resources-uc3-beam-samza"
+ files:
+ - "uc3-beam-samza-deployment.yaml"
+ - "beam-samza-service.yaml"
+ - "service-monitor.yaml"
+ afterActions:
+ - selector: # delete zookeeper nodes to reset zookeeper
+ pod:
+ matchLabels:
+ app: "zookeeper-client"
+ container: "zookeeper-client"
+ exec:
+ command: ["bash", "-c", "bin/zkCli.sh -server $ZOOKEEPER_SERVER deleteall /app-theodolite-uc3-application-1"]
+ timeoutSeconds: 60
+ loadGenerator:
+ resources:
+ - configMap:
+ name: "benchmark-resources-uc3-load-generator"
+ files:
+ - "uc3-load-generator-deployment.yaml"
+ - "uc3-load-generator-service.yaml"
+ resourceTypes:
+ - typeName: "Instances"
+ patchers:
+ - type: "ReplicaPatcher"
+ resource: "uc3-beam-samza-deployment.yaml"
+ loadTypes:
+ - typeName: "NumSensors"
+ patchers:
+ - type: "EnvVarPatcher"
+ resource: "uc3-load-generator-deployment.yaml"
+ properties:
+ container: "workload-generator"
+ variableName: "NUM_SENSORS"
+ - type: NumSensorsLoadGeneratorReplicaPatcher
+ resource: "uc3-load-generator-deployment.yaml"
+ properties:
+ loadGenMaxRecords: "150000"
+ kafkaConfig:
+ bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092"
+ topics:
+ - name: "input"
+ numPartitions: 40
+ replicationFactor: 1
+ - name: "output"
+ numPartitions: 40
+ replicationFactor: 1
+ - name: ".*samza.*"
+ removeOnly: true
+ - name: ".*uc3-application.*"
+ removeOnly: true
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc4-beam-flink/resources/flink-configuration-configmap.yaml b/theodolite-benchmarks/definitions/uc4-beam-flink/resources/flink-configuration-configmap.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..36178e2bebdac96b8648bd6c299009aa49d3fff6
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc4-beam-flink/resources/flink-configuration-configmap.yaml
@@ -0,0 +1,66 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: flink-config
+ labels:
+ app: flink
+data:
+ flink-conf.yaml: |+
+ jobmanager.rpc.address: flink-jobmanager
+ taskmanager.numberOfTaskSlots: 1 #TODO
+ #blob.server.port: 6124
+ #jobmanager.rpc.port: 6123
+ #taskmanager.rpc.port: 6122
+ #queryable-state.proxy.ports: 6125
+ #jobmanager.memory.process.size: 4Gb
+ #taskmanager.memory.process.size: 4Gb
+ #parallelism.default: 1 #TODO
+ metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
+ metrics.reporter.prom.interval: 10 SECONDS
+ taskmanager.network.detailed-metrics: true
+ # -> gives metrics about inbound/outbound network queue lengths
+ log4j-console.properties: |+
+ # This affects logging for both user code and Flink
+ rootLogger.level = INFO
+ rootLogger.appenderRef.console.ref = ConsoleAppender
+ rootLogger.appenderRef.rolling.ref = RollingFileAppender
+
+ # Uncomment this if you want to _only_ change Flink's logging
+ #logger.flink.name = org.apache.flink
+ #logger.flink.level = INFO
+
+ # The following lines keep the log level of common libraries/connectors on
+ # log level INFO. The root logger does not override this. You have to manually
+ # change the log levels here.
+ logger.akka.name = akka
+ logger.akka.level = INFO
+ logger.kafka.name= org.apache.kafka
+ logger.kafka.level = INFO
+ logger.hadoop.name = org.apache.hadoop
+ logger.hadoop.level = INFO
+ logger.zookeeper.name = org.apache.zookeeper
+ logger.zookeeper.level = INFO
+
+ # Log all infos to the console
+ appender.console.name = ConsoleAppender
+ appender.console.type = CONSOLE
+ appender.console.layout.type = PatternLayout
+ appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+
+ # Log all infos in the given rolling file
+ appender.rolling.name = RollingFileAppender
+ appender.rolling.type = RollingFile
+ appender.rolling.append = false
+ appender.rolling.fileName = ${sys:log.file}
+ appender.rolling.filePattern = ${sys:log.file}.%i
+ appender.rolling.layout.type = PatternLayout
+ appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+ appender.rolling.policies.type = Policies
+ appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
+ appender.rolling.policies.size.size=100MB
+ appender.rolling.strategy.type = DefaultRolloverStrategy
+ appender.rolling.strategy.max = 10
+
+ # Suppress the irrelevant (wrong) warnings from the Netty channel handler
+ logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
+ logger.netty.level = OFF
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc4-beam-flink/resources/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc4-beam-flink/resources/jobmanager-deployment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..90e81074c875fd27474224aaf43f4e1d5096c0d8
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc4-beam-flink/resources/jobmanager-deployment.yaml
@@ -0,0 +1,69 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: flink-jobmanager
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: flink
+ component: jobmanager
+ template:
+ metadata:
+ labels:
+ app: flink
+ component: jobmanager
+ spec:
+ terminationGracePeriodSeconds: 0
+ containers:
+ - name: jobmanager
+ image: ghcr.io/cau-se/theodolite-uc4-beam-flink:latest
+ args: ["standalone-job", "--job-classname", "application.Uc4BeamFlink",
+ "--parallelism=$(PARALLELISM)",
+ "--disableMetrics=true",
+ "--fasterCopy"]
+ # optional arguments: ["--job-id", "<job id>", "--fromSavepoint", "/path/to/savepoint", "--allowNonRestoredState"]
+ env:
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-kafka-kafka-bootstrap:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
+ - name: COMMIT_INTERVAL_MS
+ value: "100"
+ - name: CHECKPOINTING
+ value: "false"
+ - name: PARALLELISM
+ value: "1"
+ - name: "FLINK_STATE_BACKEND"
+ value: "rocksdb"
+ - name: JOB_MANAGER_RPC_ADDRESS
+ value: "flink-jobmanager"
+ - name: FLINK_PROPERTIES
+ value: |+
+ blob.server.port: 6124
+ jobmanager.rpc.port: 6123
+ taskmanager.rpc.port: 6122
+ queryable-state.proxy.ports: 6125
+ jobmanager.memory.process.size: 4Gb
+ taskmanager.memory.process.size: 4Gb
+ parallelism.default: 1 #TODO
+ resources:
+ limits:
+ memory: 4Gi
+ cpu: 1000m
+ ports:
+ - containerPort: 6123
+ name: rpc
+ - containerPort: 6124
+ name: blob-server
+ - containerPort: 8081
+ name: webui
+ - containerPort: 9249
+ name: metrics
+ livenessProbe:
+ tcpSocket:
+ port: 6123
+ initialDelaySeconds: 30
+ periodSeconds: 60
+ securityContext:
+ runAsUser: 9999
diff --git a/theodolite-benchmarks/definitions/uc4-beam-flink/resources/jobmanager-rest-service.yaml b/theodolite-benchmarks/definitions/uc4-beam-flink/resources/jobmanager-rest-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..3d74aaf7f625c6922e2e1b4f20c19e50a39b68ac
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc4-beam-flink/resources/jobmanager-rest-service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flink-jobmanager-rest
+spec:
+ type: NodePort
+ ports:
+ - name: rest
+ port: 8081
+ targetPort: 8081
+ nodePort: 30081
+ selector:
+ app: flink
+ component: jobmanager
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc4-beam-flink/resources/jobmanager-service.yaml b/theodolite-benchmarks/definitions/uc4-beam-flink/resources/jobmanager-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..e2ff5d9898eb1ebf5db9a827472a47514ab1473c
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc4-beam-flink/resources/jobmanager-service.yaml
@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flink-jobmanager
+ labels:
+ app: flink
+spec:
+ type: ClusterIP
+ ports:
+ - name: rpc
+ port: 6123
+ - name: blob-server
+ port: 6124
+ - name: webui
+ port: 8081
+ - name: metrics
+ port: 9249
+ selector:
+ app: flink
+ component: jobmanager
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc4-beam-flink/resources/service-monitor.yaml b/theodolite-benchmarks/definitions/uc4-beam-flink/resources/service-monitor.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..02f78823c627e27ddfe1db5eac3f6a7f7a7f1bf8
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc4-beam-flink/resources/service-monitor.yaml
@@ -0,0 +1,14 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+ labels:
+ app: flink
+ appScope: titan-ccp
+ name: flink
+spec:
+ selector:
+ matchLabels:
+ app: flink
+ endpoints:
+ - port: metrics
+ interval: 10s
diff --git a/theodolite-benchmarks/definitions/uc4-beam-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc4-beam-flink/resources/taskmanager-deployment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5eabd7d52875b8ea556a4b47fde18cf859a279b7
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc4-beam-flink/resources/taskmanager-deployment.yaml
@@ -0,0 +1,64 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: flink-taskmanager
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: flink
+ component: taskmanager
+ template:
+ metadata:
+ labels:
+ app: flink
+ component: taskmanager
+ spec:
+ terminationGracePeriodSeconds: 0
+ containers:
+ - name: taskmanager
+ image: ghcr.io/cau-se/theodolite-uc4-beam-flink:latest
+ args: ["taskmanager"]
+ env:
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-kafka-kafka-bootstrap:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
+ - name: COMMIT_INTERVAL_MS
+ value: "100"
+ - name: CHECKPOINTING
+ value: "false"
+ - name: "FLINK_STATE_BACKEND"
+ value: "rocksdb"
+ - name: JOB_MANAGER_RPC_ADDRESS
+ value: "flink-jobmanager"
+ # - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS
+ # value: "1" #TODO
+ # - name: FLINK_PROPERTIES
+ # value: |+
+ # blob.server.port: 6124
+ # jobmanager.rpc.port: 6123
+ # taskmanager.rpc.port: 6122
+ # queryable-state.proxy.ports: 6125
+ # jobmanager.memory.process.size: 4Gb
+ # taskmanager.memory.process.size: 4Gb
+ # #parallelism.default: 1 #TODO
+ resources:
+ limits:
+ memory: 4Gi
+ cpu: 1000m
+ ports:
+ - containerPort: 6122
+ name: rpc
+ - containerPort: 6125
+ name: query-state
+ - containerPort: 9249
+ name: metrics
+ # livenessProbe:
+ # tcpSocket:
+ # port: 6122
+ # initialDelaySeconds: 30
+ # periodSeconds: 60
+ securityContext:
+ runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
+
diff --git a/theodolite-benchmarks/definitions/uc4-beam-flink/resources/taskmanager-service.yaml b/theodolite-benchmarks/definitions/uc4-beam-flink/resources/taskmanager-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a2e27f64af1cfd1a26da142b8a50bb41c8ba5fcb
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc4-beam-flink/resources/taskmanager-service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flink-taskmanager
+ labels:
+ app: flink
+spec:
+ type: ClusterIP
+ ports:
+ - name: metrics
+ port: 9249
+ selector:
+ app: flink
+ component: taskmanager
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc4-beam-flink/uc4-beam-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc4-beam-flink/uc4-beam-flink-benchmark-operator.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..738b1c837469a9272bf27d4df08d6e5242b1a4c2
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc4-beam-flink/uc4-beam-flink-benchmark-operator.yaml
@@ -0,0 +1,69 @@
+apiVersion: theodolite.com/v1
+kind: benchmark
+metadata:
+ name: uc4-beam-flink
+spec:
+ sut:
+ resources:
+ - configMap:
+ name: "benchmark-resources-uc4-beam-flink"
+ files:
+ - "flink-configuration-configmap.yaml"
+ - "taskmanager-deployment.yaml"
+ - "taskmanager-service.yaml"
+ - "service-monitor.yaml"
+ - "jobmanager-service.yaml"
+ - "jobmanager-deployment.yaml"
+ #- "jobmanager-rest-service.yaml"
+ loadGenerator:
+ resources:
+ - configMap:
+ name: "benchmark-resources-uc4-load-generator"
+ files:
+ - "uc4-load-generator-deployment.yaml"
+ - "uc4-load-generator-service.yaml"
+ resourceTypes:
+ - typeName: "Instances"
+ patchers:
+ - type: "ReplicaPatcher"
+ resource: "taskmanager-deployment.yaml"
+ - type: "EnvVarPatcher"
+ resource: "jobmanager-deployment.yaml"
+ properties:
+ container: "jobmanager"
+ variableName: "PARALLELISM"
+ - type: "EnvVarPatcher" # required?
+ resource: "taskmanager-deployment.yaml"
+ properties:
+ container: "taskmanager"
+ variableName: "PARALLELISM"
+ loadTypes:
+ - typeName: "NumNestedGroups"
+ patchers:
+ - type: "EnvVarPatcher"
+ resource: "uc4-load-generator-deployment.yaml"
+ properties:
+ container: "workload-generator"
+ variableName: "NUM_NESTED_GROUPS"
+ - type: NumNestedGroupsLoadGeneratorReplicaPatcher
+ resource: "uc4-load-generator-deployment.yaml"
+ properties:
+ loadGenMaxRecords: "150000"
+ numSensors: "4.0"
+ kafkaConfig:
+ bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092"
+ topics:
+ - name: "input"
+ numPartitions: 40
+ replicationFactor: 1
+ - name: "output"
+ numPartitions: 40
+ replicationFactor: 1
+ - name: "configuration"
+ numPartitions: 40
+ replicationFactor: 1
+ - name: "aggregation-feedback"
+ numPartitions: 40
+ replicationFactor: 1
+ - name: "theodolite-.*"
+ removeOnly: True
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc4-beam-samza/resources/beam-samza-service.yaml b/theodolite-benchmarks/definitions/uc4-beam-samza/resources/beam-samza-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..6317caf9fe624e42449b8f630d040a068709cda3
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc4-beam-samza/resources/beam-samza-service.yaml
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: titan-ccp-aggregation
+ labels:
+ app: titan-ccp-aggregation
+spec:
+ #type: NodePort
+ selector:
+ app: titan-ccp-aggregation
+ ports:
+ - name: http
+ port: 80
+ targetPort: 80
+ protocol: TCP
+ - name: metrics
+ port: 5556
diff --git a/theodolite-benchmarks/definitions/uc4-beam-samza/resources/service-monitor.yaml b/theodolite-benchmarks/definitions/uc4-beam-samza/resources/service-monitor.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..4e7e758cacb5086305efa26292ddef2afc958096
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc4-beam-samza/resources/service-monitor.yaml
@@ -0,0 +1,14 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+ labels:
+ app: titan-ccp-aggregation
+ appScope: titan-ccp
+ name: titan-ccp-aggregation
+spec:
+ selector:
+ matchLabels:
+ app: titan-ccp-aggregation
+ endpoints:
+ - port: metrics
+ interval: 10s
diff --git a/theodolite-benchmarks/definitions/uc4-beam-samza/resources/uc4-beam-samza-deployment.yaml b/theodolite-benchmarks/definitions/uc4-beam-samza/resources/uc4-beam-samza-deployment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..c0602d86207d2e63d3eb549b068b0afe2692df21
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc4-beam-samza/resources/uc4-beam-samza-deployment.yaml
@@ -0,0 +1,37 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: titan-ccp-aggregation
+spec:
+ selector:
+ matchLabels:
+ app: titan-ccp-aggregation
+ replicas: 1
+ template:
+ metadata:
+ labels:
+ app: titan-ccp-aggregation
+ spec:
+ terminationGracePeriodSeconds: 0
+ containers:
+ - name: uc4-beam-samza
+ image: ghcr.io/cau-se/theodolite-uc4-beam-samza:latest
+ env:
+ - name: SAMZA_JOB_COORDINATOR_ZK_CONNECT
+ value: "theodolite-kafka-zoo-entrance:2181"
+ - name: SAMZA_SYSTEMS_KAFKA_PRODUCER_BOOTSTRAP_SERVERS
+ value: "theodolite-kafka-kafka-bootstrap:9092"
+ - name: SAMZA_SYSTEMS_KAFKA_CONSUMER_BOOTSTRAP_SERVERS
+ value: "theodolite-kafka-kafka-bootstrap:9092"
+ - name: MAX_SOURCE_PARALLELISM
+ value: "1024"
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-kafka-kafka-bootstrap:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
+ - name: COMMIT_INTERVAL_MS # Set as default for the applications
+ value: "100"
+ resources:
+ limits:
+ memory: 4Gi
+ cpu: 1000m
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc4-beam-samza/uc4-beam-samza-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc4-beam-samza/uc4-beam-samza-benchmark-operator.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..12690cec23761a38c948b0cb8757550ded54acef
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc4-beam-samza/uc4-beam-samza-benchmark-operator.yaml
@@ -0,0 +1,68 @@
+apiVersion: theodolite.com/v1
+kind: benchmark
+metadata:
+ name: uc4-beam-samza
+spec:
+ sut:
+ resources:
+ - configMap:
+ name: "benchmark-resources-uc4-beam-samza"
+ files:
+ - "uc4-beam-samza-deployment.yaml"
+ - "beam-samza-service.yaml"
+ - "service-monitor.yaml"
+ afterActions:
+ - selector: # delete zookeeper nodes to reset zookeeper
+ pod:
+ matchLabels:
+ app: "zookeeper-client"
+ container: "zookeeper-client"
+ exec:
+ command: ["bash", "-c", "bin/zkCli.sh -server $ZOOKEEPER_SERVER deleteall /app-theodolite-uc4-application-1"]
+ timeoutSeconds: 60
+ loadGenerator:
+ resources:
+ - configMap:
+ name: "benchmark-resources-uc4-load-generator"
+ files:
+ - "uc4-load-generator-deployment.yaml"
+ - "uc4-load-generator-service.yaml"
+ resourceTypes:
+ - typeName: "Instances"
+ patchers:
+ - type: "ReplicaPatcher"
+ resource: "uc4-beam-samza-deployment.yaml"
+ loadTypes:
+ - typeName: "NumNestedGroups"
+ patchers:
+ - type: "EnvVarPatcher"
+ resource: "uc4-load-generator-deployment.yaml"
+ properties:
+ container: "workload-generator"
+ variableName: "NUM_NESTED_GROUPS"
+ - type: NumNestedGroupsLoadGeneratorReplicaPatcher
+ resource: "uc4-load-generator-deployment.yaml"
+ properties:
+ loadGenMaxRecords: "150000"
+ numSensors: "4.0"
+ kafkaConfig:
+ bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092"
+ topics:
+ - name: "input"
+ numPartitions: 40
+ replicationFactor: 1
+ - name: "output"
+ numPartitions: 40
+ replicationFactor: 1
+ - name: "configuration"
+ numPartitions: 40
+ replicationFactor: 1
+ - name: "aggregation-feedback"
+ numPartitions: 40
+ replicationFactor: 1
+ - name: "theodolite-.*"
+ removeOnly: True
+ - name: ".*samza.*"
+ removeOnly: true
+ - name: ".*uc4-application.*"
+ removeOnly: true
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc4-load-generator/resources/uc4-load-generator-deployment.yaml b/theodolite-benchmarks/definitions/uc4-load-generator/resources/uc4-load-generator-deployment.yaml
index 0f64fc881b5beb31a6dad4ff2a755413049b96fa..70674757ff97b4377902395868eac2e03070c31a 100644
--- a/theodolite-benchmarks/definitions/uc4-load-generator/resources/uc4-load-generator-deployment.yaml
+++ b/theodolite-benchmarks/definitions/uc4-load-generator/resources/uc4-load-generator-deployment.yaml
@@ -30,5 +30,7 @@ spec:
value: "theodolite-kafka-kafka-bootstrap:9092"
- name: SCHEMA_REGISTRY_URL
value: "http://theodolite-cp-schema-registry:8081"
+ - name: NUM_SENSORS
+ value: "4"
- name: NUM_NESTED_GROUPS
value: "5"
diff --git a/theodolite/src/main/kotlin/theodolite/benchmark/KubernetesBenchmarkDeployment.kt b/theodolite/src/main/kotlin/theodolite/benchmark/KubernetesBenchmarkDeployment.kt
index 3331444a17b4c2a1aa4411c1e27b3d1e087f8841..2b3cf0fa13d894424e6a0546993e2fd9998b8620 100644
--- a/theodolite/src/main/kotlin/theodolite/benchmark/KubernetesBenchmarkDeployment.kt
+++ b/theodolite/src/main/kotlin/theodolite/benchmark/KubernetesBenchmarkDeployment.kt
@@ -38,7 +38,7 @@ class KubernetesBenchmarkDeployment(
private val kafkaController = TopicManager(this.kafkaConfig)
private val kubernetesManager = K8sManager(client)
private val LAG_EXPORTER_POD_LABEL_NAME = "app.kubernetes.io/name"
- private val LAG_EXPORTER_POD_LABEL_VALUE = "kafka-lag-exporter"
+ private val LAG_EXPORTER_POD_LABEL_VALUE = "kafka-exporter"
/**
* Setup a [KubernetesBenchmark] using the [TopicManager] and the [K8sManager]: