diff --git a/theodolite-benchmarks/definitions/uc1-flink/flink-configuration-configmap.yaml b/theodolite-benchmarks/definitions/uc1-flink/flink-configuration-configmap.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..36178e2bebdac96b8648bd6c299009aa49d3fff6
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc1-flink/flink-configuration-configmap.yaml
@@ -0,0 +1,66 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: flink-config
+ labels:
+ app: flink
+data:
+ flink-conf.yaml: |+
+ jobmanager.rpc.address: flink-jobmanager
+ taskmanager.numberOfTaskSlots: 1 #TODO
+ #blob.server.port: 6124
+ #jobmanager.rpc.port: 6123
+ #taskmanager.rpc.port: 6122
+ #queryable-state.proxy.ports: 6125
+ #jobmanager.memory.process.size: 4Gb
+ #taskmanager.memory.process.size: 4Gb
+ #parallelism.default: 1 #TODO
+ metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
+ metrics.reporter.prom.interval: 10 SECONDS
+ taskmanager.network.detailed-metrics: true
+ # -> gives metrics about inbound/outbound network queue lengths
+ log4j-console.properties: |+
+ # This affects logging for both user code and Flink
+ rootLogger.level = INFO
+ rootLogger.appenderRef.console.ref = ConsoleAppender
+ rootLogger.appenderRef.rolling.ref = RollingFileAppender
+
+ # Uncomment this if you want to _only_ change Flink's logging
+ #logger.flink.name = org.apache.flink
+ #logger.flink.level = INFO
+
+ # The following lines keep the log level of common libraries/connectors on
+ # log level INFO. The root logger does not override this. You have to manually
+ # change the log levels here.
+ logger.akka.name = akka
+ logger.akka.level = INFO
+ logger.kafka.name= org.apache.kafka
+ logger.kafka.level = INFO
+ logger.hadoop.name = org.apache.hadoop
+ logger.hadoop.level = INFO
+ logger.zookeeper.name = org.apache.zookeeper
+ logger.zookeeper.level = INFO
+
+ # Log all infos to the console
+ appender.console.name = ConsoleAppender
+ appender.console.type = CONSOLE
+ appender.console.layout.type = PatternLayout
+ appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+
+ # Log all infos in the given rolling file
+ appender.rolling.name = RollingFileAppender
+ appender.rolling.type = RollingFile
+ appender.rolling.append = false
+ appender.rolling.fileName = ${sys:log.file}
+ appender.rolling.filePattern = ${sys:log.file}.%i
+ appender.rolling.layout.type = PatternLayout
+ appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+ appender.rolling.policies.type = Policies
+ appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
+ appender.rolling.policies.size.size=100MB
+ appender.rolling.strategy.type = DefaultRolloverStrategy
+ appender.rolling.strategy.max = 10
+
+ # Suppress the irrelevant (wrong) warnings from the Netty channel handler
+ logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
+ logger.netty.level = OFF
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc1-flink/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc1-flink/jobmanager-deployment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..512d4fe3c786e1b2c44e6ec57fccadf41a2e2eeb
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc1-flink/jobmanager-deployment.yaml
@@ -0,0 +1,93 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: flink-jobmanager
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: flink
+ component: jobmanager
+ template:
+ metadata:
+ labels:
+ app: flink
+ component: jobmanager
+ spec:
+ containers:
+ - name: jobmanager
+ image: ghcr.io/cau-se/theodolite-uc1-flink:latest
+ env:
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-cp-kafka:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
+ - name: COMMIT_INTERVAL_MS
+ value: "100"
+ - name: CHECKPOINTING
+ value: "false"
+ - name: PARALLELISM
+ value: "1"
+ - name: "FLINK_STATE_BACKEND"
+ value: "rocksdb"
+ - name: JOB_MANAGER_RPC_ADDRESS
+ value: "flink-jobmanager"
+ - name: FLINK_PROPERTIES
+ value: |+
+ blob.server.port: 6124
+ jobmanager.rpc.port: 6123
+ taskmanager.rpc.port: 6122
+ queryable-state.proxy.ports: 6125
+ jobmanager.memory.process.size: 4Gb
+ taskmanager.memory.process.size: 4Gb
+ #parallelism.default: 1 #TODO
+ resources:
+ limits:
+ memory: 4Gi
+ cpu: 1000m
+ args: ["standalone-job", "--job-classname", "theodolite.uc1.application.HistoryServiceFlinkJob"] # optional arguments: ["--job-id", "<job id>", "--fromSavepoint", "/path/to/savepoint", "--allowNonRestoredState"]
+ #command: ['sleep', '60m']
+ ports:
+ - containerPort: 6123
+ name: rpc
+ - containerPort: 6124
+ name: blob-server
+ - containerPort: 8081
+ name: webui
+ - containerPort: 9249
+ name: metrics
+ livenessProbe:
+ tcpSocket:
+ port: 6123
+ initialDelaySeconds: 30
+ periodSeconds: 60
+ volumeMounts:
+ - name: flink-config-volume-rw
+ mountPath: /opt/flink/conf
+# - name: job-artifacts-volume
+# mountPath: /opt/flink/usrlib
+ securityContext:
+ runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
+ initContainers:
+ - name: init-jobmanager
+ image: busybox:1.28
+ command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
+ volumeMounts:
+ - name: flink-config-volume
+ mountPath: /flink-config/
+ - name: flink-config-volume-rw
+ mountPath: /flink-config-rw/
+ volumes:
+ - name: flink-config-volume
+ configMap:
+ name: flink-config
+ items:
+ - key: flink-conf.yaml
+ path: flink-conf.yaml
+ - key: log4j-console.properties
+ path: log4j-console.properties
+ - name: flink-config-volume-rw
+ emptyDir: {}
+# - name: job-artifacts-volume
+# hostPath:
+# path: /host/path/to/job/artifacts
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc1-flink/jobmanager-rest-service.yaml b/theodolite-benchmarks/definitions/uc1-flink/jobmanager-rest-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..3d74aaf7f625c6922e2e1b4f20c19e50a39b68ac
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc1-flink/jobmanager-rest-service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flink-jobmanager-rest
+spec:
+ type: NodePort
+ ports:
+ - name: rest
+ port: 8081
+ targetPort: 8081
+ nodePort: 30081
+ selector:
+ app: flink
+ component: jobmanager
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc1-flink/jobmanager-service.yaml b/theodolite-benchmarks/definitions/uc1-flink/jobmanager-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..e2ff5d9898eb1ebf5db9a827472a47514ab1473c
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc1-flink/jobmanager-service.yaml
@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flink-jobmanager
+ labels:
+ app: flink
+spec:
+ type: ClusterIP
+ ports:
+ - name: rpc
+ port: 6123
+ - name: blob-server
+ port: 6124
+ - name: webui
+ port: 8081
+ - name: metrics
+ port: 9249
+ selector:
+ app: flink
+ component: jobmanager
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc1-flink/service-monitor.yaml b/theodolite-benchmarks/definitions/uc1-flink/service-monitor.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..02f78823c627e27ddfe1db5eac3f6a7f7a7f1bf8
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc1-flink/service-monitor.yaml
@@ -0,0 +1,14 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+ labels:
+ app: flink
+ appScope: titan-ccp
+ name: flink
+spec:
+ selector:
+ matchLabels:
+ app: flink
+ endpoints:
+ - port: metrics
+ interval: 10s
diff --git a/theodolite-benchmarks/definitions/uc1-flink/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc1-flink/taskmanager-deployment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..7d46554692696b194736df6023eed5686040497d
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc1-flink/taskmanager-deployment.yaml
@@ -0,0 +1,87 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: flink-taskmanager
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: flink
+ component: taskmanager
+ template:
+ metadata:
+ labels:
+ app: flink
+ component: taskmanager
+ spec:
+ containers:
+ - name: taskmanager
+ image: ghcr.io/cau-se/theodolite-uc1-flink:latest
+ env:
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-cp-kafka:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
+ - name: COMMIT_INTERVAL_MS
+ value: "100"
+ - name: CHECKPOINTING
+ value: "false"
+ - name: PARALLELISM
+ value: "1"
+ - name: "FLINK_STATE_BACKEND"
+ value: "rocksdb"
+ - name: JOB_MANAGER_RPC_ADDRESS
+ value: "flink-jobmanager"
+ - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS
+ value: "1" #TODO
+ - name: FLINK_PROPERTIES
+ value: |+
+ blob.server.port: 6124
+ jobmanager.rpc.port: 6123
+ taskmanager.rpc.port: 6122
+ queryable-state.proxy.ports: 6125
+ jobmanager.memory.process.size: 4Gb
+ taskmanager.memory.process.size: 4Gb
+ #parallelism.default: 1 #TODO
+ resources:
+ limits:
+ memory: 4Gi
+ cpu: 1000m
+ args: ["taskmanager"]
+ ports:
+ - containerPort: 6122
+ name: rpc
+ - containerPort: 6125
+ name: query-state
+ - containerPort: 9249
+ name: metrics
+ livenessProbe:
+ tcpSocket:
+ port: 6122
+ initialDelaySeconds: 30
+ periodSeconds: 60
+ volumeMounts:
+ - name: flink-config-volume-rw
+ mountPath: /opt/flink/conf/
+ securityContext:
+ runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
+ initContainers:
+ - name: init-taskmanager
+ image: busybox:1.28
+ command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
+ volumeMounts:
+ - name: flink-config-volume
+ mountPath: /flink-config/
+ - name: flink-config-volume-rw
+ mountPath: /flink-config-rw/
+ volumes:
+ - name: flink-config-volume
+ configMap:
+ name: flink-config
+ items:
+ - key: flink-conf.yaml
+ path: flink-conf.yaml
+ - key: log4j-console.properties
+ path: log4j-console.properties
+ - name: flink-config-volume-rw
+ emptyDir: {}
diff --git a/theodolite-benchmarks/definitions/uc1-flink/taskmanager-service.yaml b/theodolite-benchmarks/definitions/uc1-flink/taskmanager-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a2e27f64af1cfd1a26da142b8a50bb41c8ba5fcb
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc1-flink/taskmanager-service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flink-taskmanager
+ labels:
+ app: flink
+spec:
+ type: ClusterIP
+ ports:
+ - name: metrics
+ port: 9249
+ selector:
+ app: flink
+ component: taskmanager
\ No newline at end of file
diff --git a/theodolite-quarkus/config/example-operator-benchmark.yaml b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-benchmark-operator.yaml
similarity index 53%
rename from theodolite-quarkus/config/example-operator-benchmark.yaml
rename to theodolite-benchmarks/definitions/uc1-kstreams/uc1-benchmark-operator.yaml
index 8f9af3e955fe18de26326b0094578a71517a5aef..b80b572dfd30e9c056d3c01ba17cc662d70fc749 100644
--- a/theodolite-quarkus/config/example-operator-benchmark.yaml
+++ b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-benchmark-operator.yaml
@@ -5,8 +5,8 @@ metadata:
spec:
appResource:
- "uc1-kstreams-deployment.yaml"
- - "aggregation-service.yaml"
- - "jmx-configmap.yaml"
+ - "uc1-kstreams-service.yaml"
+ - "uc1-jmx-configmap.yaml"
- "uc1-service-monitor.yaml"
loadGenResource:
- "uc1-load-generator-deployment.yaml"
@@ -17,17 +17,17 @@ spec:
- type: "ReplicaPatcher"
resource: "uc1-kstreams-deployment.yaml"
loadTypes:
- - typeName: "NumSensors"
- patchers:
- - type: "EnvVarPatcher"
- resource: "uc1-load-generator-deployment.yaml"
- properties:
- variableName: "NUM_SENSORS"
- container: "workload-generator"
- - type: "NumSensorsLoadGeneratorReplicaPatcher"
- resource: "uc1-load-generator-deployment.yaml"
- properties:
- loadGenMaxRecords: "15000"
+ - typeName: "NumSensors"
+ patchers:
+ - type: "EnvVarPatcher"
+ resource: "uc1-load-generator-deployment.yaml"
+ properties:
+ container: "workload-generator"
+ variableName: "NUM_SENSORS"
+ - type: "NumSensorsLoadGeneratorReplicaPatcher"
+ resource: "uc1-load-generator-deployment.yaml"
+ properties:
+ loadGenMaxRecords: "15000"
kafkaConfig:
bootstrapServer: "theodolite-cp-kafka:9092"
topics:
@@ -35,4 +35,4 @@ spec:
numPartitions: 40
replicationFactor: 1
- name: "theodolite-.*"
- removeOnly: True
\ No newline at end of file
+ removeOnly: True
diff --git a/theodolite-benchmarks/definitions/uc1-kstreams/uc1-benchmark-standalone.yaml b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-benchmark-standalone.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..12cbd8ea310423d28e35de8185288b27257c15ec
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-benchmark-standalone.yaml
@@ -0,0 +1,31 @@
+name: "uc1-kstreams"
+appResource:
+ - "uc1-kstreams-deployment.yaml"
+ - "uc1-kstreams-service.yaml"
+ - "uc1-jmx-configmap.yaml"
+ - "uc1-service-monitor.yaml"
+loadGenResource:
+ - "uc1-load-generator-deployment.yaml"
+ - "uc1-load-generator-service.yaml"
+resourceTypes:
+ - typeName: "Instances"
+ patchers:
+ - type: "ReplicaPatcher"
+ resource: "uc1-kstreams-deployment.yaml"
+loadTypes:
+ - typeName: "NumSensors"
+ patchers:
+ - type: "EnvVarPatcher"
+ resource: "uc1-load-generator-deployment.yaml"
+ container: "workload-generator"
+ variableName: "NUM_SENSORS"
+ - type: NumSensorsLoadGeneratorReplicaPatcher
+ resource: "uc1-load-generator-deployment.yaml"
+ properties:
+ loadGenMaxRecords: "15000"
+kafkaConfig:
+ bootstrapServer: "theodolite-cp-kafka:9092"
+ topics:
+ - name: "input"
+ numPartitions: 40
+ replicationFactor: 1
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc1-kstreams/uc1-jmx-configmap.yaml b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-jmx-configmap.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..78496a86b1242a89b9e844ead3e700fd0b9a9667
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-jmx-configmap.yaml
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: aggregation-jmx-configmap
+data:
+ jmx-kafka-prometheus.yml: |+
+ jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi
+ lowercaseOutputName: true
+ lowercaseOutputLabelNames: true
+ ssl: false
diff --git a/theodolite-benchmarks/definitions/uc1-kstreams/uc1-kstreams-deployment.yaml b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-kstreams-deployment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..171c3446db2719ee91bd8954233015316851fcf9
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-kstreams-deployment.yaml
@@ -0,0 +1,55 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: titan-ccp-aggregation
+spec:
+ selector:
+ matchLabels:
+ app: titan-ccp-aggregation
+ replicas: 1
+ template:
+ metadata:
+ labels:
+ app: titan-ccp-aggregation
+ spec:
+ terminationGracePeriodSeconds: 0
+ containers:
+ - name: uc-application
+ image: ghcr.io/cau-se/theodolite-uc1-kstreams-app:latest
+ ports:
+ - containerPort: 5555
+ name: jmx
+ env:
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-cp-kafka:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
+ - name: JAVA_OPTS
+ value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555"
+ - name: COMMIT_INTERVAL_MS # Set as default for the applications
+ value: "100"
+ resources:
+ limits:
+ memory: 4Gi
+ cpu: 1000m
+ - name: prometheus-jmx-exporter
+ image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143"
+ command:
+ - java
+ - -XX:+UnlockExperimentalVMOptions
+ - -XX:+UseCGroupMemoryLimitForHeap
+ - -XX:MaxRAMFraction=1
+ - -XshowSettings:vm
+ - -jar
+ - jmx_prometheus_httpserver.jar
+ - "5556"
+ - /etc/jmx-aggregation/jmx-kafka-prometheus.yml
+ ports:
+ - containerPort: 5556
+ volumeMounts:
+ - name: jmx-config
+ mountPath: /etc/jmx-aggregation
+ volumes:
+ - name: jmx-config
+ configMap:
+ name: aggregation-jmx-configmap
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc1-kstreams/uc1-kstreams-service.yaml b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-kstreams-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..85432d04f225c30469f3232153ef6bd72bd02bdf
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-kstreams-service.yaml
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: titan-ccp-aggregation
+ labels:
+ app: titan-ccp-aggregation
+spec:
+ #type: NodePort
+ selector:
+ app: titan-ccp-aggregation
+ ports:
+ - name: http
+ port: 80
+ targetPort: 80
+ protocol: TCP
+ - name: metrics
+ port: 5556
diff --git a/theodolite-benchmarks/definitions/uc1-kstreams/uc1-load-generator-deployment.yaml b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-load-generator-deployment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..9f9ccc6ae39407bb1f027e1e23cb152944b869e0
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-load-generator-deployment.yaml
@@ -0,0 +1,32 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: titan-ccp-load-generator
+spec:
+ selector:
+ matchLabels:
+ app: titan-ccp-load-generator
+ replicas: 1
+ template:
+ metadata:
+ labels:
+ app: titan-ccp-load-generator
+ spec:
+ terminationGracePeriodSeconds: 0
+ containers:
+ - name: workload-generator
+ image: ghcr.io/cau-se/theodolite-uc1-workload-generator:latest
+ ports:
+ - containerPort: 5701
+ name: coordination
+ env:
+ - name: KUBERNETES_NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ - name: KUBERNETES_DNS_NAME
+ value: "titan-ccp-load-generator.$(KUBERNETES_NAMESPACE).svc.cluster.local"
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-cp-kafka:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
diff --git a/theodolite-benchmarks/definitions/uc1-kstreams/uc1-load-generator-service.yaml b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-load-generator-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..f8b26b3f6dece427f9c1ad4db94e351b042749b3
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-load-generator-service.yaml
@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: titan-ccp-load-generator
+ labels:
+ app: titan-ccp-load-generator
+spec:
+ type: ClusterIP
+ clusterIP: None
+ selector:
+ app: titan-ccp-load-generator
+ ports:
+ - name: coordination
+ port: 5701
+ targetPort: 5701
+ protocol: TCP
diff --git a/theodolite-benchmarks/definitions/uc1-kstreams/uc1-service-monitor.yaml b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-service-monitor.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..4e7e758cacb5086305efa26292ddef2afc958096
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-service-monitor.yaml
@@ -0,0 +1,14 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+ labels:
+ app: titan-ccp-aggregation
+ appScope: titan-ccp
+ name: titan-ccp-aggregation
+spec:
+ selector:
+ matchLabels:
+ app: titan-ccp-aggregation
+ endpoints:
+ - port: metrics
+ interval: 10s
diff --git a/theodolite-benchmarks/definitions/uc2-flink/flink-configuration-configmap.yaml b/theodolite-benchmarks/definitions/uc2-flink/flink-configuration-configmap.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..321541f6ac8715b8546b964d8ad2b7c28552fbcd
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc2-flink/flink-configuration-configmap.yaml
@@ -0,0 +1,66 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: flink-config
+ labels:
+ app: flink
+data:
+ flink-conf.yaml: |+
+ #jobmanager.rpc.address: flink-jobmanager
+ #taskmanager.numberOfTaskSlots: 1 #TODO
+ #blob.server.port: 6124
+ #jobmanager.rpc.port: 6123
+ #taskmanager.rpc.port: 6122
+ #queryable-state.proxy.ports: 6125
+ #jobmanager.memory.process.size: 4Gb
+ #taskmanager.memory.process.size: 4Gb
+ #parallelism.default: 1 #TODO
+ metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
+ metrics.reporter.prom.interval: 10 SECONDS
+ taskmanager.network.detailed-metrics: true
+ # -> gives metrics about inbound/outbound network queue lengths
+ log4j-console.properties: |+
+ # This affects logging for both user code and Flink
+ rootLogger.level = INFO
+ rootLogger.appenderRef.console.ref = ConsoleAppender
+ rootLogger.appenderRef.rolling.ref = RollingFileAppender
+
+ # Uncomment this if you want to _only_ change Flink's logging
+ #logger.flink.name = org.apache.flink
+ #logger.flink.level = INFO
+
+ # The following lines keep the log level of common libraries/connectors on
+ # log level INFO. The root logger does not override this. You have to manually
+ # change the log levels here.
+ logger.akka.name = akka
+ logger.akka.level = INFO
+ logger.kafka.name= org.apache.kafka
+ logger.kafka.level = INFO
+ logger.hadoop.name = org.apache.hadoop
+ logger.hadoop.level = INFO
+ logger.zookeeper.name = org.apache.zookeeper
+ logger.zookeeper.level = INFO
+
+ # Log all infos to the console
+ appender.console.name = ConsoleAppender
+ appender.console.type = CONSOLE
+ appender.console.layout.type = PatternLayout
+ appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+
+ # Log all infos in the given rolling file
+ appender.rolling.name = RollingFileAppender
+ appender.rolling.type = RollingFile
+ appender.rolling.append = false
+ appender.rolling.fileName = ${sys:log.file}
+ appender.rolling.filePattern = ${sys:log.file}.%i
+ appender.rolling.layout.type = PatternLayout
+ appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+ appender.rolling.policies.type = Policies
+ appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
+ appender.rolling.policies.size.size=100MB
+ appender.rolling.strategy.type = DefaultRolloverStrategy
+ appender.rolling.strategy.max = 10
+
+ # Suppress the irrelevant (wrong) warnings from the Netty channel handler
+ logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
+ logger.netty.level = OFF
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc2-flink/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc2-flink/jobmanager-deployment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..cece4286d49a3f6ff139ca7f1e01c647acd5d9f3
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc2-flink/jobmanager-deployment.yaml
@@ -0,0 +1,93 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: flink-jobmanager
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: flink
+ component: jobmanager
+ template:
+ metadata:
+ labels:
+ app: flink
+ component: jobmanager
+ spec:
+ containers:
+ - name: jobmanager
+ image: ghcr.io/cau-se/theodolite-uc2-flink:latest
+ env:
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-cp-kafka:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
+ - name: COMMIT_INTERVAL_MS
+ value: "100"
+ - name: CHECKPOINTING
+ value: "false"
+ - name: PARALLELISM
+ value: "1"
+ - name: "FLINK_STATE_BACKEND"
+ value: "rocksdb"
+ - name: JOB_MANAGER_RPC_ADDRESS
+ value: "flink-jobmanager"
+ - name: FLINK_PROPERTIES
+ value: |+
+ blob.server.port: 6124
+ jobmanager.rpc.port: 6123
+ taskmanager.rpc.port: 6122
+ queryable-state.proxy.ports: 6125
+ jobmanager.memory.process.size: 4Gb
+ taskmanager.memory.process.size: 4Gb
+ #parallelism.default: 1 #TODO
+ resources:
+ limits:
+ memory: 4Gi
+ cpu: 1000m
+ args: ["standalone-job", "--job-classname", "theodolite.uc2.application.HistoryServiceFlinkJob"] # optional arguments: ["--job-id", "<job id>", "--fromSavepoint", "/path/to/savepoint", "--allowNonRestoredState"]
+ #command: ['sleep', '60m']
+ ports:
+ - containerPort: 6123
+ name: rpc
+ - containerPort: 6124
+ name: blob-server
+ - containerPort: 8081
+ name: webui
+ - containerPort: 9249
+ name: metrics
+ livenessProbe:
+ tcpSocket:
+ port: 6123
+ initialDelaySeconds: 30
+ periodSeconds: 60
+ volumeMounts:
+ - name: flink-config-volume-rw
+ mountPath: /opt/flink/conf
+# - name: job-artifacts-volume
+# mountPath: /opt/flink/usrlib
+ securityContext:
+ runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
+ initContainers:
+ - name: init-jobmanager
+ image: busybox:1.28
+ command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
+ volumeMounts:
+ - name: flink-config-volume
+ mountPath: /flink-config/
+ - name: flink-config-volume-rw
+ mountPath: /flink-config-rw/
+ volumes:
+ - name: flink-config-volume
+ configMap:
+ name: flink-config
+ items:
+ - key: flink-conf.yaml
+ path: flink-conf.yaml
+ - key: log4j-console.properties
+ path: log4j-console.properties
+ - name: flink-config-volume-rw
+ emptyDir: {}
+# - name: job-artifacts-volume
+# hostPath:
+# path: /host/path/to/job/artifacts
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc2-flink/jobmanager-rest-service.yaml b/theodolite-benchmarks/definitions/uc2-flink/jobmanager-rest-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..3d74aaf7f625c6922e2e1b4f20c19e50a39b68ac
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc2-flink/jobmanager-rest-service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flink-jobmanager-rest
+spec:
+ type: NodePort
+ ports:
+ - name: rest
+ port: 8081
+ targetPort: 8081
+ nodePort: 30081
+ selector:
+ app: flink
+ component: jobmanager
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc2-flink/jobmanager-service.yaml b/theodolite-benchmarks/definitions/uc2-flink/jobmanager-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..e2ff5d9898eb1ebf5db9a827472a47514ab1473c
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc2-flink/jobmanager-service.yaml
@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flink-jobmanager
+ labels:
+ app: flink
+spec:
+ type: ClusterIP
+ ports:
+ - name: rpc
+ port: 6123
+ - name: blob-server
+ port: 6124
+ - name: webui
+ port: 8081
+ - name: metrics
+ port: 9249
+ selector:
+ app: flink
+ component: jobmanager
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc2-flink/service-monitor.yaml b/theodolite-benchmarks/definitions/uc2-flink/service-monitor.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..02f78823c627e27ddfe1db5eac3f6a7f7a7f1bf8
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc2-flink/service-monitor.yaml
@@ -0,0 +1,14 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+ labels:
+ app: flink
+ appScope: titan-ccp
+ name: flink
+spec:
+ selector:
+ matchLabels:
+ app: flink
+ endpoints:
+ - port: metrics
+ interval: 10s
diff --git a/theodolite-benchmarks/definitions/uc2-flink/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc2-flink/taskmanager-deployment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..c0a10f65aae92e4ac1fd8fb92bae97794c142232
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc2-flink/taskmanager-deployment.yaml
@@ -0,0 +1,87 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: flink-taskmanager
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: flink
+ component: taskmanager
+ template:
+ metadata:
+ labels:
+ app: flink
+ component: taskmanager
+ spec:
+ containers:
+ - name: taskmanager
+ image: ghcr.io/cau-se/theodolite-uc2-flink:latest
+ env:
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-cp-kafka:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
+ - name: COMMIT_INTERVAL_MS
+ value: "100"
+ - name: CHECKPOINTING
+ value: "false"
+ - name: PARALLELISM
+ value: "1"
+ - name: "FLINK_STATE_BACKEND"
+ value: "rocksdb"
+ - name: JOB_MANAGER_RPC_ADDRESS
+ value: "flink-jobmanager"
+ - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS
+ value: "1" #TODO
+ - name: FLINK_PROPERTIES
+ value: |+
+ blob.server.port: 6124
+ jobmanager.rpc.port: 6123
+ taskmanager.rpc.port: 6122
+ queryable-state.proxy.ports: 6125
+ jobmanager.memory.process.size: 4Gb
+ taskmanager.memory.process.size: 4Gb
+ #parallelism.default: 1 #TODO
+ resources:
+ limits:
+ memory: 4Gi
+ cpu: 1000m
+ args: ["taskmanager"]
+ ports:
+ - containerPort: 6122
+ name: rpc
+ - containerPort: 6125
+ name: query-state
+ - containerPort: 9249
+ name: metrics
+ livenessProbe:
+ tcpSocket:
+ port: 6122
+ initialDelaySeconds: 30
+ periodSeconds: 60
+ volumeMounts:
+ - name: flink-config-volume-rw
+ mountPath: /opt/flink/conf/
+ securityContext:
+ runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
+ initContainers:
+ - name: init-taskmanager
+ image: busybox:1.28
+ command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
+ volumeMounts:
+ - name: flink-config-volume
+ mountPath: /flink-config/
+ - name: flink-config-volume-rw
+ mountPath: /flink-config-rw/
+ volumes:
+ - name: flink-config-volume
+ configMap:
+ name: flink-config
+ items:
+ - key: flink-conf.yaml
+ path: flink-conf.yaml
+ - key: log4j-console.properties
+ path: log4j-console.properties
+ - name: flink-config-volume-rw
+ emptyDir: {}
diff --git a/theodolite-benchmarks/definitions/uc2-flink/taskmanager-service.yaml b/theodolite-benchmarks/definitions/uc2-flink/taskmanager-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a2e27f64af1cfd1a26da142b8a50bb41c8ba5fcb
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc2-flink/taskmanager-service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flink-taskmanager
+ labels:
+ app: flink
+spec:
+ type: ClusterIP
+ ports:
+ - name: metrics
+ port: 9249
+ selector:
+ app: flink
+ component: taskmanager
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc2-kstreams/uc2-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-benchmark-operator.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b65bbdedb055c206c1ebcd7ab6a450318ee8c00f
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-benchmark-operator.yaml
@@ -0,0 +1,41 @@
+apiVersion: theodolite.com/v1
+kind: benchmark
+spec:
+ metadata:
+ name: uc2-kstreams
+ appResource:
+ - "uc2-kstreams-deployment.yaml"
+ - "uc2-kstreams-service.yaml"
+ - "uc2-jmx-configmap.yaml"
+ - "uc2-service-monitor.yaml"
+ loadGenResource:
+ - "uc2-load-generator-deployment.yaml"
+ - "uc2-load-generator-service.yaml"
+ resourceTypes:
+ - typeName: "Instances"
+ patchers:
+ - type: "ReplicaPatcher"
+ resource: "uc2-kstreams-deployment.yaml"
+ loadTypes:
+ - typeName: "NumSensors"
+ patchers:
+ - type: "EnvVarPatcher"
+ resource: "uc2-load-generator-deployment.yaml"
+ properties:
+ container: "workload-generator"
+ variableName: "NUM_SENSORS"
+ - type: NumSensorsLoadGeneratorReplicaPatcher
+ resource: "uc2-load-generator-deployment.yaml"
+ properties:
+ loadGenMaxRecords: "15000"
+ kafkaConfig:
+ bootstrapServer: "theodolite-cp-kafka:9092"
+ topics:
+ - name: "input"
+ numPartitions: 40
+ replicationFactor: 1
+ - name: "output"
+ numPartitions: 40
+ replicationFactor: 1
+ - name: "theodolite-.*"
+ removeOnly: True
diff --git a/theodolite-benchmarks/definitions/uc2-kstreams/uc2-benchmark-standalone.yaml b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-benchmark-standalone.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..e38f83f5b05d05febb59c2f775a29b2d545acf0e
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-benchmark-standalone.yaml
@@ -0,0 +1,37 @@
+name: "uc2-kstreams"
+appResource:
+ - "uc2-kstreams-deployment.yaml"
+ - "uc2-kstreams-service.yaml"
+ - "uc2-jmx-configmap.yaml"
+ - "uc2-service-monitor.yaml"
+loadGenResource:
+ - "uc2-load-generator-deployment.yaml"
+ - "uc2-load-generator-service.yaml"
+resourceTypes:
+ - typeName: "Instances"
+ patchers:
+ - type: "ReplicaPatcher"
+ resource: "uc2-kstreams-deployment.yaml"
+loadTypes:
+ - typeName: "NumSensors"
+ patchers:
+ - type: "EnvVarPatcher"
+ resource: "uc2-load-generator-deployment.yaml"
+ properties:
+ container: "workload-generator"
+ variableName: "NUM_SENSORS"
+ - type: NumSensorsLoadGeneratorReplicaPatcher
+ resource: "uc2-load-generator-deployment.yaml"
+ properties:
+ loadGenMaxRecords: "15000"
+kafkaConfig:
+ bootstrapServer: "theodolite-cp-kafka:9092"
+ topics:
+ - name: "input"
+ numPartitions: 40
+ replicationFactor: 1
+ - name: "output"
+ numPartitions: 40
+ replicationFactor: 1
+ - name: "theodolite-.*"
+ removeOnly: True
diff --git a/theodolite-benchmarks/definitions/uc2-kstreams/uc2-jmx-configmap.yaml b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-jmx-configmap.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..78496a86b1242a89b9e844ead3e700fd0b9a9667
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-jmx-configmap.yaml
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: aggregation-jmx-configmap
+data:
+ jmx-kafka-prometheus.yml: |+
+ jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi
+ lowercaseOutputName: true
+ lowercaseOutputLabelNames: true
+ ssl: false
diff --git a/theodolite-benchmarks/definitions/uc2-kstreams/uc2-kstreams-deployment.yaml b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-kstreams-deployment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..e07bb3f9e536655712c06a004c5d1fb60ffa67e0
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-kstreams-deployment.yaml
@@ -0,0 +1,55 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: titan-ccp-aggregation
+spec:
+ selector:
+ matchLabels:
+ app: titan-ccp-aggregation
+ replicas: 1
+ template:
+ metadata:
+ labels:
+ app: titan-ccp-aggregation
+ spec:
+ terminationGracePeriodSeconds: 0
+ containers:
+ - name: uc-application
+ image: ghcr.io/cau-se/theodolite-uc2-kstreams-app:latest
+ ports:
+ - containerPort: 5555
+ name: jmx
+ env:
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-cp-kafka:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
+ - name: JAVA_OPTS
+ value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555"
+ - name: COMMIT_INTERVAL_MS # Set as default for the applications
+ value: "100"
+ resources:
+ limits:
+ memory: 4Gi
+ cpu: 1000m
+ - name: prometheus-jmx-exporter
+ image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143"
+ command:
+ - java
+ - -XX:+UnlockExperimentalVMOptions
+ - -XX:+UseCGroupMemoryLimitForHeap
+ - -XX:MaxRAMFraction=1
+ - -XshowSettings:vm
+ - -jar
+ - jmx_prometheus_httpserver.jar
+ - "5556"
+ - /etc/jmx-aggregation/jmx-kafka-prometheus.yml
+ ports:
+ - containerPort: 5556
+ volumeMounts:
+ - name: jmx-config
+ mountPath: /etc/jmx-aggregation
+ volumes:
+ - name: jmx-config
+ configMap:
+ name: aggregation-jmx-configmap
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc2-kstreams/uc2-kstreams-service.yaml b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-kstreams-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..85432d04f225c30469f3232153ef6bd72bd02bdf
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-kstreams-service.yaml
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: titan-ccp-aggregation
+ labels:
+ app: titan-ccp-aggregation
+spec:
+ #type: NodePort
+ selector:
+ app: titan-ccp-aggregation
+ ports:
+ - name: http
+ port: 80
+ targetPort: 80
+ protocol: TCP
+ - name: metrics
+ port: 5556
diff --git a/theodolite-benchmarks/definitions/uc2-kstreams/uc2-load-generator-deployment.yaml b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-load-generator-deployment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..dfc0af71543c15b12b5c850919feb0e0a4f52f28
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-load-generator-deployment.yaml
@@ -0,0 +1,32 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: titan-ccp-load-generator
+spec:
+ selector:
+ matchLabels:
+ app: titan-ccp-load-generator
+ replicas: 1
+ template:
+ metadata:
+ labels:
+ app: titan-ccp-load-generator
+ spec:
+ terminationGracePeriodSeconds: 0
+ containers:
+ - name: workload-generator
+ image: ghcr.io/cau-se/theodolite-uc2-workload-generator:latest
+ ports:
+ - containerPort: 5701
+ name: coordination
+ env:
+ - name: KUBERNETES_NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ - name: KUBERNETES_DNS_NAME
+ value: "titan-ccp-load-generator.$(KUBERNETES_NAMESPACE).svc.cluster.local"
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-cp-kafka:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
diff --git a/theodolite-benchmarks/definitions/uc2-kstreams/uc2-load-generator-service.yaml b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-load-generator-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..f8b26b3f6dece427f9c1ad4db94e351b042749b3
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-load-generator-service.yaml
@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: titan-ccp-load-generator
+ labels:
+ app: titan-ccp-load-generator
+spec:
+ type: ClusterIP
+ clusterIP: None
+ selector:
+ app: titan-ccp-load-generator
+ ports:
+ - name: coordination
+ port: 5701
+ targetPort: 5701
+ protocol: TCP
diff --git a/theodolite-benchmarks/definitions/uc2-kstreams/uc2-service-monitor.yaml b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-service-monitor.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..4e7e758cacb5086305efa26292ddef2afc958096
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-service-monitor.yaml
@@ -0,0 +1,14 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+ labels:
+ app: titan-ccp-aggregation
+ appScope: titan-ccp
+ name: titan-ccp-aggregation
+spec:
+ selector:
+ matchLabels:
+ app: titan-ccp-aggregation
+ endpoints:
+ - port: metrics
+ interval: 10s
diff --git a/theodolite-benchmarks/definitions/uc3-flink/flink-configuration-configmap.yaml b/theodolite-benchmarks/definitions/uc3-flink/flink-configuration-configmap.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..321541f6ac8715b8546b964d8ad2b7c28552fbcd
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc3-flink/flink-configuration-configmap.yaml
@@ -0,0 +1,66 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: flink-config
+ labels:
+ app: flink
+data:
+ flink-conf.yaml: |+
+ #jobmanager.rpc.address: flink-jobmanager
+ #taskmanager.numberOfTaskSlots: 1 #TODO
+ #blob.server.port: 6124
+ #jobmanager.rpc.port: 6123
+ #taskmanager.rpc.port: 6122
+ #queryable-state.proxy.ports: 6125
+ #jobmanager.memory.process.size: 4Gb
+ #taskmanager.memory.process.size: 4Gb
+ #parallelism.default: 1 #TODO
+ metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
+ metrics.reporter.prom.interval: 10 SECONDS
+ taskmanager.network.detailed-metrics: true
+ # -> gives metrics about inbound/outbound network queue lengths
+ log4j-console.properties: |+
+ # This affects logging for both user code and Flink
+ rootLogger.level = INFO
+ rootLogger.appenderRef.console.ref = ConsoleAppender
+ rootLogger.appenderRef.rolling.ref = RollingFileAppender
+
+ # Uncomment this if you want to _only_ change Flink's logging
+ #logger.flink.name = org.apache.flink
+ #logger.flink.level = INFO
+
+ # The following lines keep the log level of common libraries/connectors on
+ # log level INFO. The root logger does not override this. You have to manually
+ # change the log levels here.
+ logger.akka.name = akka
+ logger.akka.level = INFO
+ logger.kafka.name= org.apache.kafka
+ logger.kafka.level = INFO
+ logger.hadoop.name = org.apache.hadoop
+ logger.hadoop.level = INFO
+ logger.zookeeper.name = org.apache.zookeeper
+ logger.zookeeper.level = INFO
+
+ # Log all infos to the console
+ appender.console.name = ConsoleAppender
+ appender.console.type = CONSOLE
+ appender.console.layout.type = PatternLayout
+ appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+
+ # Log all infos in the given rolling file
+ appender.rolling.name = RollingFileAppender
+ appender.rolling.type = RollingFile
+ appender.rolling.append = false
+ appender.rolling.fileName = ${sys:log.file}
+ appender.rolling.filePattern = ${sys:log.file}.%i
+ appender.rolling.layout.type = PatternLayout
+ appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+ appender.rolling.policies.type = Policies
+ appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
+ appender.rolling.policies.size.size=100MB
+ appender.rolling.strategy.type = DefaultRolloverStrategy
+ appender.rolling.strategy.max = 10
+
+ # Suppress the irrelevant (wrong) warnings from the Netty channel handler
+ logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
+ logger.netty.level = OFF
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc3-flink/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc3-flink/jobmanager-deployment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..33bf1f1121a9764785db7a504799314a7ed40cf3
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc3-flink/jobmanager-deployment.yaml
@@ -0,0 +1,93 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: flink-jobmanager
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: flink
+ component: jobmanager
+ template:
+ metadata:
+ labels:
+ app: flink
+ component: jobmanager
+ spec:
+ containers:
+ - name: jobmanager
+ image: ghcr.io/cau-se/theodolite-uc3-flink:latest
+ env:
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-cp-kafka:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
+ - name: COMMIT_INTERVAL_MS
+ value: "100"
+ - name: CHECKPOINTING
+ value: "false"
+ - name: PARALLELISM
+ value: "1"
+ - name: "FLINK_STATE_BACKEND"
+ value: "rocksdb"
+ - name: JOB_MANAGER_RPC_ADDRESS
+ value: "flink-jobmanager"
+ - name: FLINK_PROPERTIES
+ value: |+
+ blob.server.port: 6124
+ jobmanager.rpc.port: 6123
+ taskmanager.rpc.port: 6122
+ queryable-state.proxy.ports: 6125
+ jobmanager.memory.process.size: 4Gb
+ taskmanager.memory.process.size: 4Gb
+ #parallelism.default: 1 #TODO
+ resources:
+ limits:
+ memory: 4Gi
+ cpu: 1000m
+ args: ["standalone-job", "--job-classname", "theodolite.uc3.application.HistoryServiceFlinkJob"] # optional arguments: ["--job-id", "<job id>", "--fromSavepoint", "/path/to/savepoint", "--allowNonRestoredState"]
+ #command: ['sleep', '60m']
+ ports:
+ - containerPort: 6123
+ name: rpc
+ - containerPort: 6124
+ name: blob-server
+ - containerPort: 8081
+ name: webui
+ - containerPort: 9249
+ name: metrics
+ livenessProbe:
+ tcpSocket:
+ port: 6123
+ initialDelaySeconds: 30
+ periodSeconds: 60
+ volumeMounts:
+ - name: flink-config-volume-rw
+ mountPath: /opt/flink/conf
+# - name: job-artifacts-volume
+# mountPath: /opt/flink/usrlib
+ securityContext:
+ runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
+ initContainers:
+ - name: init-jobmanager
+ image: busybox:1.28
+ command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
+ volumeMounts:
+ - name: flink-config-volume
+ mountPath: /flink-config/
+ - name: flink-config-volume-rw
+ mountPath: /flink-config-rw/
+ volumes:
+ - name: flink-config-volume
+ configMap:
+ name: flink-config
+ items:
+ - key: flink-conf.yaml
+ path: flink-conf.yaml
+ - key: log4j-console.properties
+ path: log4j-console.properties
+ - name: flink-config-volume-rw
+ emptyDir: {}
+# - name: job-artifacts-volume
+# hostPath:
+# path: /host/path/to/job/artifacts
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc3-flink/jobmanager-rest-service.yaml b/theodolite-benchmarks/definitions/uc3-flink/jobmanager-rest-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..3d74aaf7f625c6922e2e1b4f20c19e50a39b68ac
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc3-flink/jobmanager-rest-service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flink-jobmanager-rest
+spec:
+ type: NodePort
+ ports:
+ - name: rest
+ port: 8081
+ targetPort: 8081
+ nodePort: 30081
+ selector:
+ app: flink
+ component: jobmanager
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc3-flink/jobmanager-service.yaml b/theodolite-benchmarks/definitions/uc3-flink/jobmanager-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..e2ff5d9898eb1ebf5db9a827472a47514ab1473c
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc3-flink/jobmanager-service.yaml
@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flink-jobmanager
+ labels:
+ app: flink
+spec:
+ type: ClusterIP
+ ports:
+ - name: rpc
+ port: 6123
+ - name: blob-server
+ port: 6124
+ - name: webui
+ port: 8081
+ - name: metrics
+ port: 9249
+ selector:
+ app: flink
+ component: jobmanager
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc3-flink/service-monitor.yaml b/theodolite-benchmarks/definitions/uc3-flink/service-monitor.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..02f78823c627e27ddfe1db5eac3f6a7f7a7f1bf8
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc3-flink/service-monitor.yaml
@@ -0,0 +1,14 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+ labels:
+ app: flink
+ appScope: titan-ccp
+ name: flink
+spec:
+ selector:
+ matchLabels:
+ app: flink
+ endpoints:
+ - port: metrics
+ interval: 10s
diff --git a/theodolite-benchmarks/definitions/uc3-flink/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc3-flink/taskmanager-deployment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..8f70b7308429f79cfd8f8bda7a7a96e2bc8d8689
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc3-flink/taskmanager-deployment.yaml
@@ -0,0 +1,87 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: flink-taskmanager
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: flink
+ component: taskmanager
+ template:
+ metadata:
+ labels:
+ app: flink
+ component: taskmanager
+ spec:
+ containers:
+ - name: taskmanager
+ image: ghcr.io/cau-se/theodolite-uc3-flink:latest
+ env:
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-cp-kafka:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
+ - name: COMMIT_INTERVAL_MS
+ value: "100"
+ - name: CHECKPOINTING
+ value: "false"
+ - name: PARALLELISM
+ value: "1"
+ - name: "FLINK_STATE_BACKEND"
+ value: "rocksdb"
+ - name: JOB_MANAGER_RPC_ADDRESS
+ value: "flink-jobmanager"
+ - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS
+ value: "1" #TODO
+ - name: FLINK_PROPERTIES
+ value: |+
+ blob.server.port: 6124
+ jobmanager.rpc.port: 6123
+ taskmanager.rpc.port: 6122
+ queryable-state.proxy.ports: 6125
+ jobmanager.memory.process.size: 4Gb
+ taskmanager.memory.process.size: 4Gb
+ #parallelism.default: 1 #TODO
+ resources:
+ limits:
+ memory: 4Gi
+ cpu: 1000m
+ args: ["taskmanager"]
+ ports:
+ - containerPort: 6122
+ name: rpc
+ - containerPort: 6125
+ name: query-state
+ - containerPort: 9249
+ name: metrics
+ livenessProbe:
+ tcpSocket:
+ port: 6122
+ initialDelaySeconds: 30
+ periodSeconds: 60
+ volumeMounts:
+ - name: flink-config-volume-rw
+ mountPath: /opt/flink/conf/
+ securityContext:
+ runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
+ initContainers:
+ - name: init-taskmanager
+ image: busybox:1.28
+ command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
+ volumeMounts:
+ - name: flink-config-volume
+ mountPath: /flink-config/
+ - name: flink-config-volume-rw
+ mountPath: /flink-config-rw/
+ volumes:
+ - name: flink-config-volume
+ configMap:
+ name: flink-config
+ items:
+ - key: flink-conf.yaml
+ path: flink-conf.yaml
+ - key: log4j-console.properties
+ path: log4j-console.properties
+ - name: flink-config-volume-rw
+ emptyDir: {}
diff --git a/theodolite-benchmarks/definitions/uc3-flink/taskmanager-service.yaml b/theodolite-benchmarks/definitions/uc3-flink/taskmanager-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a2e27f64af1cfd1a26da142b8a50bb41c8ba5fcb
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc3-flink/taskmanager-service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flink-taskmanager
+ labels:
+ app: flink
+spec:
+ type: ClusterIP
+ ports:
+ - name: metrics
+ port: 9249
+ selector:
+ app: flink
+ component: taskmanager
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc3-kstreams/uc3-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-benchmark-operator.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..bfbd7191c5f4a315db29100bcc05341f88cffec2
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-benchmark-operator.yaml
@@ -0,0 +1,41 @@
+apiVersion: theodolite.com/v1
+kind: benchmark
+spec:
+ metadata:
+ name: uc3-kstreams
+ appResource:
+ - "uc3-kstreams-deployment.yaml"
+ - "uc3-kstreams-service.yaml"
+ - "uc3-jmx-configmap.yaml"
+ - "uc3-service-monitor.yaml"
+ loadGenResource:
+ - "uc3-load-generator-deployment.yaml"
+ - "uc3-load-generator-service.yaml"
+ resourceTypes:
+ - typeName: "Instances"
+ patchers:
+ - type: "ReplicaPatcher"
+ resource: "uc3-kstreams-deployment.yaml"
+ loadTypes:
+ - typeName: "NumSensors"
+ patchers:
+ - type: "EnvVarPatcher"
+ resource: "uc3-load-generator-deployment.yaml"
+ properties:
+ container: "workload-generator"
+ variableName: "NUM_SENSORS"
+ - type: NumSensorsLoadGeneratorReplicaPatcher
+ resource: "uc3-load-generator-deployment.yaml"
+ properties:
+ loadGenMaxRecords: "15000"
+ kafkaConfig:
+ bootstrapServer: "theodolite-cp-kafka:9092"
+ topics:
+ - name: "input"
+ numPartitions: 40
+ replicationFactor: 1
+ - name: "output"
+ numPartitions: 40
+ replicationFactor: 1
+ - name: "theodolite-.*"
+ removeOnly: True
diff --git a/theodolite-benchmarks/definitions/uc3-kstreams/uc3-benchmark-standalone.yaml b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-benchmark-standalone.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..e00c1672c4a5a02128c2618b525573a4cddd6c72
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-benchmark-standalone.yaml
@@ -0,0 +1,37 @@
+name: "uc3-kstreams"
+appResource:
+ - "uc3-kstreams-deployment.yaml"
+ - "uc3-kstreams-service.yaml"
+ - "uc3-jmx-configmap.yaml"
+ - "uc3-service-monitor.yaml"
+loadGenResource:
+ - "uc3-load-generator-deployment.yaml"
+ - "uc3-load-generator-service.yaml"
+resourceTypes:
+ - typeName: "Instances"
+ patchers:
+ - type: "ReplicaPatcher"
+ resource: "uc3-kstreams-deployment.yaml"
+loadTypes:
+ - typeName: "NumSensors"
+ patchers:
+ - type: "EnvVarPatcher"
+ resource: "uc3-load-generator-deployment.yaml"
+ properties:
+ container: "workload-generator"
+ variableName: "NUM_SENSORS"
+ - type: NumSensorsLoadGeneratorReplicaPatcher
+ resource: "uc3-load-generator-deployment.yaml"
+ properties:
+ loadGenMaxRecords: "15000"
+kafkaConfig:
+ bootstrapServer: "theodolite-cp-kafka:9092"
+ topics:
+ - name: "input"
+ numPartitions: 40
+ replicationFactor: 1
+ - name: "output"
+ numPartitions: 40
+ replicationFactor: 1
+ - name: "theodolite-.*"
+ removeOnly: True
diff --git a/theodolite-benchmarks/definitions/uc3-kstreams/uc3-jmx-configmap.yaml b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-jmx-configmap.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..78496a86b1242a89b9e844ead3e700fd0b9a9667
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-jmx-configmap.yaml
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: aggregation-jmx-configmap
+data:
+ jmx-kafka-prometheus.yml: |+
+ jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi
+ lowercaseOutputName: true
+ lowercaseOutputLabelNames: true
+ ssl: false
diff --git a/theodolite-benchmarks/definitions/uc3-kstreams/uc3-kstreams-deployment.yaml b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-kstreams-deployment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..e3f63fae9e245e6116e0fe451480d9bc74b36433
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-kstreams-deployment.yaml
@@ -0,0 +1,55 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: titan-ccp-aggregation
+spec:
+ selector:
+ matchLabels:
+ app: titan-ccp-aggregation
+ replicas: 1
+ template:
+ metadata:
+ labels:
+ app: titan-ccp-aggregation
+ spec:
+ terminationGracePeriodSeconds: 0
+ containers:
+ - name: uc-application
+ image: ghcr.io/cau-se/theodolite-uc3-kstreams-app:latest
+ ports:
+ - containerPort: 5555
+ name: jmx
+ env:
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-cp-kafka:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
+ - name: JAVA_OPTS
+ value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555"
+ - name: COMMIT_INTERVAL_MS # Set as default for the applications
+ value: "100"
+ resources:
+ limits:
+ memory: 4Gi
+ cpu: 1000m
+ - name: prometheus-jmx-exporter
+ image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143"
+ command:
+ - java
+ - -XX:+UnlockExperimentalVMOptions
+ - -XX:+UseCGroupMemoryLimitForHeap
+ - -XX:MaxRAMFraction=1
+ - -XshowSettings:vm
+ - -jar
+ - jmx_prometheus_httpserver.jar
+ - "5556"
+ - /etc/jmx-aggregation/jmx-kafka-prometheus.yml
+ ports:
+ - containerPort: 5556
+ volumeMounts:
+ - name: jmx-config
+ mountPath: /etc/jmx-aggregation
+ volumes:
+ - name: jmx-config
+ configMap:
+ name: aggregation-jmx-configmap
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc3-kstreams/uc3-kstreams-service.yaml b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-kstreams-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..85432d04f225c30469f3232153ef6bd72bd02bdf
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-kstreams-service.yaml
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: titan-ccp-aggregation
+ labels:
+ app: titan-ccp-aggregation
+spec:
+ #type: NodePort
+ selector:
+ app: titan-ccp-aggregation
+ ports:
+ - name: http
+ port: 80
+ targetPort: 80
+ protocol: TCP
+ - name: metrics
+ port: 5556
diff --git a/theodolite-benchmarks/definitions/uc3-kstreams/uc3-load-generator-deployment.yaml b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-load-generator-deployment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..c1cad0b70fd82a5bbb43792ee79f9cf5cc71d95f
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-load-generator-deployment.yaml
@@ -0,0 +1,32 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: titan-ccp-load-generator
+spec:
+ selector:
+ matchLabels:
+ app: titan-ccp-load-generator
+ replicas: 1
+ template:
+ metadata:
+ labels:
+ app: titan-ccp-load-generator
+ spec:
+ terminationGracePeriodSeconds: 0
+ containers:
+ - name: workload-generator
+ image: ghcr.io/cau-se/theodolite-uc3-workload-generator:latest
+ ports:
+ - containerPort: 5701
+ name: coordination
+ env:
+ - name: KUBERNETES_NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ - name: KUBERNETES_DNS_NAME
+ value: "titan-ccp-load-generator.$(KUBERNETES_NAMESPACE).svc.cluster.local"
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-cp-kafka:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
diff --git a/theodolite-benchmarks/definitions/uc3-kstreams/uc3-load-generator-service.yaml b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-load-generator-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..f8b26b3f6dece427f9c1ad4db94e351b042749b3
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-load-generator-service.yaml
@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: titan-ccp-load-generator
+ labels:
+ app: titan-ccp-load-generator
+spec:
+ type: ClusterIP
+ clusterIP: None
+ selector:
+ app: titan-ccp-load-generator
+ ports:
+ - name: coordination
+ port: 5701
+ targetPort: 5701
+ protocol: TCP
diff --git a/theodolite-benchmarks/definitions/uc3-kstreams/uc3-service-monitor.yaml b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-service-monitor.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..4e7e758cacb5086305efa26292ddef2afc958096
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-service-monitor.yaml
@@ -0,0 +1,14 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+ labels:
+ app: titan-ccp-aggregation
+ appScope: titan-ccp
+ name: titan-ccp-aggregation
+spec:
+ selector:
+ matchLabels:
+ app: titan-ccp-aggregation
+ endpoints:
+ - port: metrics
+ interval: 10s
diff --git a/theodolite-benchmarks/definitions/uc4-flink/flink-configuration-configmap.yaml b/theodolite-benchmarks/definitions/uc4-flink/flink-configuration-configmap.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..321541f6ac8715b8546b964d8ad2b7c28552fbcd
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc4-flink/flink-configuration-configmap.yaml
@@ -0,0 +1,66 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: flink-config
+ labels:
+ app: flink
+data:
+ flink-conf.yaml: |+
+ #jobmanager.rpc.address: flink-jobmanager
+ #taskmanager.numberOfTaskSlots: 1 #TODO
+ #blob.server.port: 6124
+ #jobmanager.rpc.port: 6123
+ #taskmanager.rpc.port: 6122
+ #queryable-state.proxy.ports: 6125
+ #jobmanager.memory.process.size: 4Gb
+ #taskmanager.memory.process.size: 4Gb
+ #parallelism.default: 1 #TODO
+ metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
+ metrics.reporter.prom.interval: 10 SECONDS
+ taskmanager.network.detailed-metrics: true
+ # -> gives metrics about inbound/outbound network queue lengths
+ log4j-console.properties: |+
+ # This affects logging for both user code and Flink
+ rootLogger.level = INFO
+ rootLogger.appenderRef.console.ref = ConsoleAppender
+ rootLogger.appenderRef.rolling.ref = RollingFileAppender
+
+ # Uncomment this if you want to _only_ change Flink's logging
+ #logger.flink.name = org.apache.flink
+ #logger.flink.level = INFO
+
+ # The following lines keep the log level of common libraries/connectors on
+ # log level INFO. The root logger does not override this. You have to manually
+ # change the log levels here.
+ logger.akka.name = akka
+ logger.akka.level = INFO
+ logger.kafka.name= org.apache.kafka
+ logger.kafka.level = INFO
+ logger.hadoop.name = org.apache.hadoop
+ logger.hadoop.level = INFO
+ logger.zookeeper.name = org.apache.zookeeper
+ logger.zookeeper.level = INFO
+
+ # Log all infos to the console
+ appender.console.name = ConsoleAppender
+ appender.console.type = CONSOLE
+ appender.console.layout.type = PatternLayout
+ appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+
+ # Log all infos in the given rolling file
+ appender.rolling.name = RollingFileAppender
+ appender.rolling.type = RollingFile
+ appender.rolling.append = false
+ appender.rolling.fileName = ${sys:log.file}
+ appender.rolling.filePattern = ${sys:log.file}.%i
+ appender.rolling.layout.type = PatternLayout
+ appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+ appender.rolling.policies.type = Policies
+ appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
+ appender.rolling.policies.size.size=100MB
+ appender.rolling.strategy.type = DefaultRolloverStrategy
+ appender.rolling.strategy.max = 10
+
+ # Suppress the irrelevant (wrong) warnings from the Netty channel handler
+ logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
+ logger.netty.level = OFF
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc4-flink/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc4-flink/jobmanager-deployment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b6533a2c4355e227a16aeface2080253bce19958
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc4-flink/jobmanager-deployment.yaml
@@ -0,0 +1,93 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: flink-jobmanager
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: flink
+ component: jobmanager
+ template:
+ metadata:
+ labels:
+ app: flink
+ component: jobmanager
+ spec:
+ containers:
+ - name: jobmanager
+ image: ghcr.io/cau-se/theodolite-uc4-flink:latest
+ env:
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-cp-kafka:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
+ - name: COMMIT_INTERVAL_MS
+ value: "100"
+ - name: CHECKPOINTING
+ value: "false"
+ - name: PARALLELISM
+ value: "1"
+ - name: "FLINK_STATE_BACKEND"
+ value: "rocksdb"
+ - name: JOB_MANAGER_RPC_ADDRESS
+ value: "flink-jobmanager"
+ - name: FLINK_PROPERTIES
+ value: |+
+ blob.server.port: 6124
+ jobmanager.rpc.port: 6123
+ taskmanager.rpc.port: 6122
+ queryable-state.proxy.ports: 6125
+ jobmanager.memory.process.size: 4Gb
+ taskmanager.memory.process.size: 4Gb
+ #parallelism.default: 1 #TODO
+ resources:
+ limits:
+ memory: 4Gi
+ cpu: 1000m
+ args: ["standalone-job", "--job-classname", "theodolite.uc4.application.AggregationServiceFlinkJob"] # optional arguments: ["--job-id", "<job id>", "--fromSavepoint", "/path/to/savepoint", "--allowNonRestoredState"]
+ #command: ['sleep', '60m']
+ ports:
+ - containerPort: 6123
+ name: rpc
+ - containerPort: 6124
+ name: blob-server
+ - containerPort: 8081
+ name: webui
+ - containerPort: 9249
+ name: metrics
+ livenessProbe:
+ tcpSocket:
+ port: 6123
+ initialDelaySeconds: 30
+ periodSeconds: 60
+ volumeMounts:
+ - name: flink-config-volume-rw
+ mountPath: /opt/flink/conf
+# - name: job-artifacts-volume
+# mountPath: /opt/flink/usrlib
+ securityContext:
+ runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
+ initContainers:
+ - name: init-jobmanager
+ image: busybox:1.28
+ command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
+ volumeMounts:
+ - name: flink-config-volume
+ mountPath: /flink-config/
+ - name: flink-config-volume-rw
+ mountPath: /flink-config-rw/
+ volumes:
+ - name: flink-config-volume
+ configMap:
+ name: flink-config
+ items:
+ - key: flink-conf.yaml
+ path: flink-conf.yaml
+ - key: log4j-console.properties
+ path: log4j-console.properties
+ - name: flink-config-volume-rw
+ emptyDir: {}
+# - name: job-artifacts-volume
+# hostPath:
+# path: /host/path/to/job/artifacts
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc4-flink/jobmanager-rest-service.yaml b/theodolite-benchmarks/definitions/uc4-flink/jobmanager-rest-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..3d74aaf7f625c6922e2e1b4f20c19e50a39b68ac
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc4-flink/jobmanager-rest-service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flink-jobmanager-rest
+spec:
+ type: NodePort
+ ports:
+ - name: rest
+ port: 8081
+ targetPort: 8081
+ nodePort: 30081
+ selector:
+ app: flink
+ component: jobmanager
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc4-flink/jobmanager-service.yaml b/theodolite-benchmarks/definitions/uc4-flink/jobmanager-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..e2ff5d9898eb1ebf5db9a827472a47514ab1473c
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc4-flink/jobmanager-service.yaml
@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flink-jobmanager
+ labels:
+ app: flink
+spec:
+ type: ClusterIP
+ ports:
+ - name: rpc
+ port: 6123
+ - name: blob-server
+ port: 6124
+ - name: webui
+ port: 8081
+ - name: metrics
+ port: 9249
+ selector:
+ app: flink
+ component: jobmanager
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc4-flink/service-monitor.yaml b/theodolite-benchmarks/definitions/uc4-flink/service-monitor.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..02f78823c627e27ddfe1db5eac3f6a7f7a7f1bf8
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc4-flink/service-monitor.yaml
@@ -0,0 +1,14 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+ labels:
+ app: flink
+ appScope: titan-ccp
+ name: flink
+spec:
+ selector:
+ matchLabels:
+ app: flink
+ endpoints:
+ - port: metrics
+ interval: 10s
diff --git a/theodolite-benchmarks/definitions/uc4-flink/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc4-flink/taskmanager-deployment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..7363b013b21ad29b481e449113ccf31538505634
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc4-flink/taskmanager-deployment.yaml
@@ -0,0 +1,87 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: flink-taskmanager
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: flink
+ component: taskmanager
+ template:
+ metadata:
+ labels:
+ app: flink
+ component: taskmanager
+ spec:
+ containers:
+ - name: taskmanager
+ image: ghcr.io/cau-se/theodolite-uc4-flink:latest
+ env:
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-cp-kafka:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
+ - name: COMMIT_INTERVAL_MS
+ value: "100"
+ - name: CHECKPOINTING
+ value: "false"
+ - name: PARALLELISM
+ value: "1"
+ - name: "FLINK_STATE_BACKEND"
+ value: "rocksdb"
+ - name: JOB_MANAGER_RPC_ADDRESS
+ value: "flink-jobmanager"
+ - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS
+ value: "1" #TODO
+ - name: FLINK_PROPERTIES
+ value: |+
+ blob.server.port: 6124
+ jobmanager.rpc.port: 6123
+ taskmanager.rpc.port: 6122
+ queryable-state.proxy.ports: 6125
+ jobmanager.memory.process.size: 4Gb
+ taskmanager.memory.process.size: 4Gb
+ #parallelism.default: 1 #TODO
+ resources:
+ limits:
+ memory: 4Gi
+ cpu: 1000m
+ args: ["taskmanager"]
+ ports:
+ - containerPort: 6122
+ name: rpc
+ - containerPort: 6125
+ name: query-state
+ - containerPort: 9249
+ name: metrics
+ livenessProbe:
+ tcpSocket:
+ port: 6122
+ initialDelaySeconds: 30
+ periodSeconds: 60
+ volumeMounts:
+ - name: flink-config-volume-rw
+ mountPath: /opt/flink/conf/
+ securityContext:
+ runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
+ initContainers:
+ - name: init-taskmanager
+ image: busybox:1.28
+ command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
+ volumeMounts:
+ - name: flink-config-volume
+ mountPath: /flink-config/
+ - name: flink-config-volume-rw
+ mountPath: /flink-config-rw/
+ volumes:
+ - name: flink-config-volume
+ configMap:
+ name: flink-config
+ items:
+ - key: flink-conf.yaml
+ path: flink-conf.yaml
+ - key: log4j-console.properties
+ path: log4j-console.properties
+ - name: flink-config-volume-rw
+ emptyDir: {}
diff --git a/theodolite-benchmarks/definitions/uc4-flink/taskmanager-service.yaml b/theodolite-benchmarks/definitions/uc4-flink/taskmanager-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a2e27f64af1cfd1a26da142b8a50bb41c8ba5fcb
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc4-flink/taskmanager-service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flink-taskmanager
+ labels:
+ app: flink
+spec:
+ type: ClusterIP
+ ports:
+ - name: metrics
+ port: 9249
+ selector:
+ app: flink
+ component: taskmanager
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc4-kstreams/uc4-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-benchmark-operator.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..0ed48c9afd0d8d02493f7afc2df3e440d0ffabdd
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-benchmark-operator.yaml
@@ -0,0 +1,48 @@
+apiVersion: theodolite.com/v1
+kind: benchmark
+spec:
+ metadata:
+ name: uc4-kstreams
+ appResource:
+ - "uc4-kstreams-deployment.yaml"
+ - "uc4-kstreams-service.yaml"
+ - "uc4-jmx-configmap.yaml"
+ - "uc4-service-monitor.yaml"
+ loadGenResource:
+ - "uc4-load-generator-deployment.yaml"
+ - "uc4-load-generator-service.yaml"
+ resourceTypes:
+ - typeName: "Instances"
+ patchers:
+ - type: "ReplicaPatcher"
+ resource: "uc4-kstreams-deployment.yaml"
+ loadTypes:
+ - typeName: "NumNestedGroups"
+ patchers:
+ - type: "EnvVarPatcher"
+ resource: "uc4-load-generator-deployment.yaml"
+ properties:
+ container: "workload-generator"
+ variableName: "NUM_SENSORS"
+ - type: NumNestedGroupsLoadGeneratorReplicaPatcher
+ resource: "uc4-load-generator-deployment.yaml"
+ properties:
+ loadGenMaxRecords: "15000"
+ numSensors: "4.0"
+ kafkaConfig:
+ bootstrapServer: "theodolite-cp-kafka:9092"
+ topics:
+ - name: "input"
+ numPartitions: 40
+ replicationFactor: 1
+ - name: "output"
+ numPartitions: 40
+ replicationFactor: 1
+ - name: "configuration"
+ numPartitions: 40
+ replicationFactor: 1
+ - name: "aggregation-feedback"
+ numPartitions: 40
+ replicationFactor: 1
+ - name: "theodolite-.*"
+ removeOnly: True
diff --git a/theodolite-benchmarks/definitions/uc4-kstreams/uc4-benchmark-standalone.yaml b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-benchmark-standalone.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..96e72c9b6d726267044464cce6deb32f60442e96
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-benchmark-standalone.yaml
@@ -0,0 +1,44 @@
+name: "uc4-kstreams"
+appResource:
+ - "uc4-kstreams-deployment.yaml"
+ - "uc4-kstreams-service.yaml"
+ - "uc4-jmx-configmap.yaml"
+ - "uc4-service-monitor.yaml"
+loadGenResource:
+ - "uc4-load-generator-deployment.yaml"
+ - "uc4-load-generator-service.yaml"
+resourceTypes:
+ - typeName: "Instances"
+ patchers:
+ - type: "ReplicaPatcher"
+ resource: "uc4-kstreams-deployment.yaml"
+loadTypes:
+ - typeName: "NumNestedGroups"
+ patchers:
+ - type: "EnvVarPatcher"
+ resource: "uc4-load-generator-deployment.yaml"
+ properties:
+ container: "workload-generator"
+ variableName: "NUM_NESTED_GROUPS"
+ - type: "NumNestedGroupsLoadGeneratorReplicaPatcher"
+ resource: "uc4-load-generator-deployment.yaml"
+ properties:
+ loadGenMaxRecords: "15000"
+ numSensors: "4.0"
+kafkaConfig:
+ bootstrapServer: "theodolite-cp-kafka:9092"
+ topics:
+ - name: "input"
+ numPartitions: 40
+ replicationFactor: 1
+ - name: "output"
+ numPartitions: 40
+ replicationFactor: 1
+ - name: "configuration"
+ numPartitions: 40
+ replicationFactor: 1
+ - name: "aggregation-feedback"
+ numPartitions: 40
+ replicationFactor: 1
+ - name: "theodolite-.*"
+ removeOnly: True
diff --git a/theodolite-benchmarks/definitions/uc4-kstreams/uc4-jmx-configmap.yaml b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-jmx-configmap.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..78496a86b1242a89b9e844ead3e700fd0b9a9667
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-jmx-configmap.yaml
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: aggregation-jmx-configmap
+data:
+ jmx-kafka-prometheus.yml: |+
+ jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi
+ lowercaseOutputName: true
+ lowercaseOutputLabelNames: true
+ ssl: false
diff --git a/theodolite-benchmarks/definitions/uc4-kstreams/uc4-kstreams-deployment.yaml b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-kstreams-deployment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..20e0872d262df46b5c213d9d529983f5f4155735
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-kstreams-deployment.yaml
@@ -0,0 +1,55 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: titan-ccp-aggregation
+spec:
+ selector:
+ matchLabels:
+ app: titan-ccp-aggregation
+ replicas: 1
+ template:
+ metadata:
+ labels:
+ app: titan-ccp-aggregation
+ spec:
+ terminationGracePeriodSeconds: 0
+ containers:
+ - name: uc-application
+ image: ghcr.io/cau-se/theodolite-uc4-kstreams-app:latest
+ ports:
+ - containerPort: 5555
+ name: jmx
+ env:
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-cp-kafka:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
+ - name: JAVA_OPTS
+ value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555"
+ - name: COMMIT_INTERVAL_MS # Set as default for the applications
+ value: "100"
+ resources:
+ limits:
+ memory: 4Gi
+ cpu: 1000m
+ - name: prometheus-jmx-exporter
+ image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143"
+ command:
+ - java
+ - -XX:+UnlockExperimentalVMOptions
+ - -XX:+UseCGroupMemoryLimitForHeap
+ - -XX:MaxRAMFraction=1
+ - -XshowSettings:vm
+ - -jar
+ - jmx_prometheus_httpserver.jar
+ - "5556"
+ - /etc/jmx-aggregation/jmx-kafka-prometheus.yml
+ ports:
+ - containerPort: 5556
+ volumeMounts:
+ - name: jmx-config
+ mountPath: /etc/jmx-aggregation
+ volumes:
+ - name: jmx-config
+ configMap:
+ name: aggregation-jmx-configmap
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc4-kstreams/uc4-kstreams-service.yaml b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-kstreams-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..85432d04f225c30469f3232153ef6bd72bd02bdf
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-kstreams-service.yaml
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: titan-ccp-aggregation
+ labels:
+ app: titan-ccp-aggregation
+spec:
+ #type: NodePort
+ selector:
+ app: titan-ccp-aggregation
+ ports:
+ - name: http
+ port: 80
+ targetPort: 80
+ protocol: TCP
+ - name: metrics
+ port: 5556
diff --git a/theodolite-benchmarks/definitions/uc4-kstreams/uc4-load-generator-deployment.yaml b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-load-generator-deployment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..7a69d13daae57b06c77f316da9aa953b21ac096b
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-load-generator-deployment.yaml
@@ -0,0 +1,34 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: titan-ccp-load-generator
+spec:
+ selector:
+ matchLabels:
+ app: titan-ccp-load-generator
+ replicas: 1
+ template:
+ metadata:
+ labels:
+ app: titan-ccp-load-generator
+ spec:
+ terminationGracePeriodSeconds: 0
+ containers:
+ - name: workload-generator
+ image: ghcr.io/cau-se/theodolite-uc4-workload-generator:latest
+ ports:
+ - containerPort: 5701
+ name: coordination
+ env:
+ - name: KUBERNETES_NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ - name: KUBERNETES_DNS_NAME
+ value: "titan-ccp-load-generator.$(KUBERNETES_NAMESPACE).svc.cluster.local"
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-cp-kafka:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
+ - name: NUM_NESTED_GROUPS
+ value: "5"
diff --git a/theodolite-benchmarks/definitions/uc4-kstreams/uc4-load-generator-service.yaml b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-load-generator-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..f8b26b3f6dece427f9c1ad4db94e351b042749b3
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-load-generator-service.yaml
@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: titan-ccp-load-generator
+ labels:
+ app: titan-ccp-load-generator
+spec:
+ type: ClusterIP
+ clusterIP: None
+ selector:
+ app: titan-ccp-load-generator
+ ports:
+ - name: coordination
+ port: 5701
+ targetPort: 5701
+ protocol: TCP
diff --git a/theodolite-benchmarks/definitions/uc4-kstreams/uc4-service-monitor.yaml b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-service-monitor.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..4e7e758cacb5086305efa26292ddef2afc958096
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-service-monitor.yaml
@@ -0,0 +1,14 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+ labels:
+ app: titan-ccp-aggregation
+ appScope: titan-ccp
+ name: titan-ccp-aggregation
+spec:
+ selector:
+ matchLabels:
+ app: titan-ccp-aggregation
+ endpoints:
+ - port: metrics
+ interval: 10s
diff --git a/theodolite-benchmarks/uc1-flink/src/main/java/theodolite/uc1/application/HistoryServiceFlinkJob.java b/theodolite-benchmarks/uc1-flink/src/main/java/theodolite/uc1/application/HistoryServiceFlinkJob.java
index 8d9832e40253fe9e3178bfc25047ed2b376abe76..0cb132e526486e71409736b843dd25bdfa52da4a 100644
--- a/theodolite-benchmarks/uc1-flink/src/main/java/theodolite/uc1/application/HistoryServiceFlinkJob.java
+++ b/theodolite-benchmarks/uc1-flink/src/main/java/theodolite/uc1/application/HistoryServiceFlinkJob.java
@@ -47,7 +47,7 @@ public final class HistoryServiceFlinkJob {
// Parallelism
final Integer parallelism = this.config.getInteger(ConfigurationKeys.PARALLELISM, null);
if (parallelism != null) {
- LOGGER.error("Set parallelism: {}.", parallelism);
+ LOGGER.info("Set parallelism: {}.", parallelism);
this.env.setParallelism(parallelism);
}
@@ -68,7 +68,7 @@ public final class HistoryServiceFlinkJob {
final DataStream<ActivePowerRecord> stream = this.env.addSource(kafkaConsumer);
stream
- .rebalance()
+ // .rebalance()
.map(new GsonMapper())
.flatMap((record, c) -> LOGGER.info("Record: {}", record))
.returns(Types.GENERIC(Object.class)); // Will never be used
diff --git a/theodolite-benchmarks/uc2-flink/src/main/java/theodolite/uc2/application/HistoryServiceFlinkJob.java b/theodolite-benchmarks/uc2-flink/src/main/java/theodolite/uc2/application/HistoryServiceFlinkJob.java
index 1068267086892c4538001b6afc670b3b0cd043ef..d156d895d86bb01a31f96e08764df8b8df743c4d 100644
--- a/theodolite-benchmarks/uc2-flink/src/main/java/theodolite/uc2/application/HistoryServiceFlinkJob.java
+++ b/theodolite-benchmarks/uc2-flink/src/main/java/theodolite/uc2/application/HistoryServiceFlinkJob.java
@@ -59,7 +59,7 @@ public final class HistoryServiceFlinkJob {
// Parallelism
final Integer parallelism = this.config.getInteger(ConfigurationKeys.PARALLELISM, null);
if (parallelism != null) {
- LOGGER.error("Set parallelism: {}.", parallelism);
+ LOGGER.info("Set parallelism: {}.", parallelism);
this.env.setParallelism(parallelism);
}
@@ -83,7 +83,9 @@ public final class HistoryServiceFlinkJob {
final String schemaRegistryUrl = this.config.getString(ConfigurationKeys.SCHEMA_REGISTRY_URL);
final String inputTopic = this.config.getString(ConfigurationKeys.KAFKA_INPUT_TOPIC);
final String outputTopic = this.config.getString(ConfigurationKeys.KAFKA_OUTPUT_TOPIC);
- final int windowDuration = this.config.getInt(ConfigurationKeys.KAFKA_WINDOW_DURATION_MINUTES);
+ final int windowDurationMinutes =
+ this.config.getInt(ConfigurationKeys.KAFKA_WINDOW_DURATION_MINUTES);
+ final Time windowDuration = Time.minutes(windowDurationMinutes);
final boolean checkpointing = this.config.getBoolean(ConfigurationKeys.CHECKPOINTING, true);
final KafkaConnectorFactory kafkaConnector = new KafkaConnectorFactory(
@@ -100,9 +102,9 @@ public final class HistoryServiceFlinkJob {
this.env
.addSource(kafkaSource).name("[Kafka Consumer] Topic: " + inputTopic)
- .rebalance()
+ // .rebalance()
.keyBy(ActivePowerRecord::getIdentifier)
- .window(TumblingEventTimeWindows.of(Time.minutes(windowDuration)))
+ .window(TumblingEventTimeWindows.of(windowDuration))
.aggregate(new StatsAggregateFunction(), new StatsProcessWindowFunction())
.map(t -> {
final String key = t.f0;
diff --git a/theodolite-benchmarks/uc3-flink/src/main/java/theodolite/uc3/application/HistoryServiceFlinkJob.java b/theodolite-benchmarks/uc3-flink/src/main/java/theodolite/uc3/application/HistoryServiceFlinkJob.java
index d69ee47d8c831f2e5e74abdd8c33393c8ee6e07e..091b25674a2a31671ca68bd2076c694da9533d77 100644
--- a/theodolite-benchmarks/uc3-flink/src/main/java/theodolite/uc3/application/HistoryServiceFlinkJob.java
+++ b/theodolite-benchmarks/uc3-flink/src/main/java/theodolite/uc3/application/HistoryServiceFlinkJob.java
@@ -117,9 +117,8 @@ public final class HistoryServiceFlinkJob {
// Streaming topology
final StatsKeyFactory<HourOfDayKey> keyFactory = new HourOfDayKeyFactory();
this.env
- .addSource(kafkaSource)
- .name("[Kafka Consumer] Topic: " + inputTopic)
- .rebalance()
+ .addSource(kafkaSource).name("[Kafka Consumer] Topic: " + inputTopic)
+ // .rebalance()
.keyBy((KeySelector<ActivePowerRecord, HourOfDayKey>) record -> {
final Instant instant = Instant.ofEpochMilli(record.getTimestamp());
final LocalDateTime dateTime = LocalDateTime.ofInstant(instant, timeZone);
diff --git a/theodolite-benchmarks/uc4-flink/src/main/java/theodolite/uc4/application/AggregationServiceFlinkJob.java b/theodolite-benchmarks/uc4-flink/src/main/java/theodolite/uc4/application/AggregationServiceFlinkJob.java
index 45c7ff1ad1faeec6357e4ac3871dec7a51306698..3e2878a893057024de00333492462f5029eb6d77 100644
--- a/theodolite-benchmarks/uc4-flink/src/main/java/theodolite/uc4/application/AggregationServiceFlinkJob.java
+++ b/theodolite-benchmarks/uc4-flink/src/main/java/theodolite/uc4/application/AggregationServiceFlinkJob.java
@@ -79,7 +79,7 @@ public final class AggregationServiceFlinkJob {
// Parallelism
final Integer parallelism = this.config.getInteger(ConfigurationKeys.PARALLELISM, null);
if (parallelism != null) {
- LOGGER.error("Set parallelism: {}.", parallelism);
+ LOGGER.info("Set parallelism: {}.", parallelism);
this.env.setParallelism(parallelism);
}
@@ -152,7 +152,7 @@ public final class AggregationServiceFlinkJob {
// Build input stream
final DataStream<ActivePowerRecord> inputStream = this.env.addSource(kafkaInputSource)
.name("[Kafka Consumer] Topic: " + inputTopic)// NOCS
- .rebalance()
+ // .rebalance()
.map(r -> r)
.name("[Map] Rebalance Forward");
@@ -160,7 +160,7 @@ public final class AggregationServiceFlinkJob {
final DataStream<ActivePowerRecord> aggregationsInputStream =
this.env.addSource(kafkaOutputSource)
.name("[Kafka Consumer] Topic: " + outputTopic) // NOCS
- .rebalance()
+ // .rebalance()
.map(r -> new ActivePowerRecord(r.getIdentifier(), r.getTimestamp(), r.getSumInW()))
.name("[Map] AggregatedActivePowerRecord -> ActivePowerRecord");
diff --git a/theodolite-quarkus/config/thedolite-operator.yaml b/theodolite-quarkus/config/thedolite-operator.yaml
deleted file mode 100644
index 1e0e60248c2474cc8493179c003b806030f79f8c..0000000000000000000000000000000000000000
--- a/theodolite-quarkus/config/thedolite-operator.yaml
+++ /dev/null
@@ -1,24 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: theodolite
-spec:
- selector:
- matchLabels:
- app: theodolite
- replicas: 1
- template:
- metadata:
- labels:
- app: theodolite
- spec:
- terminationGracePeriodSeconds: 0
- serviceAccountName: theodolite
- containers:
- - name: thedolite
- image: ghcr.io/cau-se/theodolite:theodolite-kotlin-latest
- env:
- - name: KUBECONFIG
- value: "~/.kube/config"
- - name: NAMESPACE
- value: "default"
\ No newline at end of file
diff --git a/theodolite-quarkus/src/main/resources/operator/benchmarkCRD.yaml b/theodolite-quarkus/crd/crd-benchmark.yaml
similarity index 100%
rename from theodolite-quarkus/src/main/resources/operator/benchmarkCRD.yaml
rename to theodolite-quarkus/crd/crd-benchmark.yaml
diff --git a/theodolite-quarkus/src/main/resources/operator/executionCRD.yaml b/theodolite-quarkus/crd/crd-execution.yaml
similarity index 100%
rename from theodolite-quarkus/src/main/resources/operator/executionCRD.yaml
rename to theodolite-quarkus/crd/crd-execution.yaml
diff --git a/theodolite-quarkus/src/main/resources/operator/example-benchmark-k8s-resource.yaml b/theodolite-quarkus/examples/operator/example-benchmark.yaml
similarity index 91%
rename from theodolite-quarkus/src/main/resources/operator/example-benchmark-k8s-resource.yaml
rename to theodolite-quarkus/examples/operator/example-benchmark.yaml
index 7a728d97c52a84d81bcc44fd9f20c56e6b574527..91d9f8f1f7dfed31d9edcb59947af4e832ca2843 100644
--- a/theodolite-quarkus/src/main/resources/operator/example-benchmark-k8s-resource.yaml
+++ b/theodolite-quarkus/examples/operator/example-benchmark.yaml
@@ -3,15 +3,14 @@ kind: benchmark
metadata:
name: uc1-kstreams
spec:
- name: test
appResource:
- "uc1-kstreams-deployment.yaml"
- "aggregation-service.yaml"
- "jmx-configmap.yaml"
+ - "uc1-service-monitor.yaml"
loadGenResource:
- "uc1-load-generator-deployment.yaml"
- "uc1-load-generator-service.yaml"
- - "uc1-load-generator-service.yaml"
resourceTypes:
- typeName: "Instances"
patchers:
@@ -23,14 +22,14 @@ spec:
- type: "EnvVarPatcher"
resource: "uc1-load-generator-deployment.yaml"
properties:
- container: "workload-generator"
variableName: "NUM_SENSORS"
+ container: "workload-generator"
- type: "NumSensorsLoadGeneratorReplicaPatcher"
resource: "uc1-load-generator-deployment.yaml"
properties:
loadGenMaxRecords: "15000"
kafkaConfig:
- bootstrapServer: "localhost:31290"
+ bootstrapServer: "theodolite-cp-kafka:9092"
topics:
- name: "input"
numPartitions: 40
diff --git a/theodolite-quarkus/examples/operator/example-execution.yaml b/theodolite-quarkus/examples/operator/example-execution.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5386fd7c8665e01302067da81c5dd4caf87fc602
--- /dev/null
+++ b/theodolite-quarkus/examples/operator/example-execution.yaml
@@ -0,0 +1,57 @@
+apiVersion: theodolite.com/v1
+kind: execution
+metadata:
+ name: theodolite-example-execution
+spec:
+ benchmark: "uc1-kstreams"
+ load:
+ loadType: "NumSensors"
+ loadValues: [25000, 50000, 75000, 100000, 125000, 150000]
+ resources:
+ resourceType: "Instances"
+ resourceValues: [1, 2, 3, 4, 5]
+ slos:
+ - sloType: "lag trend"
+ threshold: 2000
+ prometheusUrl: "http://prometheus-operated:9090"
+ externalSloUrl: "http://localhost:80/evaluate-slope"
+ offset: 0
+ warmup: 60 # in seconds
+ execution:
+ strategy: "LinearSearch"
+ duration: 300 # in seconds
+ repetitions: 1
+ loadGenerationDelay: 30 # in seconds
+ restrictions:
+ - "LowerBound"
+ configOverrides:
+ # - patcher:
+ # type: "NodeSelectorPatcher"
+ # resource: "uc1-load-generator-deployment.yaml"
+ # properties:
+ # variableName: "env"
+ # value: "prod"
+ # - patcher:
+ # type: "NodeSelectorPatcher"
+ # resource: "uc1-kstreams-deployment.yaml"
+ # properties:
+ # variableName: "env"
+ # value: "prod"
+ # - patcher:
+ # type: "ResourceLimitPatcher"
+ # resource: "uc1-kstreams-deployment.yaml"
+ # properties:
+ # container: "uc-application"
+ # limitedResource: "cpu"
+ # value: "1000m"
+ # - patcher:
+ # type: "ResourceLimitPatcher"
+ # resource: "uc1-kstreams-deployment.yaml"
+ # properties:
+ # container: "uc-application"
+ # limitedResource: "memory"
+ # value: "2Gi"
+ # - patcher:
+ # type: "SchedulerNamePatcher"
+ # resource: "uc1-kstreams-deployment.yaml"
+ # value: "random-scheduler"
diff --git a/theodolite-quarkus/config/example-benchmark-yaml-resource.yaml b/theodolite-quarkus/examples/standalone/example-benchmark.yaml
similarity index 97%
rename from theodolite-quarkus/config/example-benchmark-yaml-resource.yaml
rename to theodolite-quarkus/examples/standalone/example-benchmark.yaml
index 623df71121b11b670584a83e153b4dd2962525e7..83edce93834ca9b8eef5606c1e5884ce40bdd7d8 100644
--- a/theodolite-quarkus/config/example-benchmark-yaml-resource.yaml
+++ b/theodolite-quarkus/examples/standalone/example-benchmark.yaml
@@ -31,4 +31,4 @@ kafkaConfig:
numPartitions: 40
replicationFactor: 1
- name: "theodolite-.*"
- removeOnly: True
\ No newline at end of file
+ removeOnly: True
diff --git a/theodolite-quarkus/examples/standalone/example-execution.yaml b/theodolite-quarkus/examples/standalone/example-execution.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..24b2b7f32e803553a4a13b76869ccf4cf3f6e5a5
--- /dev/null
+++ b/theodolite-quarkus/examples/standalone/example-execution.yaml
@@ -0,0 +1,23 @@
+name: example-execution
+benchmark: "uc1-kstreams"
+load:
+ loadType: "NumSensors"
+ loadValues: [25000, 50000, 75000, 100000, 125000, 150000]
+resources:
+ resourceType: "Instances"
+ resourceValues: [1, 2, 3, 4, 5]
+slos:
+ - sloType: "lag trend"
+ threshold: 2000
+ prometheusUrl: "http://prometheus-operated:9090"
+ externalSloUrl: "http://localhost:80/evaluate-slope"
+ offset: 0
+ warmup: 60 # in seconds
+execution:
+ strategy: "LinearSearch"
+ duration: 300 # in seconds
+ repetitions: 1
+ loadGenerationDelay: 30 # in seconds, optional field, default is 0 seconds
+ restrictions:
+ - "LowerBound"
+configOverrides: []
\ No newline at end of file