From 28798d93b9ba300d3a9a12cd5ef5e05bc6485739 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=B6ren=20Henning?= <soeren.henning@email.uni-kiel.de>
Date: Wed, 28 Apr 2021 19:01:29 +0200
Subject: [PATCH] Add Flink benchmark resources
Co-authored-by: Nico Biernat <stu209212@mail.uni-kiel.de>
---
.../flink-configuration-configmap.yaml | 66 +++++++++++++
.../uc1-flink/jobmanager-deployment.yaml | 93 +++++++++++++++++++
.../uc1-flink/jobmanager-rest-service.yaml | 14 +++
.../uc1-flink/jobmanager-service.yaml | 20 ++++
.../uc1-flink/service-monitor.yaml | 14 +++
.../uc1-flink/taskmanager-deployment.yaml | 87 +++++++++++++++++
.../uc1-flink/taskmanager-service.yaml | 14 +++
.../flink-configuration-configmap.yaml | 66 +++++++++++++
.../uc2-flink/jobmanager-deployment.yaml | 93 +++++++++++++++++++
.../uc2-flink/jobmanager-rest-service.yaml | 14 +++
.../uc2-flink/jobmanager-service.yaml | 20 ++++
.../uc2-flink/service-monitor.yaml | 14 +++
.../uc2-flink/taskmanager-deployment.yaml | 87 +++++++++++++++++
.../uc2-flink/taskmanager-service.yaml | 14 +++
.../flink-configuration-configmap.yaml | 66 +++++++++++++
.../uc3-flink/jobmanager-deployment.yaml | 93 +++++++++++++++++++
.../uc3-flink/jobmanager-rest-service.yaml | 14 +++
.../uc3-flink/jobmanager-service.yaml | 20 ++++
.../uc3-flink/service-monitor.yaml | 14 +++
.../uc3-flink/taskmanager-deployment.yaml | 87 +++++++++++++++++
.../uc3-flink/taskmanager-service.yaml | 14 +++
.../flink-configuration-configmap.yaml | 66 +++++++++++++
.../uc4-flink/jobmanager-deployment.yaml | 93 +++++++++++++++++++
.../uc4-flink/jobmanager-rest-service.yaml | 14 +++
.../uc4-flink/jobmanager-service.yaml | 20 ++++
.../uc4-flink/service-monitor.yaml | 14 +++
.../uc4-flink/taskmanager-deployment.yaml | 87 +++++++++++++++++
.../uc4-flink/taskmanager-service.yaml | 14 +++
28 files changed, 1232 insertions(+)
create mode 100644 theodolite-benchmarks/definitions/uc1-flink/flink-configuration-configmap.yaml
create mode 100644 theodolite-benchmarks/definitions/uc1-flink/jobmanager-deployment.yaml
create mode 100644 theodolite-benchmarks/definitions/uc1-flink/jobmanager-rest-service.yaml
create mode 100644 theodolite-benchmarks/definitions/uc1-flink/jobmanager-service.yaml
create mode 100644 theodolite-benchmarks/definitions/uc1-flink/service-monitor.yaml
create mode 100644 theodolite-benchmarks/definitions/uc1-flink/taskmanager-deployment.yaml
create mode 100644 theodolite-benchmarks/definitions/uc1-flink/taskmanager-service.yaml
create mode 100644 theodolite-benchmarks/definitions/uc2-flink/flink-configuration-configmap.yaml
create mode 100644 theodolite-benchmarks/definitions/uc2-flink/jobmanager-deployment.yaml
create mode 100644 theodolite-benchmarks/definitions/uc2-flink/jobmanager-rest-service.yaml
create mode 100644 theodolite-benchmarks/definitions/uc2-flink/jobmanager-service.yaml
create mode 100644 theodolite-benchmarks/definitions/uc2-flink/service-monitor.yaml
create mode 100644 theodolite-benchmarks/definitions/uc2-flink/taskmanager-deployment.yaml
create mode 100644 theodolite-benchmarks/definitions/uc2-flink/taskmanager-service.yaml
create mode 100644 theodolite-benchmarks/definitions/uc3-flink/flink-configuration-configmap.yaml
create mode 100644 theodolite-benchmarks/definitions/uc3-flink/jobmanager-deployment.yaml
create mode 100644 theodolite-benchmarks/definitions/uc3-flink/jobmanager-rest-service.yaml
create mode 100644 theodolite-benchmarks/definitions/uc3-flink/jobmanager-service.yaml
create mode 100644 theodolite-benchmarks/definitions/uc3-flink/service-monitor.yaml
create mode 100644 theodolite-benchmarks/definitions/uc3-flink/taskmanager-deployment.yaml
create mode 100644 theodolite-benchmarks/definitions/uc3-flink/taskmanager-service.yaml
create mode 100644 theodolite-benchmarks/definitions/uc4-flink/flink-configuration-configmap.yaml
create mode 100644 theodolite-benchmarks/definitions/uc4-flink/jobmanager-deployment.yaml
create mode 100644 theodolite-benchmarks/definitions/uc4-flink/jobmanager-rest-service.yaml
create mode 100644 theodolite-benchmarks/definitions/uc4-flink/jobmanager-service.yaml
create mode 100644 theodolite-benchmarks/definitions/uc4-flink/service-monitor.yaml
create mode 100644 theodolite-benchmarks/definitions/uc4-flink/taskmanager-deployment.yaml
create mode 100644 theodolite-benchmarks/definitions/uc4-flink/taskmanager-service.yaml
diff --git a/theodolite-benchmarks/definitions/uc1-flink/flink-configuration-configmap.yaml b/theodolite-benchmarks/definitions/uc1-flink/flink-configuration-configmap.yaml
new file mode 100644
index 000000000..36178e2be
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc1-flink/flink-configuration-configmap.yaml
@@ -0,0 +1,66 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: flink-config
+ labels:
+ app: flink
+data:
+ flink-conf.yaml: |+
+ jobmanager.rpc.address: flink-jobmanager
+ taskmanager.numberOfTaskSlots: 1 #TODO
+ #blob.server.port: 6124
+ #jobmanager.rpc.port: 6123
+ #taskmanager.rpc.port: 6122
+ #queryable-state.proxy.ports: 6125
+ #jobmanager.memory.process.size: 4Gb
+ #taskmanager.memory.process.size: 4Gb
+ #parallelism.default: 1 #TODO
+ metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
+ metrics.reporter.prom.interval: 10 SECONDS
+ taskmanager.network.detailed-metrics: true
+ # -> gives metrics about inbound/outbound network queue lengths
+ log4j-console.properties: |+
+ # This affects logging for both user code and Flink
+ rootLogger.level = INFO
+ rootLogger.appenderRef.console.ref = ConsoleAppender
+ rootLogger.appenderRef.rolling.ref = RollingFileAppender
+
+ # Uncomment this if you want to _only_ change Flink's logging
+ #logger.flink.name = org.apache.flink
+ #logger.flink.level = INFO
+
+ # The following lines keep the log level of common libraries/connectors on
+ # log level INFO. The root logger does not override this. You have to manually
+ # change the log levels here.
+ logger.akka.name = akka
+ logger.akka.level = INFO
+ logger.kafka.name= org.apache.kafka
+ logger.kafka.level = INFO
+ logger.hadoop.name = org.apache.hadoop
+ logger.hadoop.level = INFO
+ logger.zookeeper.name = org.apache.zookeeper
+ logger.zookeeper.level = INFO
+
+ # Log all infos to the console
+ appender.console.name = ConsoleAppender
+ appender.console.type = CONSOLE
+ appender.console.layout.type = PatternLayout
+ appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+
+ # Log all infos in the given rolling file
+ appender.rolling.name = RollingFileAppender
+ appender.rolling.type = RollingFile
+ appender.rolling.append = false
+ appender.rolling.fileName = ${sys:log.file}
+ appender.rolling.filePattern = ${sys:log.file}.%i
+ appender.rolling.layout.type = PatternLayout
+ appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+ appender.rolling.policies.type = Policies
+ appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
+ appender.rolling.policies.size.size=100MB
+ appender.rolling.strategy.type = DefaultRolloverStrategy
+ appender.rolling.strategy.max = 10
+
+ # Suppress the irrelevant (wrong) warnings from the Netty channel handler
+ logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
+ logger.netty.level = OFF
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc1-flink/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc1-flink/jobmanager-deployment.yaml
new file mode 100644
index 000000000..512d4fe3c
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc1-flink/jobmanager-deployment.yaml
@@ -0,0 +1,93 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: flink-jobmanager
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: flink
+ component: jobmanager
+ template:
+ metadata:
+ labels:
+ app: flink
+ component: jobmanager
+ spec:
+ containers:
+ - name: jobmanager
+ image: ghcr.io/cau-se/theodolite-uc1-flink:latest
+ env:
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-cp-kafka:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
+ - name: COMMIT_INTERVAL_MS
+ value: "100"
+ - name: CHECKPOINTING
+ value: "false"
+ - name: PARALLELISM
+ value: "1"
+ - name: "FLINK_STATE_BACKEND"
+ value: "rocksdb"
+ - name: JOB_MANAGER_RPC_ADDRESS
+ value: "flink-jobmanager"
+ - name: FLINK_PROPERTIES
+ value: |+
+ blob.server.port: 6124
+ jobmanager.rpc.port: 6123
+ taskmanager.rpc.port: 6122
+ queryable-state.proxy.ports: 6125
+ jobmanager.memory.process.size: 4Gb
+ taskmanager.memory.process.size: 4Gb
+ #parallelism.default: 1 #TODO
+ resources:
+ limits:
+ memory: 4Gi
+ cpu: 1000m
+ args: ["standalone-job", "--job-classname", "theodolite.uc1.application.HistoryServiceFlinkJob"] # optional arguments: ["--job-id", "<job id>", "--fromSavepoint", "/path/to/savepoint", "--allowNonRestoredState"]
+ #command: ['sleep', '60m']
+ ports:
+ - containerPort: 6123
+ name: rpc
+ - containerPort: 6124
+ name: blob-server
+ - containerPort: 8081
+ name: webui
+ - containerPort: 9249
+ name: metrics
+ livenessProbe:
+ tcpSocket:
+ port: 6123
+ initialDelaySeconds: 30
+ periodSeconds: 60
+ volumeMounts:
+ - name: flink-config-volume-rw
+ mountPath: /opt/flink/conf
+# - name: job-artifacts-volume
+# mountPath: /opt/flink/usrlib
+ securityContext:
+ runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
+ initContainers:
+ - name: init-jobmanager
+ image: busybox:1.28
+ command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
+ volumeMounts:
+ - name: flink-config-volume
+ mountPath: /flink-config/
+ - name: flink-config-volume-rw
+ mountPath: /flink-config-rw/
+ volumes:
+ - name: flink-config-volume
+ configMap:
+ name: flink-config
+ items:
+ - key: flink-conf.yaml
+ path: flink-conf.yaml
+ - key: log4j-console.properties
+ path: log4j-console.properties
+ - name: flink-config-volume-rw
+ emptyDir: {}
+# - name: job-artifacts-volume
+# hostPath:
+# path: /host/path/to/job/artifacts
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc1-flink/jobmanager-rest-service.yaml b/theodolite-benchmarks/definitions/uc1-flink/jobmanager-rest-service.yaml
new file mode 100644
index 000000000..3d74aaf7f
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc1-flink/jobmanager-rest-service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flink-jobmanager-rest
+spec:
+ type: NodePort
+ ports:
+ - name: rest
+ port: 8081
+ targetPort: 8081
+ nodePort: 30081
+ selector:
+ app: flink
+ component: jobmanager
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc1-flink/jobmanager-service.yaml b/theodolite-benchmarks/definitions/uc1-flink/jobmanager-service.yaml
new file mode 100644
index 000000000..e2ff5d989
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc1-flink/jobmanager-service.yaml
@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flink-jobmanager
+ labels:
+ app: flink
+spec:
+ type: ClusterIP
+ ports:
+ - name: rpc
+ port: 6123
+ - name: blob-server
+ port: 6124
+ - name: webui
+ port: 8081
+ - name: metrics
+ port: 9249
+ selector:
+ app: flink
+ component: jobmanager
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc1-flink/service-monitor.yaml b/theodolite-benchmarks/definitions/uc1-flink/service-monitor.yaml
new file mode 100644
index 000000000..02f78823c
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc1-flink/service-monitor.yaml
@@ -0,0 +1,14 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+ labels:
+ app: flink
+ appScope: titan-ccp
+ name: flink
+spec:
+ selector:
+ matchLabels:
+ app: flink
+ endpoints:
+ - port: metrics
+ interval: 10s
diff --git a/theodolite-benchmarks/definitions/uc1-flink/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc1-flink/taskmanager-deployment.yaml
new file mode 100644
index 000000000..7d4655469
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc1-flink/taskmanager-deployment.yaml
@@ -0,0 +1,87 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: flink-taskmanager
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: flink
+ component: taskmanager
+ template:
+ metadata:
+ labels:
+ app: flink
+ component: taskmanager
+ spec:
+ containers:
+ - name: taskmanager
+ image: ghcr.io/cau-se/theodolite-uc1-flink:latest
+ env:
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-cp-kafka:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
+ - name: COMMIT_INTERVAL_MS
+ value: "100"
+ - name: CHECKPOINTING
+ value: "false"
+ - name: PARALLELISM
+ value: "1"
+ - name: "FLINK_STATE_BACKEND"
+ value: "rocksdb"
+ - name: JOB_MANAGER_RPC_ADDRESS
+ value: "flink-jobmanager"
+ - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS
+ value: "1" #TODO
+ - name: FLINK_PROPERTIES
+ value: |+
+ blob.server.port: 6124
+ jobmanager.rpc.port: 6123
+ taskmanager.rpc.port: 6122
+ queryable-state.proxy.ports: 6125
+ jobmanager.memory.process.size: 4Gb
+ taskmanager.memory.process.size: 4Gb
+ #parallelism.default: 1 #TODO
+ resources:
+ limits:
+ memory: 4Gi
+ cpu: 1000m
+ args: ["taskmanager"]
+ ports:
+ - containerPort: 6122
+ name: rpc
+ - containerPort: 6125
+ name: query-state
+ - containerPort: 9249
+ name: metrics
+ livenessProbe:
+ tcpSocket:
+ port: 6122
+ initialDelaySeconds: 30
+ periodSeconds: 60
+ volumeMounts:
+ - name: flink-config-volume-rw
+ mountPath: /opt/flink/conf/
+ securityContext:
+ runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
+ initContainers:
+ - name: init-taskmanager
+ image: busybox:1.28
+ command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
+ volumeMounts:
+ - name: flink-config-volume
+ mountPath: /flink-config/
+ - name: flink-config-volume-rw
+ mountPath: /flink-config-rw/
+ volumes:
+ - name: flink-config-volume
+ configMap:
+ name: flink-config
+ items:
+ - key: flink-conf.yaml
+ path: flink-conf.yaml
+ - key: log4j-console.properties
+ path: log4j-console.properties
+ - name: flink-config-volume-rw
+ emptyDir: {}
diff --git a/theodolite-benchmarks/definitions/uc1-flink/taskmanager-service.yaml b/theodolite-benchmarks/definitions/uc1-flink/taskmanager-service.yaml
new file mode 100644
index 000000000..a2e27f64a
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc1-flink/taskmanager-service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flink-taskmanager
+ labels:
+ app: flink
+spec:
+ type: ClusterIP
+ ports:
+ - name: metrics
+ port: 9249
+ selector:
+ app: flink
+ component: taskmanager
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc2-flink/flink-configuration-configmap.yaml b/theodolite-benchmarks/definitions/uc2-flink/flink-configuration-configmap.yaml
new file mode 100644
index 000000000..321541f6a
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc2-flink/flink-configuration-configmap.yaml
@@ -0,0 +1,66 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: flink-config
+ labels:
+ app: flink
+data:
+ flink-conf.yaml: |+
+ #jobmanager.rpc.address: flink-jobmanager
+ #taskmanager.numberOfTaskSlots: 1 #TODO
+ #blob.server.port: 6124
+ #jobmanager.rpc.port: 6123
+ #taskmanager.rpc.port: 6122
+ #queryable-state.proxy.ports: 6125
+ #jobmanager.memory.process.size: 4Gb
+ #taskmanager.memory.process.size: 4Gb
+ #parallelism.default: 1 #TODO
+ metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
+ metrics.reporter.prom.interval: 10 SECONDS
+ taskmanager.network.detailed-metrics: true
+ # -> gives metrics about inbound/outbound network queue lengths
+ log4j-console.properties: |+
+ # This affects logging for both user code and Flink
+ rootLogger.level = INFO
+ rootLogger.appenderRef.console.ref = ConsoleAppender
+ rootLogger.appenderRef.rolling.ref = RollingFileAppender
+
+ # Uncomment this if you want to _only_ change Flink's logging
+ #logger.flink.name = org.apache.flink
+ #logger.flink.level = INFO
+
+ # The following lines keep the log level of common libraries/connectors on
+ # log level INFO. The root logger does not override this. You have to manually
+ # change the log levels here.
+ logger.akka.name = akka
+ logger.akka.level = INFO
+ logger.kafka.name= org.apache.kafka
+ logger.kafka.level = INFO
+ logger.hadoop.name = org.apache.hadoop
+ logger.hadoop.level = INFO
+ logger.zookeeper.name = org.apache.zookeeper
+ logger.zookeeper.level = INFO
+
+ # Log all infos to the console
+ appender.console.name = ConsoleAppender
+ appender.console.type = CONSOLE
+ appender.console.layout.type = PatternLayout
+ appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+
+ # Log all infos in the given rolling file
+ appender.rolling.name = RollingFileAppender
+ appender.rolling.type = RollingFile
+ appender.rolling.append = false
+ appender.rolling.fileName = ${sys:log.file}
+ appender.rolling.filePattern = ${sys:log.file}.%i
+ appender.rolling.layout.type = PatternLayout
+ appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+ appender.rolling.policies.type = Policies
+ appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
+ appender.rolling.policies.size.size=100MB
+ appender.rolling.strategy.type = DefaultRolloverStrategy
+ appender.rolling.strategy.max = 10
+
+ # Suppress the irrelevant (wrong) warnings from the Netty channel handler
+ logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
+ logger.netty.level = OFF
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc2-flink/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc2-flink/jobmanager-deployment.yaml
new file mode 100644
index 000000000..cece4286d
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc2-flink/jobmanager-deployment.yaml
@@ -0,0 +1,93 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: flink-jobmanager
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: flink
+ component: jobmanager
+ template:
+ metadata:
+ labels:
+ app: flink
+ component: jobmanager
+ spec:
+ containers:
+ - name: jobmanager
+ image: ghcr.io/cau-se/theodolite-uc2-flink:latest
+ env:
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-cp-kafka:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
+ - name: COMMIT_INTERVAL_MS
+ value: "100"
+ - name: CHECKPOINTING
+ value: "false"
+ - name: PARALLELISM
+ value: "1"
+ - name: "FLINK_STATE_BACKEND"
+ value: "rocksdb"
+ - name: JOB_MANAGER_RPC_ADDRESS
+ value: "flink-jobmanager"
+ - name: FLINK_PROPERTIES
+ value: |+
+ blob.server.port: 6124
+ jobmanager.rpc.port: 6123
+ taskmanager.rpc.port: 6122
+ queryable-state.proxy.ports: 6125
+ jobmanager.memory.process.size: 4Gb
+ taskmanager.memory.process.size: 4Gb
+ #parallelism.default: 1 #TODO
+ resources:
+ limits:
+ memory: 4Gi
+ cpu: 1000m
+ args: ["standalone-job", "--job-classname", "theodolite.uc2.application.HistoryServiceFlinkJob"] # optional arguments: ["--job-id", "<job id>", "--fromSavepoint", "/path/to/savepoint", "--allowNonRestoredState"]
+ #command: ['sleep', '60m']
+ ports:
+ - containerPort: 6123
+ name: rpc
+ - containerPort: 6124
+ name: blob-server
+ - containerPort: 8081
+ name: webui
+ - containerPort: 9249
+ name: metrics
+ livenessProbe:
+ tcpSocket:
+ port: 6123
+ initialDelaySeconds: 30
+ periodSeconds: 60
+ volumeMounts:
+ - name: flink-config-volume-rw
+ mountPath: /opt/flink/conf
+# - name: job-artifacts-volume
+# mountPath: /opt/flink/usrlib
+ securityContext:
+ runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
+ initContainers:
+ - name: init-jobmanager
+ image: busybox:1.28
+ command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
+ volumeMounts:
+ - name: flink-config-volume
+ mountPath: /flink-config/
+ - name: flink-config-volume-rw
+ mountPath: /flink-config-rw/
+ volumes:
+ - name: flink-config-volume
+ configMap:
+ name: flink-config
+ items:
+ - key: flink-conf.yaml
+ path: flink-conf.yaml
+ - key: log4j-console.properties
+ path: log4j-console.properties
+ - name: flink-config-volume-rw
+ emptyDir: {}
+# - name: job-artifacts-volume
+# hostPath:
+# path: /host/path/to/job/artifacts
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc2-flink/jobmanager-rest-service.yaml b/theodolite-benchmarks/definitions/uc2-flink/jobmanager-rest-service.yaml
new file mode 100644
index 000000000..3d74aaf7f
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc2-flink/jobmanager-rest-service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flink-jobmanager-rest
+spec:
+ type: NodePort
+ ports:
+ - name: rest
+ port: 8081
+ targetPort: 8081
+ nodePort: 30081
+ selector:
+ app: flink
+ component: jobmanager
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc2-flink/jobmanager-service.yaml b/theodolite-benchmarks/definitions/uc2-flink/jobmanager-service.yaml
new file mode 100644
index 000000000..e2ff5d989
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc2-flink/jobmanager-service.yaml
@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flink-jobmanager
+ labels:
+ app: flink
+spec:
+ type: ClusterIP
+ ports:
+ - name: rpc
+ port: 6123
+ - name: blob-server
+ port: 6124
+ - name: webui
+ port: 8081
+ - name: metrics
+ port: 9249
+ selector:
+ app: flink
+ component: jobmanager
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc2-flink/service-monitor.yaml b/theodolite-benchmarks/definitions/uc2-flink/service-monitor.yaml
new file mode 100644
index 000000000..02f78823c
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc2-flink/service-monitor.yaml
@@ -0,0 +1,14 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+ labels:
+ app: flink
+ appScope: titan-ccp
+ name: flink
+spec:
+ selector:
+ matchLabels:
+ app: flink
+ endpoints:
+ - port: metrics
+ interval: 10s
diff --git a/theodolite-benchmarks/definitions/uc2-flink/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc2-flink/taskmanager-deployment.yaml
new file mode 100644
index 000000000..c0a10f65a
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc2-flink/taskmanager-deployment.yaml
@@ -0,0 +1,87 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: flink-taskmanager
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: flink
+ component: taskmanager
+ template:
+ metadata:
+ labels:
+ app: flink
+ component: taskmanager
+ spec:
+ containers:
+ - name: taskmanager
+ image: ghcr.io/cau-se/theodolite-uc2-flink:latest
+ env:
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-cp-kafka:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
+ - name: COMMIT_INTERVAL_MS
+ value: "100"
+ - name: CHECKPOINTING
+ value: "false"
+ - name: PARALLELISM
+ value: "1"
+ - name: "FLINK_STATE_BACKEND"
+ value: "rocksdb"
+ - name: JOB_MANAGER_RPC_ADDRESS
+ value: "flink-jobmanager"
+ - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS
+ value: "1" #TODO
+ - name: FLINK_PROPERTIES
+ value: |+
+ blob.server.port: 6124
+ jobmanager.rpc.port: 6123
+ taskmanager.rpc.port: 6122
+ queryable-state.proxy.ports: 6125
+ jobmanager.memory.process.size: 4Gb
+ taskmanager.memory.process.size: 4Gb
+ #parallelism.default: 1 #TODO
+ resources:
+ limits:
+ memory: 4Gi
+ cpu: 1000m
+ args: ["taskmanager"]
+ ports:
+ - containerPort: 6122
+ name: rpc
+ - containerPort: 6125
+ name: query-state
+ - containerPort: 9249
+ name: metrics
+ livenessProbe:
+ tcpSocket:
+ port: 6122
+ initialDelaySeconds: 30
+ periodSeconds: 60
+ volumeMounts:
+ - name: flink-config-volume-rw
+ mountPath: /opt/flink/conf/
+ securityContext:
+ runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
+ initContainers:
+ - name: init-taskmanager
+ image: busybox:1.28
+ command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
+ volumeMounts:
+ - name: flink-config-volume
+ mountPath: /flink-config/
+ - name: flink-config-volume-rw
+ mountPath: /flink-config-rw/
+ volumes:
+ - name: flink-config-volume
+ configMap:
+ name: flink-config
+ items:
+ - key: flink-conf.yaml
+ path: flink-conf.yaml
+ - key: log4j-console.properties
+ path: log4j-console.properties
+ - name: flink-config-volume-rw
+ emptyDir: {}
diff --git a/theodolite-benchmarks/definitions/uc2-flink/taskmanager-service.yaml b/theodolite-benchmarks/definitions/uc2-flink/taskmanager-service.yaml
new file mode 100644
index 000000000..a2e27f64a
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc2-flink/taskmanager-service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flink-taskmanager
+ labels:
+ app: flink
+spec:
+ type: ClusterIP
+ ports:
+ - name: metrics
+ port: 9249
+ selector:
+ app: flink
+ component: taskmanager
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc3-flink/flink-configuration-configmap.yaml b/theodolite-benchmarks/definitions/uc3-flink/flink-configuration-configmap.yaml
new file mode 100644
index 000000000..321541f6a
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc3-flink/flink-configuration-configmap.yaml
@@ -0,0 +1,66 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: flink-config
+ labels:
+ app: flink
+data:
+ flink-conf.yaml: |+
+ #jobmanager.rpc.address: flink-jobmanager
+ #taskmanager.numberOfTaskSlots: 1 #TODO
+ #blob.server.port: 6124
+ #jobmanager.rpc.port: 6123
+ #taskmanager.rpc.port: 6122
+ #queryable-state.proxy.ports: 6125
+ #jobmanager.memory.process.size: 4Gb
+ #taskmanager.memory.process.size: 4Gb
+ #parallelism.default: 1 #TODO
+ metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
+ metrics.reporter.prom.interval: 10 SECONDS
+ taskmanager.network.detailed-metrics: true
+ # -> gives metrics about inbound/outbound network queue lengths
+ log4j-console.properties: |+
+ # This affects logging for both user code and Flink
+ rootLogger.level = INFO
+ rootLogger.appenderRef.console.ref = ConsoleAppender
+ rootLogger.appenderRef.rolling.ref = RollingFileAppender
+
+ # Uncomment this if you want to _only_ change Flink's logging
+ #logger.flink.name = org.apache.flink
+ #logger.flink.level = INFO
+
+ # The following lines keep the log level of common libraries/connectors on
+ # log level INFO. The root logger does not override this. You have to manually
+ # change the log levels here.
+ logger.akka.name = akka
+ logger.akka.level = INFO
+ logger.kafka.name= org.apache.kafka
+ logger.kafka.level = INFO
+ logger.hadoop.name = org.apache.hadoop
+ logger.hadoop.level = INFO
+ logger.zookeeper.name = org.apache.zookeeper
+ logger.zookeeper.level = INFO
+
+ # Log all infos to the console
+ appender.console.name = ConsoleAppender
+ appender.console.type = CONSOLE
+ appender.console.layout.type = PatternLayout
+ appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+
+ # Log all infos in the given rolling file
+ appender.rolling.name = RollingFileAppender
+ appender.rolling.type = RollingFile
+ appender.rolling.append = false
+ appender.rolling.fileName = ${sys:log.file}
+ appender.rolling.filePattern = ${sys:log.file}.%i
+ appender.rolling.layout.type = PatternLayout
+ appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+ appender.rolling.policies.type = Policies
+ appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
+ appender.rolling.policies.size.size=100MB
+ appender.rolling.strategy.type = DefaultRolloverStrategy
+ appender.rolling.strategy.max = 10
+
+ # Suppress the irrelevant (wrong) warnings from the Netty channel handler
+ logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
+ logger.netty.level = OFF
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc3-flink/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc3-flink/jobmanager-deployment.yaml
new file mode 100644
index 000000000..33bf1f112
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc3-flink/jobmanager-deployment.yaml
@@ -0,0 +1,93 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: flink-jobmanager
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: flink
+ component: jobmanager
+ template:
+ metadata:
+ labels:
+ app: flink
+ component: jobmanager
+ spec:
+ containers:
+ - name: jobmanager
+ image: ghcr.io/cau-se/theodolite-uc3-flink:latest
+ env:
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-cp-kafka:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
+ - name: COMMIT_INTERVAL_MS
+ value: "100"
+ - name: CHECKPOINTING
+ value: "false"
+ - name: PARALLELISM
+ value: "1"
+ - name: "FLINK_STATE_BACKEND"
+ value: "rocksdb"
+ - name: JOB_MANAGER_RPC_ADDRESS
+ value: "flink-jobmanager"
+ - name: FLINK_PROPERTIES
+ value: |+
+ blob.server.port: 6124
+ jobmanager.rpc.port: 6123
+ taskmanager.rpc.port: 6122
+ queryable-state.proxy.ports: 6125
+ jobmanager.memory.process.size: 4Gb
+ taskmanager.memory.process.size: 4Gb
+ #parallelism.default: 1 #TODO
+ resources:
+ limits:
+ memory: 4Gi
+ cpu: 1000m
+ args: ["standalone-job", "--job-classname", "theodolite.uc3.application.HistoryServiceFlinkJob"] # optional arguments: ["--job-id", "<job id>", "--fromSavepoint", "/path/to/savepoint", "--allowNonRestoredState"]
+ #command: ['sleep', '60m']
+ ports:
+ - containerPort: 6123
+ name: rpc
+ - containerPort: 6124
+ name: blob-server
+ - containerPort: 8081
+ name: webui
+ - containerPort: 9249
+ name: metrics
+ livenessProbe:
+ tcpSocket:
+ port: 6123
+ initialDelaySeconds: 30
+ periodSeconds: 60
+ volumeMounts:
+ - name: flink-config-volume-rw
+ mountPath: /opt/flink/conf
+# - name: job-artifacts-volume
+# mountPath: /opt/flink/usrlib
+ securityContext:
+ runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
+ initContainers:
+ - name: init-jobmanager
+ image: busybox:1.28
+ command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
+ volumeMounts:
+ - name: flink-config-volume
+ mountPath: /flink-config/
+ - name: flink-config-volume-rw
+ mountPath: /flink-config-rw/
+ volumes:
+ - name: flink-config-volume
+ configMap:
+ name: flink-config
+ items:
+ - key: flink-conf.yaml
+ path: flink-conf.yaml
+ - key: log4j-console.properties
+ path: log4j-console.properties
+ - name: flink-config-volume-rw
+ emptyDir: {}
+# - name: job-artifacts-volume
+# hostPath:
+# path: /host/path/to/job/artifacts
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc3-flink/jobmanager-rest-service.yaml b/theodolite-benchmarks/definitions/uc3-flink/jobmanager-rest-service.yaml
new file mode 100644
index 000000000..3d74aaf7f
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc3-flink/jobmanager-rest-service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flink-jobmanager-rest
+spec:
+ type: NodePort
+ ports:
+ - name: rest
+ port: 8081
+ targetPort: 8081
+ nodePort: 30081
+ selector:
+ app: flink
+ component: jobmanager
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc3-flink/jobmanager-service.yaml b/theodolite-benchmarks/definitions/uc3-flink/jobmanager-service.yaml
new file mode 100644
index 000000000..e2ff5d989
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc3-flink/jobmanager-service.yaml
@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flink-jobmanager
+ labels:
+ app: flink
+spec:
+ type: ClusterIP
+ ports:
+ - name: rpc
+ port: 6123
+ - name: blob-server
+ port: 6124
+ - name: webui
+ port: 8081
+ - name: metrics
+ port: 9249
+ selector:
+ app: flink
+ component: jobmanager
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc3-flink/service-monitor.yaml b/theodolite-benchmarks/definitions/uc3-flink/service-monitor.yaml
new file mode 100644
index 000000000..02f78823c
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc3-flink/service-monitor.yaml
@@ -0,0 +1,14 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+ labels:
+ app: flink
+ appScope: titan-ccp
+ name: flink
+spec:
+ selector:
+ matchLabels:
+ app: flink
+ endpoints:
+ - port: metrics
+ interval: 10s
diff --git a/theodolite-benchmarks/definitions/uc3-flink/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc3-flink/taskmanager-deployment.yaml
new file mode 100644
index 000000000..8f70b7308
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc3-flink/taskmanager-deployment.yaml
@@ -0,0 +1,87 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: flink-taskmanager
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: flink
+ component: taskmanager
+ template:
+ metadata:
+ labels:
+ app: flink
+ component: taskmanager
+ spec:
+ containers:
+ - name: taskmanager
+ image: ghcr.io/cau-se/theodolite-uc3-flink:latest
+ env:
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-cp-kafka:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
+ - name: COMMIT_INTERVAL_MS
+ value: "100"
+ - name: CHECKPOINTING
+ value: "false"
+ - name: PARALLELISM
+ value: "1"
+ - name: "FLINK_STATE_BACKEND"
+ value: "rocksdb"
+ - name: JOB_MANAGER_RPC_ADDRESS
+ value: "flink-jobmanager"
+ - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS
+ value: "1" #TODO
+ - name: FLINK_PROPERTIES
+ value: |+
+ blob.server.port: 6124
+ jobmanager.rpc.port: 6123
+ taskmanager.rpc.port: 6122
+ queryable-state.proxy.ports: 6125
+ jobmanager.memory.process.size: 4Gb
+ taskmanager.memory.process.size: 4Gb
+ #parallelism.default: 1 #TODO
+ resources:
+ limits:
+ memory: 4Gi
+ cpu: 1000m
+ args: ["taskmanager"]
+ ports:
+ - containerPort: 6122
+ name: rpc
+ - containerPort: 6125
+ name: query-state
+ - containerPort: 9249
+ name: metrics
+ livenessProbe:
+ tcpSocket:
+ port: 6122
+ initialDelaySeconds: 30
+ periodSeconds: 60
+ volumeMounts:
+ - name: flink-config-volume-rw
+ mountPath: /opt/flink/conf/
+ securityContext:
+ runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
+ initContainers:
+ - name: init-taskmanager
+ image: busybox:1.28
+ command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
+ volumeMounts:
+ - name: flink-config-volume
+ mountPath: /flink-config/
+ - name: flink-config-volume-rw
+ mountPath: /flink-config-rw/
+ volumes:
+ - name: flink-config-volume
+ configMap:
+ name: flink-config
+ items:
+ - key: flink-conf.yaml
+ path: flink-conf.yaml
+ - key: log4j-console.properties
+ path: log4j-console.properties
+ - name: flink-config-volume-rw
+ emptyDir: {}
diff --git a/theodolite-benchmarks/definitions/uc3-flink/taskmanager-service.yaml b/theodolite-benchmarks/definitions/uc3-flink/taskmanager-service.yaml
new file mode 100644
index 000000000..a2e27f64a
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc3-flink/taskmanager-service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flink-taskmanager
+ labels:
+ app: flink
+spec:
+ type: ClusterIP
+ ports:
+ - name: metrics
+ port: 9249
+ selector:
+ app: flink
+ component: taskmanager
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc4-flink/flink-configuration-configmap.yaml b/theodolite-benchmarks/definitions/uc4-flink/flink-configuration-configmap.yaml
new file mode 100644
index 000000000..321541f6a
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc4-flink/flink-configuration-configmap.yaml
@@ -0,0 +1,66 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: flink-config
+ labels:
+ app: flink
+data:
+ flink-conf.yaml: |+
+ #jobmanager.rpc.address: flink-jobmanager
+ #taskmanager.numberOfTaskSlots: 1 #TODO
+ #blob.server.port: 6124
+ #jobmanager.rpc.port: 6123
+ #taskmanager.rpc.port: 6122
+ #queryable-state.proxy.ports: 6125
+ #jobmanager.memory.process.size: 4Gb
+ #taskmanager.memory.process.size: 4Gb
+ #parallelism.default: 1 #TODO
+ metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
+ metrics.reporter.prom.interval: 10 SECONDS
+ taskmanager.network.detailed-metrics: true
+ # -> gives metrics about inbound/outbound network queue lengths
+ log4j-console.properties: |+
+ # This affects logging for both user code and Flink
+ rootLogger.level = INFO
+ rootLogger.appenderRef.console.ref = ConsoleAppender
+ rootLogger.appenderRef.rolling.ref = RollingFileAppender
+
+ # Uncomment this if you want to _only_ change Flink's logging
+ #logger.flink.name = org.apache.flink
+ #logger.flink.level = INFO
+
+ # The following lines keep the log level of common libraries/connectors on
+ # log level INFO. The root logger does not override this. You have to manually
+ # change the log levels here.
+ logger.akka.name = akka
+ logger.akka.level = INFO
+ logger.kafka.name= org.apache.kafka
+ logger.kafka.level = INFO
+ logger.hadoop.name = org.apache.hadoop
+ logger.hadoop.level = INFO
+ logger.zookeeper.name = org.apache.zookeeper
+ logger.zookeeper.level = INFO
+
+ # Log all infos to the console
+ appender.console.name = ConsoleAppender
+ appender.console.type = CONSOLE
+ appender.console.layout.type = PatternLayout
+ appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+
+ # Log all infos in the given rolling file
+ appender.rolling.name = RollingFileAppender
+ appender.rolling.type = RollingFile
+ appender.rolling.append = false
+ appender.rolling.fileName = ${sys:log.file}
+ appender.rolling.filePattern = ${sys:log.file}.%i
+ appender.rolling.layout.type = PatternLayout
+ appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+ appender.rolling.policies.type = Policies
+ appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
+ appender.rolling.policies.size.size=100MB
+ appender.rolling.strategy.type = DefaultRolloverStrategy
+ appender.rolling.strategy.max = 10
+
+ # Suppress the irrelevant (wrong) warnings from the Netty channel handler
+ logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
+ logger.netty.level = OFF
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc4-flink/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc4-flink/jobmanager-deployment.yaml
new file mode 100644
index 000000000..b6533a2c4
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc4-flink/jobmanager-deployment.yaml
@@ -0,0 +1,93 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: flink-jobmanager
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: flink
+ component: jobmanager
+ template:
+ metadata:
+ labels:
+ app: flink
+ component: jobmanager
+ spec:
+ containers:
+ - name: jobmanager
+ image: ghcr.io/cau-se/theodolite-uc4-flink:latest
+ env:
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-cp-kafka:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
+ - name: COMMIT_INTERVAL_MS
+ value: "100"
+ - name: CHECKPOINTING
+ value: "false"
+ - name: PARALLELISM
+ value: "1"
+ - name: "FLINK_STATE_BACKEND"
+ value: "rocksdb"
+ - name: JOB_MANAGER_RPC_ADDRESS
+ value: "flink-jobmanager"
+ - name: FLINK_PROPERTIES
+ value: |+
+ blob.server.port: 6124
+ jobmanager.rpc.port: 6123
+ taskmanager.rpc.port: 6122
+ queryable-state.proxy.ports: 6125
+ jobmanager.memory.process.size: 4Gb
+ taskmanager.memory.process.size: 4Gb
+ #parallelism.default: 1 #TODO
+ resources:
+ limits:
+ memory: 4Gi
+ cpu: 1000m
+ args: ["standalone-job", "--job-classname", "theodolite.uc4.application.AggregationServiceFlinkJob"] # optional arguments: ["--job-id", "<job id>", "--fromSavepoint", "/path/to/savepoint", "--allowNonRestoredState"]
+ #command: ['sleep', '60m']
+ ports:
+ - containerPort: 6123
+ name: rpc
+ - containerPort: 6124
+ name: blob-server
+ - containerPort: 8081
+ name: webui
+ - containerPort: 9249
+ name: metrics
+ livenessProbe:
+ tcpSocket:
+ port: 6123
+ initialDelaySeconds: 30
+ periodSeconds: 60
+ volumeMounts:
+ - name: flink-config-volume-rw
+ mountPath: /opt/flink/conf
+# - name: job-artifacts-volume
+# mountPath: /opt/flink/usrlib
+ securityContext:
+ runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
+ initContainers:
+ - name: init-jobmanager
+ image: busybox:1.28
+ command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
+ volumeMounts:
+ - name: flink-config-volume
+ mountPath: /flink-config/
+ - name: flink-config-volume-rw
+ mountPath: /flink-config-rw/
+ volumes:
+ - name: flink-config-volume
+ configMap:
+ name: flink-config
+ items:
+ - key: flink-conf.yaml
+ path: flink-conf.yaml
+ - key: log4j-console.properties
+ path: log4j-console.properties
+ - name: flink-config-volume-rw
+ emptyDir: {}
+# - name: job-artifacts-volume
+# hostPath:
+# path: /host/path/to/job/artifacts
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc4-flink/jobmanager-rest-service.yaml b/theodolite-benchmarks/definitions/uc4-flink/jobmanager-rest-service.yaml
new file mode 100644
index 000000000..3d74aaf7f
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc4-flink/jobmanager-rest-service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flink-jobmanager-rest
+spec:
+ type: NodePort
+ ports:
+ - name: rest
+ port: 8081
+ targetPort: 8081
+ nodePort: 30081
+ selector:
+ app: flink
+ component: jobmanager
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc4-flink/jobmanager-service.yaml b/theodolite-benchmarks/definitions/uc4-flink/jobmanager-service.yaml
new file mode 100644
index 000000000..e2ff5d989
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc4-flink/jobmanager-service.yaml
@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flink-jobmanager
+ labels:
+ app: flink
+spec:
+ type: ClusterIP
+ ports:
+ - name: rpc
+ port: 6123
+ - name: blob-server
+ port: 6124
+ - name: webui
+ port: 8081
+ - name: metrics
+ port: 9249
+ selector:
+ app: flink
+ component: jobmanager
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc4-flink/service-monitor.yaml b/theodolite-benchmarks/definitions/uc4-flink/service-monitor.yaml
new file mode 100644
index 000000000..02f78823c
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc4-flink/service-monitor.yaml
@@ -0,0 +1,14 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+ labels:
+ app: flink
+ appScope: titan-ccp
+ name: flink
+spec:
+ selector:
+ matchLabels:
+ app: flink
+ endpoints:
+ - port: metrics
+ interval: 10s
diff --git a/theodolite-benchmarks/definitions/uc4-flink/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc4-flink/taskmanager-deployment.yaml
new file mode 100644
index 000000000..8f70b7308
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc4-flink/taskmanager-deployment.yaml
@@ -0,0 +1,87 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: flink-taskmanager
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: flink
+ component: taskmanager
+ template:
+ metadata:
+ labels:
+ app: flink
+ component: taskmanager
+ spec:
+ containers:
+ - name: taskmanager
+ image: ghcr.io/cau-se/theodolite-uc3-flink:latest
+ env:
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-cp-kafka:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
+ - name: COMMIT_INTERVAL_MS
+ value: "100"
+ - name: CHECKPOINTING
+ value: "false"
+ - name: PARALLELISM
+ value: "1"
+ - name: "FLINK_STATE_BACKEND"
+ value: "rocksdb"
+ - name: JOB_MANAGER_RPC_ADDRESS
+ value: "flink-jobmanager"
+ - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS
+ value: "1" #TODO
+ - name: FLINK_PROPERTIES
+ value: |+
+ blob.server.port: 6124
+ jobmanager.rpc.port: 6123
+ taskmanager.rpc.port: 6122
+ queryable-state.proxy.ports: 6125
+ jobmanager.memory.process.size: 4Gb
+ taskmanager.memory.process.size: 4Gb
+ #parallelism.default: 1 #TODO
+ resources:
+ limits:
+ memory: 4Gi
+ cpu: 1000m
+ args: ["taskmanager"]
+ ports:
+ - containerPort: 6122
+ name: rpc
+ - containerPort: 6125
+ name: query-state
+ - containerPort: 9249
+ name: metrics
+ livenessProbe:
+ tcpSocket:
+ port: 6122
+ initialDelaySeconds: 30
+ periodSeconds: 60
+ volumeMounts:
+ - name: flink-config-volume-rw
+ mountPath: /opt/flink/conf/
+ securityContext:
+ runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
+ initContainers:
+ - name: init-taskmanager
+ image: busybox:1.28
+ command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
+ volumeMounts:
+ - name: flink-config-volume
+ mountPath: /flink-config/
+ - name: flink-config-volume-rw
+ mountPath: /flink-config-rw/
+ volumes:
+ - name: flink-config-volume
+ configMap:
+ name: flink-config
+ items:
+ - key: flink-conf.yaml
+ path: flink-conf.yaml
+ - key: log4j-console.properties
+ path: log4j-console.properties
+ - name: flink-config-volume-rw
+ emptyDir: {}
diff --git a/theodolite-benchmarks/definitions/uc4-flink/taskmanager-service.yaml b/theodolite-benchmarks/definitions/uc4-flink/taskmanager-service.yaml
new file mode 100644
index 000000000..a2e27f64a
--- /dev/null
+++ b/theodolite-benchmarks/definitions/uc4-flink/taskmanager-service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: flink-taskmanager
+ labels:
+ app: flink
+spec:
+ type: ClusterIP
+ ports:
+ - name: metrics
+ port: 9249
+ selector:
+ app: flink
+ component: taskmanager
\ No newline at end of file
--
GitLab