diff --git a/docs/api-reference/patchers.md b/docs/api-reference/patchers.md
index bea63ccd23decef5654f257221ce0358b4f68e45..b0c99437fd003072b22b10df884ece88e3234c52 100644
--- a/docs/api-reference/patchers.md
+++ b/docs/api-reference/patchers.md
@@ -27,6 +27,18 @@ Patchers can be seen as functions which take a value as input and modify a Kuber
* **properties**:
* loadGenMaxRecords: 150000
+* **DataVolumeLoadGeneratorReplicaPatcher**: Takes the total load that should be generated and computes the number of instances needed for this load based on the `maxVolume` ((load + maxVolume - 1) / maxVolume) and calculates the load per instance (loadPerInstance = load / instances). The number of instances are set for the load generator and the given variable is set to the load per instance.
+ * **type**: "DataVolumeLoadGeneratorReplicaPatcher"
+ * **resource**: "osp-load-generator-deployment.yaml"
+ * **properties**:
+ * maxVolume: "50"
+ * container: "workload-generator"
+ * variableName: "DATA_VOLUME"
+
+* **ReplicaPatcher**: Allows to modify the number of Replicas for a kubernetes deployment.
+ * **type**: "ReplicaPatcher"
+ * **resource**: "uc1-kstreams-deployment.yaml"
+
* **EnvVarPatcher**: Modifies the value of an environment variable for a container in a Kubernetes deployment.
* **type**: "EnvVarPatcher"
* **resource**: "uc1-load-generator-deployment.yaml"
@@ -34,6 +46,14 @@ Patchers can be seen as functions which take a value as input and modify a Kuber
* container: "workload-generator"
* variableName: "NUM_SENSORS"
+* **ConfigMapYamlPatcher**: allows to add/modify a key-value pair in a YAML file of a ConfigMap
+ * **type**: "ConfigMapYamlPatcher"
+ * **resource**: "flink-configuration-configmap.yaml"
+ * **properties**:
+ * fileName: "flink-conf.yaml"
+ * variableName: "jobmanager.memory.process.size"
+ * **value**: "4Gb"
+
* **NodeSelectorPatcher**: Changes the node selection field in Kubernetes resources.
* **type**: "NodeSelectorPatcher"
* **resource**: "uc1-load-generator-deployment.yaml"
diff --git a/theodolite-benchmarks/definitions/uc1-beam-flink/resources/flink-configuration-configmap.yaml b/theodolite-benchmarks/definitions/uc1-beam-flink/resources/flink-configuration-configmap.yaml
deleted file mode 100644
index 36178e2bebdac96b8648bd6c299009aa49d3fff6..0000000000000000000000000000000000000000
--- a/theodolite-benchmarks/definitions/uc1-beam-flink/resources/flink-configuration-configmap.yaml
+++ /dev/null
@@ -1,66 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: flink-config
- labels:
- app: flink
-data:
- flink-conf.yaml: |+
- jobmanager.rpc.address: flink-jobmanager
- taskmanager.numberOfTaskSlots: 1 #TODO
- #blob.server.port: 6124
- #jobmanager.rpc.port: 6123
- #taskmanager.rpc.port: 6122
- #queryable-state.proxy.ports: 6125
- #jobmanager.memory.process.size: 4Gb
- #taskmanager.memory.process.size: 4Gb
- #parallelism.default: 1 #TODO
- metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
- metrics.reporter.prom.interval: 10 SECONDS
- taskmanager.network.detailed-metrics: true
- # -> gives metrics about inbound/outbound network queue lengths
- log4j-console.properties: |+
- # This affects logging for both user code and Flink
- rootLogger.level = INFO
- rootLogger.appenderRef.console.ref = ConsoleAppender
- rootLogger.appenderRef.rolling.ref = RollingFileAppender
-
- # Uncomment this if you want to _only_ change Flink's logging
- #logger.flink.name = org.apache.flink
- #logger.flink.level = INFO
-
- # The following lines keep the log level of common libraries/connectors on
- # log level INFO. The root logger does not override this. You have to manually
- # change the log levels here.
- logger.akka.name = akka
- logger.akka.level = INFO
- logger.kafka.name= org.apache.kafka
- logger.kafka.level = INFO
- logger.hadoop.name = org.apache.hadoop
- logger.hadoop.level = INFO
- logger.zookeeper.name = org.apache.zookeeper
- logger.zookeeper.level = INFO
-
- # Log all infos to the console
- appender.console.name = ConsoleAppender
- appender.console.type = CONSOLE
- appender.console.layout.type = PatternLayout
- appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
-
- # Log all infos in the given rolling file
- appender.rolling.name = RollingFileAppender
- appender.rolling.type = RollingFile
- appender.rolling.append = false
- appender.rolling.fileName = ${sys:log.file}
- appender.rolling.filePattern = ${sys:log.file}.%i
- appender.rolling.layout.type = PatternLayout
- appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
- appender.rolling.policies.type = Policies
- appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
- appender.rolling.policies.size.size=100MB
- appender.rolling.strategy.type = DefaultRolloverStrategy
- appender.rolling.strategy.max = 10
-
- # Suppress the irrelevant (wrong) warnings from the Netty channel handler
- logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
- logger.netty.level = OFF
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc1-beam-flink/resources/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc1-beam-flink/resources/jobmanager-deployment.yaml
index 343b7e7729ca5cdc1c78150e8b77896dcc59eb81..03647569555bcf84a84a97cb21e783269b59029e 100644
--- a/theodolite-benchmarks/definitions/uc1-beam-flink/resources/jobmanager-deployment.yaml
+++ b/theodolite-benchmarks/definitions/uc1-beam-flink/resources/jobmanager-deployment.yaml
@@ -63,33 +63,5 @@ spec:
port: 6123
initialDelaySeconds: 30
periodSeconds: 60
- # volumeMounts:
- # - name: flink-config-volume-rw
- # mountPath: /opt/flink/conf
-# - name: job-artifacts-volume
-# mountPath: /opt/flink/usrlib
securityContext:
runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
- # initContainers:
- # - name: init-jobmanager
- # image: busybox:1.28
- # command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
- # volumeMounts:
- # - name: flink-config-volume
- # mountPath: /flink-config/
- # - name: flink-config-volume-rw
- # mountPath: /flink-config-rw/
- # volumes:
- # - name: flink-config-volume
- # configMap:
- # name: flink-config
- # items:
- # - key: flink-conf.yaml
- # path: flink-conf.yaml
- # - key: log4j-console.properties
- # path: log4j-console.properties
- # - name: flink-config-volume-rw
- # emptyDir: {}
-# - name: job-artifacts-volume
-# hostPath:
-# path: /host/path/to/job/artifacts
diff --git a/theodolite-benchmarks/definitions/uc1-beam-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc1-beam-flink/resources/taskmanager-deployment.yaml
index 3af767a83266287e984241be52197c8ad92f7a63..9e6e17ea26802d026255aefc6a64a0ea60a632f5 100644
--- a/theodolite-benchmarks/definitions/uc1-beam-flink/resources/taskmanager-deployment.yaml
+++ b/theodolite-benchmarks/definitions/uc1-beam-flink/resources/taskmanager-deployment.yaml
@@ -20,29 +20,8 @@ spec:
image: ghcr.io/cau-se/theodolite-uc1-beam-flink:latest
args: ["taskmanager"]
env:
- - name: KAFKA_BOOTSTRAP_SERVERS
- value: "theodolite-kafka-kafka-bootstrap:9092"
- - name: SCHEMA_REGISTRY_URL
- value: "http://theodolite-kafka-schema-registry:8081"
- - name: CHECKPOINTING
- value: "false"
- # - name: PARALLELISM
- # value: "1"
- - name: "FLINK_STATE_BACKEND"
- value: "rocksdb"
- name: JOB_MANAGER_RPC_ADDRESS
value: "flink-jobmanager"
- # - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS
- # value: "1" #TODO
- # - name: FLINK_PROPERTIES
- # value: |+
- # blob.server.port: 6124
- # jobmanager.rpc.port: 6123
- # taskmanager.rpc.port: 6122
- # queryable-state.proxy.ports: 6125
- # jobmanager.memory.process.size: 4Gb
- # taskmanager.memory.process.size: 4Gb
- # #parallelism.default: 1 #TODO
resources:
limits:
memory: 4Gi
@@ -54,33 +33,5 @@ spec:
name: query-state
- containerPort: 9249
name: metrics
- # livenessProbe:
- # tcpSocket:
- # port: 6122
- # initialDelaySeconds: 30
- # periodSeconds: 60
- # volumeMounts:
- # - name: flink-config-volume-rw
- # mountPath: /opt/flink/conf/
securityContext:
runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
- # initContainers:
- # - name: init-taskmanager
- # image: busybox:1.28
- # command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
- # volumeMounts:
- # - name: flink-config-volume
- # mountPath: /flink-config/
- # - name: flink-config-volume-rw
- # mountPath: /flink-config-rw/
- # volumes:
- # - name: flink-config-volume
- # configMap:
- # name: flink-config
- # items:
- # - key: flink-conf.yaml
- # path: flink-conf.yaml
- # - key: log4j-console.properties
- # path: log4j-console.properties
- # - name: flink-config-volume-rw
- # emptyDir: {}
diff --git a/theodolite-benchmarks/definitions/uc1-beam-flink/uc1-beam-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc1-beam-flink/uc1-beam-flink-benchmark-operator.yaml
index 12ef870fdc8e18f93fd83c689ca46b5d82c92201..d9946c9c3d4e5b30647f228b9cda3e1523d539f3 100644
--- a/theodolite-benchmarks/definitions/uc1-beam-flink/uc1-beam-flink-benchmark-operator.yaml
+++ b/theodolite-benchmarks/definitions/uc1-beam-flink/uc1-beam-flink-benchmark-operator.yaml
@@ -36,11 +36,6 @@ spec:
properties:
container: "jobmanager"
variableName: "PARALLELISM"
- - type: "EnvVarPatcher" # required?
- resource: "taskmanager-deployment.yaml"
- properties:
- container: "taskmanager"
- variableName: "PARALLELISM"
loadTypes:
- typeName: "NumSensors"
patchers:
@@ -68,4 +63,4 @@ spec:
topics:
- name: "input"
numPartitions: 40
- replicationFactor: 1
\ No newline at end of file
+ replicationFactor: 1
diff --git a/theodolite-benchmarks/definitions/uc1-flink/resources/flink-configuration-configmap.yaml b/theodolite-benchmarks/definitions/uc1-flink/resources/flink-configuration-configmap.yaml
index 36178e2bebdac96b8648bd6c299009aa49d3fff6..ecda4025087f103613b9700ca733962709c48836 100644
--- a/theodolite-benchmarks/definitions/uc1-flink/resources/flink-configuration-configmap.yaml
+++ b/theodolite-benchmarks/definitions/uc1-flink/resources/flink-configuration-configmap.yaml
@@ -7,18 +7,16 @@ metadata:
data:
flink-conf.yaml: |+
jobmanager.rpc.address: flink-jobmanager
- taskmanager.numberOfTaskSlots: 1 #TODO
- #blob.server.port: 6124
- #jobmanager.rpc.port: 6123
- #taskmanager.rpc.port: 6122
- #queryable-state.proxy.ports: 6125
- #jobmanager.memory.process.size: 4Gb
- #taskmanager.memory.process.size: 4Gb
- #parallelism.default: 1 #TODO
+ blob.server.port: 6124
+ jobmanager.rpc.port: 6123
+ taskmanager.rpc.port: 6122
+ queryable-state.proxy.ports: 6125
+ jobmanager.memory.process.size: 4Gb
+ taskmanager.memory.process.size: 4Gb
metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
metrics.reporter.prom.interval: 10 SECONDS
+ # gives metrics about inbound/outbound network queue lengths
taskmanager.network.detailed-metrics: true
- # -> gives metrics about inbound/outbound network queue lengths
log4j-console.properties: |+
# This affects logging for both user code and Flink
rootLogger.level = INFO
@@ -63,4 +61,4 @@ data:
# Suppress the irrelevant (wrong) warnings from the Netty channel handler
logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
- logger.netty.level = OFF
\ No newline at end of file
+ logger.netty.level = OFF
diff --git a/theodolite-benchmarks/definitions/uc1-flink/resources/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc1-flink/resources/jobmanager-deployment.yaml
index f247a5d9b2fcc21825d2ac1a5164edba0d0d840b..e6112c476602cad9f6805a118f253f2800a945f1 100644
--- a/theodolite-benchmarks/definitions/uc1-flink/resources/jobmanager-deployment.yaml
+++ b/theodolite-benchmarks/definitions/uc1-flink/resources/jobmanager-deployment.yaml
@@ -29,17 +29,6 @@ spec:
value: "1"
- name: "FLINK_STATE_BACKEND"
value: "rocksdb"
- - name: JOB_MANAGER_RPC_ADDRESS
- value: "flink-jobmanager"
- - name: FLINK_PROPERTIES
- value: |+
- blob.server.port: 6124
- jobmanager.rpc.port: 6123
- taskmanager.rpc.port: 6122
- queryable-state.proxy.ports: 6125
- jobmanager.memory.process.size: 4Gb
- taskmanager.memory.process.size: 4Gb
- #parallelism.default: 1 #TODO
resources:
limits:
memory: 4Gi
@@ -61,21 +50,10 @@ spec:
initialDelaySeconds: 30
periodSeconds: 60
volumeMounts:
- - name: flink-config-volume-rw
+ - name: flink-config-volume
mountPath: /opt/flink/conf
-# - name: job-artifacts-volume
-# mountPath: /opt/flink/usrlib
securityContext:
runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
- initContainers:
- - name: init-jobmanager
- image: busybox:1.28
- command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
- volumeMounts:
- - name: flink-config-volume
- mountPath: /flink-config/
- - name: flink-config-volume-rw
- mountPath: /flink-config-rw/
volumes:
- name: flink-config-volume
configMap:
@@ -85,8 +63,3 @@ spec:
path: flink-conf.yaml
- key: log4j-console.properties
path: log4j-console.properties
- - name: flink-config-volume-rw
- emptyDir: {}
-# - name: job-artifacts-volume
-# hostPath:
-# path: /host/path/to/job/artifacts
diff --git a/theodolite-benchmarks/definitions/uc1-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc1-flink/resources/taskmanager-deployment.yaml
index a940f1ee30a1bc705c19329cfd25db7af48d27b0..c945b036d26161c849a51246bcdae0e0f15398ff 100644
--- a/theodolite-benchmarks/definitions/uc1-flink/resources/taskmanager-deployment.yaml
+++ b/theodolite-benchmarks/definitions/uc1-flink/resources/taskmanager-deployment.yaml
@@ -18,30 +18,6 @@ spec:
containers:
- name: taskmanager
image: ghcr.io/cau-se/theodolite-uc1-flink:latest
- env:
- - name: KAFKA_BOOTSTRAP_SERVERS
- value: "theodolite-kafka-kafka-bootstrap:9092"
- - name: SCHEMA_REGISTRY_URL
- value: "http://theodolite-kafka-schema-registry:8081"
- - name: CHECKPOINTING
- value: "false"
- - name: PARALLELISM
- value: "1"
- - name: "FLINK_STATE_BACKEND"
- value: "rocksdb"
- - name: JOB_MANAGER_RPC_ADDRESS
- value: "flink-jobmanager"
- - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS
- value: "1" #TODO
- - name: FLINK_PROPERTIES
- value: |+
- blob.server.port: 6124
- jobmanager.rpc.port: 6123
- taskmanager.rpc.port: 6122
- queryable-state.proxy.ports: 6125
- jobmanager.memory.process.size: 4Gb
- taskmanager.memory.process.size: 4Gb
- #parallelism.default: 1 #TODO
resources:
limits:
memory: 4Gi
@@ -60,19 +36,10 @@ spec:
initialDelaySeconds: 30
periodSeconds: 60
volumeMounts:
- - name: flink-config-volume-rw
+ - name: flink-config-volume
mountPath: /opt/flink/conf/
securityContext:
runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
- initContainers:
- - name: init-taskmanager
- image: busybox:1.28
- command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
- volumeMounts:
- - name: flink-config-volume
- mountPath: /flink-config/
- - name: flink-config-volume-rw
- mountPath: /flink-config-rw/
volumes:
- name: flink-config-volume
configMap:
@@ -82,5 +49,3 @@ spec:
path: flink-conf.yaml
- key: log4j-console.properties
path: log4j-console.properties
- - name: flink-config-volume-rw
- emptyDir: {}
diff --git a/theodolite-benchmarks/definitions/uc1-flink/uc1-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc1-flink/uc1-flink-benchmark-operator.yaml
index ca372f52fea462abdf1e28ef4d1f7b8d399ab896..b7d33a11d597db3f6371353f8783dc0a7d2252fe 100644
--- a/theodolite-benchmarks/definitions/uc1-flink/uc1-flink-benchmark-operator.yaml
+++ b/theodolite-benchmarks/definitions/uc1-flink/uc1-flink-benchmark-operator.yaml
@@ -36,11 +36,6 @@ spec:
properties:
container: "jobmanager"
variableName: "PARALLELISM"
- - type: "EnvVarPatcher" # required?
- resource: "taskmanager-deployment.yaml"
- properties:
- container: "taskmanager"
- variableName: "PARALLELISM"
loadTypes:
- typeName: "NumSensors"
patchers:
@@ -68,4 +63,4 @@ spec:
topics:
- name: "input"
numPartitions: 40
- replicationFactor: 1
\ No newline at end of file
+ replicationFactor: 1
diff --git a/theodolite-benchmarks/definitions/uc2-beam-flink/resources/flink-configuration-configmap.yaml b/theodolite-benchmarks/definitions/uc2-beam-flink/resources/flink-configuration-configmap.yaml
deleted file mode 100644
index 36178e2bebdac96b8648bd6c299009aa49d3fff6..0000000000000000000000000000000000000000
--- a/theodolite-benchmarks/definitions/uc2-beam-flink/resources/flink-configuration-configmap.yaml
+++ /dev/null
@@ -1,66 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: flink-config
- labels:
- app: flink
-data:
- flink-conf.yaml: |+
- jobmanager.rpc.address: flink-jobmanager
- taskmanager.numberOfTaskSlots: 1 #TODO
- #blob.server.port: 6124
- #jobmanager.rpc.port: 6123
- #taskmanager.rpc.port: 6122
- #queryable-state.proxy.ports: 6125
- #jobmanager.memory.process.size: 4Gb
- #taskmanager.memory.process.size: 4Gb
- #parallelism.default: 1 #TODO
- metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
- metrics.reporter.prom.interval: 10 SECONDS
- taskmanager.network.detailed-metrics: true
- # -> gives metrics about inbound/outbound network queue lengths
- log4j-console.properties: |+
- # This affects logging for both user code and Flink
- rootLogger.level = INFO
- rootLogger.appenderRef.console.ref = ConsoleAppender
- rootLogger.appenderRef.rolling.ref = RollingFileAppender
-
- # Uncomment this if you want to _only_ change Flink's logging
- #logger.flink.name = org.apache.flink
- #logger.flink.level = INFO
-
- # The following lines keep the log level of common libraries/connectors on
- # log level INFO. The root logger does not override this. You have to manually
- # change the log levels here.
- logger.akka.name = akka
- logger.akka.level = INFO
- logger.kafka.name= org.apache.kafka
- logger.kafka.level = INFO
- logger.hadoop.name = org.apache.hadoop
- logger.hadoop.level = INFO
- logger.zookeeper.name = org.apache.zookeeper
- logger.zookeeper.level = INFO
-
- # Log all infos to the console
- appender.console.name = ConsoleAppender
- appender.console.type = CONSOLE
- appender.console.layout.type = PatternLayout
- appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
-
- # Log all infos in the given rolling file
- appender.rolling.name = RollingFileAppender
- appender.rolling.type = RollingFile
- appender.rolling.append = false
- appender.rolling.fileName = ${sys:log.file}
- appender.rolling.filePattern = ${sys:log.file}.%i
- appender.rolling.layout.type = PatternLayout
- appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
- appender.rolling.policies.type = Policies
- appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
- appender.rolling.policies.size.size=100MB
- appender.rolling.strategy.type = DefaultRolloverStrategy
- appender.rolling.strategy.max = 10
-
- # Suppress the irrelevant (wrong) warnings from the Netty channel handler
- logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
- logger.netty.level = OFF
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc2-beam-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc2-beam-flink/resources/taskmanager-deployment.yaml
index 7da3d46483fca9a011b62faa1cc6166df6c04c03..b52d4198f6dc290d15c7496ba5c0f8b741dc6303 100644
--- a/theodolite-benchmarks/definitions/uc2-beam-flink/resources/taskmanager-deployment.yaml
+++ b/theodolite-benchmarks/definitions/uc2-beam-flink/resources/taskmanager-deployment.yaml
@@ -20,27 +20,8 @@ spec:
image: ghcr.io/cau-se/theodolite-uc2-beam-flink:latest
args: ["taskmanager"]
env:
- - name: KAFKA_BOOTSTRAP_SERVERS
- value: "theodolite-kafka-kafka-bootstrap:9092"
- - name: SCHEMA_REGISTRY_URL
- value: "http://theodolite-kafka-schema-registry:8081"
- - name: CHECKPOINTING
- value: "false"
- - name: "FLINK_STATE_BACKEND"
- value: "rocksdb"
- name: JOB_MANAGER_RPC_ADDRESS
value: "flink-jobmanager"
- # - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS
- # value: "1" #TODO
- # - name: FLINK_PROPERTIES
- # value: |+
- # blob.server.port: 6124
- # jobmanager.rpc.port: 6123
- # taskmanager.rpc.port: 6122
- # queryable-state.proxy.ports: 6125
- # jobmanager.memory.process.size: 4Gb
- # taskmanager.memory.process.size: 4Gb
- # #parallelism.default: 1 #TODO
resources:
limits:
memory: 4Gi
@@ -52,11 +33,5 @@ spec:
name: query-state
- containerPort: 9249
name: metrics
- # livenessProbe:
- # tcpSocket:
- # port: 6122
- # initialDelaySeconds: 30
- # periodSeconds: 60
securityContext:
runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
-
diff --git a/theodolite-benchmarks/definitions/uc2-beam-flink/uc2-beam-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc2-beam-flink/uc2-beam-flink-benchmark-operator.yaml
index 8b34b41637e16bfc579e239524aba329a4138433..dbb1a308accc261e22c0a0be0fb2918454d422c0 100644
--- a/theodolite-benchmarks/definitions/uc2-beam-flink/uc2-beam-flink-benchmark-operator.yaml
+++ b/theodolite-benchmarks/definitions/uc2-beam-flink/uc2-beam-flink-benchmark-operator.yaml
@@ -36,11 +36,6 @@ spec:
properties:
container: "jobmanager"
variableName: "PARALLELISM"
- - type: "EnvVarPatcher" # required?
- resource: "taskmanager-deployment.yaml"
- properties:
- container: "taskmanager"
- variableName: "PARALLELISM"
loadTypes:
- typeName: "NumSensors"
patchers:
@@ -73,4 +68,4 @@ spec:
numPartitions: 40
replicationFactor: 1
- name: "theodolite-.*"
- removeOnly: True
\ No newline at end of file
+ removeOnly: True
diff --git a/theodolite-benchmarks/definitions/uc2-flink/resources/flink-configuration-configmap.yaml b/theodolite-benchmarks/definitions/uc2-flink/resources/flink-configuration-configmap.yaml
index 321541f6ac8715b8546b964d8ad2b7c28552fbcd..ecda4025087f103613b9700ca733962709c48836 100644
--- a/theodolite-benchmarks/definitions/uc2-flink/resources/flink-configuration-configmap.yaml
+++ b/theodolite-benchmarks/definitions/uc2-flink/resources/flink-configuration-configmap.yaml
@@ -6,19 +6,17 @@ metadata:
app: flink
data:
flink-conf.yaml: |+
- #jobmanager.rpc.address: flink-jobmanager
- #taskmanager.numberOfTaskSlots: 1 #TODO
- #blob.server.port: 6124
- #jobmanager.rpc.port: 6123
- #taskmanager.rpc.port: 6122
- #queryable-state.proxy.ports: 6125
- #jobmanager.memory.process.size: 4Gb
- #taskmanager.memory.process.size: 4Gb
- #parallelism.default: 1 #TODO
+ jobmanager.rpc.address: flink-jobmanager
+ blob.server.port: 6124
+ jobmanager.rpc.port: 6123
+ taskmanager.rpc.port: 6122
+ queryable-state.proxy.ports: 6125
+ jobmanager.memory.process.size: 4Gb
+ taskmanager.memory.process.size: 4Gb
metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
metrics.reporter.prom.interval: 10 SECONDS
+ # gives metrics about inbound/outbound network queue lengths
taskmanager.network.detailed-metrics: true
- # -> gives metrics about inbound/outbound network queue lengths
log4j-console.properties: |+
# This affects logging for both user code and Flink
rootLogger.level = INFO
@@ -63,4 +61,4 @@ data:
# Suppress the irrelevant (wrong) warnings from the Netty channel handler
logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
- logger.netty.level = OFF
\ No newline at end of file
+ logger.netty.level = OFF
diff --git a/theodolite-benchmarks/definitions/uc2-flink/resources/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc2-flink/resources/jobmanager-deployment.yaml
index d96701b891c725c08dfc7c1e786c2618c8755df6..512667e756bfe78ca68286b396fb1e867ab5915b 100644
--- a/theodolite-benchmarks/definitions/uc2-flink/resources/jobmanager-deployment.yaml
+++ b/theodolite-benchmarks/definitions/uc2-flink/resources/jobmanager-deployment.yaml
@@ -29,17 +29,6 @@ spec:
value: "1"
- name: "FLINK_STATE_BACKEND"
value: "rocksdb"
- - name: JOB_MANAGER_RPC_ADDRESS
- value: "flink-jobmanager"
- - name: FLINK_PROPERTIES
- value: |+
- blob.server.port: 6124
- jobmanager.rpc.port: 6123
- taskmanager.rpc.port: 6122
- queryable-state.proxy.ports: 6125
- jobmanager.memory.process.size: 4Gb
- taskmanager.memory.process.size: 4Gb
- #parallelism.default: 1 #TODO
resources:
limits:
memory: 4Gi
@@ -61,21 +50,10 @@ spec:
initialDelaySeconds: 30
periodSeconds: 60
volumeMounts:
- - name: flink-config-volume-rw
+ - name: flink-config-volume
mountPath: /opt/flink/conf
-# - name: job-artifacts-volume
-# mountPath: /opt/flink/usrlib
securityContext:
runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
- initContainers:
- - name: init-jobmanager
- image: busybox:1.28
- command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
- volumeMounts:
- - name: flink-config-volume
- mountPath: /flink-config/
- - name: flink-config-volume-rw
- mountPath: /flink-config-rw/
volumes:
- name: flink-config-volume
configMap:
@@ -85,8 +63,3 @@ spec:
path: flink-conf.yaml
- key: log4j-console.properties
path: log4j-console.properties
- - name: flink-config-volume-rw
- emptyDir: {}
-# - name: job-artifacts-volume
-# hostPath:
-# path: /host/path/to/job/artifacts
diff --git a/theodolite-benchmarks/definitions/uc2-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc2-flink/resources/taskmanager-deployment.yaml
index bea93f4539888189a3ca85008d26513c6d5f8793..66f81b143cbdbb733e5a81c00de0d20aca13af39 100644
--- a/theodolite-benchmarks/definitions/uc2-flink/resources/taskmanager-deployment.yaml
+++ b/theodolite-benchmarks/definitions/uc2-flink/resources/taskmanager-deployment.yaml
@@ -18,30 +18,6 @@ spec:
containers:
- name: taskmanager
image: ghcr.io/cau-se/theodolite-uc2-flink:latest
- env:
- - name: KAFKA_BOOTSTRAP_SERVERS
- value: "theodolite-kafka-kafka-bootstrap:9092"
- - name: SCHEMA_REGISTRY_URL
- value: "http://theodolite-kafka-schema-registry:8081"
- - name: CHECKPOINTING
- value: "false"
- - name: PARALLELISM
- value: "1"
- - name: "FLINK_STATE_BACKEND"
- value: "rocksdb"
- - name: JOB_MANAGER_RPC_ADDRESS
- value: "flink-jobmanager"
- - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS
- value: "1" #TODO
- - name: FLINK_PROPERTIES
- value: |+
- blob.server.port: 6124
- jobmanager.rpc.port: 6123
- taskmanager.rpc.port: 6122
- queryable-state.proxy.ports: 6125
- jobmanager.memory.process.size: 4Gb
- taskmanager.memory.process.size: 4Gb
- #parallelism.default: 1 #TODO
resources:
limits:
memory: 4Gi
@@ -60,19 +36,10 @@ spec:
initialDelaySeconds: 30
periodSeconds: 60
volumeMounts:
- - name: flink-config-volume-rw
+ - name: flink-config-volume
mountPath: /opt/flink/conf/
securityContext:
runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
- initContainers:
- - name: init-taskmanager
- image: busybox:1.28
- command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
- volumeMounts:
- - name: flink-config-volume
- mountPath: /flink-config/
- - name: flink-config-volume-rw
- mountPath: /flink-config-rw/
volumes:
- name: flink-config-volume
configMap:
@@ -82,5 +49,3 @@ spec:
path: flink-conf.yaml
- key: log4j-console.properties
path: log4j-console.properties
- - name: flink-config-volume-rw
- emptyDir: {}
diff --git a/theodolite-benchmarks/definitions/uc2-flink/uc2-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc2-flink/uc2-flink-benchmark-operator.yaml
index 9e89335dc81c87a6e8517eb14820fad3e8667a23..d1cf9e5200f47a6a3317fe114b31231dbb9c3767 100644
--- a/theodolite-benchmarks/definitions/uc2-flink/uc2-flink-benchmark-operator.yaml
+++ b/theodolite-benchmarks/definitions/uc2-flink/uc2-flink-benchmark-operator.yaml
@@ -18,7 +18,7 @@ spec:
- "service-monitor.yaml"
- "jobmanager-service.yaml"
- "jobmanager-deployment.yaml"
- #- "jobmanager-rest-service.yaml"
+ # - "jobmanager-rest-service.yaml"
loadGenerator:
resources:
- configMap:
@@ -36,11 +36,6 @@ spec:
properties:
container: "jobmanager"
variableName: "PARALLELISM"
- - type: "EnvVarPatcher" # required?
- resource: "taskmanager-deployment.yaml"
- properties:
- container: "taskmanager"
- variableName: "PARALLELISM"
loadTypes:
- typeName: "NumSensors"
patchers:
diff --git a/theodolite-benchmarks/definitions/uc3-beam-flink/resources/flink-configuration-configmap.yaml b/theodolite-benchmarks/definitions/uc3-beam-flink/resources/flink-configuration-configmap.yaml
deleted file mode 100644
index 36178e2bebdac96b8648bd6c299009aa49d3fff6..0000000000000000000000000000000000000000
--- a/theodolite-benchmarks/definitions/uc3-beam-flink/resources/flink-configuration-configmap.yaml
+++ /dev/null
@@ -1,66 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: flink-config
- labels:
- app: flink
-data:
- flink-conf.yaml: |+
- jobmanager.rpc.address: flink-jobmanager
- taskmanager.numberOfTaskSlots: 1 #TODO
- #blob.server.port: 6124
- #jobmanager.rpc.port: 6123
- #taskmanager.rpc.port: 6122
- #queryable-state.proxy.ports: 6125
- #jobmanager.memory.process.size: 4Gb
- #taskmanager.memory.process.size: 4Gb
- #parallelism.default: 1 #TODO
- metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
- metrics.reporter.prom.interval: 10 SECONDS
- taskmanager.network.detailed-metrics: true
- # -> gives metrics about inbound/outbound network queue lengths
- log4j-console.properties: |+
- # This affects logging for both user code and Flink
- rootLogger.level = INFO
- rootLogger.appenderRef.console.ref = ConsoleAppender
- rootLogger.appenderRef.rolling.ref = RollingFileAppender
-
- # Uncomment this if you want to _only_ change Flink's logging
- #logger.flink.name = org.apache.flink
- #logger.flink.level = INFO
-
- # The following lines keep the log level of common libraries/connectors on
- # log level INFO. The root logger does not override this. You have to manually
- # change the log levels here.
- logger.akka.name = akka
- logger.akka.level = INFO
- logger.kafka.name= org.apache.kafka
- logger.kafka.level = INFO
- logger.hadoop.name = org.apache.hadoop
- logger.hadoop.level = INFO
- logger.zookeeper.name = org.apache.zookeeper
- logger.zookeeper.level = INFO
-
- # Log all infos to the console
- appender.console.name = ConsoleAppender
- appender.console.type = CONSOLE
- appender.console.layout.type = PatternLayout
- appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
-
- # Log all infos in the given rolling file
- appender.rolling.name = RollingFileAppender
- appender.rolling.type = RollingFile
- appender.rolling.append = false
- appender.rolling.fileName = ${sys:log.file}
- appender.rolling.filePattern = ${sys:log.file}.%i
- appender.rolling.layout.type = PatternLayout
- appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
- appender.rolling.policies.type = Policies
- appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
- appender.rolling.policies.size.size=100MB
- appender.rolling.strategy.type = DefaultRolloverStrategy
- appender.rolling.strategy.max = 10
-
- # Suppress the irrelevant (wrong) warnings from the Netty channel handler
- logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
- logger.netty.level = OFF
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc3-beam-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc3-beam-flink/resources/taskmanager-deployment.yaml
index ce94e22ee97f6a002913d6d6ef1fa1c74ab0f93b..13028d5ab33fcbce47f095403302260fd8ab31bd 100644
--- a/theodolite-benchmarks/definitions/uc3-beam-flink/resources/taskmanager-deployment.yaml
+++ b/theodolite-benchmarks/definitions/uc3-beam-flink/resources/taskmanager-deployment.yaml
@@ -20,27 +20,8 @@ spec:
image: ghcr.io/cau-se/theodolite-uc3-beam-flink:latest
args: ["taskmanager"]
env:
- - name: KAFKA_BOOTSTRAP_SERVERS
- value: "theodolite-kafka-kafka-bootstrap:9092"
- - name: SCHEMA_REGISTRY_URL
- value: "http://theodolite-kafka-schema-registry:8081"
- - name: CHECKPOINTING
- value: "false"
- - name: "FLINK_STATE_BACKEND"
- value: "rocksdb"
- name: JOB_MANAGER_RPC_ADDRESS
value: "flink-jobmanager"
- # - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS
- # value: "1" #TODO
- # - name: FLINK_PROPERTIES
- # value: |+
- # blob.server.port: 6124
- # jobmanager.rpc.port: 6123
- # taskmanager.rpc.port: 6122
- # queryable-state.proxy.ports: 6125
- # jobmanager.memory.process.size: 4Gb
- # taskmanager.memory.process.size: 4Gb
- # #parallelism.default: 1 #TODO
resources:
limits:
memory: 4Gi
@@ -52,11 +33,5 @@ spec:
name: query-state
- containerPort: 9249
name: metrics
- # livenessProbe:
- # tcpSocket:
- # port: 6122
- # initialDelaySeconds: 30
- # periodSeconds: 60
securityContext:
runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
-
diff --git a/theodolite-benchmarks/definitions/uc3-beam-flink/uc3-beam-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc3-beam-flink/uc3-beam-flink-benchmark-operator.yaml
index 0250e735442a997edb3511500b5a7ec136827394..9cfa992411dee608c3bf34b28d9f7dfea936b82f 100644
--- a/theodolite-benchmarks/definitions/uc3-beam-flink/uc3-beam-flink-benchmark-operator.yaml
+++ b/theodolite-benchmarks/definitions/uc3-beam-flink/uc3-beam-flink-benchmark-operator.yaml
@@ -36,11 +36,6 @@ spec:
properties:
container: "jobmanager"
variableName: "PARALLELISM"
- - type: "EnvVarPatcher" # required?
- resource: "taskmanager-deployment.yaml"
- properties:
- container: "taskmanager"
- variableName: "PARALLELISM"
loadTypes:
- typeName: "NumSensors"
patchers:
@@ -73,4 +68,4 @@ spec:
numPartitions: 40
replicationFactor: 1
- name: "theodolite-.*"
- removeOnly: True
\ No newline at end of file
+ removeOnly: True
diff --git a/theodolite-benchmarks/definitions/uc3-flink/resources/flink-configuration-configmap.yaml b/theodolite-benchmarks/definitions/uc3-flink/resources/flink-configuration-configmap.yaml
index 321541f6ac8715b8546b964d8ad2b7c28552fbcd..ecda4025087f103613b9700ca733962709c48836 100644
--- a/theodolite-benchmarks/definitions/uc3-flink/resources/flink-configuration-configmap.yaml
+++ b/theodolite-benchmarks/definitions/uc3-flink/resources/flink-configuration-configmap.yaml
@@ -6,19 +6,17 @@ metadata:
app: flink
data:
flink-conf.yaml: |+
- #jobmanager.rpc.address: flink-jobmanager
- #taskmanager.numberOfTaskSlots: 1 #TODO
- #blob.server.port: 6124
- #jobmanager.rpc.port: 6123
- #taskmanager.rpc.port: 6122
- #queryable-state.proxy.ports: 6125
- #jobmanager.memory.process.size: 4Gb
- #taskmanager.memory.process.size: 4Gb
- #parallelism.default: 1 #TODO
+ jobmanager.rpc.address: flink-jobmanager
+ blob.server.port: 6124
+ jobmanager.rpc.port: 6123
+ taskmanager.rpc.port: 6122
+ queryable-state.proxy.ports: 6125
+ jobmanager.memory.process.size: 4Gb
+ taskmanager.memory.process.size: 4Gb
metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
metrics.reporter.prom.interval: 10 SECONDS
+ # gives metrics about inbound/outbound network queue lengths
taskmanager.network.detailed-metrics: true
- # -> gives metrics about inbound/outbound network queue lengths
log4j-console.properties: |+
# This affects logging for both user code and Flink
rootLogger.level = INFO
@@ -63,4 +61,4 @@ data:
# Suppress the irrelevant (wrong) warnings from the Netty channel handler
logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
- logger.netty.level = OFF
\ No newline at end of file
+ logger.netty.level = OFF
diff --git a/theodolite-benchmarks/definitions/uc3-flink/resources/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc3-flink/resources/jobmanager-deployment.yaml
index 8aea2eda64cc155140b7344401809ceff8ebefd4..133205b13e1b5da6938ee0e2532849ec28038929 100644
--- a/theodolite-benchmarks/definitions/uc3-flink/resources/jobmanager-deployment.yaml
+++ b/theodolite-benchmarks/definitions/uc3-flink/resources/jobmanager-deployment.yaml
@@ -29,17 +29,6 @@ spec:
value: "1"
- name: "FLINK_STATE_BACKEND"
value: "rocksdb"
- - name: JOB_MANAGER_RPC_ADDRESS
- value: "flink-jobmanager"
- - name: FLINK_PROPERTIES
- value: |+
- blob.server.port: 6124
- jobmanager.rpc.port: 6123
- taskmanager.rpc.port: 6122
- queryable-state.proxy.ports: 6125
- jobmanager.memory.process.size: 4Gb
- taskmanager.memory.process.size: 4Gb
- #parallelism.default: 1 #TODO
resources:
limits:
memory: 4Gi
@@ -61,21 +50,10 @@ spec:
initialDelaySeconds: 30
periodSeconds: 60
volumeMounts:
- - name: flink-config-volume-rw
+ - name: flink-config-volume
mountPath: /opt/flink/conf
-# - name: job-artifacts-volume
-# mountPath: /opt/flink/usrlib
securityContext:
runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
- initContainers:
- - name: init-jobmanager
- image: busybox:1.28
- command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
- volumeMounts:
- - name: flink-config-volume
- mountPath: /flink-config/
- - name: flink-config-volume-rw
- mountPath: /flink-config-rw/
volumes:
- name: flink-config-volume
configMap:
@@ -85,8 +63,3 @@ spec:
path: flink-conf.yaml
- key: log4j-console.properties
path: log4j-console.properties
- - name: flink-config-volume-rw
- emptyDir: {}
-# - name: job-artifacts-volume
-# hostPath:
-# path: /host/path/to/job/artifacts
diff --git a/theodolite-benchmarks/definitions/uc3-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc3-flink/resources/taskmanager-deployment.yaml
index 861e8cfeebef4c7fcb367c2f6c719fe50a93039d..a9d7da30b0285ac440ac1d4ca3d09bc98c4af37c 100644
--- a/theodolite-benchmarks/definitions/uc3-flink/resources/taskmanager-deployment.yaml
+++ b/theodolite-benchmarks/definitions/uc3-flink/resources/taskmanager-deployment.yaml
@@ -18,30 +18,6 @@ spec:
containers:
- name: taskmanager
image: ghcr.io/cau-se/theodolite-uc3-flink:latest
- env:
- - name: KAFKA_BOOTSTRAP_SERVERS
- value: "theodolite-kafka-kafka-bootstrap:9092"
- - name: SCHEMA_REGISTRY_URL
- value: "http://theodolite-kafka-schema-registry:8081"
- - name: CHECKPOINTING
- value: "false"
- - name: PARALLELISM
- value: "1"
- - name: "FLINK_STATE_BACKEND"
- value: "rocksdb"
- - name: JOB_MANAGER_RPC_ADDRESS
- value: "flink-jobmanager"
- - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS
- value: "1" #TODO
- - name: FLINK_PROPERTIES
- value: |+
- blob.server.port: 6124
- jobmanager.rpc.port: 6123
- taskmanager.rpc.port: 6122
- queryable-state.proxy.ports: 6125
- jobmanager.memory.process.size: 4Gb
- taskmanager.memory.process.size: 4Gb
- #parallelism.default: 1 #TODO
resources:
limits:
memory: 4Gi
@@ -60,19 +36,10 @@ spec:
initialDelaySeconds: 30
periodSeconds: 60
volumeMounts:
- - name: flink-config-volume-rw
+ - name: flink-config-volume
mountPath: /opt/flink/conf/
securityContext:
runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
- initContainers:
- - name: init-taskmanager
- image: busybox:1.28
- command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
- volumeMounts:
- - name: flink-config-volume
- mountPath: /flink-config/
- - name: flink-config-volume-rw
- mountPath: /flink-config-rw/
volumes:
- name: flink-config-volume
configMap:
@@ -82,5 +49,3 @@ spec:
path: flink-conf.yaml
- key: log4j-console.properties
path: log4j-console.properties
- - name: flink-config-volume-rw
- emptyDir: {}
diff --git a/theodolite-benchmarks/definitions/uc3-flink/uc3-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc3-flink/uc3-flink-benchmark-operator.yaml
index d7b12e708dcd4f376a51207550411c089b367d01..ce0bfc5ba193b8fce2a057f50aa0d2eb284afb3f 100644
--- a/theodolite-benchmarks/definitions/uc3-flink/uc3-flink-benchmark-operator.yaml
+++ b/theodolite-benchmarks/definitions/uc3-flink/uc3-flink-benchmark-operator.yaml
@@ -18,7 +18,7 @@ spec:
- "service-monitor.yaml"
- "jobmanager-service.yaml"
- "jobmanager-deployment.yaml"
- #- "jobmanager-rest-service.yaml"
+ # - "jobmanager-rest-service.yaml"
loadGenerator:
resources:
- configMap:
@@ -36,11 +36,6 @@ spec:
properties:
container: "jobmanager"
variableName: "PARALLELISM"
- - type: "EnvVarPatcher" # required?
- resource: "taskmanager-deployment.yaml"
- properties:
- container: "taskmanager"
- variableName: "PARALLELISM"
loadTypes:
- typeName: "NumSensors"
patchers:
diff --git a/theodolite-benchmarks/definitions/uc4-beam-flink/resources/flink-configuration-configmap.yaml b/theodolite-benchmarks/definitions/uc4-beam-flink/resources/flink-configuration-configmap.yaml
deleted file mode 100644
index 36178e2bebdac96b8648bd6c299009aa49d3fff6..0000000000000000000000000000000000000000
--- a/theodolite-benchmarks/definitions/uc4-beam-flink/resources/flink-configuration-configmap.yaml
+++ /dev/null
@@ -1,66 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: flink-config
- labels:
- app: flink
-data:
- flink-conf.yaml: |+
- jobmanager.rpc.address: flink-jobmanager
- taskmanager.numberOfTaskSlots: 1 #TODO
- #blob.server.port: 6124
- #jobmanager.rpc.port: 6123
- #taskmanager.rpc.port: 6122
- #queryable-state.proxy.ports: 6125
- #jobmanager.memory.process.size: 4Gb
- #taskmanager.memory.process.size: 4Gb
- #parallelism.default: 1 #TODO
- metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
- metrics.reporter.prom.interval: 10 SECONDS
- taskmanager.network.detailed-metrics: true
- # -> gives metrics about inbound/outbound network queue lengths
- log4j-console.properties: |+
- # This affects logging for both user code and Flink
- rootLogger.level = INFO
- rootLogger.appenderRef.console.ref = ConsoleAppender
- rootLogger.appenderRef.rolling.ref = RollingFileAppender
-
- # Uncomment this if you want to _only_ change Flink's logging
- #logger.flink.name = org.apache.flink
- #logger.flink.level = INFO
-
- # The following lines keep the log level of common libraries/connectors on
- # log level INFO. The root logger does not override this. You have to manually
- # change the log levels here.
- logger.akka.name = akka
- logger.akka.level = INFO
- logger.kafka.name= org.apache.kafka
- logger.kafka.level = INFO
- logger.hadoop.name = org.apache.hadoop
- logger.hadoop.level = INFO
- logger.zookeeper.name = org.apache.zookeeper
- logger.zookeeper.level = INFO
-
- # Log all infos to the console
- appender.console.name = ConsoleAppender
- appender.console.type = CONSOLE
- appender.console.layout.type = PatternLayout
- appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
-
- # Log all infos in the given rolling file
- appender.rolling.name = RollingFileAppender
- appender.rolling.type = RollingFile
- appender.rolling.append = false
- appender.rolling.fileName = ${sys:log.file}
- appender.rolling.filePattern = ${sys:log.file}.%i
- appender.rolling.layout.type = PatternLayout
- appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
- appender.rolling.policies.type = Policies
- appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
- appender.rolling.policies.size.size=100MB
- appender.rolling.strategy.type = DefaultRolloverStrategy
- appender.rolling.strategy.max = 10
-
- # Suppress the irrelevant (wrong) warnings from the Netty channel handler
- logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
- logger.netty.level = OFF
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc4-beam-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc4-beam-flink/resources/taskmanager-deployment.yaml
index 2bd37ed9b2e511c4ed4a0024690b719867d11857..062ef653bbdf280c0aa31aa7ce476bc02c9993a1 100644
--- a/theodolite-benchmarks/definitions/uc4-beam-flink/resources/taskmanager-deployment.yaml
+++ b/theodolite-benchmarks/definitions/uc4-beam-flink/resources/taskmanager-deployment.yaml
@@ -20,27 +20,8 @@ spec:
image: ghcr.io/cau-se/theodolite-uc4-beam-flink:latest
args: ["taskmanager"]
env:
- - name: KAFKA_BOOTSTRAP_SERVERS
- value: "theodolite-kafka-kafka-bootstrap:9092"
- - name: SCHEMA_REGISTRY_URL
- value: "http://theodolite-kafka-schema-registry:8081"
- - name: CHECKPOINTING
- value: "false"
- - name: "FLINK_STATE_BACKEND"
- value: "rocksdb"
- name: JOB_MANAGER_RPC_ADDRESS
value: "flink-jobmanager"
- # - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS
- # value: "1" #TODO
- # - name: FLINK_PROPERTIES
- # value: |+
- # blob.server.port: 6124
- # jobmanager.rpc.port: 6123
- # taskmanager.rpc.port: 6122
- # queryable-state.proxy.ports: 6125
- # jobmanager.memory.process.size: 4Gb
- # taskmanager.memory.process.size: 4Gb
- # #parallelism.default: 1 #TODO
resources:
limits:
memory: 4Gi
@@ -52,11 +33,5 @@ spec:
name: query-state
- containerPort: 9249
name: metrics
- # livenessProbe:
- # tcpSocket:
- # port: 6122
- # initialDelaySeconds: 30
- # periodSeconds: 60
securityContext:
runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
-
diff --git a/theodolite-benchmarks/definitions/uc4-beam-flink/uc4-beam-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc4-beam-flink/uc4-beam-flink-benchmark-operator.yaml
index aab7d75588266c68fcc6484bd8f588038c7cd820..ddb7b574dda436ba703b10090838aae30dd8bacd 100644
--- a/theodolite-benchmarks/definitions/uc4-beam-flink/uc4-beam-flink-benchmark-operator.yaml
+++ b/theodolite-benchmarks/definitions/uc4-beam-flink/uc4-beam-flink-benchmark-operator.yaml
@@ -36,11 +36,6 @@ spec:
properties:
container: "jobmanager"
variableName: "PARALLELISM"
- - type: "EnvVarPatcher" # required?
- resource: "taskmanager-deployment.yaml"
- properties:
- container: "taskmanager"
- variableName: "PARALLELISM"
loadTypes:
- typeName: "NumNestedGroups"
patchers:
@@ -80,4 +75,4 @@ spec:
numPartitions: 40
replicationFactor: 1
- name: "theodolite-.*"
- removeOnly: True
\ No newline at end of file
+ removeOnly: True
diff --git a/theodolite-benchmarks/definitions/uc4-flink/resources/flink-configuration-configmap.yaml b/theodolite-benchmarks/definitions/uc4-flink/resources/flink-configuration-configmap.yaml
index 321541f6ac8715b8546b964d8ad2b7c28552fbcd..ecda4025087f103613b9700ca733962709c48836 100644
--- a/theodolite-benchmarks/definitions/uc4-flink/resources/flink-configuration-configmap.yaml
+++ b/theodolite-benchmarks/definitions/uc4-flink/resources/flink-configuration-configmap.yaml
@@ -6,19 +6,17 @@ metadata:
app: flink
data:
flink-conf.yaml: |+
- #jobmanager.rpc.address: flink-jobmanager
- #taskmanager.numberOfTaskSlots: 1 #TODO
- #blob.server.port: 6124
- #jobmanager.rpc.port: 6123
- #taskmanager.rpc.port: 6122
- #queryable-state.proxy.ports: 6125
- #jobmanager.memory.process.size: 4Gb
- #taskmanager.memory.process.size: 4Gb
- #parallelism.default: 1 #TODO
+ jobmanager.rpc.address: flink-jobmanager
+ blob.server.port: 6124
+ jobmanager.rpc.port: 6123
+ taskmanager.rpc.port: 6122
+ queryable-state.proxy.ports: 6125
+ jobmanager.memory.process.size: 4Gb
+ taskmanager.memory.process.size: 4Gb
metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
metrics.reporter.prom.interval: 10 SECONDS
+ # gives metrics about inbound/outbound network queue lengths
taskmanager.network.detailed-metrics: true
- # -> gives metrics about inbound/outbound network queue lengths
log4j-console.properties: |+
# This affects logging for both user code and Flink
rootLogger.level = INFO
@@ -63,4 +61,4 @@ data:
# Suppress the irrelevant (wrong) warnings from the Netty channel handler
logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
- logger.netty.level = OFF
\ No newline at end of file
+ logger.netty.level = OFF
diff --git a/theodolite-benchmarks/definitions/uc4-flink/resources/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc4-flink/resources/jobmanager-deployment.yaml
index d76bcf72eae1036ec2f1d5bad0038af1b8fe72a1..56c8066e209a1c3a41924ad320cc2fe0a9282fa0 100644
--- a/theodolite-benchmarks/definitions/uc4-flink/resources/jobmanager-deployment.yaml
+++ b/theodolite-benchmarks/definitions/uc4-flink/resources/jobmanager-deployment.yaml
@@ -29,17 +29,6 @@ spec:
value: "1"
- name: "FLINK_STATE_BACKEND"
value: "rocksdb"
- - name: JOB_MANAGER_RPC_ADDRESS
- value: "flink-jobmanager"
- - name: FLINK_PROPERTIES
- value: |+
- blob.server.port: 6124
- jobmanager.rpc.port: 6123
- taskmanager.rpc.port: 6122
- queryable-state.proxy.ports: 6125
- jobmanager.memory.process.size: 4Gb
- taskmanager.memory.process.size: 4Gb
- #parallelism.default: 1 #TODO
resources:
limits:
memory: 4Gi
@@ -61,21 +50,10 @@ spec:
initialDelaySeconds: 30
periodSeconds: 60
volumeMounts:
- - name: flink-config-volume-rw
+ - name: flink-config-volume
mountPath: /opt/flink/conf
-# - name: job-artifacts-volume
-# mountPath: /opt/flink/usrlib
securityContext:
runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
- initContainers:
- - name: init-jobmanager
- image: busybox:1.28
- command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
- volumeMounts:
- - name: flink-config-volume
- mountPath: /flink-config/
- - name: flink-config-volume-rw
- mountPath: /flink-config-rw/
volumes:
- name: flink-config-volume
configMap:
@@ -85,8 +63,3 @@ spec:
path: flink-conf.yaml
- key: log4j-console.properties
path: log4j-console.properties
- - name: flink-config-volume-rw
- emptyDir: {}
-# - name: job-artifacts-volume
-# hostPath:
-# path: /host/path/to/job/artifacts
diff --git a/theodolite-benchmarks/definitions/uc4-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc4-flink/resources/taskmanager-deployment.yaml
index 861e8cfeebef4c7fcb367c2f6c719fe50a93039d..a9d7da30b0285ac440ac1d4ca3d09bc98c4af37c 100644
--- a/theodolite-benchmarks/definitions/uc4-flink/resources/taskmanager-deployment.yaml
+++ b/theodolite-benchmarks/definitions/uc4-flink/resources/taskmanager-deployment.yaml
@@ -18,30 +18,6 @@ spec:
containers:
- name: taskmanager
image: ghcr.io/cau-se/theodolite-uc3-flink:latest
- env:
- - name: KAFKA_BOOTSTRAP_SERVERS
- value: "theodolite-kafka-kafka-bootstrap:9092"
- - name: SCHEMA_REGISTRY_URL
- value: "http://theodolite-kafka-schema-registry:8081"
- - name: CHECKPOINTING
- value: "false"
- - name: PARALLELISM
- value: "1"
- - name: "FLINK_STATE_BACKEND"
- value: "rocksdb"
- - name: JOB_MANAGER_RPC_ADDRESS
- value: "flink-jobmanager"
- - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS
- value: "1" #TODO
- - name: FLINK_PROPERTIES
- value: |+
- blob.server.port: 6124
- jobmanager.rpc.port: 6123
- taskmanager.rpc.port: 6122
- queryable-state.proxy.ports: 6125
- jobmanager.memory.process.size: 4Gb
- taskmanager.memory.process.size: 4Gb
- #parallelism.default: 1 #TODO
resources:
limits:
memory: 4Gi
@@ -60,19 +36,10 @@ spec:
initialDelaySeconds: 30
periodSeconds: 60
volumeMounts:
- - name: flink-config-volume-rw
+ - name: flink-config-volume
mountPath: /opt/flink/conf/
securityContext:
runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
- initContainers:
- - name: init-taskmanager
- image: busybox:1.28
- command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
- volumeMounts:
- - name: flink-config-volume
- mountPath: /flink-config/
- - name: flink-config-volume-rw
- mountPath: /flink-config-rw/
volumes:
- name: flink-config-volume
configMap:
@@ -82,5 +49,3 @@ spec:
path: flink-conf.yaml
- key: log4j-console.properties
path: log4j-console.properties
- - name: flink-config-volume-rw
- emptyDir: {}
diff --git a/theodolite-benchmarks/definitions/uc4-flink/uc4-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc4-flink/uc4-flink-benchmark-operator.yaml
index dd90e517f16995db1dcef53ce911c493b58f6775..b1b563bb415deea9978f9887a581f02b0e086497 100644
--- a/theodolite-benchmarks/definitions/uc4-flink/uc4-flink-benchmark-operator.yaml
+++ b/theodolite-benchmarks/definitions/uc4-flink/uc4-flink-benchmark-operator.yaml
@@ -36,11 +36,6 @@ spec:
properties:
container: "jobmanager"
variableName: "PARALLELISM"
- - type: "EnvVarPatcher" # required?
- resource: "taskmanager-deployment.yaml"
- properties:
- container: "taskmanager"
- variableName: "PARALLELISM"
loadTypes:
- typeName: "NumNestedGroups"
patchers:
diff --git a/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/patcher/ConfigMapYamlPatcher.kt b/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/patcher/ConfigMapYamlPatcher.kt
new file mode 100644
index 0000000000000000000000000000000000000000..272e3785e274eecbb85e3618113ae5137d5314bc
--- /dev/null
+++ b/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/patcher/ConfigMapYamlPatcher.kt
@@ -0,0 +1,38 @@
+package rocks.theodolite.kubernetes.patcher
+
+import io.fabric8.kubernetes.api.model.ConfigMap
+import io.fabric8.kubernetes.api.model.HasMetadata
+import org.yaml.snakeyaml.DumperOptions
+import org.yaml.snakeyaml.Yaml
+
+/**
+ * The ConfigMapYamlPatcher allows to add/modify a key-value pair in a YAML file of a ConfigMap
+ *
+ * @property fileName of the YAML file in the configmap that should be modified.
+ * @property variableName Name of the environment variable to be patched.
+ */
+class ConfigMapYamlPatcher(
+ private val fileName: String,
+ private val variableName: String
+) : AbstractPatcher() {
+
+ override fun patchSingleResource(resource: HasMetadata, value: String): HasMetadata {
+ if (resource is ConfigMap) {
+ val yamlFile = resource.data[fileName]
+
+ // Configure YAML parser
+ val dumperOptions = DumperOptions()
+ // used to omit curly braces around and new lines for every property
+ dumperOptions.defaultFlowStyle = DumperOptions.FlowStyle.BLOCK
+ val parser = Yaml(dumperOptions)
+
+ // Change value
+ val yaml = parser.loadAs(yamlFile, LinkedHashMap<String, String>()::class.java)
+ yaml[variableName] = value
+
+ // Convert back to String and set in Kubernetes resource
+ resource.data[fileName] = parser.dump(yaml)
+ }
+ return resource
+ }
+}
diff --git a/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/patcher/PatcherFactory.kt b/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/patcher/PatcherFactory.kt
index b5513b5b796c4ab8119db2a2da17a7cbe7231e57..69a2528bbcd2a39d535d77571f7ca520b8c81488 100644
--- a/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/patcher/PatcherFactory.kt
+++ b/theodolite/src/main/kotlin/rocks/theodolite/kubernetes/patcher/PatcherFactory.kt
@@ -73,6 +73,10 @@ class PatcherFactory {
"ImagePatcher" -> ImagePatcher(
container = patcherDefinition.properties["container"]!!
)
+ "ConfigMapYamlPatcher" -> ConfigMapYamlPatcher(
+ fileName = patcherDefinition.properties["fileName"]!!,
+ variableName = patcherDefinition.properties["variableName"]!!
+ )
"NamePatcher" -> NamePatcher()
"ServiceSelectorPatcher" -> ServiceSelectorPatcher(
variableName = patcherDefinition.properties["label"]!!