From 8df89beac11c3b05d51dbbf2ed2930d7133b287d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=B6ren=20Henning?= <soeren.henning@email.uni-kiel.de>
Date: Sat, 26 Nov 2022 23:49:30 +0100
Subject: [PATCH] Clean up Beam/Flink deployment
---
.../flink-configuration-configmap.yaml | 66 -------------------
.../resources/jobmanager-deployment.yaml | 28 --------
.../resources/taskmanager-deployment.yaml | 49 --------------
.../uc1-beam-flink-benchmark-operator.yaml | 7 +-
.../flink-configuration-configmap.yaml | 66 -------------------
.../resources/taskmanager-deployment.yaml | 25 -------
.../uc2-beam-flink-benchmark-operator.yaml | 7 +-
.../flink-configuration-configmap.yaml | 66 -------------------
.../resources/taskmanager-deployment.yaml | 25 -------
.../uc3-beam-flink-benchmark-operator.yaml | 7 +-
.../flink-configuration-configmap.yaml | 66 -------------------
.../resources/taskmanager-deployment.yaml | 25 -------
.../uc4-beam-flink-benchmark-operator.yaml | 7 +-
13 files changed, 4 insertions(+), 440 deletions(-)
delete mode 100644 theodolite-benchmarks/definitions/uc1-beam-flink/resources/flink-configuration-configmap.yaml
delete mode 100644 theodolite-benchmarks/definitions/uc2-beam-flink/resources/flink-configuration-configmap.yaml
delete mode 100644 theodolite-benchmarks/definitions/uc3-beam-flink/resources/flink-configuration-configmap.yaml
delete mode 100644 theodolite-benchmarks/definitions/uc4-beam-flink/resources/flink-configuration-configmap.yaml
diff --git a/theodolite-benchmarks/definitions/uc1-beam-flink/resources/flink-configuration-configmap.yaml b/theodolite-benchmarks/definitions/uc1-beam-flink/resources/flink-configuration-configmap.yaml
deleted file mode 100644
index 36178e2be..000000000
--- a/theodolite-benchmarks/definitions/uc1-beam-flink/resources/flink-configuration-configmap.yaml
+++ /dev/null
@@ -1,66 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: flink-config
- labels:
- app: flink
-data:
- flink-conf.yaml: |+
- jobmanager.rpc.address: flink-jobmanager
- taskmanager.numberOfTaskSlots: 1 #TODO
- #blob.server.port: 6124
- #jobmanager.rpc.port: 6123
- #taskmanager.rpc.port: 6122
- #queryable-state.proxy.ports: 6125
- #jobmanager.memory.process.size: 4Gb
- #taskmanager.memory.process.size: 4Gb
- #parallelism.default: 1 #TODO
- metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
- metrics.reporter.prom.interval: 10 SECONDS
- taskmanager.network.detailed-metrics: true
- # -> gives metrics about inbound/outbound network queue lengths
- log4j-console.properties: |+
- # This affects logging for both user code and Flink
- rootLogger.level = INFO
- rootLogger.appenderRef.console.ref = ConsoleAppender
- rootLogger.appenderRef.rolling.ref = RollingFileAppender
-
- # Uncomment this if you want to _only_ change Flink's logging
- #logger.flink.name = org.apache.flink
- #logger.flink.level = INFO
-
- # The following lines keep the log level of common libraries/connectors on
- # log level INFO. The root logger does not override this. You have to manually
- # change the log levels here.
- logger.akka.name = akka
- logger.akka.level = INFO
- logger.kafka.name= org.apache.kafka
- logger.kafka.level = INFO
- logger.hadoop.name = org.apache.hadoop
- logger.hadoop.level = INFO
- logger.zookeeper.name = org.apache.zookeeper
- logger.zookeeper.level = INFO
-
- # Log all infos to the console
- appender.console.name = ConsoleAppender
- appender.console.type = CONSOLE
- appender.console.layout.type = PatternLayout
- appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
-
- # Log all infos in the given rolling file
- appender.rolling.name = RollingFileAppender
- appender.rolling.type = RollingFile
- appender.rolling.append = false
- appender.rolling.fileName = ${sys:log.file}
- appender.rolling.filePattern = ${sys:log.file}.%i
- appender.rolling.layout.type = PatternLayout
- appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
- appender.rolling.policies.type = Policies
- appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
- appender.rolling.policies.size.size=100MB
- appender.rolling.strategy.type = DefaultRolloverStrategy
- appender.rolling.strategy.max = 10
-
- # Suppress the irrelevant (wrong) warnings from the Netty channel handler
- logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
- logger.netty.level = OFF
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc1-beam-flink/resources/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc1-beam-flink/resources/jobmanager-deployment.yaml
index 343b7e772..036475695 100644
--- a/theodolite-benchmarks/definitions/uc1-beam-flink/resources/jobmanager-deployment.yaml
+++ b/theodolite-benchmarks/definitions/uc1-beam-flink/resources/jobmanager-deployment.yaml
@@ -63,33 +63,5 @@ spec:
port: 6123
initialDelaySeconds: 30
periodSeconds: 60
- # volumeMounts:
- # - name: flink-config-volume-rw
- # mountPath: /opt/flink/conf
-# - name: job-artifacts-volume
-# mountPath: /opt/flink/usrlib
securityContext:
runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
- # initContainers:
- # - name: init-jobmanager
- # image: busybox:1.28
- # command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
- # volumeMounts:
- # - name: flink-config-volume
- # mountPath: /flink-config/
- # - name: flink-config-volume-rw
- # mountPath: /flink-config-rw/
- # volumes:
- # - name: flink-config-volume
- # configMap:
- # name: flink-config
- # items:
- # - key: flink-conf.yaml
- # path: flink-conf.yaml
- # - key: log4j-console.properties
- # path: log4j-console.properties
- # - name: flink-config-volume-rw
- # emptyDir: {}
-# - name: job-artifacts-volume
-# hostPath:
-# path: /host/path/to/job/artifacts
diff --git a/theodolite-benchmarks/definitions/uc1-beam-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc1-beam-flink/resources/taskmanager-deployment.yaml
index 3af767a83..9e6e17ea2 100644
--- a/theodolite-benchmarks/definitions/uc1-beam-flink/resources/taskmanager-deployment.yaml
+++ b/theodolite-benchmarks/definitions/uc1-beam-flink/resources/taskmanager-deployment.yaml
@@ -20,29 +20,8 @@ spec:
image: ghcr.io/cau-se/theodolite-uc1-beam-flink:latest
args: ["taskmanager"]
env:
- - name: KAFKA_BOOTSTRAP_SERVERS
- value: "theodolite-kafka-kafka-bootstrap:9092"
- - name: SCHEMA_REGISTRY_URL
- value: "http://theodolite-kafka-schema-registry:8081"
- - name: CHECKPOINTING
- value: "false"
- # - name: PARALLELISM
- # value: "1"
- - name: "FLINK_STATE_BACKEND"
- value: "rocksdb"
- name: JOB_MANAGER_RPC_ADDRESS
value: "flink-jobmanager"
- # - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS
- # value: "1" #TODO
- # - name: FLINK_PROPERTIES
- # value: |+
- # blob.server.port: 6124
- # jobmanager.rpc.port: 6123
- # taskmanager.rpc.port: 6122
- # queryable-state.proxy.ports: 6125
- # jobmanager.memory.process.size: 4Gb
- # taskmanager.memory.process.size: 4Gb
- # #parallelism.default: 1 #TODO
resources:
limits:
memory: 4Gi
@@ -54,33 +33,5 @@ spec:
name: query-state
- containerPort: 9249
name: metrics
- # livenessProbe:
- # tcpSocket:
- # port: 6122
- # initialDelaySeconds: 30
- # periodSeconds: 60
- # volumeMounts:
- # - name: flink-config-volume-rw
- # mountPath: /opt/flink/conf/
securityContext:
runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
- # initContainers:
- # - name: init-taskmanager
- # image: busybox:1.28
- # command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
- # volumeMounts:
- # - name: flink-config-volume
- # mountPath: /flink-config/
- # - name: flink-config-volume-rw
- # mountPath: /flink-config-rw/
- # volumes:
- # - name: flink-config-volume
- # configMap:
- # name: flink-config
- # items:
- # - key: flink-conf.yaml
- # path: flink-conf.yaml
- # - key: log4j-console.properties
- # path: log4j-console.properties
- # - name: flink-config-volume-rw
- # emptyDir: {}
diff --git a/theodolite-benchmarks/definitions/uc1-beam-flink/uc1-beam-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc1-beam-flink/uc1-beam-flink-benchmark-operator.yaml
index 12ef870fd..d9946c9c3 100644
--- a/theodolite-benchmarks/definitions/uc1-beam-flink/uc1-beam-flink-benchmark-operator.yaml
+++ b/theodolite-benchmarks/definitions/uc1-beam-flink/uc1-beam-flink-benchmark-operator.yaml
@@ -36,11 +36,6 @@ spec:
properties:
container: "jobmanager"
variableName: "PARALLELISM"
- - type: "EnvVarPatcher" # required?
- resource: "taskmanager-deployment.yaml"
- properties:
- container: "taskmanager"
- variableName: "PARALLELISM"
loadTypes:
- typeName: "NumSensors"
patchers:
@@ -68,4 +63,4 @@ spec:
topics:
- name: "input"
numPartitions: 40
- replicationFactor: 1
\ No newline at end of file
+ replicationFactor: 1
diff --git a/theodolite-benchmarks/definitions/uc2-beam-flink/resources/flink-configuration-configmap.yaml b/theodolite-benchmarks/definitions/uc2-beam-flink/resources/flink-configuration-configmap.yaml
deleted file mode 100644
index 36178e2be..000000000
--- a/theodolite-benchmarks/definitions/uc2-beam-flink/resources/flink-configuration-configmap.yaml
+++ /dev/null
@@ -1,66 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: flink-config
- labels:
- app: flink
-data:
- flink-conf.yaml: |+
- jobmanager.rpc.address: flink-jobmanager
- taskmanager.numberOfTaskSlots: 1 #TODO
- #blob.server.port: 6124
- #jobmanager.rpc.port: 6123
- #taskmanager.rpc.port: 6122
- #queryable-state.proxy.ports: 6125
- #jobmanager.memory.process.size: 4Gb
- #taskmanager.memory.process.size: 4Gb
- #parallelism.default: 1 #TODO
- metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
- metrics.reporter.prom.interval: 10 SECONDS
- taskmanager.network.detailed-metrics: true
- # -> gives metrics about inbound/outbound network queue lengths
- log4j-console.properties: |+
- # This affects logging for both user code and Flink
- rootLogger.level = INFO
- rootLogger.appenderRef.console.ref = ConsoleAppender
- rootLogger.appenderRef.rolling.ref = RollingFileAppender
-
- # Uncomment this if you want to _only_ change Flink's logging
- #logger.flink.name = org.apache.flink
- #logger.flink.level = INFO
-
- # The following lines keep the log level of common libraries/connectors on
- # log level INFO. The root logger does not override this. You have to manually
- # change the log levels here.
- logger.akka.name = akka
- logger.akka.level = INFO
- logger.kafka.name= org.apache.kafka
- logger.kafka.level = INFO
- logger.hadoop.name = org.apache.hadoop
- logger.hadoop.level = INFO
- logger.zookeeper.name = org.apache.zookeeper
- logger.zookeeper.level = INFO
-
- # Log all infos to the console
- appender.console.name = ConsoleAppender
- appender.console.type = CONSOLE
- appender.console.layout.type = PatternLayout
- appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
-
- # Log all infos in the given rolling file
- appender.rolling.name = RollingFileAppender
- appender.rolling.type = RollingFile
- appender.rolling.append = false
- appender.rolling.fileName = ${sys:log.file}
- appender.rolling.filePattern = ${sys:log.file}.%i
- appender.rolling.layout.type = PatternLayout
- appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
- appender.rolling.policies.type = Policies
- appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
- appender.rolling.policies.size.size=100MB
- appender.rolling.strategy.type = DefaultRolloverStrategy
- appender.rolling.strategy.max = 10
-
- # Suppress the irrelevant (wrong) warnings from the Netty channel handler
- logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
- logger.netty.level = OFF
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc2-beam-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc2-beam-flink/resources/taskmanager-deployment.yaml
index 7da3d4648..b52d4198f 100644
--- a/theodolite-benchmarks/definitions/uc2-beam-flink/resources/taskmanager-deployment.yaml
+++ b/theodolite-benchmarks/definitions/uc2-beam-flink/resources/taskmanager-deployment.yaml
@@ -20,27 +20,8 @@ spec:
image: ghcr.io/cau-se/theodolite-uc2-beam-flink:latest
args: ["taskmanager"]
env:
- - name: KAFKA_BOOTSTRAP_SERVERS
- value: "theodolite-kafka-kafka-bootstrap:9092"
- - name: SCHEMA_REGISTRY_URL
- value: "http://theodolite-kafka-schema-registry:8081"
- - name: CHECKPOINTING
- value: "false"
- - name: "FLINK_STATE_BACKEND"
- value: "rocksdb"
- name: JOB_MANAGER_RPC_ADDRESS
value: "flink-jobmanager"
- # - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS
- # value: "1" #TODO
- # - name: FLINK_PROPERTIES
- # value: |+
- # blob.server.port: 6124
- # jobmanager.rpc.port: 6123
- # taskmanager.rpc.port: 6122
- # queryable-state.proxy.ports: 6125
- # jobmanager.memory.process.size: 4Gb
- # taskmanager.memory.process.size: 4Gb
- # #parallelism.default: 1 #TODO
resources:
limits:
memory: 4Gi
@@ -52,11 +33,5 @@ spec:
name: query-state
- containerPort: 9249
name: metrics
- # livenessProbe:
- # tcpSocket:
- # port: 6122
- # initialDelaySeconds: 30
- # periodSeconds: 60
securityContext:
runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
-
diff --git a/theodolite-benchmarks/definitions/uc2-beam-flink/uc2-beam-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc2-beam-flink/uc2-beam-flink-benchmark-operator.yaml
index 8b34b4163..dbb1a308a 100644
--- a/theodolite-benchmarks/definitions/uc2-beam-flink/uc2-beam-flink-benchmark-operator.yaml
+++ b/theodolite-benchmarks/definitions/uc2-beam-flink/uc2-beam-flink-benchmark-operator.yaml
@@ -36,11 +36,6 @@ spec:
properties:
container: "jobmanager"
variableName: "PARALLELISM"
- - type: "EnvVarPatcher" # required?
- resource: "taskmanager-deployment.yaml"
- properties:
- container: "taskmanager"
- variableName: "PARALLELISM"
loadTypes:
- typeName: "NumSensors"
patchers:
@@ -73,4 +68,4 @@ spec:
numPartitions: 40
replicationFactor: 1
- name: "theodolite-.*"
- removeOnly: True
\ No newline at end of file
+ removeOnly: True
diff --git a/theodolite-benchmarks/definitions/uc3-beam-flink/resources/flink-configuration-configmap.yaml b/theodolite-benchmarks/definitions/uc3-beam-flink/resources/flink-configuration-configmap.yaml
deleted file mode 100644
index 36178e2be..000000000
--- a/theodolite-benchmarks/definitions/uc3-beam-flink/resources/flink-configuration-configmap.yaml
+++ /dev/null
@@ -1,66 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: flink-config
- labels:
- app: flink
-data:
- flink-conf.yaml: |+
- jobmanager.rpc.address: flink-jobmanager
- taskmanager.numberOfTaskSlots: 1 #TODO
- #blob.server.port: 6124
- #jobmanager.rpc.port: 6123
- #taskmanager.rpc.port: 6122
- #queryable-state.proxy.ports: 6125
- #jobmanager.memory.process.size: 4Gb
- #taskmanager.memory.process.size: 4Gb
- #parallelism.default: 1 #TODO
- metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
- metrics.reporter.prom.interval: 10 SECONDS
- taskmanager.network.detailed-metrics: true
- # -> gives metrics about inbound/outbound network queue lengths
- log4j-console.properties: |+
- # This affects logging for both user code and Flink
- rootLogger.level = INFO
- rootLogger.appenderRef.console.ref = ConsoleAppender
- rootLogger.appenderRef.rolling.ref = RollingFileAppender
-
- # Uncomment this if you want to _only_ change Flink's logging
- #logger.flink.name = org.apache.flink
- #logger.flink.level = INFO
-
- # The following lines keep the log level of common libraries/connectors on
- # log level INFO. The root logger does not override this. You have to manually
- # change the log levels here.
- logger.akka.name = akka
- logger.akka.level = INFO
- logger.kafka.name= org.apache.kafka
- logger.kafka.level = INFO
- logger.hadoop.name = org.apache.hadoop
- logger.hadoop.level = INFO
- logger.zookeeper.name = org.apache.zookeeper
- logger.zookeeper.level = INFO
-
- # Log all infos to the console
- appender.console.name = ConsoleAppender
- appender.console.type = CONSOLE
- appender.console.layout.type = PatternLayout
- appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
-
- # Log all infos in the given rolling file
- appender.rolling.name = RollingFileAppender
- appender.rolling.type = RollingFile
- appender.rolling.append = false
- appender.rolling.fileName = ${sys:log.file}
- appender.rolling.filePattern = ${sys:log.file}.%i
- appender.rolling.layout.type = PatternLayout
- appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
- appender.rolling.policies.type = Policies
- appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
- appender.rolling.policies.size.size=100MB
- appender.rolling.strategy.type = DefaultRolloverStrategy
- appender.rolling.strategy.max = 10
-
- # Suppress the irrelevant (wrong) warnings from the Netty channel handler
- logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
- logger.netty.level = OFF
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc3-beam-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc3-beam-flink/resources/taskmanager-deployment.yaml
index ce94e22ee..13028d5ab 100644
--- a/theodolite-benchmarks/definitions/uc3-beam-flink/resources/taskmanager-deployment.yaml
+++ b/theodolite-benchmarks/definitions/uc3-beam-flink/resources/taskmanager-deployment.yaml
@@ -20,27 +20,8 @@ spec:
image: ghcr.io/cau-se/theodolite-uc3-beam-flink:latest
args: ["taskmanager"]
env:
- - name: KAFKA_BOOTSTRAP_SERVERS
- value: "theodolite-kafka-kafka-bootstrap:9092"
- - name: SCHEMA_REGISTRY_URL
- value: "http://theodolite-kafka-schema-registry:8081"
- - name: CHECKPOINTING
- value: "false"
- - name: "FLINK_STATE_BACKEND"
- value: "rocksdb"
- name: JOB_MANAGER_RPC_ADDRESS
value: "flink-jobmanager"
- # - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS
- # value: "1" #TODO
- # - name: FLINK_PROPERTIES
- # value: |+
- # blob.server.port: 6124
- # jobmanager.rpc.port: 6123
- # taskmanager.rpc.port: 6122
- # queryable-state.proxy.ports: 6125
- # jobmanager.memory.process.size: 4Gb
- # taskmanager.memory.process.size: 4Gb
- # #parallelism.default: 1 #TODO
resources:
limits:
memory: 4Gi
@@ -52,11 +33,5 @@ spec:
name: query-state
- containerPort: 9249
name: metrics
- # livenessProbe:
- # tcpSocket:
- # port: 6122
- # initialDelaySeconds: 30
- # periodSeconds: 60
securityContext:
runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
-
diff --git a/theodolite-benchmarks/definitions/uc3-beam-flink/uc3-beam-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc3-beam-flink/uc3-beam-flink-benchmark-operator.yaml
index 0250e7354..9cfa99241 100644
--- a/theodolite-benchmarks/definitions/uc3-beam-flink/uc3-beam-flink-benchmark-operator.yaml
+++ b/theodolite-benchmarks/definitions/uc3-beam-flink/uc3-beam-flink-benchmark-operator.yaml
@@ -36,11 +36,6 @@ spec:
properties:
container: "jobmanager"
variableName: "PARALLELISM"
- - type: "EnvVarPatcher" # required?
- resource: "taskmanager-deployment.yaml"
- properties:
- container: "taskmanager"
- variableName: "PARALLELISM"
loadTypes:
- typeName: "NumSensors"
patchers:
@@ -73,4 +68,4 @@ spec:
numPartitions: 40
replicationFactor: 1
- name: "theodolite-.*"
- removeOnly: True
\ No newline at end of file
+ removeOnly: True
diff --git a/theodolite-benchmarks/definitions/uc4-beam-flink/resources/flink-configuration-configmap.yaml b/theodolite-benchmarks/definitions/uc4-beam-flink/resources/flink-configuration-configmap.yaml
deleted file mode 100644
index 36178e2be..000000000
--- a/theodolite-benchmarks/definitions/uc4-beam-flink/resources/flink-configuration-configmap.yaml
+++ /dev/null
@@ -1,66 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: flink-config
- labels:
- app: flink
-data:
- flink-conf.yaml: |+
- jobmanager.rpc.address: flink-jobmanager
- taskmanager.numberOfTaskSlots: 1 #TODO
- #blob.server.port: 6124
- #jobmanager.rpc.port: 6123
- #taskmanager.rpc.port: 6122
- #queryable-state.proxy.ports: 6125
- #jobmanager.memory.process.size: 4Gb
- #taskmanager.memory.process.size: 4Gb
- #parallelism.default: 1 #TODO
- metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
- metrics.reporter.prom.interval: 10 SECONDS
- taskmanager.network.detailed-metrics: true
- # -> gives metrics about inbound/outbound network queue lengths
- log4j-console.properties: |+
- # This affects logging for both user code and Flink
- rootLogger.level = INFO
- rootLogger.appenderRef.console.ref = ConsoleAppender
- rootLogger.appenderRef.rolling.ref = RollingFileAppender
-
- # Uncomment this if you want to _only_ change Flink's logging
- #logger.flink.name = org.apache.flink
- #logger.flink.level = INFO
-
- # The following lines keep the log level of common libraries/connectors on
- # log level INFO. The root logger does not override this. You have to manually
- # change the log levels here.
- logger.akka.name = akka
- logger.akka.level = INFO
- logger.kafka.name= org.apache.kafka
- logger.kafka.level = INFO
- logger.hadoop.name = org.apache.hadoop
- logger.hadoop.level = INFO
- logger.zookeeper.name = org.apache.zookeeper
- logger.zookeeper.level = INFO
-
- # Log all infos to the console
- appender.console.name = ConsoleAppender
- appender.console.type = CONSOLE
- appender.console.layout.type = PatternLayout
- appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
-
- # Log all infos in the given rolling file
- appender.rolling.name = RollingFileAppender
- appender.rolling.type = RollingFile
- appender.rolling.append = false
- appender.rolling.fileName = ${sys:log.file}
- appender.rolling.filePattern = ${sys:log.file}.%i
- appender.rolling.layout.type = PatternLayout
- appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
- appender.rolling.policies.type = Policies
- appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
- appender.rolling.policies.size.size=100MB
- appender.rolling.strategy.type = DefaultRolloverStrategy
- appender.rolling.strategy.max = 10
-
- # Suppress the irrelevant (wrong) warnings from the Netty channel handler
- logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
- logger.netty.level = OFF
\ No newline at end of file
diff --git a/theodolite-benchmarks/definitions/uc4-beam-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc4-beam-flink/resources/taskmanager-deployment.yaml
index 2bd37ed9b..062ef653b 100644
--- a/theodolite-benchmarks/definitions/uc4-beam-flink/resources/taskmanager-deployment.yaml
+++ b/theodolite-benchmarks/definitions/uc4-beam-flink/resources/taskmanager-deployment.yaml
@@ -20,27 +20,8 @@ spec:
image: ghcr.io/cau-se/theodolite-uc4-beam-flink:latest
args: ["taskmanager"]
env:
- - name: KAFKA_BOOTSTRAP_SERVERS
- value: "theodolite-kafka-kafka-bootstrap:9092"
- - name: SCHEMA_REGISTRY_URL
- value: "http://theodolite-kafka-schema-registry:8081"
- - name: CHECKPOINTING
- value: "false"
- - name: "FLINK_STATE_BACKEND"
- value: "rocksdb"
- name: JOB_MANAGER_RPC_ADDRESS
value: "flink-jobmanager"
- # - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS
- # value: "1" #TODO
- # - name: FLINK_PROPERTIES
- # value: |+
- # blob.server.port: 6124
- # jobmanager.rpc.port: 6123
- # taskmanager.rpc.port: 6122
- # queryable-state.proxy.ports: 6125
- # jobmanager.memory.process.size: 4Gb
- # taskmanager.memory.process.size: 4Gb
- # #parallelism.default: 1 #TODO
resources:
limits:
memory: 4Gi
@@ -52,11 +33,5 @@ spec:
name: query-state
- containerPort: 9249
name: metrics
- # livenessProbe:
- # tcpSocket:
- # port: 6122
- # initialDelaySeconds: 30
- # periodSeconds: 60
securityContext:
runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
-
diff --git a/theodolite-benchmarks/definitions/uc4-beam-flink/uc4-beam-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc4-beam-flink/uc4-beam-flink-benchmark-operator.yaml
index aab7d7558..ddb7b574d 100644
--- a/theodolite-benchmarks/definitions/uc4-beam-flink/uc4-beam-flink-benchmark-operator.yaml
+++ b/theodolite-benchmarks/definitions/uc4-beam-flink/uc4-beam-flink-benchmark-operator.yaml
@@ -36,11 +36,6 @@ spec:
properties:
container: "jobmanager"
variableName: "PARALLELISM"
- - type: "EnvVarPatcher" # required?
- resource: "taskmanager-deployment.yaml"
- properties:
- container: "taskmanager"
- variableName: "PARALLELISM"
loadTypes:
- typeName: "NumNestedGroups"
patchers:
@@ -80,4 +75,4 @@ spec:
numPartitions: 40
replicationFactor: 1
- name: "theodolite-.*"
- removeOnly: True
\ No newline at end of file
+ removeOnly: True
--
GitLab