Clean up Beam/Flink deployment

theodolite-benchmarks/definitions/uc1-beam-flink/resources/flink-configuration-configmap.yaml

-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: flink-config
-  labels:
-    app: flink
-data:
-  flink-conf.yaml: |+
-    jobmanager.rpc.address: flink-jobmanager
-    taskmanager.numberOfTaskSlots: 1 #TODO
-    #blob.server.port: 6124
-    #jobmanager.rpc.port: 6123
-    #taskmanager.rpc.port: 6122
-    #queryable-state.proxy.ports: 6125
-    #jobmanager.memory.process.size: 4Gb
-    #taskmanager.memory.process.size: 4Gb
-    #parallelism.default: 1 #TODO
-    metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
-    metrics.reporter.prom.interval: 10 SECONDS
-    taskmanager.network.detailed-metrics: true
-  # -> gives metrics about inbound/outbound network queue lengths
-  log4j-console.properties: |+
-    # This affects logging for both user code and Flink
-    rootLogger.level = INFO
-    rootLogger.appenderRef.console.ref = ConsoleAppender
-    rootLogger.appenderRef.rolling.ref = RollingFileAppender
-
-    # Uncomment this if you want to _only_ change Flink's logging
-    #logger.flink.name = org.apache.flink
-    #logger.flink.level = INFO
-
-    # The following lines keep the log level of common libraries/connectors on
-    # log level INFO. The root logger does not override this. You have to manually
-    # change the log levels here.
-    logger.akka.name = akka
-    logger.akka.level = INFO
-    logger.kafka.name= org.apache.kafka
-    logger.kafka.level = INFO
-    logger.hadoop.name = org.apache.hadoop
-    logger.hadoop.level = INFO
-    logger.zookeeper.name = org.apache.zookeeper
-    logger.zookeeper.level = INFO
-
-    # Log all infos to the console
-    appender.console.name = ConsoleAppender
-    appender.console.type = CONSOLE
-    appender.console.layout.type = PatternLayout
-    appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
-
-    # Log all infos in the given rolling file
-    appender.rolling.name = RollingFileAppender
-    appender.rolling.type = RollingFile
-    appender.rolling.append = false
-    appender.rolling.fileName = ${sys:log.file}
-    appender.rolling.filePattern = ${sys:log.file}.%i
-    appender.rolling.layout.type = PatternLayout
-    appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
-    appender.rolling.policies.type = Policies
-    appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
-    appender.rolling.policies.size.size=100MB
-    appender.rolling.strategy.type = DefaultRolloverStrategy
-    appender.rolling.strategy.max = 10
-
-    # Suppress the irrelevant (wrong) warnings from the Netty channel handler
-    logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
-    logger.netty.level = OFF
\ No newline at end of file

theodolite-benchmarks/definitions/uc1-beam-flink/resources/jobmanager-deployment.yaml

@@ -63,33 +63,5 @@ spec:
               port: 6123
             initialDelaySeconds: 30
             periodSeconds: 60
-          # volumeMounts:
-          #   - name: flink-config-volume-rw
-          #     mountPath: /opt/flink/conf
-#            - name: job-artifacts-volume
-#              mountPath: /opt/flink/usrlib
           securityContext:
             runAsUser: 9999  # refers to user _flink_ from official flink image, change if necessary
-      # initContainers:
-      #   - name: init-jobmanager
-      #     image: busybox:1.28
-      #     command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
-      #     volumeMounts:
-      #       - name: flink-config-volume
-      #         mountPath: /flink-config/
-      #       - name: flink-config-volume-rw
-      #         mountPath: /flink-config-rw/
-      # volumes:
-      #   - name: flink-config-volume
-      #     configMap:
-      #       name: flink-config
-      #       items:
-      #         - key: flink-conf.yaml
-      #           path: flink-conf.yaml
-      #         - key: log4j-console.properties
-      #           path: log4j-console.properties
-      #   - name: flink-config-volume-rw
-      #     emptyDir: {}
-#        - name: job-artifacts-volume
-#          hostPath:
-#            path: /host/path/to/job/artifacts

theodolite-benchmarks/definitions/uc1-beam-flink/resources/taskmanager-deployment.yaml

@@ -20,29 +20,8 @@ spec:
           image: ghcr.io/cau-se/theodolite-uc1-beam-flink:latest
           args: ["taskmanager"]
           env:
-            - name: KAFKA_BOOTSTRAP_SERVERS
-              value: "theodolite-kafka-kafka-bootstrap:9092"
-            - name: SCHEMA_REGISTRY_URL
-              value: "http://theodolite-kafka-schema-registry:8081"
-            - name: CHECKPOINTING
-              value: "false"
-            # - name: PARALLELISM
-            #   value: "1"
-            - name: "FLINK_STATE_BACKEND"
-              value: "rocksdb"
             - name: JOB_MANAGER_RPC_ADDRESS
               value: "flink-jobmanager"
-            # - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS
-            #   value: "1" #TODO
-            # - name: FLINK_PROPERTIES
-            #   value: |+
-            #     blob.server.port: 6124
-            #     jobmanager.rpc.port: 6123
-            #     taskmanager.rpc.port: 6122
-            #     queryable-state.proxy.ports: 6125
-            #     jobmanager.memory.process.size: 4Gb
-            #     taskmanager.memory.process.size: 4Gb
-            #     #parallelism.default: 1 #TODO
           resources:
             limits:
               memory: 4Gi
@@ -54,33 +33,5 @@ spec:
               name: query-state
             - containerPort: 9249
               name: metrics
-          # livenessProbe:
-          #   tcpSocket:
-          #     port: 6122
-          #   initialDelaySeconds: 30
-          #   periodSeconds: 60
-          # volumeMounts:
-          #   - name: flink-config-volume-rw
-          #     mountPath: /opt/flink/conf/
           securityContext:
             runAsUser: 9999  # refers to user _flink_ from official flink image, change if necessary
-      # initContainers:
-      #   - name: init-taskmanager
-      #     image: busybox:1.28
-      #     command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
-      #     volumeMounts:
-      #       - name: flink-config-volume
-      #         mountPath: /flink-config/
-      #       - name: flink-config-volume-rw
-      #         mountPath: /flink-config-rw/
-      # volumes:
-      #   - name: flink-config-volume
-      #     configMap:
-      #       name: flink-config
-      #       items:
-      #         - key: flink-conf.yaml
-      #           path: flink-conf.yaml
-      #         - key: log4j-console.properties
-      #           path: log4j-console.properties
-      #   - name: flink-config-volume-rw
-      #     emptyDir: {}

theodolite-benchmarks/definitions/uc1-beam-flink/uc1-beam-flink-benchmark-operator.yaml

@@ -36,11 +36,6 @@ spec:
           properties:
             container: "jobmanager"
             variableName: "PARALLELISM"
-        - type: "EnvVarPatcher" # required?
-          resource: "taskmanager-deployment.yaml"
-          properties:
-            container: "taskmanager"
-            variableName: "PARALLELISM"
   loadTypes:
     - typeName: "NumSensors"
       patchers:

theodolite-benchmarks/definitions/uc2-beam-flink/resources/flink-configuration-configmap.yaml

-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: flink-config
-  labels:
-    app: flink
-data:
-  flink-conf.yaml: |+
-    jobmanager.rpc.address: flink-jobmanager
-    taskmanager.numberOfTaskSlots: 1 #TODO
-    #blob.server.port: 6124
-    #jobmanager.rpc.port: 6123
-    #taskmanager.rpc.port: 6122
-    #queryable-state.proxy.ports: 6125
-    #jobmanager.memory.process.size: 4Gb
-    #taskmanager.memory.process.size: 4Gb
-    #parallelism.default: 1 #TODO
-    metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
-    metrics.reporter.prom.interval: 10 SECONDS
-    taskmanager.network.detailed-metrics: true
-  # -> gives metrics about inbound/outbound network queue lengths
-  log4j-console.properties: |+
-    # This affects logging for both user code and Flink
-    rootLogger.level = INFO
-    rootLogger.appenderRef.console.ref = ConsoleAppender
-    rootLogger.appenderRef.rolling.ref = RollingFileAppender
-
-    # Uncomment this if you want to _only_ change Flink's logging
-    #logger.flink.name = org.apache.flink
-    #logger.flink.level = INFO
-
-    # The following lines keep the log level of common libraries/connectors on
-    # log level INFO. The root logger does not override this. You have to manually
-    # change the log levels here.
-    logger.akka.name = akka
-    logger.akka.level = INFO
-    logger.kafka.name= org.apache.kafka
-    logger.kafka.level = INFO
-    logger.hadoop.name = org.apache.hadoop
-    logger.hadoop.level = INFO
-    logger.zookeeper.name = org.apache.zookeeper
-    logger.zookeeper.level = INFO
-
-    # Log all infos to the console
-    appender.console.name = ConsoleAppender
-    appender.console.type = CONSOLE
-    appender.console.layout.type = PatternLayout
-    appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
-
-    # Log all infos in the given rolling file
-    appender.rolling.name = RollingFileAppender
-    appender.rolling.type = RollingFile
-    appender.rolling.append = false
-    appender.rolling.fileName = ${sys:log.file}
-    appender.rolling.filePattern = ${sys:log.file}.%i
-    appender.rolling.layout.type = PatternLayout
-    appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
-    appender.rolling.policies.type = Policies
-    appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
-    appender.rolling.policies.size.size=100MB
-    appender.rolling.strategy.type = DefaultRolloverStrategy
-    appender.rolling.strategy.max = 10
-
-    # Suppress the irrelevant (wrong) warnings from the Netty channel handler
-    logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
-    logger.netty.level = OFF
\ No newline at end of file

theodolite-benchmarks/definitions/uc2-beam-flink/resources/taskmanager-deployment.yaml

@@ -20,27 +20,8 @@ spec:
           image: ghcr.io/cau-se/theodolite-uc2-beam-flink:latest
           args: ["taskmanager"]
           env:
-            - name: KAFKA_BOOTSTRAP_SERVERS
-              value: "theodolite-kafka-kafka-bootstrap:9092"
-            - name: SCHEMA_REGISTRY_URL
-              value: "http://theodolite-kafka-schema-registry:8081"
-            - name: CHECKPOINTING
-              value: "false"
-            - name: "FLINK_STATE_BACKEND"
-              value: "rocksdb"
             - name: JOB_MANAGER_RPC_ADDRESS
               value: "flink-jobmanager"
-            # - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS
-            #   value: "1" #TODO
-            # - name: FLINK_PROPERTIES
-            #   value: |+
-            #     blob.server.port: 6124
-            #     jobmanager.rpc.port: 6123
-            #     taskmanager.rpc.port: 6122
-            #     queryable-state.proxy.ports: 6125
-            #     jobmanager.memory.process.size: 4Gb
-            #     taskmanager.memory.process.size: 4Gb
-            #     #parallelism.default: 1 #TODO
           resources:
             limits:
               memory: 4Gi
@@ -52,11 +33,5 @@ spec:
               name: query-state
             - containerPort: 9249
               name: metrics
-          # livenessProbe:
-          #   tcpSocket:
-          #     port: 6122
-          #   initialDelaySeconds: 30
-          #   periodSeconds: 60
           securityContext:
             runAsUser: 9999  # refers to user _flink_ from official flink image, change if necessary
-

theodolite-benchmarks/definitions/uc2-beam-flink/uc2-beam-flink-benchmark-operator.yaml

@@ -36,11 +36,6 @@ spec:
           properties:
             container: "jobmanager"
             variableName: "PARALLELISM"
-        - type: "EnvVarPatcher" # required?
-          resource: "taskmanager-deployment.yaml"
-          properties:
-            container: "taskmanager"
-            variableName: "PARALLELISM"
   loadTypes:
     - typeName: "NumSensors"
       patchers:

theodolite-benchmarks/definitions/uc3-beam-flink/resources/flink-configuration-configmap.yaml

-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: flink-config
-  labels:
-    app: flink
-data:
-  flink-conf.yaml: |+
-    jobmanager.rpc.address: flink-jobmanager
-    taskmanager.numberOfTaskSlots: 1 #TODO
-    #blob.server.port: 6124
-    #jobmanager.rpc.port: 6123
-    #taskmanager.rpc.port: 6122
-    #queryable-state.proxy.ports: 6125
-    #jobmanager.memory.process.size: 4Gb
-    #taskmanager.memory.process.size: 4Gb
-    #parallelism.default: 1 #TODO
-    metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
-    metrics.reporter.prom.interval: 10 SECONDS
-    taskmanager.network.detailed-metrics: true
-  # -> gives metrics about inbound/outbound network queue lengths
-  log4j-console.properties: |+
-    # This affects logging for both user code and Flink
-    rootLogger.level = INFO
-    rootLogger.appenderRef.console.ref = ConsoleAppender
-    rootLogger.appenderRef.rolling.ref = RollingFileAppender
-
-    # Uncomment this if you want to _only_ change Flink's logging
-    #logger.flink.name = org.apache.flink
-    #logger.flink.level = INFO
-
-    # The following lines keep the log level of common libraries/connectors on
-    # log level INFO. The root logger does not override this. You have to manually
-    # change the log levels here.
-    logger.akka.name = akka
-    logger.akka.level = INFO
-    logger.kafka.name= org.apache.kafka
-    logger.kafka.level = INFO
-    logger.hadoop.name = org.apache.hadoop
-    logger.hadoop.level = INFO
-    logger.zookeeper.name = org.apache.zookeeper
-    logger.zookeeper.level = INFO
-
-    # Log all infos to the console
-    appender.console.name = ConsoleAppender
-    appender.console.type = CONSOLE
-    appender.console.layout.type = PatternLayout
-    appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
-
-    # Log all infos in the given rolling file
-    appender.rolling.name = RollingFileAppender
-    appender.rolling.type = RollingFile
-    appender.rolling.append = false
-    appender.rolling.fileName = ${sys:log.file}
-    appender.rolling.filePattern = ${sys:log.file}.%i
-    appender.rolling.layout.type = PatternLayout
-    appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
-    appender.rolling.policies.type = Policies
-    appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
-    appender.rolling.policies.size.size=100MB
-    appender.rolling.strategy.type = DefaultRolloverStrategy
-    appender.rolling.strategy.max = 10
-
-    # Suppress the irrelevant (wrong) warnings from the Netty channel handler
-    logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
-    logger.netty.level = OFF
\ No newline at end of file

theodolite-benchmarks/definitions/uc3-beam-flink/resources/taskmanager-deployment.yaml

@@ -20,27 +20,8 @@ spec:
           image: ghcr.io/cau-se/theodolite-uc3-beam-flink:latest
           args: ["taskmanager"]
           env:
-            - name: KAFKA_BOOTSTRAP_SERVERS
-              value: "theodolite-kafka-kafka-bootstrap:9092"
-            - name: SCHEMA_REGISTRY_URL
-              value: "http://theodolite-kafka-schema-registry:8081"
-            - name: CHECKPOINTING
-              value: "false"
-            - name: "FLINK_STATE_BACKEND"
-              value: "rocksdb"
             - name: JOB_MANAGER_RPC_ADDRESS
               value: "flink-jobmanager"
-            # - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS
-            #   value: "1" #TODO
-            # - name: FLINK_PROPERTIES
-            #   value: |+
-            #     blob.server.port: 6124
-            #     jobmanager.rpc.port: 6123
-            #     taskmanager.rpc.port: 6122
-            #     queryable-state.proxy.ports: 6125
-            #     jobmanager.memory.process.size: 4Gb
-            #     taskmanager.memory.process.size: 4Gb
-            #     #parallelism.default: 1 #TODO
           resources:
             limits:
               memory: 4Gi
@@ -52,11 +33,5 @@ spec:
               name: query-state
             - containerPort: 9249
               name: metrics
-          # livenessProbe:
-          #   tcpSocket:
-          #     port: 6122
-          #   initialDelaySeconds: 30
-          #   periodSeconds: 60
           securityContext:
             runAsUser: 9999  # refers to user _flink_ from official flink image, change if necessary
-

theodolite-benchmarks/definitions/uc3-beam-flink/uc3-beam-flink-benchmark-operator.yaml

@@ -36,11 +36,6 @@ spec:
           properties:
             container: "jobmanager"
             variableName: "PARALLELISM"
-        - type: "EnvVarPatcher" # required?
-          resource: "taskmanager-deployment.yaml"
-          properties:
-            container: "taskmanager"
-            variableName: "PARALLELISM"
   loadTypes:
     - typeName: "NumSensors"
       patchers:

theodolite-benchmarks/definitions/uc4-beam-flink/resources/flink-configuration-configmap.yaml

-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: flink-config
-  labels:
-    app: flink
-data:
-  flink-conf.yaml: |+
-    jobmanager.rpc.address: flink-jobmanager
-    taskmanager.numberOfTaskSlots: 1 #TODO
-    #blob.server.port: 6124
-    #jobmanager.rpc.port: 6123
-    #taskmanager.rpc.port: 6122
-    #queryable-state.proxy.ports: 6125
-    #jobmanager.memory.process.size: 4Gb
-    #taskmanager.memory.process.size: 4Gb
-    #parallelism.default: 1 #TODO
-    metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
-    metrics.reporter.prom.interval: 10 SECONDS
-    taskmanager.network.detailed-metrics: true
-  # -> gives metrics about inbound/outbound network queue lengths
-  log4j-console.properties: |+
-    # This affects logging for both user code and Flink
-    rootLogger.level = INFO
-    rootLogger.appenderRef.console.ref = ConsoleAppender
-    rootLogger.appenderRef.rolling.ref = RollingFileAppender
-
-    # Uncomment this if you want to _only_ change Flink's logging
-    #logger.flink.name = org.apache.flink
-    #logger.flink.level = INFO
-
-    # The following lines keep the log level of common libraries/connectors on
-    # log level INFO. The root logger does not override this. You have to manually
-    # change the log levels here.
-    logger.akka.name = akka
-    logger.akka.level = INFO
-    logger.kafka.name= org.apache.kafka
-    logger.kafka.level = INFO
-    logger.hadoop.name = org.apache.hadoop
-    logger.hadoop.level = INFO
-    logger.zookeeper.name = org.apache.zookeeper
-    logger.zookeeper.level = INFO
-
-    # Log all infos to the console
-    appender.console.name = ConsoleAppender
-    appender.console.type = CONSOLE
-    appender.console.layout.type = PatternLayout
-    appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
-
-    # Log all infos in the given rolling file
-    appender.rolling.name = RollingFileAppender
-    appender.rolling.type = RollingFile
-    appender.rolling.append = false
-    appender.rolling.fileName = ${sys:log.file}
-    appender.rolling.filePattern = ${sys:log.file}.%i
-    appender.rolling.layout.type = PatternLayout
-    appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
-    appender.rolling.policies.type = Policies
-    appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
-    appender.rolling.policies.size.size=100MB
-    appender.rolling.strategy.type = DefaultRolloverStrategy
-    appender.rolling.strategy.max = 10
-
-    # Suppress the irrelevant (wrong) warnings from the Netty channel handler
-    logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
-    logger.netty.level = OFF
\ No newline at end of file

theodolite-benchmarks/definitions/uc4-beam-flink/resources/taskmanager-deployment.yaml

@@ -20,27 +20,8 @@ spec:
           image: ghcr.io/cau-se/theodolite-uc4-beam-flink:latest
           args: ["taskmanager"]
           env:
-            - name: KAFKA_BOOTSTRAP_SERVERS
-              value: "theodolite-kafka-kafka-bootstrap:9092"
-            - name: SCHEMA_REGISTRY_URL
-              value: "http://theodolite-kafka-schema-registry:8081"
-            - name: CHECKPOINTING
-              value: "false"
-            - name: "FLINK_STATE_BACKEND"
-              value: "rocksdb"
             - name: JOB_MANAGER_RPC_ADDRESS
               value: "flink-jobmanager"
-            # - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS
-            #   value: "1" #TODO
-            # - name: FLINK_PROPERTIES
-            #   value: |+
-            #     blob.server.port: 6124
-            #     jobmanager.rpc.port: 6123
-            #     taskmanager.rpc.port: 6122
-            #     queryable-state.proxy.ports: 6125
-            #     jobmanager.memory.process.size: 4Gb
-            #     taskmanager.memory.process.size: 4Gb
-            #     #parallelism.default: 1 #TODO
           resources:
             limits:
               memory: 4Gi
@@ -52,11 +33,5 @@ spec:
               name: query-state
             - containerPort: 9249
               name: metrics
-          # livenessProbe:
-          #   tcpSocket:
-          #     port: 6122
-          #   initialDelaySeconds: 30
-          #   periodSeconds: 60
           securityContext:
             runAsUser: 9999  # refers to user _flink_ from official flink image, change if necessary
-

theodolite-benchmarks/definitions/uc4-beam-flink/uc4-beam-flink-benchmark-operator.yaml

@@ -36,11 +36,6 @@ spec:
           properties:
             container: "jobmanager"
             variableName: "PARALLELISM"
-        - type: "EnvVarPatcher" # required?
-          resource: "taskmanager-deployment.yaml"
-          properties:
-            container: "taskmanager"
-            variableName: "PARALLELISM"
   loadTypes:
     - typeName: "NumNestedGroups"
       patchers: