From 002b6c10107aaf8c71a322f54efeaf49ed5b44cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Vonheiden?= <bjoern.vonheiden@hotmail.de> Date: Mon, 20 Sep 2021 10:45:58 +0200 Subject: [PATCH] Clen up definitions files for flink use cases The flink use cases now uses the configmap instead of environment variables for configuring the flink job and taskmanager. This allows a single point for configuration and removes uncertainty about where to set the configurations. --- .../flink-configuration-configmap.yaml | 18 ++++----- .../resources/jobmanager-deployment.yaml | 29 +------------- .../resources/taskmanager-deployment.yaml | 38 +------------------ .../uc1-flink-benchmark-operator.yaml | 7 +--- .../flink-configuration-configmap.yaml | 20 +++++----- .../resources/jobmanager-deployment.yaml | 29 +------------- .../resources/taskmanager-deployment.yaml | 38 +------------------ .../uc2-flink-benchmark-operator.yaml | 5 --- .../flink-configuration-configmap.yaml | 2 +- .../flink-configuration-configmap.yaml | 20 +++++----- .../resources/jobmanager-deployment.yaml | 29 +------------- .../resources/taskmanager-deployment.yaml | 38 +------------------ .../uc4-flink-benchmark-operator.yaml | 5 --- 13 files changed, 34 insertions(+), 244 deletions(-) diff --git a/theodolite-benchmarks/definitions/uc1-flink/resources/flink-configuration-configmap.yaml b/theodolite-benchmarks/definitions/uc1-flink/resources/flink-configuration-configmap.yaml index 36178e2be..ecda40250 100644 --- a/theodolite-benchmarks/definitions/uc1-flink/resources/flink-configuration-configmap.yaml +++ b/theodolite-benchmarks/definitions/uc1-flink/resources/flink-configuration-configmap.yaml @@ -7,18 +7,16 @@ metadata: data: flink-conf.yaml: |+ jobmanager.rpc.address: flink-jobmanager - taskmanager.numberOfTaskSlots: 1 #TODO - #blob.server.port: 6124 - #jobmanager.rpc.port: 6123 - #taskmanager.rpc.port: 6122 - #queryable-state.proxy.ports: 6125 - #jobmanager.memory.process.size: 4Gb - #taskmanager.memory.process.size: 4Gb - #parallelism.default: 1 #TODO + blob.server.port: 6124 + jobmanager.rpc.port: 6123 + taskmanager.rpc.port: 6122 + queryable-state.proxy.ports: 6125 + jobmanager.memory.process.size: 4Gb + taskmanager.memory.process.size: 4Gb metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter metrics.reporter.prom.interval: 10 SECONDS + # gives metrics about inbound/outbound network queue lengths taskmanager.network.detailed-metrics: true - # -> gives metrics about inbound/outbound network queue lengths log4j-console.properties: |+ # This affects logging for both user code and Flink rootLogger.level = INFO @@ -63,4 +61,4 @@ data: # Suppress the irrelevant (wrong) warnings from the Netty channel handler logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline - logger.netty.level = OFF \ No newline at end of file + logger.netty.level = OFF diff --git a/theodolite-benchmarks/definitions/uc1-flink/resources/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc1-flink/resources/jobmanager-deployment.yaml index 1f328b1cd..6502281a1 100644 --- a/theodolite-benchmarks/definitions/uc1-flink/resources/jobmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc1-flink/resources/jobmanager-deployment.yaml @@ -31,17 +31,6 @@ spec: value: "1" - name: "FLINK_STATE_BACKEND" value: "rocksdb" - - name: JOB_MANAGER_RPC_ADDRESS - value: "flink-jobmanager" - - name: FLINK_PROPERTIES - value: |+ - blob.server.port: 6124 - jobmanager.rpc.port: 6123 - taskmanager.rpc.port: 6122 - queryable-state.proxy.ports: 6125 - jobmanager.memory.process.size: 4Gb - taskmanager.memory.process.size: 4Gb - #parallelism.default: 1 #TODO resources: limits: memory: 4Gi @@ -63,21 +52,10 @@ spec: initialDelaySeconds: 30 periodSeconds: 60 volumeMounts: - - name: flink-config-volume-rw + - name: flink-config-volume mountPath: /opt/flink/conf -# - name: job-artifacts-volume -# mountPath: /opt/flink/usrlib securityContext: runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary - initContainers: - - name: init-jobmanager - image: busybox:1.28 - command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/'] - volumeMounts: - - name: flink-config-volume - mountPath: /flink-config/ - - name: flink-config-volume-rw - mountPath: /flink-config-rw/ volumes: - name: flink-config-volume configMap: @@ -87,8 +65,3 @@ spec: path: flink-conf.yaml - key: log4j-console.properties path: log4j-console.properties - - name: flink-config-volume-rw - emptyDir: {} -# - name: job-artifacts-volume -# hostPath: -# path: /host/path/to/job/artifacts diff --git a/theodolite-benchmarks/definitions/uc1-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc1-flink/resources/taskmanager-deployment.yaml index c2266a4ae..15c4bdad7 100644 --- a/theodolite-benchmarks/definitions/uc1-flink/resources/taskmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc1-flink/resources/taskmanager-deployment.yaml @@ -19,31 +19,6 @@ spec: - name: taskmanager image: ghcr.io/cau-se/theodolite-uc1-flink:latest env: - - name: KAFKA_BOOTSTRAP_SERVERS - value: "theodolite-cp-kafka:9092" - - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" - - name: COMMIT_INTERVAL_MS - value: "100" - - name: CHECKPOINTING - value: "false" - - name: PARALLELISM - value: "1" - - name: "FLINK_STATE_BACKEND" - value: "rocksdb" - - name: JOB_MANAGER_RPC_ADDRESS - value: "flink-jobmanager" - - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS - value: "1" #TODO - - name: FLINK_PROPERTIES - value: |+ - blob.server.port: 6124 - jobmanager.rpc.port: 6123 - taskmanager.rpc.port: 6122 - queryable-state.proxy.ports: 6125 - jobmanager.memory.process.size: 4Gb - taskmanager.memory.process.size: 4Gb - #parallelism.default: 1 #TODO resources: limits: memory: 4Gi @@ -62,19 +37,10 @@ spec: initialDelaySeconds: 30 periodSeconds: 60 volumeMounts: - - name: flink-config-volume-rw + - name: flink-config-volume mountPath: /opt/flink/conf/ securityContext: runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary - initContainers: - - name: init-taskmanager - image: busybox:1.28 - command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/'] - volumeMounts: - - name: flink-config-volume - mountPath: /flink-config/ - - name: flink-config-volume-rw - mountPath: /flink-config-rw/ volumes: - name: flink-config-volume configMap: @@ -84,5 +50,3 @@ spec: path: flink-conf.yaml - key: log4j-console.properties path: log4j-console.properties - - name: flink-config-volume-rw - emptyDir: {} diff --git a/theodolite-benchmarks/definitions/uc1-flink/uc1-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc1-flink/uc1-flink-benchmark-operator.yaml index 3e16e4864..0f3dbe83b 100644 --- a/theodolite-benchmarks/definitions/uc1-flink/uc1-flink-benchmark-operator.yaml +++ b/theodolite-benchmarks/definitions/uc1-flink/uc1-flink-benchmark-operator.yaml @@ -24,11 +24,6 @@ spec: properties: container: "jobmanager" variableName: "PARALLELISM" - - type: "EnvVarPatcher" # required? - resource: "uc1-flink/taskmanager-deployment.yaml" - properties: - container: "taskmanager" - variableName: "PARALLELISM" loadTypes: - typeName: "NumSensors" patchers: @@ -46,4 +41,4 @@ spec: topics: - name: "input" numPartitions: 40 - replicationFactor: 1 \ No newline at end of file + replicationFactor: 1 diff --git a/theodolite-benchmarks/definitions/uc2-flink/resources/flink-configuration-configmap.yaml b/theodolite-benchmarks/definitions/uc2-flink/resources/flink-configuration-configmap.yaml index 321541f6a..ecda40250 100644 --- a/theodolite-benchmarks/definitions/uc2-flink/resources/flink-configuration-configmap.yaml +++ b/theodolite-benchmarks/definitions/uc2-flink/resources/flink-configuration-configmap.yaml @@ -6,19 +6,17 @@ metadata: app: flink data: flink-conf.yaml: |+ - #jobmanager.rpc.address: flink-jobmanager - #taskmanager.numberOfTaskSlots: 1 #TODO - #blob.server.port: 6124 - #jobmanager.rpc.port: 6123 - #taskmanager.rpc.port: 6122 - #queryable-state.proxy.ports: 6125 - #jobmanager.memory.process.size: 4Gb - #taskmanager.memory.process.size: 4Gb - #parallelism.default: 1 #TODO + jobmanager.rpc.address: flink-jobmanager + blob.server.port: 6124 + jobmanager.rpc.port: 6123 + taskmanager.rpc.port: 6122 + queryable-state.proxy.ports: 6125 + jobmanager.memory.process.size: 4Gb + taskmanager.memory.process.size: 4Gb metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter metrics.reporter.prom.interval: 10 SECONDS + # gives metrics about inbound/outbound network queue lengths taskmanager.network.detailed-metrics: true - # -> gives metrics about inbound/outbound network queue lengths log4j-console.properties: |+ # This affects logging for both user code and Flink rootLogger.level = INFO @@ -63,4 +61,4 @@ data: # Suppress the irrelevant (wrong) warnings from the Netty channel handler logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline - logger.netty.level = OFF \ No newline at end of file + logger.netty.level = OFF diff --git a/theodolite-benchmarks/definitions/uc2-flink/resources/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc2-flink/resources/jobmanager-deployment.yaml index 87ea174f7..0f9e0c3b2 100644 --- a/theodolite-benchmarks/definitions/uc2-flink/resources/jobmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc2-flink/resources/jobmanager-deployment.yaml @@ -31,17 +31,6 @@ spec: value: "1" - name: "FLINK_STATE_BACKEND" value: "rocksdb" - - name: JOB_MANAGER_RPC_ADDRESS - value: "flink-jobmanager" - - name: FLINK_PROPERTIES - value: |+ - blob.server.port: 6124 - jobmanager.rpc.port: 6123 - taskmanager.rpc.port: 6122 - queryable-state.proxy.ports: 6125 - jobmanager.memory.process.size: 4Gb - taskmanager.memory.process.size: 4Gb - #parallelism.default: 1 #TODO resources: limits: memory: 4Gi @@ -63,21 +52,10 @@ spec: initialDelaySeconds: 30 periodSeconds: 60 volumeMounts: - - name: flink-config-volume-rw + - name: flink-config-volume mountPath: /opt/flink/conf -# - name: job-artifacts-volume -# mountPath: /opt/flink/usrlib securityContext: runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary - initContainers: - - name: init-jobmanager - image: busybox:1.28 - command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/'] - volumeMounts: - - name: flink-config-volume - mountPath: /flink-config/ - - name: flink-config-volume-rw - mountPath: /flink-config-rw/ volumes: - name: flink-config-volume configMap: @@ -87,8 +65,3 @@ spec: path: flink-conf.yaml - key: log4j-console.properties path: log4j-console.properties - - name: flink-config-volume-rw - emptyDir: {} -# - name: job-artifacts-volume -# hostPath: -# path: /host/path/to/job/artifacts diff --git a/theodolite-benchmarks/definitions/uc2-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc2-flink/resources/taskmanager-deployment.yaml index c37df972a..5a0d6847a 100644 --- a/theodolite-benchmarks/definitions/uc2-flink/resources/taskmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc2-flink/resources/taskmanager-deployment.yaml @@ -19,31 +19,6 @@ spec: - name: taskmanager image: ghcr.io/cau-se/theodolite-uc2-flink:latest env: - - name: KAFKA_BOOTSTRAP_SERVERS - value: "theodolite-cp-kafka:9092" - - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" - - name: COMMIT_INTERVAL_MS - value: "100" - - name: CHECKPOINTING - value: "false" - - name: PARALLELISM - value: "1" - - name: "FLINK_STATE_BACKEND" - value: "rocksdb" - - name: JOB_MANAGER_RPC_ADDRESS - value: "flink-jobmanager" - - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS - value: "1" #TODO - - name: FLINK_PROPERTIES - value: |+ - blob.server.port: 6124 - jobmanager.rpc.port: 6123 - taskmanager.rpc.port: 6122 - queryable-state.proxy.ports: 6125 - jobmanager.memory.process.size: 4Gb - taskmanager.memory.process.size: 4Gb - #parallelism.default: 1 #TODO resources: limits: memory: 4Gi @@ -62,19 +37,10 @@ spec: initialDelaySeconds: 30 periodSeconds: 60 volumeMounts: - - name: flink-config-volume-rw + - name: flink-config-volume mountPath: /opt/flink/conf/ securityContext: runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary - initContainers: - - name: init-taskmanager - image: busybox:1.28 - command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/'] - volumeMounts: - - name: flink-config-volume - mountPath: /flink-config/ - - name: flink-config-volume-rw - mountPath: /flink-config-rw/ volumes: - name: flink-config-volume configMap: @@ -84,5 +50,3 @@ spec: path: flink-conf.yaml - key: log4j-console.properties path: log4j-console.properties - - name: flink-config-volume-rw - emptyDir: {} diff --git a/theodolite-benchmarks/definitions/uc2-flink/uc2-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc2-flink/uc2-flink-benchmark-operator.yaml index ab335cc4f..71f479bdd 100644 --- a/theodolite-benchmarks/definitions/uc2-flink/uc2-flink-benchmark-operator.yaml +++ b/theodolite-benchmarks/definitions/uc2-flink/uc2-flink-benchmark-operator.yaml @@ -24,11 +24,6 @@ spec: properties: container: "jobmanager" variableName: "PARALLELISM" - - type: "EnvVarPatcher" # required? - resource: "uc2-flink/taskmanager-deployment.yaml" - properties: - container: "taskmanager" - variableName: "PARALLELISM" loadTypes: - typeName: "NumSensors" patchers: diff --git a/theodolite-benchmarks/definitions/uc3-flink/resources/flink-configuration-configmap.yaml b/theodolite-benchmarks/definitions/uc3-flink/resources/flink-configuration-configmap.yaml index 04dbd0d00..ecda40250 100644 --- a/theodolite-benchmarks/definitions/uc3-flink/resources/flink-configuration-configmap.yaml +++ b/theodolite-benchmarks/definitions/uc3-flink/resources/flink-configuration-configmap.yaml @@ -6,7 +6,7 @@ metadata: app: flink data: flink-conf.yaml: |+ - jobmanager.rpc.address: flink-jobmanager # must be set also in deployments + jobmanager.rpc.address: flink-jobmanager blob.server.port: 6124 jobmanager.rpc.port: 6123 taskmanager.rpc.port: 6122 diff --git a/theodolite-benchmarks/definitions/uc4-flink/resources/flink-configuration-configmap.yaml b/theodolite-benchmarks/definitions/uc4-flink/resources/flink-configuration-configmap.yaml index 321541f6a..ecda40250 100644 --- a/theodolite-benchmarks/definitions/uc4-flink/resources/flink-configuration-configmap.yaml +++ b/theodolite-benchmarks/definitions/uc4-flink/resources/flink-configuration-configmap.yaml @@ -6,19 +6,17 @@ metadata: app: flink data: flink-conf.yaml: |+ - #jobmanager.rpc.address: flink-jobmanager - #taskmanager.numberOfTaskSlots: 1 #TODO - #blob.server.port: 6124 - #jobmanager.rpc.port: 6123 - #taskmanager.rpc.port: 6122 - #queryable-state.proxy.ports: 6125 - #jobmanager.memory.process.size: 4Gb - #taskmanager.memory.process.size: 4Gb - #parallelism.default: 1 #TODO + jobmanager.rpc.address: flink-jobmanager + blob.server.port: 6124 + jobmanager.rpc.port: 6123 + taskmanager.rpc.port: 6122 + queryable-state.proxy.ports: 6125 + jobmanager.memory.process.size: 4Gb + taskmanager.memory.process.size: 4Gb metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter metrics.reporter.prom.interval: 10 SECONDS + # gives metrics about inbound/outbound network queue lengths taskmanager.network.detailed-metrics: true - # -> gives metrics about inbound/outbound network queue lengths log4j-console.properties: |+ # This affects logging for both user code and Flink rootLogger.level = INFO @@ -63,4 +61,4 @@ data: # Suppress the irrelevant (wrong) warnings from the Netty channel handler logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline - logger.netty.level = OFF \ No newline at end of file + logger.netty.level = OFF diff --git a/theodolite-benchmarks/definitions/uc4-flink/resources/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc4-flink/resources/jobmanager-deployment.yaml index 032499ea4..ccd17a9e4 100644 --- a/theodolite-benchmarks/definitions/uc4-flink/resources/jobmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc4-flink/resources/jobmanager-deployment.yaml @@ -31,17 +31,6 @@ spec: value: "1" - name: "FLINK_STATE_BACKEND" value: "rocksdb" - - name: JOB_MANAGER_RPC_ADDRESS - value: "flink-jobmanager" - - name: FLINK_PROPERTIES - value: |+ - blob.server.port: 6124 - jobmanager.rpc.port: 6123 - taskmanager.rpc.port: 6122 - queryable-state.proxy.ports: 6125 - jobmanager.memory.process.size: 4Gb - taskmanager.memory.process.size: 4Gb - #parallelism.default: 1 #TODO resources: limits: memory: 4Gi @@ -63,21 +52,10 @@ spec: initialDelaySeconds: 30 periodSeconds: 60 volumeMounts: - - name: flink-config-volume-rw + - name: flink-config-volume mountPath: /opt/flink/conf -# - name: job-artifacts-volume -# mountPath: /opt/flink/usrlib securityContext: runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary - initContainers: - - name: init-jobmanager - image: busybox:1.28 - command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/'] - volumeMounts: - - name: flink-config-volume - mountPath: /flink-config/ - - name: flink-config-volume-rw - mountPath: /flink-config-rw/ volumes: - name: flink-config-volume configMap: @@ -87,8 +65,3 @@ spec: path: flink-conf.yaml - key: log4j-console.properties path: log4j-console.properties - - name: flink-config-volume-rw - emptyDir: {} -# - name: job-artifacts-volume -# hostPath: -# path: /host/path/to/job/artifacts diff --git a/theodolite-benchmarks/definitions/uc4-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc4-flink/resources/taskmanager-deployment.yaml index 7af13f20b..c131af806 100644 --- a/theodolite-benchmarks/definitions/uc4-flink/resources/taskmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc4-flink/resources/taskmanager-deployment.yaml @@ -19,31 +19,6 @@ spec: - name: taskmanager image: ghcr.io/cau-se/theodolite-uc4-flink:latest env: - - name: KAFKA_BOOTSTRAP_SERVERS - value: "theodolite-cp-kafka:9092" - - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" - - name: COMMIT_INTERVAL_MS - value: "100" - - name: CHECKPOINTING - value: "false" - - name: PARALLELISM - value: "1" - - name: "FLINK_STATE_BACKEND" - value: "rocksdb" - - name: JOB_MANAGER_RPC_ADDRESS - value: "flink-jobmanager" - - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS - value: "1" #TODO - - name: FLINK_PROPERTIES - value: |+ - blob.server.port: 6124 - jobmanager.rpc.port: 6123 - taskmanager.rpc.port: 6122 - queryable-state.proxy.ports: 6125 - jobmanager.memory.process.size: 4Gb - taskmanager.memory.process.size: 4Gb - #parallelism.default: 1 #TODO resources: limits: memory: 4Gi @@ -62,19 +37,10 @@ spec: initialDelaySeconds: 30 periodSeconds: 60 volumeMounts: - - name: flink-config-volume-rw + - name: flink-config-volume mountPath: /opt/flink/conf/ securityContext: runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary - initContainers: - - name: init-taskmanager - image: busybox:1.28 - command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/'] - volumeMounts: - - name: flink-config-volume - mountPath: /flink-config/ - - name: flink-config-volume-rw - mountPath: /flink-config-rw/ volumes: - name: flink-config-volume configMap: @@ -84,5 +50,3 @@ spec: path: flink-conf.yaml - key: log4j-console.properties path: log4j-console.properties - - name: flink-config-volume-rw - emptyDir: {} diff --git a/theodolite-benchmarks/definitions/uc4-flink/uc4-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc4-flink/uc4-flink-benchmark-operator.yaml index 9979f540a..cff11fc56 100644 --- a/theodolite-benchmarks/definitions/uc4-flink/uc4-flink-benchmark-operator.yaml +++ b/theodolite-benchmarks/definitions/uc4-flink/uc4-flink-benchmark-operator.yaml @@ -24,11 +24,6 @@ spec: properties: container: "jobmanager" variableName: "PARALLELISM" - - type: "EnvVarPatcher" # required? - resource: "uc4-flink/taskmanager-deployment.yaml" - properties: - container: "taskmanager" - variableName: "PARALLELISM" loadTypes: - typeName: "NumNestedGroups" patchers: -- GitLab