From 1ff9f56d08c92a073d15e194e25184da1aaafcdb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=B6ren=20Henning?= <soeren.henning@email.uni-kiel.de>
Date: Thu, 13 Jan 2022 11:03:35 +0100
Subject: [PATCH] Clean up Beam/Flink deployment
---
.../resources/jobmanager-deployment.yaml | 64 +++++++--------
.../resources/taskmanager-deployment.yaml | 77 ++++++++++---------
2 files changed, 71 insertions(+), 70 deletions(-)
diff --git a/theodolite-benchmarks/definitions/uc1-beam-flink/resources/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc1-beam-flink/resources/jobmanager-deployment.yaml
index 73d276d96..a6f57b513 100644
--- a/theodolite-benchmarks/definitions/uc1-beam-flink/resources/jobmanager-deployment.yaml
+++ b/theodolite-benchmarks/definitions/uc1-beam-flink/resources/jobmanager-deployment.yaml
@@ -17,11 +17,13 @@ spec:
terminationGracePeriodSeconds: 0
containers:
- name: jobmanager
- image: uc1-beam-flink:latest
- imagePullPolicy: Never
+ image: uc1-beam-flink:latest # TODO Change when ready
+ imagePullPolicy: Never # TODO Change when ready
+ args: ["standalone-job", "--job-classname", "application.Uc1BeamFlink", "--disableMetrics=true", "--fasterCopy"]
+ # optional arguments: ["--job-id", "<job id>", "--fromSavepoint", "/path/to/savepoint", "--allowNonRestoredState"]
env:
- - name: maxSourceParallelism
- value: "1024"
+ # - name: maxSourceParallelism
+ # value: "1024"
- name: KAFKA_BOOTSTRAP_SERVERS
value: "theodolite-cp-kafka:9092"
- name: SCHEMA_REGISTRY_URL
@@ -30,8 +32,8 @@ spec:
value: "100"
- name: CHECKPOINTING
value: "false"
- - name: PARALLELISM
- value: "1"
+ # - name: PARALLELISM
+ # value: "2"
- name: "FLINK_STATE_BACKEND"
value: "rocksdb"
- name: JOB_MANAGER_RPC_ADDRESS
@@ -44,13 +46,11 @@ spec:
queryable-state.proxy.ports: 6125
jobmanager.memory.process.size: 4Gb
taskmanager.memory.process.size: 4Gb
- #parallelism.default: 1 #TODO
+ parallelism.default: 2 #TODO
resources:
limits:
memory: 4Gi
cpu: 1000m
- args: ["standalone-job", "--job-classname", "application.Uc1BeamFlink", "--streaming", "--disableMetrics=true", "--fasterCopy"]
- # optional arguments: ["--job-id", "<job id>", "--fromSavepoint", "/path/to/savepoint", "--allowNonRestoredState"]
ports:
- containerPort: 6123
name: rpc
@@ -65,33 +65,33 @@ spec:
port: 6123
initialDelaySeconds: 30
periodSeconds: 60
- volumeMounts:
- - name: flink-config-volume-rw
- mountPath: /opt/flink/conf
+ # volumeMounts:
+ # - name: flink-config-volume-rw
+ # mountPath: /opt/flink/conf
# - name: job-artifacts-volume
# mountPath: /opt/flink/usrlib
securityContext:
runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
- initContainers:
- - name: init-jobmanager
- image: busybox:1.28
- command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
- volumeMounts:
- - name: flink-config-volume
- mountPath: /flink-config/
- - name: flink-config-volume-rw
- mountPath: /flink-config-rw/
- volumes:
- - name: flink-config-volume
- configMap:
- name: flink-config
- items:
- - key: flink-conf.yaml
- path: flink-conf.yaml
- - key: log4j-console.properties
- path: log4j-console.properties
- - name: flink-config-volume-rw
- emptyDir: {}
+ # initContainers:
+ # - name: init-jobmanager
+ # image: busybox:1.28
+ # command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
+ # volumeMounts:
+ # - name: flink-config-volume
+ # mountPath: /flink-config/
+ # - name: flink-config-volume-rw
+ # mountPath: /flink-config-rw/
+ # volumes:
+ # - name: flink-config-volume
+ # configMap:
+ # name: flink-config
+ # items:
+ # - key: flink-conf.yaml
+ # path: flink-conf.yaml
+ # - key: log4j-console.properties
+ # path: log4j-console.properties
+ # - name: flink-config-volume-rw
+ # emptyDir: {}
# - name: job-artifacts-volume
# hostPath:
# path: /host/path/to/job/artifacts
diff --git a/theodolite-benchmarks/definitions/uc1-beam-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc1-beam-flink/resources/taskmanager-deployment.yaml
index 7958713f8..8019b5d94 100644
--- a/theodolite-benchmarks/definitions/uc1-beam-flink/resources/taskmanager-deployment.yaml
+++ b/theodolite-benchmarks/definitions/uc1-beam-flink/resources/taskmanager-deployment.yaml
@@ -17,7 +17,9 @@ spec:
terminationGracePeriodSeconds: 0
containers:
- name: taskmanager
- image: uc1-beam-flink:latest
+ image: uc1-beam-flink:latest # TODO Change when ready
+ imagePullPolicy: Never # TODO Change when ready
+ args: ["taskmanager"]
env:
- name: KAFKA_BOOTSTRAP_SERVERS
value: "theodolite-cp-kafka:9092"
@@ -27,28 +29,27 @@ spec:
value: "100"
- name: CHECKPOINTING
value: "false"
- - name: PARALLELISM
- value: "1"
+ # - name: PARALLELISM
+ # value: "1"
- name: "FLINK_STATE_BACKEND"
value: "rocksdb"
- name: JOB_MANAGER_RPC_ADDRESS
value: "flink-jobmanager"
- - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS
- value: "1" #TODO
- - name: FLINK_PROPERTIES
- value: |+
- blob.server.port: 6124
- jobmanager.rpc.port: 6123
- taskmanager.rpc.port: 6122
- queryable-state.proxy.ports: 6125
- jobmanager.memory.process.size: 4Gb
- taskmanager.memory.process.size: 4Gb
- #parallelism.default: 1 #TODO
+ # - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS
+ # value: "1" #TODO
+ # - name: FLINK_PROPERTIES
+ # value: |+
+ # blob.server.port: 6124
+ # jobmanager.rpc.port: 6123
+ # taskmanager.rpc.port: 6122
+ # queryable-state.proxy.ports: 6125
+ # jobmanager.memory.process.size: 4Gb
+ # taskmanager.memory.process.size: 4Gb
+ # #parallelism.default: 1 #TODO
resources:
limits:
memory: 4Gi
cpu: 1000m
- args: ["taskmanager"]
ports:
- containerPort: 6122
name: rpc
@@ -61,28 +62,28 @@ spec:
port: 6122
initialDelaySeconds: 30
periodSeconds: 60
- volumeMounts:
- - name: flink-config-volume-rw
- mountPath: /opt/flink/conf/
+ # volumeMounts:
+ # - name: flink-config-volume-rw
+ # mountPath: /opt/flink/conf/
securityContext:
runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
- initContainers:
- - name: init-taskmanager
- image: busybox:1.28
- command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
- volumeMounts:
- - name: flink-config-volume
- mountPath: /flink-config/
- - name: flink-config-volume-rw
- mountPath: /flink-config-rw/
- volumes:
- - name: flink-config-volume
- configMap:
- name: flink-config
- items:
- - key: flink-conf.yaml
- path: flink-conf.yaml
- - key: log4j-console.properties
- path: log4j-console.properties
- - name: flink-config-volume-rw
- emptyDir: {}
+ # initContainers:
+ # - name: init-taskmanager
+ # image: busybox:1.28
+ # command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
+ # volumeMounts:
+ # - name: flink-config-volume
+ # mountPath: /flink-config/
+ # - name: flink-config-volume-rw
+ # mountPath: /flink-config-rw/
+ # volumes:
+ # - name: flink-config-volume
+ # configMap:
+ # name: flink-config
+ # items:
+ # - key: flink-conf.yaml
+ # path: flink-conf.yaml
+ # - key: log4j-console.properties
+ # path: log4j-console.properties
+ # - name: flink-config-volume-rw
+ # emptyDir: {}
--
GitLab