From acc594b7c6a325659d84209b689dfd646c60c72e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=B6ren=20Henning?= <soeren.henning@email.uni-kiel.de>
Date: Sun, 4 Dec 2022 15:56:24 +0100
Subject: [PATCH] Add vertical scaling for Beam/Flink + native Flink
---
.../uc1-beam-flink-benchmark-operator.yaml | 33 +++++++++++++++++++
.../uc1-flink-benchmark-operator.yaml | 33 +++++++++++++++++++
.../uc2-beam-flink-benchmark-operator.yaml | 33 +++++++++++++++++++
.../uc2-flink-benchmark-operator.yaml | 33 +++++++++++++++++++
.../uc3-beam-flink-benchmark-operator.yaml | 33 +++++++++++++++++++
.../uc3-flink-benchmark-operator.yaml | 33 +++++++++++++++++++
.../uc4-beam-flink-benchmark-operator.yaml | 33 +++++++++++++++++++
.../uc4-flink-benchmark-operator.yaml | 33 +++++++++++++++++++
8 files changed, 264 insertions(+)
diff --git a/theodolite-benchmarks/definitions/uc1-beam-flink/uc1-beam-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc1-beam-flink/uc1-beam-flink-benchmark-operator.yaml
index 328a51745..471b617c1 100644
--- a/theodolite-benchmarks/definitions/uc1-beam-flink/uc1-beam-flink-benchmark-operator.yaml
+++ b/theodolite-benchmarks/definitions/uc1-beam-flink/uc1-beam-flink-benchmark-operator.yaml
@@ -35,6 +35,39 @@ spec:
properties:
container: "jobmanager"
variableName: "PARALLELISM"
+ - typeName: TaskSlotsAndPodResources
+ patchers:
+ - type: ConfigMapYamlPatcher
+ resource: flink-configuration-configmap.yaml
+ properties:
+ fileName: flink-conf.yaml
+ variableName: taskmanager.numberOfTaskSlots
+ - type: ResourceLimitPatcher
+ resource: taskmanager-deployment.yaml
+ properties:
+ container: taskmanager
+ limitedResource: cpu
+ factor: 1000
+ format: m
+ - type: ResourceLimitPatcher
+ resource: taskmanager-deployment.yaml
+ properties:
+ container: taskmanager
+ limitedResource: memory
+ factor: 4
+ format: Gi
+ - type: ConfigMapYamlPatcher
+ resource: flink-configuration-configmap.yaml
+ properties:
+ fileName: flink-conf.yaml
+ variableName: taskmanager.memory.process.size
+ factor: 4
+ suffix: Gb
+ - type: "EnvVarPatcher"
+ resource: "jobmanager-deployment.yaml"
+ properties:
+ container: "jobmanager"
+ variableName: "PARALLELISM"
loadTypes:
- typeName: "NumSensors"
patchers:
diff --git a/theodolite-benchmarks/definitions/uc1-flink/uc1-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc1-flink/uc1-flink-benchmark-operator.yaml
index b7d33a11d..bba91daa2 100644
--- a/theodolite-benchmarks/definitions/uc1-flink/uc1-flink-benchmark-operator.yaml
+++ b/theodolite-benchmarks/definitions/uc1-flink/uc1-flink-benchmark-operator.yaml
@@ -36,6 +36,39 @@ spec:
properties:
container: "jobmanager"
variableName: "PARALLELISM"
+ - typeName: TaskSlotsAndPodResources
+ patchers:
+ - type: ConfigMapYamlPatcher
+ resource: flink-configuration-configmap.yaml
+ properties:
+ fileName: flink-conf.yaml
+ variableName: taskmanager.numberOfTaskSlots
+ - type: ResourceLimitPatcher
+ resource: taskmanager-deployment.yaml
+ properties:
+ container: taskmanager
+ limitedResource: cpu
+ factor: 1000
+ format: m
+ - type: ResourceLimitPatcher
+ resource: taskmanager-deployment.yaml
+ properties:
+ container: taskmanager
+ limitedResource: memory
+ factor: 4
+ format: Gi
+ - type: ConfigMapYamlPatcher
+ resource: flink-configuration-configmap.yaml
+ properties:
+ fileName: flink-conf.yaml
+ variableName: taskmanager.memory.process.size
+ factor: 4
+ suffix: Gb
+ - type: "EnvVarPatcher"
+ resource: "jobmanager-deployment.yaml"
+ properties:
+ container: "jobmanager"
+ variableName: "PARALLELISM"
loadTypes:
- typeName: "NumSensors"
patchers:
diff --git a/theodolite-benchmarks/definitions/uc2-beam-flink/uc2-beam-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc2-beam-flink/uc2-beam-flink-benchmark-operator.yaml
index 39ddaa38d..c7e4ec235 100644
--- a/theodolite-benchmarks/definitions/uc2-beam-flink/uc2-beam-flink-benchmark-operator.yaml
+++ b/theodolite-benchmarks/definitions/uc2-beam-flink/uc2-beam-flink-benchmark-operator.yaml
@@ -35,6 +35,39 @@ spec:
properties:
container: "jobmanager"
variableName: "PARALLELISM"
+ - typeName: TaskSlotsAndPodResources
+ patchers:
+ - type: ConfigMapYamlPatcher
+ resource: flink-configuration-configmap.yaml
+ properties:
+ fileName: flink-conf.yaml
+ variableName: taskmanager.numberOfTaskSlots
+ - type: ResourceLimitPatcher
+ resource: taskmanager-deployment.yaml
+ properties:
+ container: taskmanager
+ limitedResource: cpu
+ factor: 1000
+ format: m
+ - type: ResourceLimitPatcher
+ resource: taskmanager-deployment.yaml
+ properties:
+ container: taskmanager
+ limitedResource: memory
+ factor: 4
+ format: Gi
+ - type: ConfigMapYamlPatcher
+ resource: flink-configuration-configmap.yaml
+ properties:
+ fileName: flink-conf.yaml
+ variableName: taskmanager.memory.process.size
+ factor: 4
+ suffix: Gb
+ - type: "EnvVarPatcher"
+ resource: "jobmanager-deployment.yaml"
+ properties:
+ container: "jobmanager"
+ variableName: "PARALLELISM"
loadTypes:
- typeName: "NumSensors"
patchers:
diff --git a/theodolite-benchmarks/definitions/uc2-flink/uc2-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc2-flink/uc2-flink-benchmark-operator.yaml
index d1cf9e520..1ceebe6cd 100644
--- a/theodolite-benchmarks/definitions/uc2-flink/uc2-flink-benchmark-operator.yaml
+++ b/theodolite-benchmarks/definitions/uc2-flink/uc2-flink-benchmark-operator.yaml
@@ -36,6 +36,39 @@ spec:
properties:
container: "jobmanager"
variableName: "PARALLELISM"
+ - typeName: TaskSlotsAndPodResources
+ patchers:
+ - type: ConfigMapYamlPatcher
+ resource: flink-configuration-configmap.yaml
+ properties:
+ fileName: flink-conf.yaml
+ variableName: taskmanager.numberOfTaskSlots
+ - type: ResourceLimitPatcher
+ resource: taskmanager-deployment.yaml
+ properties:
+ container: taskmanager
+ limitedResource: cpu
+ factor: 1000
+ format: m
+ - type: ResourceLimitPatcher
+ resource: taskmanager-deployment.yaml
+ properties:
+ container: taskmanager
+ limitedResource: memory
+ factor: 4
+ format: Gi
+ - type: ConfigMapYamlPatcher
+ resource: flink-configuration-configmap.yaml
+ properties:
+ fileName: flink-conf.yaml
+ variableName: taskmanager.memory.process.size
+ factor: 4
+ suffix: Gb
+ - type: "EnvVarPatcher"
+ resource: "jobmanager-deployment.yaml"
+ properties:
+ container: "jobmanager"
+ variableName: "PARALLELISM"
loadTypes:
- typeName: "NumSensors"
patchers:
diff --git a/theodolite-benchmarks/definitions/uc3-beam-flink/uc3-beam-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc3-beam-flink/uc3-beam-flink-benchmark-operator.yaml
index 263cfe3a6..a99f99902 100644
--- a/theodolite-benchmarks/definitions/uc3-beam-flink/uc3-beam-flink-benchmark-operator.yaml
+++ b/theodolite-benchmarks/definitions/uc3-beam-flink/uc3-beam-flink-benchmark-operator.yaml
@@ -35,6 +35,39 @@ spec:
properties:
container: "jobmanager"
variableName: "PARALLELISM"
+ - typeName: TaskSlotsAndPodResources
+ patchers:
+ - type: ConfigMapYamlPatcher
+ resource: flink-configuration-configmap.yaml
+ properties:
+ fileName: flink-conf.yaml
+ variableName: taskmanager.numberOfTaskSlots
+ - type: ResourceLimitPatcher
+ resource: taskmanager-deployment.yaml
+ properties:
+ container: taskmanager
+ limitedResource: cpu
+ factor: 1000
+ format: m
+ - type: ResourceLimitPatcher
+ resource: taskmanager-deployment.yaml
+ properties:
+ container: taskmanager
+ limitedResource: memory
+ factor: 4
+ format: Gi
+ - type: ConfigMapYamlPatcher
+ resource: flink-configuration-configmap.yaml
+ properties:
+ fileName: flink-conf.yaml
+ variableName: taskmanager.memory.process.size
+ factor: 4
+ suffix: Gb
+ - type: "EnvVarPatcher"
+ resource: "jobmanager-deployment.yaml"
+ properties:
+ container: "jobmanager"
+ variableName: "PARALLELISM"
loadTypes:
- typeName: "NumSensors"
patchers:
diff --git a/theodolite-benchmarks/definitions/uc3-flink/uc3-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc3-flink/uc3-flink-benchmark-operator.yaml
index ce0bfc5ba..115dfe530 100644
--- a/theodolite-benchmarks/definitions/uc3-flink/uc3-flink-benchmark-operator.yaml
+++ b/theodolite-benchmarks/definitions/uc3-flink/uc3-flink-benchmark-operator.yaml
@@ -36,6 +36,39 @@ spec:
properties:
container: "jobmanager"
variableName: "PARALLELISM"
+ - typeName: TaskSlotsAndPodResources
+ patchers:
+ - type: ConfigMapYamlPatcher
+ resource: flink-configuration-configmap.yaml
+ properties:
+ fileName: flink-conf.yaml
+ variableName: taskmanager.numberOfTaskSlots
+ - type: ResourceLimitPatcher
+ resource: taskmanager-deployment.yaml
+ properties:
+ container: taskmanager
+ limitedResource: cpu
+ factor: 1000
+ format: m
+ - type: ResourceLimitPatcher
+ resource: taskmanager-deployment.yaml
+ properties:
+ container: taskmanager
+ limitedResource: memory
+ factor: 4
+ format: Gi
+ - type: ConfigMapYamlPatcher
+ resource: flink-configuration-configmap.yaml
+ properties:
+ fileName: flink-conf.yaml
+ variableName: taskmanager.memory.process.size
+ factor: 4
+ suffix: Gb
+ - type: "EnvVarPatcher"
+ resource: "jobmanager-deployment.yaml"
+ properties:
+ container: "jobmanager"
+ variableName: "PARALLELISM"
loadTypes:
- typeName: "NumSensors"
patchers:
diff --git a/theodolite-benchmarks/definitions/uc4-beam-flink/uc4-beam-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc4-beam-flink/uc4-beam-flink-benchmark-operator.yaml
index 6a9e33b40..4ef153879 100644
--- a/theodolite-benchmarks/definitions/uc4-beam-flink/uc4-beam-flink-benchmark-operator.yaml
+++ b/theodolite-benchmarks/definitions/uc4-beam-flink/uc4-beam-flink-benchmark-operator.yaml
@@ -35,6 +35,39 @@ spec:
properties:
container: "jobmanager"
variableName: "PARALLELISM"
+ - typeName: TaskSlotsAndPodResources
+ patchers:
+ - type: ConfigMapYamlPatcher
+ resource: flink-configuration-configmap.yaml
+ properties:
+ fileName: flink-conf.yaml
+ variableName: taskmanager.numberOfTaskSlots
+ - type: ResourceLimitPatcher
+ resource: taskmanager-deployment.yaml
+ properties:
+ container: taskmanager
+ limitedResource: cpu
+ factor: 1000
+ format: m
+ - type: ResourceLimitPatcher
+ resource: taskmanager-deployment.yaml
+ properties:
+ container: taskmanager
+ limitedResource: memory
+ factor: 4
+ format: Gi
+ - type: ConfigMapYamlPatcher
+ resource: flink-configuration-configmap.yaml
+ properties:
+ fileName: flink-conf.yaml
+ variableName: taskmanager.memory.process.size
+ factor: 4
+ suffix: Gb
+ - type: "EnvVarPatcher"
+ resource: "jobmanager-deployment.yaml"
+ properties:
+ container: "jobmanager"
+ variableName: "PARALLELISM"
loadTypes:
- typeName: "NumNestedGroups"
patchers:
diff --git a/theodolite-benchmarks/definitions/uc4-flink/uc4-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc4-flink/uc4-flink-benchmark-operator.yaml
index b1b563bb4..ea7491fa6 100644
--- a/theodolite-benchmarks/definitions/uc4-flink/uc4-flink-benchmark-operator.yaml
+++ b/theodolite-benchmarks/definitions/uc4-flink/uc4-flink-benchmark-operator.yaml
@@ -36,6 +36,39 @@ spec:
properties:
container: "jobmanager"
variableName: "PARALLELISM"
+ - typeName: TaskSlotsAndPodResources
+ patchers:
+ - type: ConfigMapYamlPatcher
+ resource: flink-configuration-configmap.yaml
+ properties:
+ fileName: flink-conf.yaml
+ variableName: taskmanager.numberOfTaskSlots
+ - type: ResourceLimitPatcher
+ resource: taskmanager-deployment.yaml
+ properties:
+ container: taskmanager
+ limitedResource: cpu
+ factor: 1000
+ format: m
+ - type: ResourceLimitPatcher
+ resource: taskmanager-deployment.yaml
+ properties:
+ container: taskmanager
+ limitedResource: memory
+ factor: 4
+ format: Gi
+ - type: ConfigMapYamlPatcher
+ resource: flink-configuration-configmap.yaml
+ properties:
+ fileName: flink-conf.yaml
+ variableName: taskmanager.memory.process.size
+ factor: 4
+ suffix: Gb
+ - type: "EnvVarPatcher"
+ resource: "jobmanager-deployment.yaml"
+ properties:
+ container: "jobmanager"
+ variableName: "PARALLELISM"
loadTypes:
- typeName: "NumNestedGroups"
patchers:
--
GitLab