Skip to content
Snippets Groups Projects
Commit acc594b7 authored by Sören Henning's avatar Sören Henning
Browse files

Add vertical scaling for Beam/Flink + native Flink

parent 864d2101
No related branches found
No related tags found
1 merge request!303Add resource type for threads (vertical scaling)
Pipeline #10344 passed
Stage: build
Stage: test
Stage: check
Stage: smoketest
Hide whitespace changes
Inline Side-by-side
Showing
with 264 additions and 0 deletions
theodolite-benchmarks/definitions/uc1-beam-flink/uc1-beam-flink-benchmark-operator.yaml
+ 33
− 0
View file @ acc594b7
...@@ -35,6 +35,39 @@ spec: ...@@ -35,6 +35,39 @@ spec:
properties: properties:
container: "jobmanager" container: "jobmanager"
variableName: "PARALLELISM" variableName: "PARALLELISM"
- typeName: TaskSlotsAndPodResources
patchers:
- type: ConfigMapYamlPatcher
resource: flink-configuration-configmap.yaml
properties:
fileName: flink-conf.yaml
variableName: taskmanager.numberOfTaskSlots
- type: ResourceLimitPatcher
resource: taskmanager-deployment.yaml
properties:
container: taskmanager
limitedResource: cpu
factor: 1000
format: m
- type: ResourceLimitPatcher
resource: taskmanager-deployment.yaml
properties:
container: taskmanager
limitedResource: memory
factor: 4
format: Gi
- type: ConfigMapYamlPatcher
resource: flink-configuration-configmap.yaml
properties:
fileName: flink-conf.yaml
variableName: taskmanager.memory.process.size
factor: 4
suffix: Gb
- type: "EnvVarPatcher"
resource: "jobmanager-deployment.yaml"
properties:
container: "jobmanager"
variableName: "PARALLELISM"
loadTypes: loadTypes:
- typeName: "NumSensors" - typeName: "NumSensors"
patchers: patchers:
......
theodolite-benchmarks/definitions/uc1-flink/uc1-flink-benchmark-operator.yaml
+ 33
− 0
View file @ acc594b7
...@@ -36,6 +36,39 @@ spec: ...@@ -36,6 +36,39 @@ spec:
properties: properties:
container: "jobmanager" container: "jobmanager"
variableName: "PARALLELISM" variableName: "PARALLELISM"
- typeName: TaskSlotsAndPodResources
patchers:
- type: ConfigMapYamlPatcher
resource: flink-configuration-configmap.yaml
properties:
fileName: flink-conf.yaml
variableName: taskmanager.numberOfTaskSlots
- type: ResourceLimitPatcher
resource: taskmanager-deployment.yaml
properties:
container: taskmanager
limitedResource: cpu
factor: 1000
format: m
- type: ResourceLimitPatcher
resource: taskmanager-deployment.yaml
properties:
container: taskmanager
limitedResource: memory
factor: 4
format: Gi
- type: ConfigMapYamlPatcher
resource: flink-configuration-configmap.yaml
properties:
fileName: flink-conf.yaml
variableName: taskmanager.memory.process.size
factor: 4
suffix: Gb
- type: "EnvVarPatcher"
resource: "jobmanager-deployment.yaml"
properties:
container: "jobmanager"
variableName: "PARALLELISM"
loadTypes: loadTypes:
- typeName: "NumSensors" - typeName: "NumSensors"
patchers: patchers:
......
theodolite-benchmarks/definitions/uc2-beam-flink/uc2-beam-flink-benchmark-operator.yaml
+ 33
− 0
View file @ acc594b7
...@@ -35,6 +35,39 @@ spec: ...@@ -35,6 +35,39 @@ spec:
properties: properties:
container: "jobmanager" container: "jobmanager"
variableName: "PARALLELISM" variableName: "PARALLELISM"
- typeName: TaskSlotsAndPodResources
patchers:
- type: ConfigMapYamlPatcher
resource: flink-configuration-configmap.yaml
properties:
fileName: flink-conf.yaml
variableName: taskmanager.numberOfTaskSlots
- type: ResourceLimitPatcher
resource: taskmanager-deployment.yaml
properties:
container: taskmanager
limitedResource: cpu
factor: 1000
format: m
- type: ResourceLimitPatcher
resource: taskmanager-deployment.yaml
properties:
container: taskmanager
limitedResource: memory
factor: 4
format: Gi
- type: ConfigMapYamlPatcher
resource: flink-configuration-configmap.yaml
properties:
fileName: flink-conf.yaml
variableName: taskmanager.memory.process.size
factor: 4
suffix: Gb
- type: "EnvVarPatcher"
resource: "jobmanager-deployment.yaml"
properties:
container: "jobmanager"
variableName: "PARALLELISM"
loadTypes: loadTypes:
- typeName: "NumSensors" - typeName: "NumSensors"
patchers: patchers:
......
theodolite-benchmarks/definitions/uc2-flink/uc2-flink-benchmark-operator.yaml
+ 33
− 0
View file @ acc594b7
...@@ -36,6 +36,39 @@ spec: ...@@ -36,6 +36,39 @@ spec:
properties: properties:
container: "jobmanager" container: "jobmanager"
variableName: "PARALLELISM" variableName: "PARALLELISM"
- typeName: TaskSlotsAndPodResources
patchers:
- type: ConfigMapYamlPatcher
resource: flink-configuration-configmap.yaml
properties:
fileName: flink-conf.yaml
variableName: taskmanager.numberOfTaskSlots
- type: ResourceLimitPatcher
resource: taskmanager-deployment.yaml
properties:
container: taskmanager
limitedResource: cpu
factor: 1000
format: m
- type: ResourceLimitPatcher
resource: taskmanager-deployment.yaml
properties:
container: taskmanager
limitedResource: memory
factor: 4
format: Gi
- type: ConfigMapYamlPatcher
resource: flink-configuration-configmap.yaml
properties:
fileName: flink-conf.yaml
variableName: taskmanager.memory.process.size
factor: 4
suffix: Gb
- type: "EnvVarPatcher"
resource: "jobmanager-deployment.yaml"
properties:
container: "jobmanager"
variableName: "PARALLELISM"
loadTypes: loadTypes:
- typeName: "NumSensors" - typeName: "NumSensors"
patchers: patchers:
......
theodolite-benchmarks/definitions/uc3-beam-flink/uc3-beam-flink-benchmark-operator.yaml
+ 33
− 0
View file @ acc594b7
...@@ -35,6 +35,39 @@ spec: ...@@ -35,6 +35,39 @@ spec:
properties: properties:
container: "jobmanager" container: "jobmanager"
variableName: "PARALLELISM" variableName: "PARALLELISM"
- typeName: TaskSlotsAndPodResources
patchers:
- type: ConfigMapYamlPatcher
resource: flink-configuration-configmap.yaml
properties:
fileName: flink-conf.yaml
variableName: taskmanager.numberOfTaskSlots
- type: ResourceLimitPatcher
resource: taskmanager-deployment.yaml
properties:
container: taskmanager
limitedResource: cpu
factor: 1000
format: m
- type: ResourceLimitPatcher
resource: taskmanager-deployment.yaml
properties:
container: taskmanager
limitedResource: memory
factor: 4
format: Gi
- type: ConfigMapYamlPatcher
resource: flink-configuration-configmap.yaml
properties:
fileName: flink-conf.yaml
variableName: taskmanager.memory.process.size
factor: 4
suffix: Gb
- type: "EnvVarPatcher"
resource: "jobmanager-deployment.yaml"
properties:
container: "jobmanager"
variableName: "PARALLELISM"
loadTypes: loadTypes:
- typeName: "NumSensors" - typeName: "NumSensors"
patchers: patchers:
......
theodolite-benchmarks/definitions/uc3-flink/uc3-flink-benchmark-operator.yaml
+ 33
− 0
View file @ acc594b7
...@@ -36,6 +36,39 @@ spec: ...@@ -36,6 +36,39 @@ spec:
properties: properties:
container: "jobmanager" container: "jobmanager"
variableName: "PARALLELISM" variableName: "PARALLELISM"
- typeName: TaskSlotsAndPodResources
patchers:
- type: ConfigMapYamlPatcher
resource: flink-configuration-configmap.yaml
properties:
fileName: flink-conf.yaml
variableName: taskmanager.numberOfTaskSlots
- type: ResourceLimitPatcher
resource: taskmanager-deployment.yaml
properties:
container: taskmanager
limitedResource: cpu
factor: 1000
format: m
- type: ResourceLimitPatcher
resource: taskmanager-deployment.yaml
properties:
container: taskmanager
limitedResource: memory
factor: 4
format: Gi
- type: ConfigMapYamlPatcher
resource: flink-configuration-configmap.yaml
properties:
fileName: flink-conf.yaml
variableName: taskmanager.memory.process.size
factor: 4
suffix: Gb
- type: "EnvVarPatcher"
resource: "jobmanager-deployment.yaml"
properties:
container: "jobmanager"
variableName: "PARALLELISM"
loadTypes: loadTypes:
- typeName: "NumSensors" - typeName: "NumSensors"
patchers: patchers:
......
theodolite-benchmarks/definitions/uc4-beam-flink/uc4-beam-flink-benchmark-operator.yaml
+ 33
− 0
View file @ acc594b7
...@@ -35,6 +35,39 @@ spec: ...@@ -35,6 +35,39 @@ spec:
properties: properties:
container: "jobmanager" container: "jobmanager"
variableName: "PARALLELISM" variableName: "PARALLELISM"
- typeName: TaskSlotsAndPodResources
patchers:
- type: ConfigMapYamlPatcher
resource: flink-configuration-configmap.yaml
properties:
fileName: flink-conf.yaml
variableName: taskmanager.numberOfTaskSlots
- type: ResourceLimitPatcher
resource: taskmanager-deployment.yaml
properties:
container: taskmanager
limitedResource: cpu
factor: 1000
format: m
- type: ResourceLimitPatcher
resource: taskmanager-deployment.yaml
properties:
container: taskmanager
limitedResource: memory
factor: 4
format: Gi
- type: ConfigMapYamlPatcher
resource: flink-configuration-configmap.yaml
properties:
fileName: flink-conf.yaml
variableName: taskmanager.memory.process.size
factor: 4
suffix: Gb
- type: "EnvVarPatcher"
resource: "jobmanager-deployment.yaml"
properties:
container: "jobmanager"
variableName: "PARALLELISM"
loadTypes: loadTypes:
- typeName: "NumNestedGroups" - typeName: "NumNestedGroups"
patchers: patchers:
......
theodolite-benchmarks/definitions/uc4-flink/uc4-flink-benchmark-operator.yaml
+ 33
− 0
View file @ acc594b7
...@@ -36,6 +36,39 @@ spec: ...@@ -36,6 +36,39 @@ spec:
properties: properties:
container: "jobmanager" container: "jobmanager"
variableName: "PARALLELISM" variableName: "PARALLELISM"
- typeName: TaskSlotsAndPodResources
patchers:
- type: ConfigMapYamlPatcher
resource: flink-configuration-configmap.yaml
properties:
fileName: flink-conf.yaml
variableName: taskmanager.numberOfTaskSlots
- type: ResourceLimitPatcher
resource: taskmanager-deployment.yaml
properties:
container: taskmanager
limitedResource: cpu
factor: 1000
format: m
- type: ResourceLimitPatcher
resource: taskmanager-deployment.yaml
properties:
container: taskmanager
limitedResource: memory
factor: 4
format: Gi
- type: ConfigMapYamlPatcher
resource: flink-configuration-configmap.yaml
properties:
fileName: flink-conf.yaml
variableName: taskmanager.memory.process.size
factor: 4
suffix: Gb
- type: "EnvVarPatcher"
resource: "jobmanager-deployment.yaml"
properties:
container: "jobmanager"
variableName: "PARALLELISM"
loadTypes: loadTypes:
- typeName: "NumNestedGroups" - typeName: "NumNestedGroups"
patchers: patchers:
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment