Merge branch 'vertical-scaling' into 'main'

Add resource type for threads (vertical scaling)

Closes #269

See merge request !303

theodolite-benchmarks/definitions/uc1-beam-flink/resources/flink-configuration-configmap.yaml

@@ -7,6 +7,7 @@ metadata:
 data:
   flink-conf.yaml: |+
     jobmanager.rpc.address: flink-jobmanager
+    taskmanager.numberOfTaskSlots: 1
     blob.server.port: 6124
     jobmanager.rpc.port: 6123
     taskmanager.rpc.port: 6122

theodolite-benchmarks/definitions/uc1-beam-flink/uc1-beam-flink-benchmark-operator.yaml

@@ -12,6 +12,7 @@ spec:
       - configMap:
           name: "benchmark-resources-uc1-beam-flink"
           files:
+          - "flink-configuration-configmap.yaml"
           - "taskmanager-deployment.yaml"
           - "taskmanager-service.yaml"
           - "service-monitor.yaml"
@@ -35,6 +36,39 @@ spec:
           properties:
             container: "jobmanager"
             variableName: "PARALLELISM"
+    - typeName: TaskSlotsAndPodResources
+      patchers:
+        - type: ConfigMapYamlPatcher
+          resource: flink-configuration-configmap.yaml
+          properties:
+            fileName: flink-conf.yaml
+            variableName: taskmanager.numberOfTaskSlots
+        - type: ResourceLimitPatcher
+          resource: taskmanager-deployment.yaml
+          properties:
+            container: taskmanager
+            limitedResource: cpu
+            factor: 1000
+            format: m
+        - type: ResourceLimitPatcher
+          resource: taskmanager-deployment.yaml
+          properties:
+            container: taskmanager
+            limitedResource: memory
+            factor: 4
+            format: Gi
+        - type: ConfigMapYamlPatcher
+          resource: flink-configuration-configmap.yaml
+          properties:
+            fileName: flink-conf.yaml
+            variableName: taskmanager.memory.process.size
+            factor: 4
+            suffix: Gb
+        - type: "EnvVarPatcher"
+          resource: "jobmanager-deployment.yaml"
+          properties:
+            container: "jobmanager"
+            variableName: "PARALLELISM"
   loadTypes:
     - typeName: "NumSensors"
       patchers:

theodolite-benchmarks/definitions/uc1-beam-samza/uc1-beam-samza-benchmark-operator.yaml

@@ -37,6 +37,27 @@ spec:
       patchers:
         - type: "ReplicaPatcher"
           resource: "uc1-beam-samza-deployment.yaml"
+    - typeName: ThreadsAndPodResources
+      patchers:
+        - type: EnvVarPatcher
+          resource: uc1-beam-samza-deployment.yaml
+          properties:
+            container: uc1-beam-samza
+            variableName: SAMZA_JOB_CONTAINER_THREADS_POOL_SIZE
+        - type: ResourceLimitPatcher
+          resource: uc1-beam-samza-deployment.yaml
+          properties:
+            container: uc1-beam-samza
+            limitedResource: cpu
+            factor: 1000
+            format: m
+        - type: ResourceLimitPatcher
+          resource: uc1-beam-samza-deployment.yaml
+          properties:
+            container: uc1-beam-samza
+            limitedResource: memory
+            factor: 4
+            format: Gi
   loadTypes:
     - typeName: "NumSensors"
       patchers:

theodolite-benchmarks/definitions/uc1-flink/resources/flink-configuration-configmap.yaml

@@ -7,6 +7,7 @@ metadata:
 data:
   flink-conf.yaml: |+
     jobmanager.rpc.address: flink-jobmanager
+    taskmanager.numberOfTaskSlots: 1
     blob.server.port: 6124
     jobmanager.rpc.port: 6123
     taskmanager.rpc.port: 6122

theodolite-benchmarks/definitions/uc1-flink/uc1-flink-benchmark-operator.yaml

@@ -36,6 +36,39 @@ spec:
           properties:
             container: "jobmanager"
             variableName: "PARALLELISM"
+    - typeName: TaskSlotsAndPodResources
+      patchers:
+        - type: ConfigMapYamlPatcher
+          resource: flink-configuration-configmap.yaml
+          properties:
+            fileName: flink-conf.yaml
+            variableName: taskmanager.numberOfTaskSlots
+        - type: ResourceLimitPatcher
+          resource: taskmanager-deployment.yaml
+          properties:
+            container: taskmanager
+            limitedResource: cpu
+            factor: 1000
+            format: m
+        - type: ResourceLimitPatcher
+          resource: taskmanager-deployment.yaml
+          properties:
+            container: taskmanager
+            limitedResource: memory
+            factor: 4
+            format: Gi
+        - type: ConfigMapYamlPatcher
+          resource: flink-configuration-configmap.yaml
+          properties:
+            fileName: flink-conf.yaml
+            variableName: taskmanager.memory.process.size
+            factor: 4
+            suffix: Gb
+        - type: "EnvVarPatcher"
+          resource: "jobmanager-deployment.yaml"
+          properties:
+            container: "jobmanager"
+            variableName: "PARALLELISM"
   loadTypes:
     - typeName: "NumSensors"
       patchers:

theodolite-benchmarks/definitions/uc1-hazelcastjet/uc1-hazelcastjet-benchmark-operator.yaml

@@ -28,6 +28,22 @@ spec:
       patchers:
         - type: "ReplicaPatcher"
           resource: "uc1-hazelcastjet-deployment.yaml"
+    - typeName: PodResources
+      patchers:
+        - type: ResourceLimitPatcher
+          resource: uc1-hazelcastjet-deployment.yaml
+          properties:
+            container: uc-application
+            limitedResource: cpu
+            factor: 1000
+            format: m
+        - type: ResourceLimitPatcher
+          resource: uc1-hazelcastjet-deployment.yaml
+          properties:
+            container: uc-application
+            limitedResource: memory
+            factor: 4
+            format: Gi
   loadTypes:
     - typeName: "NumSensors"
       patchers:

theodolite-benchmarks/definitions/uc1-kstreams/uc1-kstreams-benchmark-operator.yaml

@@ -28,6 +28,27 @@ spec:
       patchers:
         - type: "ReplicaPatcher"
           resource: "uc1-kstreams-deployment.yaml"
+    - typeName: ThreadsAndPodResources
+      patchers:
+        - type: EnvVarPatcher
+          resource: uc1-kstreams-deployment.yaml
+          properties:
+            container: uc-application
+            variableName: NUM_STREAM_THREADS
+        - type: ResourceLimitPatcher
+          resource: uc1-kstreams-deployment.yaml
+          properties:
+            container: uc-application
+            limitedResource: cpu
+            factor: 1000
+            format: m
+        - type: ResourceLimitPatcher
+          resource: uc1-kstreams-deployment.yaml
+          properties:
+            container: uc-application
+            limitedResource: memory
+            factor: 4
+            format: Gi
   loadTypes:
     - typeName: "NumSensors"
       patchers:

theodolite-benchmarks/definitions/uc2-beam-flink/resources/flink-configuration-configmap.yaml

@@ -7,6 +7,7 @@ metadata:
 data:
   flink-conf.yaml: |+
     jobmanager.rpc.address: flink-jobmanager
+    taskmanager.numberOfTaskSlots: 1
     blob.server.port: 6124
     jobmanager.rpc.port: 6123
     taskmanager.rpc.port: 6122

theodolite-benchmarks/definitions/uc2-beam-flink/uc2-beam-flink-benchmark-operator.yaml

@@ -12,6 +12,7 @@ spec:
       - configMap:
           name: "benchmark-resources-uc2-beam-flink"
           files:
+          - "flink-configuration-configmap.yaml"
           - "taskmanager-deployment.yaml"
           - "taskmanager-service.yaml"
           - "service-monitor.yaml"
@@ -35,6 +36,39 @@ spec:
           properties:
             container: "jobmanager"
             variableName: "PARALLELISM"
+    - typeName: TaskSlotsAndPodResources
+      patchers:
+        - type: ConfigMapYamlPatcher
+          resource: flink-configuration-configmap.yaml
+          properties:
+            fileName: flink-conf.yaml
+            variableName: taskmanager.numberOfTaskSlots
+        - type: ResourceLimitPatcher
+          resource: taskmanager-deployment.yaml
+          properties:
+            container: taskmanager
+            limitedResource: cpu
+            factor: 1000
+            format: m
+        - type: ResourceLimitPatcher
+          resource: taskmanager-deployment.yaml
+          properties:
+            container: taskmanager
+            limitedResource: memory
+            factor: 4
+            format: Gi
+        - type: ConfigMapYamlPatcher
+          resource: flink-configuration-configmap.yaml
+          properties:
+            fileName: flink-conf.yaml
+            variableName: taskmanager.memory.process.size
+            factor: 4
+            suffix: Gb
+        - type: "EnvVarPatcher"
+          resource: "jobmanager-deployment.yaml"
+          properties:
+            container: "jobmanager"
+            variableName: "PARALLELISM"
   loadTypes:
     - typeName: "NumSensors"
       patchers:

theodolite-benchmarks/definitions/uc2-beam-samza/uc2-beam-samza-benchmark-operator.yaml

@@ -37,6 +37,27 @@ spec:
       patchers:
         - type: "ReplicaPatcher"
           resource: "uc2-beam-samza-deployment.yaml"
+    - typeName: ThreadsAndPodResources
+      patchers:
+        - type: EnvVarPatcher
+          resource: uc2-beam-samza-deployment.yaml
+          properties:
+            container: uc2-beam-samza
+            variableName: SAMZA_JOB_CONTAINER_THREADS_POOL_SIZE
+        - type: ResourceLimitPatcher
+          resource: uc2-beam-samza-deployment.yaml
+          properties:
+            container: uc2-beam-samza
+            limitedResource: cpu
+            factor: 1000
+            format: m
+        - type: ResourceLimitPatcher
+          resource: uc2-beam-samza-deployment.yaml
+          properties:
+            container: uc2-beam-samza
+            limitedResource: memory
+            factor: 4
+            format: Gi
   loadTypes:
     - typeName: "NumSensors"
       patchers:

theodolite-benchmarks/definitions/uc2-flink/resources/flink-configuration-configmap.yaml

@@ -7,6 +7,7 @@ metadata:
 data:
   flink-conf.yaml: |+
     jobmanager.rpc.address: flink-jobmanager
+    taskmanager.numberOfTaskSlots: 1
     blob.server.port: 6124
     jobmanager.rpc.port: 6123
     taskmanager.rpc.port: 6122

theodolite-benchmarks/definitions/uc2-flink/uc2-flink-benchmark-operator.yaml

@@ -36,6 +36,39 @@ spec:
           properties:
             container: "jobmanager"
             variableName: "PARALLELISM"
+    - typeName: TaskSlotsAndPodResources
+      patchers:
+        - type: ConfigMapYamlPatcher
+          resource: flink-configuration-configmap.yaml
+          properties:
+            fileName: flink-conf.yaml
+            variableName: taskmanager.numberOfTaskSlots
+        - type: ResourceLimitPatcher
+          resource: taskmanager-deployment.yaml
+          properties:
+            container: taskmanager
+            limitedResource: cpu
+            factor: 1000
+            format: m
+        - type: ResourceLimitPatcher
+          resource: taskmanager-deployment.yaml
+          properties:
+            container: taskmanager
+            limitedResource: memory
+            factor: 4
+            format: Gi
+        - type: ConfigMapYamlPatcher
+          resource: flink-configuration-configmap.yaml
+          properties:
+            fileName: flink-conf.yaml
+            variableName: taskmanager.memory.process.size
+            factor: 4
+            suffix: Gb
+        - type: "EnvVarPatcher"
+          resource: "jobmanager-deployment.yaml"
+          properties:
+            container: "jobmanager"
+            variableName: "PARALLELISM"
   loadTypes:
     - typeName: "NumSensors"
       patchers:

theodolite-benchmarks/definitions/uc2-hazelcastjet/uc2-hazelcastjet-benchmark-operator.yaml

@@ -28,6 +28,22 @@ spec:
       patchers:
         - type: "ReplicaPatcher"
           resource: "uc2-hazelcastjet-deployment.yaml"
+    - typeName: PodResources
+      patchers:
+        - type: ResourceLimitPatcher
+          resource: uc2-hazelcastjet-deployment.yaml
+          properties:
+            container: uc-application
+            limitedResource: cpu
+            factor: 1000
+            format: m
+        - type: ResourceLimitPatcher
+          resource: uc2-hazelcastjet-deployment.yaml
+          properties:
+            container: uc-application
+            limitedResource: memory
+            factor: 4
+            format: Gi
   loadTypes:
     - typeName: "NumSensors"
       patchers:

theodolite-benchmarks/definitions/uc2-kstreams/uc2-kstreams-benchmark-operator.yaml

@@ -28,6 +28,27 @@ spec:
       patchers:
         - type: "ReplicaPatcher"
           resource: "uc2-kstreams-deployment.yaml"
+    - typeName: ThreadsAndPodResources
+      patchers:
+        - type: EnvVarPatcher
+          resource: uc2-kstreams-deployment.yaml
+          properties:
+            container: uc-application
+            variableName: NUM_STREAM_THREADS
+        - type: ResourceLimitPatcher
+          resource: uc2-kstreams-deployment.yaml
+          properties:
+            container: uc-application
+            limitedResource: cpu
+            factor: 1000
+            format: m
+        - type: ResourceLimitPatcher
+          resource: uc2-kstreams-deployment.yaml
+          properties:
+            container: uc-application
+            limitedResource: memory
+            factor: 4
+            format: Gi
   loadTypes:
     - typeName: "NumSensors"
       patchers:

theodolite-benchmarks/definitions/uc3-beam-flink/resources/flink-configuration-configmap.yaml

@@ -7,6 +7,7 @@ metadata:
 data:
   flink-conf.yaml: |+
     jobmanager.rpc.address: flink-jobmanager
+    taskmanager.numberOfTaskSlots: 1
     blob.server.port: 6124
     jobmanager.rpc.port: 6123
     taskmanager.rpc.port: 6122

theodolite-benchmarks/definitions/uc3-beam-flink/uc3-beam-flink-benchmark-operator.yaml

@@ -12,6 +12,7 @@ spec:
       - configMap:
           name: "benchmark-resources-uc3-beam-flink"
           files:
+          - "flink-configuration-configmap.yaml"
           - "taskmanager-deployment.yaml"
           - "taskmanager-service.yaml"
           - "service-monitor.yaml"
@@ -35,6 +36,39 @@ spec:
           properties:
             container: "jobmanager"
             variableName: "PARALLELISM"
+    - typeName: TaskSlotsAndPodResources
+      patchers:
+        - type: ConfigMapYamlPatcher
+          resource: flink-configuration-configmap.yaml
+          properties:
+            fileName: flink-conf.yaml
+            variableName: taskmanager.numberOfTaskSlots
+        - type: ResourceLimitPatcher
+          resource: taskmanager-deployment.yaml
+          properties:
+            container: taskmanager
+            limitedResource: cpu
+            factor: 1000
+            format: m
+        - type: ResourceLimitPatcher
+          resource: taskmanager-deployment.yaml
+          properties:
+            container: taskmanager
+            limitedResource: memory
+            factor: 4
+            format: Gi
+        - type: ConfigMapYamlPatcher
+          resource: flink-configuration-configmap.yaml
+          properties:
+            fileName: flink-conf.yaml
+            variableName: taskmanager.memory.process.size
+            factor: 4
+            suffix: Gb
+        - type: "EnvVarPatcher"
+          resource: "jobmanager-deployment.yaml"
+          properties:
+            container: "jobmanager"
+            variableName: "PARALLELISM"
   loadTypes:
     - typeName: "NumSensors"
       patchers:

theodolite-benchmarks/definitions/uc3-beam-samza/uc3-beam-samza-benchmark-operator.yaml

@@ -37,6 +37,27 @@ spec:
       patchers:
         - type: "ReplicaPatcher"
           resource: "uc3-beam-samza-deployment.yaml"
+    - typeName: ThreadsAndPodResources
+      patchers:
+        - type: EnvVarPatcher
+          resource: uc3-beam-samza-deployment.yaml
+          properties:
+            container: uc3-beam-samza
+            variableName: SAMZA_JOB_CONTAINER_THREADS_POOL_SIZE
+        - type: ResourceLimitPatcher
+          resource: uc3-beam-samza-deployment.yaml
+          properties:
+            container: uc3-beam-samza
+            limitedResource: cpu
+            factor: 1000
+            format: m
+        - type: ResourceLimitPatcher
+          resource: uc3-beam-samza-deployment.yaml
+          properties:
+            container: uc3-beam-samza
+            limitedResource: memory
+            factor: 4
+            format: Gi
   loadTypes:
     - typeName: "NumSensors"
       patchers:

theodolite-benchmarks/definitions/uc3-flink/resources/flink-configuration-configmap.yaml

@@ -7,6 +7,7 @@ metadata:
 data:
   flink-conf.yaml: |+
     jobmanager.rpc.address: flink-jobmanager
+    taskmanager.numberOfTaskSlots: 1
     blob.server.port: 6124
     jobmanager.rpc.port: 6123
     taskmanager.rpc.port: 6122

theodolite-benchmarks/definitions/uc3-flink/uc3-flink-benchmark-operator.yaml

@@ -36,6 +36,39 @@ spec:
           properties:
             container: "jobmanager"
             variableName: "PARALLELISM"
+    - typeName: TaskSlotsAndPodResources
+      patchers:
+        - type: ConfigMapYamlPatcher
+          resource: flink-configuration-configmap.yaml
+          properties:
+            fileName: flink-conf.yaml
+            variableName: taskmanager.numberOfTaskSlots
+        - type: ResourceLimitPatcher
+          resource: taskmanager-deployment.yaml
+          properties:
+            container: taskmanager
+            limitedResource: cpu
+            factor: 1000
+            format: m
+        - type: ResourceLimitPatcher
+          resource: taskmanager-deployment.yaml
+          properties:
+            container: taskmanager
+            limitedResource: memory
+            factor: 4
+            format: Gi
+        - type: ConfigMapYamlPatcher
+          resource: flink-configuration-configmap.yaml
+          properties:
+            fileName: flink-conf.yaml
+            variableName: taskmanager.memory.process.size
+            factor: 4
+            suffix: Gb
+        - type: "EnvVarPatcher"
+          resource: "jobmanager-deployment.yaml"
+          properties:
+            container: "jobmanager"
+            variableName: "PARALLELISM"
   loadTypes:
     - typeName: "NumSensors"
       patchers:

theodolite-benchmarks/definitions/uc3-hazelcastjet/uc3-hazelcastjet-benchmark-operator.yaml

@@ -28,6 +28,22 @@ spec:
       patchers:
         - type: "ReplicaPatcher"
           resource: "uc3-hazelcastjet-deployment.yaml"
+    - typeName: PodResources
+      patchers:
+        - type: ResourceLimitPatcher
+          resource: uc3-hazelcastjet-deployment.yaml
+          properties:
+            container: uc-application
+            limitedResource: cpu
+            factor: 1000
+            format: m
+        - type: ResourceLimitPatcher
+          resource: uc3-hazelcastjet-deployment.yaml
+          properties:
+            container: uc-application
+            limitedResource: memory
+            factor: 4
+            format: Gi
   loadTypes:
     - typeName: "NumSensors"
       patchers: