Merge branch 'main' into vertical-scaling

.gitlab-ci.yml

@@ -573,6 +573,16 @@ smoketest-uc3-kstreams:
     DOCKER_COMPOSE_DIR: "uc3-kstreams"
     JAVA_PROJECT_DEPS: "uc3-kstreams,kstreams-commons,uc3-load-generator,load-generator-commons"
 
+smoketest-uc3-flink:
+  extends: .smoketest-benchmarks
+  needs:
+    - deploy-uc3-flink
+    - deploy-uc3-load-generator
+  variables:
+    DOCKER_COMPOSE_DIR: "uc3-flink"
+    JAVA_PROJECT_DEPS: "uc3-flink,flink-commons,uc3-load-generator,load-generator-commons"
+
+
 smoketest-uc3-beam-flink:
   extends: .smoketest-benchmarks
   needs:

docs/api-reference/patchers.md

@@ -27,6 +27,18 @@ Patchers can be seen as functions which take a value as input and modify a Kuber
   * **properties**:
     * loadGenMaxRecords: 150000
 
+* **DataVolumeLoadGeneratorReplicaPatcher**: Takes the total load that should be generated and computes the number of instances needed for this load based on the `maxVolume` ((load + maxVolume - 1) / maxVolume) and calculates the load per instance (loadPerInstance = load / instances). The number of instances are set for the load generator and the given variable is set to the load per instance.
+  * **type**: "DataVolumeLoadGeneratorReplicaPatcher"
+  * **resource**: "osp-load-generator-deployment.yaml"
+  * **properties**:
+    * maxVolume: "50"
+    * container: "workload-generator"
+    * variableName: "DATA_VOLUME"
+
+* **ReplicaPatcher**: Allows to modify the number of Replicas for a kubernetes deployment.
+  * **type**: "ReplicaPatcher"
+  * **resource**: "uc1-kstreams-deployment.yaml"
+
 * **EnvVarPatcher**: Modifies the value of an environment variable for a container in a Kubernetes deployment. 
   * **type**: "EnvVarPatcher"
   * **resource**: "uc1-load-generator-deployment.yaml"
@@ -34,6 +46,14 @@ Patchers can be seen as functions which take a value as input and modify a Kuber
     * container: "workload-generator"
     * variableName: "NUM_SENSORS"
 
+* **ConfigMapYamlPatcher**: allows to add/modify a key-value pair in a YAML file of a ConfigMap
+  * **type**: "ConfigMapYamlPatcher"
+  * **resource**: "flink-configuration-configmap.yaml"
+  * **properties**:
+    * fileName: "flink-conf.yaml"
+    * variableName: "jobmanager.memory.process.size"
+  * **value**: "4Gb"
+
 * **NodeSelectorPatcher**: Changes the node selection field in Kubernetes resources.
   * **type**: "NodeSelectorPatcher"
   * **resource**: "uc1-load-generator-deployment.yaml"

docs/project-info.md

@@ -26,6 +26,7 @@ Theodolite's internal development including issue boards, merge requests and ext
 * [Lorenz Boguhn](https://github.com/lorenzboguhn)
 * [Simon Ehrenstein](https://github.com/sehrenstein)
 * [Willi Hasselbring](https://www.se.informatik.uni-kiel.de/en/team/prof.-dr.-wilhelm-willi-hasselbring)
+* [Luca Mertens](https://www.linkedin.com/in/luca-mertens-35a932201)
 * [Tobias Pfandzelter](https://pfandzelter.com/)
 * [Julia Rossow](https://www.linkedin.com/in/julia-rossow/)
 * [Björn Vonheiden](https://github.com/bvonheid)

helm/README.md

@@ -67,10 +67,11 @@ kubectl delete crd thanosrulers.monitoring.coreos.com
 
 The following 3rd party charts are used by Theodolite:
 
-- Kube Prometheus Stack (to install the Prometheus Operator, which is used to create a Prometheus instances)
-- Grafana (including a dashboard and a data source configuration)
-- Confluent Platform (for Kafka and Zookeeper)
-- Kafka Lag Exporter (used to collect monitoring data of the Kafka lag)
+- Kube Prometheus Stack
+  - to install the Prometheus Operator, which is used to create a Prometheus instances
+  - to deploy Grafana (including a dashboard and a data source configuration)
+- Grafana (deprecated as replaced by Kube Prometheus Stack)
+- Strimzi (for managing Kafka and Zookeeper)
 
 ### Hints
 

theodolite-benchmarks/beam-commons/src/main/java/rocks/theodolite/benchmarks/commons/beam/ConfigurationKeys.java

@@ -4,7 +4,7 @@ package rocks.theodolite.benchmarks.commons.beam;
  * Keys to access configuration parameters.
  */
 public final class ConfigurationKeys {
-  // Common keys
+
   public static final String APPLICATION_NAME = "application.name";
 
   public static final String KAFKA_BOOTSTRAP_SERVERS = "kafka.bootstrap.servers";
@@ -13,26 +13,6 @@ public final class ConfigurationKeys {
 
   public static final String KAFKA_INPUT_TOPIC = "kafka.input.topic";
 
-  // Additional topics
-  public static final String KAFKA_FEEDBACK_TOPIC = "kafka.feedback.topic";
-
-  public static final String KAFKA_OUTPUT_TOPIC = "kafka.output.topic";
-
-  public static final String KAFKA_CONFIGURATION_TOPIC = "kafka.configuration.topic";
-
-  // UC2
-  public static final String KAFKA_WINDOW_DURATION_MINUTES = "kafka.window.duration.minutes";
-
-  // UC3
-  public static final String AGGREGATION_DURATION_DAYS = "aggregation.duration.days";
-
-  public static final String AGGREGATION_ADVANCE_DAYS = "aggregation.advance.days";
-
-  // UC4
-  public static final String GRACE_PERIOD_MS = "grace.period.ms";
-
-
-  // BEAM
   public static final String ENABLE_AUTO_COMMIT = "enable.auto.commit";
 
   public static final String MAX_POLL_RECORDS = "max.poll.records";
@@ -41,9 +21,6 @@ public final class ConfigurationKeys {
 
   public static final String SPECIFIC_AVRO_READER = "specific.avro.reader";
 
-  public static final String TRIGGER_INTERVAL = "trigger.interval";
-
-
   private ConfigurationKeys() {}
 
 }

theodolite-benchmarks/definitions/uc1-beam-flink/resources/flink-configuration-configmap.yaml

-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: flink-config
-  labels:
-    app: flink
-data:
-  flink-conf.yaml: |+
-    jobmanager.rpc.address: flink-jobmanager
-    taskmanager.numberOfTaskSlots: 1 #TODO
-    #blob.server.port: 6124
-    #jobmanager.rpc.port: 6123
-    #taskmanager.rpc.port: 6122
-    #queryable-state.proxy.ports: 6125
-    #jobmanager.memory.process.size: 4Gb
-    #taskmanager.memory.process.size: 4Gb
-    #parallelism.default: 1 #TODO
-    metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
-    metrics.reporter.prom.interval: 10 SECONDS
-    taskmanager.network.detailed-metrics: true
-  # -> gives metrics about inbound/outbound network queue lengths
-  log4j-console.properties: |+
-    # This affects logging for both user code and Flink
-    rootLogger.level = INFO
-    rootLogger.appenderRef.console.ref = ConsoleAppender
-    rootLogger.appenderRef.rolling.ref = RollingFileAppender
-
-    # Uncomment this if you want to _only_ change Flink's logging
-    #logger.flink.name = org.apache.flink
-    #logger.flink.level = INFO
-
-    # The following lines keep the log level of common libraries/connectors on
-    # log level INFO. The root logger does not override this. You have to manually
-    # change the log levels here.
-    logger.akka.name = akka
-    logger.akka.level = INFO
-    logger.kafka.name= org.apache.kafka
-    logger.kafka.level = INFO
-    logger.hadoop.name = org.apache.hadoop
-    logger.hadoop.level = INFO
-    logger.zookeeper.name = org.apache.zookeeper
-    logger.zookeeper.level = INFO
-
-    # Log all infos to the console
-    appender.console.name = ConsoleAppender
-    appender.console.type = CONSOLE
-    appender.console.layout.type = PatternLayout
-    appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
-
-    # Log all infos in the given rolling file
-    appender.rolling.name = RollingFileAppender
-    appender.rolling.type = RollingFile
-    appender.rolling.append = false
-    appender.rolling.fileName = ${sys:log.file}
-    appender.rolling.filePattern = ${sys:log.file}.%i
-    appender.rolling.layout.type = PatternLayout
-    appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
-    appender.rolling.policies.type = Policies
-    appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
-    appender.rolling.policies.size.size=100MB
-    appender.rolling.strategy.type = DefaultRolloverStrategy
-    appender.rolling.strategy.max = 10
-
-    # Suppress the irrelevant (wrong) warnings from the Netty channel handler
-    logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
-    logger.netty.level = OFF
\ No newline at end of file

theodolite-benchmarks/definitions/uc1-beam-flink/resources/jobmanager-deployment.yaml

@@ -20,20 +20,22 @@ spec:
           image: ghcr.io/cau-se/theodolite-uc1-beam-flink:latest
           args: ["standalone-job", "--job-classname", "rocks.theodolite.benchmarks.uc1.beam.flink.Uc1BeamFlink",
                   "--parallelism=$(PARALLELISM)",
-                  "--disableMetrics=true",
-                  "--fasterCopy"]
+                  "--disableMetrics=$(DISABLE_METRICS)",
+                  "--fasterCopy=$(FASTER_COPY)"]
           # optional arguments: ["--job-id", "<job id>", "--fromSavepoint", "/path/to/savepoint", "--allowNonRestoredState"]
           env:
             - name: KAFKA_BOOTSTRAP_SERVERS
               value: "theodolite-kafka-kafka-bootstrap:9092"
             - name: SCHEMA_REGISTRY_URL
               value: "http://theodolite-kafka-schema-registry:8081"
-            - name: COMMIT_INTERVAL_MS
-              value: "100"
             - name: CHECKPOINTING
               value: "false"
             - name: PARALLELISM
               value: "1"
+            - name: DISABLE_METRICS
+              value: "true"
+            - name: FASTER_COPY
+              value: "true"
             - name: "FLINK_STATE_BACKEND"
               value: "rocksdb"
             - name: JOB_MANAGER_RPC_ADDRESS
@@ -65,33 +67,5 @@ spec:
               port: 6123
             initialDelaySeconds: 30
             periodSeconds: 60
-          # volumeMounts:
-          #   - name: flink-config-volume-rw
-          #     mountPath: /opt/flink/conf
-#            - name: job-artifacts-volume
-#              mountPath: /opt/flink/usrlib
           securityContext:
             runAsUser: 9999  # refers to user _flink_ from official flink image, change if necessary
-      # initContainers:
-      #   - name: init-jobmanager
-      #     image: busybox:1.28
-      #     command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
-      #     volumeMounts:
-      #       - name: flink-config-volume
-      #         mountPath: /flink-config/
-      #       - name: flink-config-volume-rw
-      #         mountPath: /flink-config-rw/
-      # volumes:
-      #   - name: flink-config-volume
-      #     configMap:
-      #       name: flink-config
-      #       items:
-      #         - key: flink-conf.yaml
-      #           path: flink-conf.yaml
-      #         - key: log4j-console.properties
-      #           path: log4j-console.properties
-      #   - name: flink-config-volume-rw
-      #     emptyDir: {}
-#        - name: job-artifacts-volume
-#          hostPath:
-#            path: /host/path/to/job/artifacts

theodolite-benchmarks/definitions/uc1-beam-flink/resources/taskmanager-deployment.yaml

@@ -20,31 +20,8 @@ spec:
           image: ghcr.io/cau-se/theodolite-uc1-beam-flink:latest
           args: ["taskmanager"]
           env:
-            - name: KAFKA_BOOTSTRAP_SERVERS
-              value: "theodolite-kafka-kafka-bootstrap:9092"
-            - name: SCHEMA_REGISTRY_URL
-              value: "http://theodolite-kafka-schema-registry:8081"
-            - name: COMMIT_INTERVAL_MS
-              value: "100"
-            - name: CHECKPOINTING
-              value: "false"
-            # - name: PARALLELISM
-            #   value: "1"
-            - name: "FLINK_STATE_BACKEND"
-              value: "rocksdb"
             - name: JOB_MANAGER_RPC_ADDRESS
               value: "flink-jobmanager"
-            # - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS
-            #   value: "1" #TODO
-            # - name: FLINK_PROPERTIES
-            #   value: |+
-            #     blob.server.port: 6124
-            #     jobmanager.rpc.port: 6123
-            #     taskmanager.rpc.port: 6122
-            #     queryable-state.proxy.ports: 6125
-            #     jobmanager.memory.process.size: 4Gb
-            #     taskmanager.memory.process.size: 4Gb
-            #     #parallelism.default: 1 #TODO
           resources:
             limits:
               memory: 4Gi
@@ -56,33 +33,5 @@ spec:
               name: query-state
             - containerPort: 9249
               name: metrics
-          # livenessProbe:
-          #   tcpSocket:
-          #     port: 6122
-          #   initialDelaySeconds: 30
-          #   periodSeconds: 60
-          # volumeMounts:
-          #   - name: flink-config-volume-rw
-          #     mountPath: /opt/flink/conf/
           securityContext:
             runAsUser: 9999  # refers to user _flink_ from official flink image, change if necessary
-      # initContainers:
-      #   - name: init-taskmanager
-      #     image: busybox:1.28
-      #     command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
-      #     volumeMounts:
-      #       - name: flink-config-volume
-      #         mountPath: /flink-config/
-      #       - name: flink-config-volume-rw
-      #         mountPath: /flink-config-rw/
-      # volumes:
-      #   - name: flink-config-volume
-      #     configMap:
-      #       name: flink-config
-      #       items:
-      #         - key: flink-conf.yaml
-      #           path: flink-conf.yaml
-      #         - key: log4j-console.properties
-      #           path: log4j-console.properties
-      #   - name: flink-config-volume-rw
-      #     emptyDir: {}

theodolite-benchmarks/definitions/uc1-beam-flink/uc1-beam-flink-benchmark-operator.yaml

@@ -2,13 +2,16 @@ apiVersion: theodolite.rocks/v1beta1
 kind: benchmark
 metadata:
   name: uc1-beam-flink
+  labels:
+    suite: theodolite-stream-processing
+    benchmark: uc1
+    sut: beam-flink
 spec:
   sut:
     resources:
       - configMap:
           name: "benchmark-resources-uc1-beam-flink"
           files:
-          - "flink-configuration-configmap.yaml"
           - "taskmanager-deployment.yaml"
           - "taskmanager-service.yaml"
           - "service-monitor.yaml"
@@ -32,11 +35,6 @@ spec:
           properties:
             container: "jobmanager"
             variableName: "PARALLELISM"
-        - type: "EnvVarPatcher" # required?
-          resource: "taskmanager-deployment.yaml"
-          properties:
-            container: "taskmanager"
-            variableName: "PARALLELISM"
   loadTypes:
     - typeName: "NumSensors"
       patchers:

theodolite-benchmarks/definitions/uc1-beam-samza/resources/uc1-beam-samza-deployment.yaml

@@ -25,14 +25,14 @@ spec:
               value: "theodolite-kafka-kafka-bootstrap:9092"
             - name: MAX_SOURCE_PARALLELISM
               value: "1024"
+            - name: ENABLE_METRICS
+              value: "false"
             - name: KAFKA_BOOTSTRAP_SERVERS
               value: "theodolite-kafka-kafka-bootstrap:9092"
             - name: SCHEMA_REGISTRY_URL
               value: "http://theodolite-kafka-schema-registry:8081"
             # - name: JAVA_OPTS
             #   value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555"
-            - name: COMMIT_INTERVAL_MS # Set as default for the applications
-              value: "100"
           resources:
             limits:
               memory: 4Gi

theodolite-benchmarks/definitions/uc1-beam-samza/uc1-beam-samza-benchmark-operator.yaml

@@ -2,6 +2,10 @@ apiVersion: theodolite.rocks/v1beta1
 kind: benchmark
 metadata:
   name: uc1-beam-samza
+  labels:
+    suite: theodolite-stream-processing
+    benchmark: uc1
+    sut: beam-samza
 spec:
   sut:
     resources:

theodolite-benchmarks/definitions/uc1-flink/resources/flink-configuration-configmap.yaml

@@ -7,18 +7,16 @@ metadata:
 data:
   flink-conf.yaml: |+
     jobmanager.rpc.address: flink-jobmanager
-    taskmanager.numberOfTaskSlots: 1 #TODO
-    #blob.server.port: 6124
-    #jobmanager.rpc.port: 6123
-    #taskmanager.rpc.port: 6122
-    #queryable-state.proxy.ports: 6125
-    #jobmanager.memory.process.size: 4Gb
-    #taskmanager.memory.process.size: 4Gb
-    #parallelism.default: 1 #TODO
+    blob.server.port: 6124
+    jobmanager.rpc.port: 6123
+    taskmanager.rpc.port: 6122
+    queryable-state.proxy.ports: 6125
+    jobmanager.memory.process.size: 4Gb
+    taskmanager.memory.process.size: 4Gb
     metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
     metrics.reporter.prom.interval: 10 SECONDS
+    # gives metrics about inbound/outbound network queue lengths
     taskmanager.network.detailed-metrics: true
-  # -> gives metrics about inbound/outbound network queue lengths
   log4j-console.properties: |+
     # This affects logging for both user code and Flink
     rootLogger.level = INFO

theodolite-benchmarks/definitions/uc1-flink/resources/jobmanager-deployment.yaml

@@ -23,25 +23,12 @@ spec:
               value: "theodolite-kafka-kafka-bootstrap:9092"
             - name: SCHEMA_REGISTRY_URL
               value: "http://theodolite-kafka-schema-registry:8081"
-            - name: COMMIT_INTERVAL_MS
-              value: "100"
             - name: CHECKPOINTING
               value: "false"
             - name: PARALLELISM
               value: "1"
             - name: "FLINK_STATE_BACKEND"
               value: "rocksdb"
-            - name: JOB_MANAGER_RPC_ADDRESS
-              value: "flink-jobmanager"
-            - name: FLINK_PROPERTIES
-              value: |+
-                blob.server.port: 6124
-                jobmanager.rpc.port: 6123
-                taskmanager.rpc.port: 6122
-                queryable-state.proxy.ports: 6125
-                jobmanager.memory.process.size: 4Gb
-                taskmanager.memory.process.size: 4Gb
-                #parallelism.default: 1 #TODO
           resources:
             limits:
               memory: 4Gi
@@ -63,21 +50,10 @@ spec:
             initialDelaySeconds: 30
             periodSeconds: 60
           volumeMounts:
-            - name: flink-config-volume-rw
+            - name: flink-config-volume
               mountPath: /opt/flink/conf
-#            - name: job-artifacts-volume
-#              mountPath: /opt/flink/usrlib
           securityContext:
             runAsUser: 9999  # refers to user _flink_ from official flink image, change if necessary
-      initContainers:
-        - name: init-jobmanager
-          image: busybox:1.28
-          command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
-          volumeMounts:
-            - name: flink-config-volume
-              mountPath: /flink-config/
-            - name: flink-config-volume-rw
-              mountPath: /flink-config-rw/
       volumes:
         - name: flink-config-volume
           configMap:
@@ -87,8 +63,3 @@ spec:
                 path: flink-conf.yaml
               - key: log4j-console.properties
                 path: log4j-console.properties
-        - name: flink-config-volume-rw
-          emptyDir: {}
-#        - name: job-artifacts-volume
-#          hostPath:
-#            path: /host/path/to/job/artifacts

theodolite-benchmarks/definitions/uc1-flink/resources/taskmanager-deployment.yaml

@@ -18,32 +18,6 @@ spec:
       containers:
         - name: taskmanager
           image: ghcr.io/cau-se/theodolite-uc1-flink:latest
-          env:
-            - name: KAFKA_BOOTSTRAP_SERVERS
-              value: "theodolite-kafka-kafka-bootstrap:9092"
-            - name: SCHEMA_REGISTRY_URL
-              value: "http://theodolite-kafka-schema-registry:8081"
-            - name: COMMIT_INTERVAL_MS
-              value: "100"
-            - name: CHECKPOINTING
-              value: "false"
-            - name: PARALLELISM
-              value: "1"
-            - name: "FLINK_STATE_BACKEND"
-              value: "rocksdb"
-            - name: JOB_MANAGER_RPC_ADDRESS
-              value: "flink-jobmanager"
-            - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS
-              value: "1" #TODO
-            - name: FLINK_PROPERTIES
-              value: |+
-                blob.server.port: 6124
-                jobmanager.rpc.port: 6123
-                taskmanager.rpc.port: 6122
-                queryable-state.proxy.ports: 6125
-                jobmanager.memory.process.size: 4Gb
-                taskmanager.memory.process.size: 4Gb
-                #parallelism.default: 1 #TODO
           resources:
             limits:
               memory: 4Gi
@@ -62,19 +36,10 @@ spec:
             initialDelaySeconds: 30
             periodSeconds: 60
           volumeMounts:
-            - name: flink-config-volume-rw
+            - name: flink-config-volume
               mountPath: /opt/flink/conf/
           securityContext:
             runAsUser: 9999  # refers to user _flink_ from official flink image, change if necessary
-      initContainers:
-        - name: init-taskmanager
-          image: busybox:1.28
-          command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
-          volumeMounts:
-            - name: flink-config-volume
-              mountPath: /flink-config/
-            - name: flink-config-volume-rw
-              mountPath: /flink-config-rw/
       volumes:
         - name: flink-config-volume
           configMap:
@@ -84,5 +49,3 @@ spec:
                 path: flink-conf.yaml
               - key: log4j-console.properties
                 path: log4j-console.properties
-        - name: flink-config-volume-rw
-          emptyDir: {}

theodolite-benchmarks/definitions/uc1-flink/uc1-flink-benchmark-operator.yaml

@@ -2,6 +2,10 @@ apiVersion: theodolite.rocks/v1beta1
 kind: benchmark
 metadata:
   name: uc1-flink
+  labels:
+    suite: theodolite-stream-processing
+    benchmark: uc1
+    sut: flink
 spec:
   sut:
     resources:
@@ -32,11 +36,6 @@ spec:
           properties:
             container: "jobmanager"
             variableName: "PARALLELISM"
-        - type: "EnvVarPatcher" # required?
-          resource: "taskmanager-deployment.yaml"
-          properties:
-            container: "taskmanager"
-            variableName: "PARALLELISM"
   loadTypes:
     - typeName: "NumSensors"
       patchers:

theodolite-benchmarks/definitions/uc1-hazelcastjet/resources/uc1-hazelcastjet-deployment.yaml

@@ -21,8 +21,6 @@ spec:
               value: "theodolite-kafka-kafka-bootstrap:9092"
             - name: SCHEMA_REGISTRY_URL
               value: "http://theodolite-kafka-schema-registry:8081"
-            - name: COMMIT_INTERVAL_MS # Set as default for the applications
-              value: "100"
             - name: KUBERNETES_NAMESPACE
               valueFrom:
                 fieldRef:

theodolite-benchmarks/definitions/uc1-hazelcastjet/uc1-hazelcastjet-benchmark-operator.yaml

@@ -2,6 +2,10 @@ apiVersion: theodolite.rocks/v1beta1
 kind: benchmark
 metadata:
   name: uc1-hazelcastjet
+  labels:
+    suite: theodolite-stream-processing
+    benchmark: uc1
+    sut: hazelcastjet
 spec:
   sut:
     resources:

theodolite-benchmarks/definitions/uc1-kstreams/resources/uc1-kstreams-deployment.yaml

@@ -26,8 +26,6 @@ spec:
               value: "http://theodolite-kafka-schema-registry:8081"
             - name: JAVA_OPTS
               value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555"
-            - name: COMMIT_INTERVAL_MS # Set as default for the applications
-              value: "100"
           resources:
             limits:
               memory: 4Gi

theodolite-benchmarks/definitions/uc1-kstreams/uc1-kstreams-benchmark-operator.yaml

@@ -2,6 +2,10 @@ apiVersion: theodolite.rocks/v1beta1
 kind: benchmark
 metadata:
   name: uc1-kstreams
+  labels:
+    suite: theodolite-stream-processing
+    benchmark: uc1
+    sut: kstreams
 spec:
   sut:
     resources:

theodolite-benchmarks/definitions/uc2-beam-flink/resources/flink-configuration-configmap.yaml

-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: flink-config
-  labels:
-    app: flink
-data:
-  flink-conf.yaml: |+
-    jobmanager.rpc.address: flink-jobmanager
-    taskmanager.numberOfTaskSlots: 1 #TODO
-    #blob.server.port: 6124
-    #jobmanager.rpc.port: 6123
-    #taskmanager.rpc.port: 6122
-    #queryable-state.proxy.ports: 6125
-    #jobmanager.memory.process.size: 4Gb
-    #taskmanager.memory.process.size: 4Gb
-    #parallelism.default: 1 #TODO
-    metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
-    metrics.reporter.prom.interval: 10 SECONDS
-    taskmanager.network.detailed-metrics: true
-  # -> gives metrics about inbound/outbound network queue lengths
-  log4j-console.properties: |+
-    # This affects logging for both user code and Flink
-    rootLogger.level = INFO
-    rootLogger.appenderRef.console.ref = ConsoleAppender
-    rootLogger.appenderRef.rolling.ref = RollingFileAppender
-
-    # Uncomment this if you want to _only_ change Flink's logging
-    #logger.flink.name = org.apache.flink
-    #logger.flink.level = INFO
-
-    # The following lines keep the log level of common libraries/connectors on
-    # log level INFO. The root logger does not override this. You have to manually
-    # change the log levels here.
-    logger.akka.name = akka
-    logger.akka.level = INFO
-    logger.kafka.name= org.apache.kafka
-    logger.kafka.level = INFO
-    logger.hadoop.name = org.apache.hadoop
-    logger.hadoop.level = INFO
-    logger.zookeeper.name = org.apache.zookeeper
-    logger.zookeeper.level = INFO
-
-    # Log all infos to the console
-    appender.console.name = ConsoleAppender
-    appender.console.type = CONSOLE
-    appender.console.layout.type = PatternLayout
-    appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
-
-    # Log all infos in the given rolling file
-    appender.rolling.name = RollingFileAppender
-    appender.rolling.type = RollingFile
-    appender.rolling.append = false
-    appender.rolling.fileName = ${sys:log.file}
-    appender.rolling.filePattern = ${sys:log.file}.%i
-    appender.rolling.layout.type = PatternLayout
-    appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
-    appender.rolling.policies.type = Policies
-    appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
-    appender.rolling.policies.size.size=100MB
-    appender.rolling.strategy.type = DefaultRolloverStrategy
-    appender.rolling.strategy.max = 10
-
-    # Suppress the irrelevant (wrong) warnings from the Netty channel handler
-    logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
-    logger.netty.level = OFF
\ No newline at end of file