From 123b3c12d905cf5ea64831c2d132f85df3245f19 Mon Sep 17 00:00:00 2001 From: lorenz <stu203404@mail.uni-kiel.de> Date: Tue, 8 Mar 2022 16:25:42 +0100 Subject: [PATCH] Update hazelcastjet benchmark resourcess --- .../definitions/install-configmaps.sh | 6 ++++ .../uc1-hazelcastjet-deployment.yaml | 34 +++---------------- .../uc1-benchmark-operator.yaml | 30 +++++++++------- .../uc2-hazelcastjet-deployment.yaml | 34 +++---------------- .../uc2-benchmark-operator.yaml | 32 ++++++++++------- .../uc3-hazelcastjet-deployment.yaml | 34 +++---------------- .../uc3-benchmark-operator.yaml | 32 ++++++++++------- .../uc4-hazelcastjet-deployment.yaml | 34 +++---------------- .../uc4-benchmark-operator.yaml | 32 ++++++++++------- 9 files changed, 100 insertions(+), 168 deletions(-) diff --git a/theodolite-benchmarks/definitions/install-configmaps.sh b/theodolite-benchmarks/definitions/install-configmaps.sh index 03d87bf97..e6630ca08 100755 --- a/theodolite-benchmarks/definitions/install-configmaps.sh +++ b/theodolite-benchmarks/definitions/install-configmaps.sh @@ -4,6 +4,12 @@ kubectl create configmap benchmark-resources-uc2-flink --from-file uc2-flink/res kubectl create configmap benchmark-resources-uc3-flink --from-file uc3-flink/resources kubectl create configmap benchmark-resources-uc4-flink --from-file uc4-flink/resources +kubectl create configmap benchmark-resources-uc1-hazelcastjet --from-file uc1-hazelcastjet/resources +kubectl create configmap benchmark-resources-uc2-hazelcastjet --from-file uc2-hazelcastjet/resources +kubectl create configmap benchmark-resources-uc3-hazelcastjet --from-file uc3-hazelcastjet/resources +kubectl create configmap benchmark-resources-uc4-hazelcastjet --from-file uc4-hazelcastjet/resources + + # Kafka Streams kubectl create configmap benchmark-resources-uc1-kstreams --from-file uc1-kstreams/resources kubectl create configmap benchmark-resources-uc2-kstreams --from-file uc2-kstreams/resources diff --git a/theodolite-benchmarks/definitions/uc1-hazelcastjet/resources/uc1-hazelcastjet-deployment.yaml b/theodolite-benchmarks/definitions/uc1-hazelcastjet/resources/uc1-hazelcastjet-deployment.yaml index 1fb8caf37..cab5a625c 100644 --- a/theodolite-benchmarks/definitions/uc1-hazelcastjet/resources/uc1-hazelcastjet-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc1-hazelcastjet/resources/uc1-hazelcastjet-deployment.yaml @@ -6,7 +6,7 @@ spec: selector: matchLabels: app: titan-ccp-aggregation - replicas: 2 + replicas: 1 template: metadata: labels: @@ -17,16 +17,11 @@ spec: - name: uc-application image: uc1-hazelcastjet imagePullPolicy: "Never" - ports: - - containerPort: 5555 - name: jmx env: - name: KAFKA_BOOTSTRAP_SERVERS - value: "my-confluent-cp-kafka:9092" + value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://my-confluent-cp-schema-registry:8081" - - name: JAVA_OPTS - value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555" + value: "http://theodolite-cp-schema-registry:8081" - name: COMMIT_INTERVAL_MS # Set as default for the applications value: "100" - name: KUBERNETES_NAMESPACE @@ -38,25 +33,4 @@ spec: resources: limits: memory: 4Gi - cpu: 1000m - #- name: prometheus-jmx-exporter - # image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143" - # command: - # - java - # - -XX:+UnlockExperimentalVMOptions - # - -XX:+UseCGroupMemoryLimitForHeap - # - -XX:MaxRAMFraction=1 - # - -XshowSettings:vm - # - -jar - # - jmx_prometheus_httpserver.jar - # - "5556" - # - /etc/jmx-aggregation/jmx-kafka-prometheus.yml - # ports: - # - containerPort: 5556 - # volumeMounts: - # - name: jmx-config - # mountPath: /etc/jmx-aggregation - volumes: - - name: jmx-config - configMap: - name: aggregation-jmx-configmap \ No newline at end of file + cpu: 1000m \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc1-hazelcastjet/uc1-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc1-hazelcastjet/uc1-benchmark-operator.yaml index 56f1b7f32..ca8994c40 100644 --- a/theodolite-benchmarks/definitions/uc1-hazelcastjet/uc1-benchmark-operator.yaml +++ b/theodolite-benchmarks/definitions/uc1-hazelcastjet/uc1-benchmark-operator.yaml @@ -3,33 +3,39 @@ kind: benchmark metadata: name: uc1-hazelcastjet spec: - appResource: - - "uc1-hazelcastjet/uc1-hazelcastjet-deployment.yaml" - - "uc1-hazelcastjet/uc1-hazelcastjet-service.yaml" - #- "uc1-hazelcastjet/uc1-jmx-configmap.yaml" - #- "uc1-hazelcastjet/uc1-service-monitor.yaml" - loadGenResource: - - "uc1-kstreams/uc1-load-generator-deployment.yaml" - - "uc1-kstreams/uc1-load-generator-service.yaml" + sut: + resources: + - configMap: + name: "benchmark-resources-uc1-hazelcastjet" + files: + - "uc1-hazelcastjet-deployment.yaml" + - "uc1-hazelcastjet-service.yaml" + loadGenerator: + resources: + - configMap: + name: "benchmark-resources-uc1-load-generator" + files: + - "uc1-load-generator-deployment.yaml" + - "uc1-load-generator-service.yaml" resourceTypes: - typeName: "Instances" patchers: - type: "ReplicaPatcher" - resource: "uc1-hazelcsatjet/uc1-hazelcastjet-deployment.yaml" + resource: "uc1-hazelcastjet-deployment.yaml" loadTypes: - typeName: "NumSensors" patchers: - type: "EnvVarPatcher" - resource: "uc1-kstreams/uc1-load-generator-deployment.yaml" + resource: "uc1-load-generator-deployment.yaml" properties: container: "workload-generator" variableName: "NUM_SENSORS" - type: NumSensorsLoadGeneratorReplicaPatcher - resource: "uc1-kstreams/uc1-load-generator-deployment.yaml" + resource: "uc1-load-generator-deployment.yaml" properties: loadGenMaxRecords: "150000" kafkaConfig: - bootstrapServer: "theodolite-cp-kafka:9092" + bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092" topics: - name: "input" numPartitions: 40 diff --git a/theodolite-benchmarks/definitions/uc2-hazelcastjet/resources/uc2-hazelcastjet-deployment.yaml b/theodolite-benchmarks/definitions/uc2-hazelcastjet/resources/uc2-hazelcastjet-deployment.yaml index a00997736..0f5000902 100644 --- a/theodolite-benchmarks/definitions/uc2-hazelcastjet/resources/uc2-hazelcastjet-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc2-hazelcastjet/resources/uc2-hazelcastjet-deployment.yaml @@ -6,7 +6,7 @@ spec: selector: matchLabels: app: titan-ccp-aggregation - replicas: 2 + replicas: 1 template: metadata: labels: @@ -17,16 +17,11 @@ spec: - name: uc-application image: uc2-hazelcastjet imagePullPolicy: "Never" - ports: - - containerPort: 5555 - name: jmx env: - name: KAFKA_BOOTSTRAP_SERVERS - value: "my-confluent-cp-kafka:9092" + value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://my-confluent-cp-schema-registry:8081" - - name: JAVA_OPTS - value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555" + value: "http://theodolite-cp-schema-registry:8081" - name: COMMIT_INTERVAL_MS # Set as default for the applications value: "100" - name: DOWNSAMPLE_INTERVAL @@ -42,25 +37,4 @@ spec: resources: limits: memory: 4Gi - cpu: 1000m - #- name: prometheus-jmx-exporter - # image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143" - # command: - # - java - # - -XX:+UnlockExperimentalVMOptions - # - -XX:+UseCGroupMemoryLimitForHeap - # - -XX:MaxRAMFraction=1 - # - -XshowSettings:vm - # - -jar - # - jmx_prometheus_httpserver.jar - # - "5556" - # - /etc/jmx-aggregation/jmx-kafka-prometheus.yml - # ports: - # - containerPort: 5556 - # volumeMounts: - # - name: jmx-config - # mountPath: /etc/jmx-aggregation - volumes: - - name: jmx-config - configMap: - name: aggregation-jmx-configmap \ No newline at end of file + cpu: 1000m \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc2-hazelcastjet/uc2-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc2-hazelcastjet/uc2-benchmark-operator.yaml index d53d0cc97..436bcc790 100644 --- a/theodolite-benchmarks/definitions/uc2-hazelcastjet/uc2-benchmark-operator.yaml +++ b/theodolite-benchmarks/definitions/uc2-hazelcastjet/uc2-benchmark-operator.yaml @@ -3,33 +3,39 @@ kind: benchmark metadata: name: uc2-hazelcastjet spec: - appResource: - - "uc2-hazelcastjet/uc2-hazelcastjet-deployment.yaml" - - "uc2-khazelcastjet/uc2-hazelcastjet-service.yaml" - #- "uc2-hazelcastjet/uc2-jmx-configmap.yaml" - #- "uc2-hazelcastjet/uc2-service-monitor.yaml" - loadGenResource: - - "uc2-kstreams/uc2-load-generator-deployment.yaml" - - "uc2-kstreams/uc2-load-generator-service.yaml" + sut: + resources: + - configMap: + name: "benchmark-resources-uc2-hazelcastjet" + files: + - "uc2-hazelcastjet-deployment.yaml" + - "uc2-hazelcastjet-service.yaml" + loadGenerator: + resources: + - configMap: + name: "benchmark-resources-uc2-load-generator" + files: + - "uc2-load-generator-deployment.yaml" + - "uc2-load-generator-service.yaml" resourceTypes: - typeName: "Instances" patchers: - type: "ReplicaPatcher" - resource: "uc2-hazelcastjet/uc2-hazelcastjet-deployment.yaml" + resource: "uc2-hazelcastjet-deployment.yaml" loadTypes: - typeName: "NumSensors" patchers: - type: "EnvVarPatcher" - resource: "uc2-kstreams/uc2-load-generator-deployment.yaml" + resource: "uc2-load-generator-deployment.yaml" properties: container: "workload-generator" variableName: "NUM_SENSORS" - type: NumSensorsLoadGeneratorReplicaPatcher - resource: "uc2-kstreams/uc2-load-generator-deployment.yaml" + resource: "uc2-load-generator-deployment.yaml" properties: loadGenMaxRecords: "150000" kafkaConfig: - bootstrapServer: "theodolite-cp-kafka:9092" + bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092" topics: - name: "input" numPartitions: 40 @@ -37,3 +43,5 @@ spec: - name: "output" numPartitions: 40 replicationFactor: 1 + - name: "theodolite-.*" + removeOnly: True diff --git a/theodolite-benchmarks/definitions/uc3-hazelcastjet/resources/uc3-hazelcastjet-deployment.yaml b/theodolite-benchmarks/definitions/uc3-hazelcastjet/resources/uc3-hazelcastjet-deployment.yaml index 2f797dc14..317d30328 100644 --- a/theodolite-benchmarks/definitions/uc3-hazelcastjet/resources/uc3-hazelcastjet-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc3-hazelcastjet/resources/uc3-hazelcastjet-deployment.yaml @@ -6,7 +6,7 @@ spec: selector: matchLabels: app: titan-ccp-aggregation - replicas: 2 + replicas: 1 template: metadata: labels: @@ -17,16 +17,11 @@ spec: - name: uc-application image: uc3-hazelcastjet imagePullPolicy: "Never" - ports: - - containerPort: 5555 - name: jmx env: - name: KAFKA_BOOTSTRAP_SERVERS - value: "my-confluent-cp-kafka:9092" + value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://my-confluent-cp-schema-registry:8081" - - name: JAVA_OPTS - value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555" + value: "http://theodolite-cp-schema-registry:8081" - name: COMMIT_INTERVAL_MS # Set as default for the applications value: "100" - name: WINDOW_SIZE_IN_SECONDS @@ -44,25 +39,4 @@ spec: resources: limits: memory: 4Gi - cpu: 1000m - #- name: prometheus-jmx-exporter - # image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143" - # command: - # - java - # - -XX:+UnlockExperimentalVMOptions - # - -XX:+UseCGroupMemoryLimitForHeap - # - -XX:MaxRAMFraction=1 - # - -XshowSettings:vm - # - -jar - # - jmx_prometheus_httpserver.jar - # - "5556" - # - /etc/jmx-aggregation/jmx-kafka-prometheus.yml - # ports: - # - containerPort: 5556 - # volumeMounts: - # - name: jmx-config - # mountPath: /etc/jmx-aggregation - volumes: - - name: jmx-config - configMap: - name: aggregation-jmx-configmap \ No newline at end of file + cpu: 1000m \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc3-hazelcastjet/uc3-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc3-hazelcastjet/uc3-benchmark-operator.yaml index 69199c629..3d9f755dc 100644 --- a/theodolite-benchmarks/definitions/uc3-hazelcastjet/uc3-benchmark-operator.yaml +++ b/theodolite-benchmarks/definitions/uc3-hazelcastjet/uc3-benchmark-operator.yaml @@ -3,33 +3,39 @@ kind: benchmark metadata: name: uc3-hazelcastjet spec: - appResource: - - "uc3-hazelcastjet/uc3-hazelcastjet-deployment.yaml" - - "uc3-hazelcastjet/uc3-hazelcastjet-service.yaml" - #- "uc3-hazelcastjet/uc3-jmx-configmap.yaml" - #- "uc3-hazelcastjet/uc3-service-monitor.yaml" - loadGenResource: - - "uc3-kstreams/uc3-load-generator-deployment.yaml" - - "uc3-kstreams/uc3-load-generator-service.yaml" + sut: + resources: + - configMap: + name: "benchmark-resources-uc3-hazelcastjet" + files: + - "uc3-hazelcastjet-deployment.yaml" + - "uc3-hazelcastjet-service.yaml" + loadGenerator: + resources: + - configMap: + name: "benchmark-resources-uc3-load-generator" + files: + - "uc3-load-generator-deployment.yaml" + - "uc3-load-generator-service.yaml" resourceTypes: - typeName: "Instances" patchers: - type: "ReplicaPatcher" - resource: "uc3-hazelcastjet/uc3-hazelcastjet-deployment.yaml" + resource: "uc3-hazelcastjet-deployment.yaml" loadTypes: - typeName: "NumSensors" patchers: - type: "EnvVarPatcher" - resource: "uc3-kstreams/uc3-load-generator-deployment.yaml" + resource: "uc3-load-generator-deployment.yaml" properties: container: "workload-generator" variableName: "NUM_SENSORS" - type: NumSensorsLoadGeneratorReplicaPatcher - resource: "uc3-kstreams/uc3-load-generator-deployment.yaml" + resource: "uc3-load-generator-deployment.yaml" properties: loadGenMaxRecords: "150000" kafkaConfig: - bootstrapServer: "theodolite-cp-kafka:9092" + bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092" topics: - name: "input" numPartitions: 40 @@ -37,3 +43,5 @@ spec: - name: "output" numPartitions: 40 replicationFactor: 1 + - name: "theodolite-.*" + removeOnly: True diff --git a/theodolite-benchmarks/definitions/uc4-hazelcastjet/resources/uc4-hazelcastjet-deployment.yaml b/theodolite-benchmarks/definitions/uc4-hazelcastjet/resources/uc4-hazelcastjet-deployment.yaml index 2e95c7499..f0736e585 100644 --- a/theodolite-benchmarks/definitions/uc4-hazelcastjet/resources/uc4-hazelcastjet-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc4-hazelcastjet/resources/uc4-hazelcastjet-deployment.yaml @@ -6,7 +6,7 @@ spec: selector: matchLabels: app: titan-ccp-aggregation - replicas: 2 + replicas: 1 template: metadata: labels: @@ -17,16 +17,11 @@ spec: - name: uc-application image: uc4-hazelcastjet imagePullPolicy: "Never" - ports: - - containerPort: 5555 - name: jmx env: - name: KAFKA_BOOTSTRAP_SERVERS - value: "my-confluent-cp-kafka:9092" + value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://my-confluent-cp-schema-registry:8081" - - name: JAVA_OPTS - value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555" + value: "http://theodolite-cp-schema-registry:8081" - name: COMMIT_INTERVAL_MS # Set as default for the applications value: "100" - name: WINDOW_SIZE @@ -42,25 +37,4 @@ spec: resources: limits: memory: 4Gi - cpu: 1000m - #- name: prometheus-jmx-exporter - # image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143" - # command: - # - java - # - -XX:+UnlockExperimentalVMOptions - # - -XX:+UseCGroupMemoryLimitForHeap - # - -XX:MaxRAMFraction=1 - # - -XshowSettings:vm - # - -jar - # - jmx_prometheus_httpserver.jar - # - "5556" - # - /etc/jmx-aggregation/jmx-kafka-prometheus.yml - # ports: - # - containerPort: 5556 - # volumeMounts: - # - name: jmx-config - # mountPath: /etc/jmx-aggregation - volumes: - - name: jmx-config - configMap: - name: aggregation-jmx-configmap \ No newline at end of file + cpu: 1000m \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc4-hazelcastjet/uc4-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc4-hazelcastjet/uc4-benchmark-operator.yaml index ce548c639..f0151969c 100644 --- a/theodolite-benchmarks/definitions/uc4-hazelcastjet/uc4-benchmark-operator.yaml +++ b/theodolite-benchmarks/definitions/uc4-hazelcastjet/uc4-benchmark-operator.yaml @@ -3,34 +3,40 @@ kind: benchmark metadata: name: uc4-hazelcastjet spec: - appResource: - - "uc4-hazelcastjet/uc4-hazelcastjet-deployment.yaml" - - "uc4-hazelcastjet/uc4-hazelcastjet-service.yaml" - #- "uc4-hazelcastjet/uc4-jmx-configmap.yaml" - #- "uc4-hazelcastjet/uc4-service-monitor.yaml" - loadGenResource: - - "uc4-kstreams/uc4-load-generator-deployment.yaml" - - "uc4-kstreams/uc4-load-generator-service.yaml" + sut: + resources: + - configMap: + name: "benchmark-resources-uc4-hazelcastjet" + files: + - "uc4-hazelcastjet-deployment.yaml" + - "uc4-hazelcastjet-service.yaml" + loadGenerator: + resources: + - configMap: + name: "benchmark-resources-uc4-load-generator" + files: + - "uc4-load-generator-deployment.yaml" + - "uc4-load-generator-service.yaml" resourceTypes: - typeName: "Instances" patchers: - type: "ReplicaPatcher" - resource: "uc4-hazelcastjet/uc4-hazelcastjet-deployment.yaml" + resource: "uc4-hazelcastjet-deployment.yaml" loadTypes: - typeName: "NumNestedGroups" patchers: - type: "EnvVarPatcher" - resource: "uc4-kstreams/uc4-load-generator-deployment.yaml" + resource: "uc4-load-generator-deployment.yaml" properties: container: "workload-generator" variableName: "NUM_SENSORS" - type: NumNestedGroupsLoadGeneratorReplicaPatcher - resource: "uc4-kstreams/uc4-load-generator-deployment.yaml" + resource: "uc4-load-generator-deployment.yaml" properties: loadGenMaxRecords: "150000" numSensors: "4.0" kafkaConfig: - bootstrapServer: "theodolite-cp-kafka:9092" + bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092" topics: - name: "input" numPartitions: 40 @@ -44,3 +50,5 @@ spec: - name: "aggregation-feedback" numPartitions: 40 replicationFactor: 1 + - name: "theodolite-.*" + removeOnly: True -- GitLab