From e7789a1f7d947a12919a6a47e61a8f800b761a16 Mon Sep 17 00:00:00 2001 From: "stu126940@mail.uni-kiel.de" <stu126940@mail.uni-kiel.de> Date: Fri, 16 Apr 2021 11:57:13 +0200 Subject: [PATCH] Add benchmark definitions for uc1-uc4 --- .../uc1-kstreams/uc-1-jmx-configmap.yaml | 10 ++++ .../uc1-kstreams/uc1-benchmark-operator.yaml | 32 +++++++++++ .../uc1-benchmark-standalone.yaml | 29 ++++++++++ .../uc1-kstreams/uc1-kstreams-deployment.yaml | 55 +++++++++++++++++++ .../uc1-kstreams/uc1-kstreams-service.yaml | 17 ++++++ .../uc1-load-generator-deployment.yaml | 32 +++++++++++ .../uc1-load-generator-service.yaml | 16 ++++++ .../uc1-kstreams/uc1-service-monitor.yaml | 14 +++++ .../uc2-kstreams/uc2-benchmark-operator.yaml | 32 +++++++++++ .../uc2-benchmark-standalone.yaml | 29 ++++++++++ .../uc2-kstreams/uc2-jmx-configmap.yaml | 10 ++++ .../uc2-kstreams/uc2-kstreams-deployment.yaml | 55 +++++++++++++++++++ .../uc2-kstreams/uc2-kstreams-service.yaml | 17 ++++++ .../uc2-load-generator-deployment.yaml | 32 +++++++++++ .../uc2-load-generator-service.yaml | 16 ++++++ .../uc2-kstreams/uc2-service-monitor.yaml | 14 +++++ .../uc3-kstreams/uc3-benchmark-operator.yaml | 32 +++++++++++ .../uc3-benchmark-standalone.yaml | 29 ++++++++++ .../uc3-kstreams/uc3-jmx-configmap.yaml | 10 ++++ .../uc3-kstreams/uc3-kstreams-deployment.yaml | 55 +++++++++++++++++++ .../uc3-kstreams/uc3-kstreams-service.yaml | 17 ++++++ .../uc3-load-generator-deployment.yaml | 32 +++++++++++ .../uc3-load-generator-service.yaml | 16 ++++++ .../uc3-kstreams/uc3-service-monitor.yaml | 14 +++++ .../uc4-kstreams/uc4-benchmark-operator.yaml | 32 +++++++++++ .../uc4-benchmark-standalone.yaml | 29 ++++++++++ .../uc4-kstreams/uc4-jmx-configmap.yaml | 10 ++++ .../uc4-kstreams/uc4-kstreams-deployment.yaml | 55 +++++++++++++++++++ .../uc4-kstreams/uc4-kstreams-service.yaml | 17 ++++++ .../uc4-load-generator-deployment.yaml | 32 +++++++++++ .../uc4-load-generator-service.yaml | 16 ++++++ .../uc4-kstreams/uc4-service-monitor.yaml | 14 +++++ 32 files changed, 820 insertions(+) create mode 100644 theodolite-benchmarks/definitions/uc1-kstreams/uc-1-jmx-configmap.yaml create mode 100644 theodolite-benchmarks/definitions/uc1-kstreams/uc1-benchmark-operator.yaml create mode 100644 theodolite-benchmarks/definitions/uc1-kstreams/uc1-benchmark-standalone.yaml create mode 100644 theodolite-benchmarks/definitions/uc1-kstreams/uc1-kstreams-deployment.yaml create mode 100644 theodolite-benchmarks/definitions/uc1-kstreams/uc1-kstreams-service.yaml create mode 100644 theodolite-benchmarks/definitions/uc1-kstreams/uc1-load-generator-deployment.yaml create mode 100644 theodolite-benchmarks/definitions/uc1-kstreams/uc1-load-generator-service.yaml create mode 100644 theodolite-benchmarks/definitions/uc1-kstreams/uc1-service-monitor.yaml create mode 100644 theodolite-benchmarks/definitions/uc2-kstreams/uc2-benchmark-operator.yaml create mode 100644 theodolite-benchmarks/definitions/uc2-kstreams/uc2-benchmark-standalone.yaml create mode 100644 theodolite-benchmarks/definitions/uc2-kstreams/uc2-jmx-configmap.yaml create mode 100644 theodolite-benchmarks/definitions/uc2-kstreams/uc2-kstreams-deployment.yaml create mode 100644 theodolite-benchmarks/definitions/uc2-kstreams/uc2-kstreams-service.yaml create mode 100644 theodolite-benchmarks/definitions/uc2-kstreams/uc2-load-generator-deployment.yaml create mode 100644 theodolite-benchmarks/definitions/uc2-kstreams/uc2-load-generator-service.yaml create mode 100644 theodolite-benchmarks/definitions/uc2-kstreams/uc2-service-monitor.yaml create mode 100644 theodolite-benchmarks/definitions/uc3-kstreams/uc3-benchmark-operator.yaml create mode 100644 theodolite-benchmarks/definitions/uc3-kstreams/uc3-benchmark-standalone.yaml create mode 100644 theodolite-benchmarks/definitions/uc3-kstreams/uc3-jmx-configmap.yaml create mode 100644 theodolite-benchmarks/definitions/uc3-kstreams/uc3-kstreams-deployment.yaml create mode 100644 theodolite-benchmarks/definitions/uc3-kstreams/uc3-kstreams-service.yaml create mode 100644 theodolite-benchmarks/definitions/uc3-kstreams/uc3-load-generator-deployment.yaml create mode 100644 theodolite-benchmarks/definitions/uc3-kstreams/uc3-load-generator-service.yaml create mode 100644 theodolite-benchmarks/definitions/uc3-kstreams/uc3-service-monitor.yaml create mode 100644 theodolite-benchmarks/definitions/uc4-kstreams/uc4-benchmark-operator.yaml create mode 100644 theodolite-benchmarks/definitions/uc4-kstreams/uc4-benchmark-standalone.yaml create mode 100644 theodolite-benchmarks/definitions/uc4-kstreams/uc4-jmx-configmap.yaml create mode 100644 theodolite-benchmarks/definitions/uc4-kstreams/uc4-kstreams-deployment.yaml create mode 100644 theodolite-benchmarks/definitions/uc4-kstreams/uc4-kstreams-service.yaml create mode 100644 theodolite-benchmarks/definitions/uc4-kstreams/uc4-load-generator-deployment.yaml create mode 100644 theodolite-benchmarks/definitions/uc4-kstreams/uc4-load-generator-service.yaml create mode 100644 theodolite-benchmarks/definitions/uc4-kstreams/uc4-service-monitor.yaml diff --git a/theodolite-benchmarks/definitions/uc1-kstreams/uc-1-jmx-configmap.yaml b/theodolite-benchmarks/definitions/uc1-kstreams/uc-1-jmx-configmap.yaml new file mode 100644 index 000000000..78496a86b --- /dev/null +++ b/theodolite-benchmarks/definitions/uc1-kstreams/uc-1-jmx-configmap.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: aggregation-jmx-configmap +data: + jmx-kafka-prometheus.yml: |+ + jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi + lowercaseOutputName: true + lowercaseOutputLabelNames: true + ssl: false diff --git a/theodolite-benchmarks/definitions/uc1-kstreams/uc1-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-benchmark-operator.yaml new file mode 100644 index 000000000..a7695eeaf --- /dev/null +++ b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-benchmark-operator.yaml @@ -0,0 +1,32 @@ +apiVersion: theodolite.com/v1alpha1 +kind: benchmark +metadata: + name: uc1-kstreams +appResource: + - "uc1-kstreams-deployment.yaml" + - "uc1-kstreams-service.yaml" + - "uc1-jmx-configmap.yaml" + - "uc1-service-monitor.yaml" +loadGenResource: + - "uc1-load-generator-deployment.yaml" + - "uc1-load-generator-service.yaml" +resourceTypes: + - typeName: "Instances" + patchers: + - type: "ReplicaPatcher" + resource: "uc1-kstreams-deployment.yaml" +loadTypes: + - typeName: "NumSensors" + patchers: + - type: "EnvVarPatcher" + resource: "uc1-load-generator-deployment.yaml" + container: "workload-generator" + variableName: "NUM_SENSORS" + - type: NumSensorsLoadGeneratorReplicaPatcher + resource: "uc1-load-generator-deployment.yaml" +kafkaConfig: + bootstrapServer: "theodolite-cp-kafka:9092" + topics: + - name: "input" + numPartitions: 40 + replicationFactor: 1 \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc1-kstreams/uc1-benchmark-standalone.yaml b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-benchmark-standalone.yaml new file mode 100644 index 000000000..02ea0ab3b --- /dev/null +++ b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-benchmark-standalone.yaml @@ -0,0 +1,29 @@ +name: "uc1-kstreams" +appResource: + - "uc1-kstreams-deployment.yaml" + - "uc1-kstreams-service.yaml" + - "uc1-jmx-configmap.yaml" + - "uc1-service-monitor.yaml" +loadGenResource: + - "uc1-load-generator-deployment.yaml" + - "uc1-load-generator-service.yaml" +resourceTypes: + - typeName: "Instances" + patchers: + - type: "ReplicaPatcher" + resource: "uc1-kstreams-deployment.yaml" +loadTypes: + - typeName: "NumSensors" + patchers: + - type: "EnvVarPatcher" + resource: "uc1-load-generator-deployment.yaml" + container: "workload-generator" + variableName: "NUM_SENSORS" + - type: NumSensorsLoadGeneratorReplicaPatcher + resource: "uc1-load-generator-deployment.yaml" +kafkaConfig: + bootstrapServer: "theodolite-cp-kafka:9092" + topics: + - name: "input" + numPartitions: 40 + replicationFactor: 1 \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc1-kstreams/uc1-kstreams-deployment.yaml b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-kstreams-deployment.yaml new file mode 100644 index 000000000..171c3446d --- /dev/null +++ b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-kstreams-deployment.yaml @@ -0,0 +1,55 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: titan-ccp-aggregation +spec: + selector: + matchLabels: + app: titan-ccp-aggregation + replicas: 1 + template: + metadata: + labels: + app: titan-ccp-aggregation + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: uc-application + image: ghcr.io/cau-se/theodolite-uc1-kstreams-app:latest + ports: + - containerPort: 5555 + name: jmx + env: + - name: KAFKA_BOOTSTRAP_SERVERS + value: "theodolite-cp-kafka:9092" + - name: SCHEMA_REGISTRY_URL + value: "http://theodolite-cp-schema-registry:8081" + - name: JAVA_OPTS + value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555" + - name: COMMIT_INTERVAL_MS # Set as default for the applications + value: "100" + resources: + limits: + memory: 4Gi + cpu: 1000m + - name: prometheus-jmx-exporter + image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143" + command: + - java + - -XX:+UnlockExperimentalVMOptions + - -XX:+UseCGroupMemoryLimitForHeap + - -XX:MaxRAMFraction=1 + - -XshowSettings:vm + - -jar + - jmx_prometheus_httpserver.jar + - "5556" + - /etc/jmx-aggregation/jmx-kafka-prometheus.yml + ports: + - containerPort: 5556 + volumeMounts: + - name: jmx-config + mountPath: /etc/jmx-aggregation + volumes: + - name: jmx-config + configMap: + name: aggregation-jmx-configmap \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc1-kstreams/uc1-kstreams-service.yaml b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-kstreams-service.yaml new file mode 100644 index 000000000..85432d04f --- /dev/null +++ b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-kstreams-service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: titan-ccp-aggregation + labels: + app: titan-ccp-aggregation +spec: + #type: NodePort + selector: + app: titan-ccp-aggregation + ports: + - name: http + port: 80 + targetPort: 80 + protocol: TCP + - name: metrics + port: 5556 diff --git a/theodolite-benchmarks/definitions/uc1-kstreams/uc1-load-generator-deployment.yaml b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-load-generator-deployment.yaml new file mode 100644 index 000000000..9f9ccc6ae --- /dev/null +++ b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-load-generator-deployment.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: titan-ccp-load-generator +spec: + selector: + matchLabels: + app: titan-ccp-load-generator + replicas: 1 + template: + metadata: + labels: + app: titan-ccp-load-generator + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: workload-generator + image: ghcr.io/cau-se/theodolite-uc1-workload-generator:latest + ports: + - containerPort: 5701 + name: coordination + env: + - name: KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: KUBERNETES_DNS_NAME + value: "titan-ccp-load-generator.$(KUBERNETES_NAMESPACE).svc.cluster.local" + - name: KAFKA_BOOTSTRAP_SERVERS + value: "theodolite-cp-kafka:9092" + - name: SCHEMA_REGISTRY_URL + value: "http://theodolite-cp-schema-registry:8081" diff --git a/theodolite-benchmarks/definitions/uc1-kstreams/uc1-load-generator-service.yaml b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-load-generator-service.yaml new file mode 100644 index 000000000..f8b26b3f6 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-load-generator-service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: titan-ccp-load-generator + labels: + app: titan-ccp-load-generator +spec: + type: ClusterIP + clusterIP: None + selector: + app: titan-ccp-load-generator + ports: + - name: coordination + port: 5701 + targetPort: 5701 + protocol: TCP diff --git a/theodolite-benchmarks/definitions/uc1-kstreams/uc1-service-monitor.yaml b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-service-monitor.yaml new file mode 100644 index 000000000..4e7e758ca --- /dev/null +++ b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-service-monitor.yaml @@ -0,0 +1,14 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + app: titan-ccp-aggregation + appScope: titan-ccp + name: titan-ccp-aggregation +spec: + selector: + matchLabels: + app: titan-ccp-aggregation + endpoints: + - port: metrics + interval: 10s diff --git a/theodolite-benchmarks/definitions/uc2-kstreams/uc2-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-benchmark-operator.yaml new file mode 100644 index 000000000..cfc8ff0e4 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-benchmark-operator.yaml @@ -0,0 +1,32 @@ +apiVersion: theodolite.com/v1alpha1 +kind: benchmark +metadata: + name: uc2-kstreams +appResource: + - "uc2-kstreams-deployment.yaml" + - "uc2-kstreams-service.yaml" + - "uc2-jmx-configmap.yaml" + - "uc2-service-monitor.yaml" +loadGenResource: + - "uc2-load-generator-deployment.yaml" + - "uc2-load-generator-service.yaml" +resourceTypes: + - typeName: "Instances" + patchers: + - type: "ReplicaPatcher" + resource: "uc2-kstreams-deployment.yaml" +loadTypes: + - typeName: "NumSensors" + patchers: + - type: "EnvVarPatcher" + resource: "uc2-load-generator-deployment.yaml" + container: "workload-generator" + variableName: "NUM_SENSORS" + - type: NumSensorsLoadGeneratorReplicaPatcher + resource: "uc2-load-generator-deployment.yaml" +kafkaConfig: + bootstrapServer: "theodolite-cp-kafka:9092" + topics: + - name: "input" + numPartitions: 40 + replicationFactor: 1 \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc2-kstreams/uc2-benchmark-standalone.yaml b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-benchmark-standalone.yaml new file mode 100644 index 000000000..3c4291530 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-benchmark-standalone.yaml @@ -0,0 +1,29 @@ +name: "uc2-kstreams" +appResource: + - "uc2-kstreams-deployment.yaml" + - "uc2-kstreams-service.yaml" + - "uc2-jmx-configmap.yaml" + - "uc2-service-monitor.yaml" +loadGenResource: + - "uc2-load-generator-deployment.yaml" + - "uc2-load-generator-service.yaml" +resourceTypes: + - typeName: "Instances" + patchers: + - type: "ReplicaPatcher" + resource: "uc2-kstreams-deployment.yaml" +loadTypes: + - typeName: "NumSensors" + patchers: + - type: "EnvVarPatcher" + resource: "uc2-load-generator-deployment.yaml" + container: "workload-generator" + variableName: "NUM_SENSORS" + - type: NumSensorsLoadGeneratorReplicaPatcher + resource: "uc2-load-generator-deployment.yaml" +kafkaConfig: + bootstrapServer: "theodolite-cp-kafka:9092" + topics: + - name: "input" + numPartitions: 40 + replicationFactor: 1 \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc2-kstreams/uc2-jmx-configmap.yaml b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-jmx-configmap.yaml new file mode 100644 index 000000000..78496a86b --- /dev/null +++ b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-jmx-configmap.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: aggregation-jmx-configmap +data: + jmx-kafka-prometheus.yml: |+ + jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi + lowercaseOutputName: true + lowercaseOutputLabelNames: true + ssl: false diff --git a/theodolite-benchmarks/definitions/uc2-kstreams/uc2-kstreams-deployment.yaml b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-kstreams-deployment.yaml new file mode 100644 index 000000000..e07bb3f9e --- /dev/null +++ b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-kstreams-deployment.yaml @@ -0,0 +1,55 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: titan-ccp-aggregation +spec: + selector: + matchLabels: + app: titan-ccp-aggregation + replicas: 1 + template: + metadata: + labels: + app: titan-ccp-aggregation + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: uc-application + image: ghcr.io/cau-se/theodolite-uc2-kstreams-app:latest + ports: + - containerPort: 5555 + name: jmx + env: + - name: KAFKA_BOOTSTRAP_SERVERS + value: "theodolite-cp-kafka:9092" + - name: SCHEMA_REGISTRY_URL + value: "http://theodolite-cp-schema-registry:8081" + - name: JAVA_OPTS + value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555" + - name: COMMIT_INTERVAL_MS # Set as default for the applications + value: "100" + resources: + limits: + memory: 4Gi + cpu: 1000m + - name: prometheus-jmx-exporter + image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143" + command: + - java + - -XX:+UnlockExperimentalVMOptions + - -XX:+UseCGroupMemoryLimitForHeap + - -XX:MaxRAMFraction=1 + - -XshowSettings:vm + - -jar + - jmx_prometheus_httpserver.jar + - "5556" + - /etc/jmx-aggregation/jmx-kafka-prometheus.yml + ports: + - containerPort: 5556 + volumeMounts: + - name: jmx-config + mountPath: /etc/jmx-aggregation + volumes: + - name: jmx-config + configMap: + name: aggregation-jmx-configmap \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc2-kstreams/uc2-kstreams-service.yaml b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-kstreams-service.yaml new file mode 100644 index 000000000..85432d04f --- /dev/null +++ b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-kstreams-service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: titan-ccp-aggregation + labels: + app: titan-ccp-aggregation +spec: + #type: NodePort + selector: + app: titan-ccp-aggregation + ports: + - name: http + port: 80 + targetPort: 80 + protocol: TCP + - name: metrics + port: 5556 diff --git a/theodolite-benchmarks/definitions/uc2-kstreams/uc2-load-generator-deployment.yaml b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-load-generator-deployment.yaml new file mode 100644 index 000000000..dfc0af715 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-load-generator-deployment.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: titan-ccp-load-generator +spec: + selector: + matchLabels: + app: titan-ccp-load-generator + replicas: 1 + template: + metadata: + labels: + app: titan-ccp-load-generator + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: workload-generator + image: ghcr.io/cau-se/theodolite-uc2-workload-generator:latest + ports: + - containerPort: 5701 + name: coordination + env: + - name: KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: KUBERNETES_DNS_NAME + value: "titan-ccp-load-generator.$(KUBERNETES_NAMESPACE).svc.cluster.local" + - name: KAFKA_BOOTSTRAP_SERVERS + value: "theodolite-cp-kafka:9092" + - name: SCHEMA_REGISTRY_URL + value: "http://theodolite-cp-schema-registry:8081" diff --git a/theodolite-benchmarks/definitions/uc2-kstreams/uc2-load-generator-service.yaml b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-load-generator-service.yaml new file mode 100644 index 000000000..f8b26b3f6 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-load-generator-service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: titan-ccp-load-generator + labels: + app: titan-ccp-load-generator +spec: + type: ClusterIP + clusterIP: None + selector: + app: titan-ccp-load-generator + ports: + - name: coordination + port: 5701 + targetPort: 5701 + protocol: TCP diff --git a/theodolite-benchmarks/definitions/uc2-kstreams/uc2-service-monitor.yaml b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-service-monitor.yaml new file mode 100644 index 000000000..4e7e758ca --- /dev/null +++ b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-service-monitor.yaml @@ -0,0 +1,14 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + app: titan-ccp-aggregation + appScope: titan-ccp + name: titan-ccp-aggregation +spec: + selector: + matchLabels: + app: titan-ccp-aggregation + endpoints: + - port: metrics + interval: 10s diff --git a/theodolite-benchmarks/definitions/uc3-kstreams/uc3-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-benchmark-operator.yaml new file mode 100644 index 000000000..6346f70d4 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-benchmark-operator.yaml @@ -0,0 +1,32 @@ +apiVersion: theodolite.com/v1alpha1 +kind: benchmark +metadata: + name: uc3-kstreams +appResource: + - "uc3-kstreams-deployment.yaml" + - "uc3-kstreams-service.yaml" + - "uc3-jmx-configmap.yaml" + - "uc3-service-monitor.yaml" +loadGenResource: + - "uc3-load-generator-deployment.yaml" + - "uc3-load-generator-service.yaml" +resourceTypes: + - typeName: "Instances" + patchers: + - type: "ReplicaPatcher" + resource: "uc3-kstreams-deployment.yaml" +loadTypes: + - typeName: "NumSensors" + patchers: + - type: "EnvVarPatcher" + resource: "uc3-load-generator-deployment.yaml" + container: "workload-generator" + variableName: "NUM_SENSORS" + - type: NumSensorsLoadGeneratorReplicaPatcher + resource: "uc3-load-generator-deployment.yaml" +kafkaConfig: + bootstrapServer: "theodolite-cp-kafka:9092" + topics: + - name: "input" + numPartitions: 40 + replicationFactor: 1 \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc3-kstreams/uc3-benchmark-standalone.yaml b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-benchmark-standalone.yaml new file mode 100644 index 000000000..cfc62cc6b --- /dev/null +++ b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-benchmark-standalone.yaml @@ -0,0 +1,29 @@ +name: "uc3-kstreams" +appResource: + - "uc3-kstreams-deployment.yaml" + - "uc3-kstreams-service.yaml" + - "uc3-jmx-configmap.yaml" + - "uc3-service-monitor.yaml" +loadGenResource: + - "uc3-load-generator-deployment.yaml" + - "uc3-load-generator-service.yaml" +resourceTypes: + - typeName: "Instances" + patchers: + - type: "ReplicaPatcher" + resource: "uc3-kstreams-deployment.yaml" +loadTypes: + - typeName: "NumSensors" + patchers: + - type: "EnvVarPatcher" + resource: "uc3-load-generator-deployment.yaml" + container: "workload-generator" + variableName: "NUM_SENSORS" + - type: NumSensorsLoadGeneratorReplicaPatcher + resource: "uc3-load-generator-deployment.yaml" +kafkaConfig: + bootstrapServer: "theodolite-cp-kafka:9092" + topics: + - name: "input" + numPartitions: 40 + replicationFactor: 1 \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc3-kstreams/uc3-jmx-configmap.yaml b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-jmx-configmap.yaml new file mode 100644 index 000000000..78496a86b --- /dev/null +++ b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-jmx-configmap.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: aggregation-jmx-configmap +data: + jmx-kafka-prometheus.yml: |+ + jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi + lowercaseOutputName: true + lowercaseOutputLabelNames: true + ssl: false diff --git a/theodolite-benchmarks/definitions/uc3-kstreams/uc3-kstreams-deployment.yaml b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-kstreams-deployment.yaml new file mode 100644 index 000000000..e3f63fae9 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-kstreams-deployment.yaml @@ -0,0 +1,55 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: titan-ccp-aggregation +spec: + selector: + matchLabels: + app: titan-ccp-aggregation + replicas: 1 + template: + metadata: + labels: + app: titan-ccp-aggregation + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: uc-application + image: ghcr.io/cau-se/theodolite-uc3-kstreams-app:latest + ports: + - containerPort: 5555 + name: jmx + env: + - name: KAFKA_BOOTSTRAP_SERVERS + value: "theodolite-cp-kafka:9092" + - name: SCHEMA_REGISTRY_URL + value: "http://theodolite-cp-schema-registry:8081" + - name: JAVA_OPTS + value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555" + - name: COMMIT_INTERVAL_MS # Set as default for the applications + value: "100" + resources: + limits: + memory: 4Gi + cpu: 1000m + - name: prometheus-jmx-exporter + image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143" + command: + - java + - -XX:+UnlockExperimentalVMOptions + - -XX:+UseCGroupMemoryLimitForHeap + - -XX:MaxRAMFraction=1 + - -XshowSettings:vm + - -jar + - jmx_prometheus_httpserver.jar + - "5556" + - /etc/jmx-aggregation/jmx-kafka-prometheus.yml + ports: + - containerPort: 5556 + volumeMounts: + - name: jmx-config + mountPath: /etc/jmx-aggregation + volumes: + - name: jmx-config + configMap: + name: aggregation-jmx-configmap \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc3-kstreams/uc3-kstreams-service.yaml b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-kstreams-service.yaml new file mode 100644 index 000000000..85432d04f --- /dev/null +++ b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-kstreams-service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: titan-ccp-aggregation + labels: + app: titan-ccp-aggregation +spec: + #type: NodePort + selector: + app: titan-ccp-aggregation + ports: + - name: http + port: 80 + targetPort: 80 + protocol: TCP + - name: metrics + port: 5556 diff --git a/theodolite-benchmarks/definitions/uc3-kstreams/uc3-load-generator-deployment.yaml b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-load-generator-deployment.yaml new file mode 100644 index 000000000..c1cad0b70 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-load-generator-deployment.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: titan-ccp-load-generator +spec: + selector: + matchLabels: + app: titan-ccp-load-generator + replicas: 1 + template: + metadata: + labels: + app: titan-ccp-load-generator + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: workload-generator + image: ghcr.io/cau-se/theodolite-uc3-workload-generator:latest + ports: + - containerPort: 5701 + name: coordination + env: + - name: KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: KUBERNETES_DNS_NAME + value: "titan-ccp-load-generator.$(KUBERNETES_NAMESPACE).svc.cluster.local" + - name: KAFKA_BOOTSTRAP_SERVERS + value: "theodolite-cp-kafka:9092" + - name: SCHEMA_REGISTRY_URL + value: "http://theodolite-cp-schema-registry:8081" diff --git a/theodolite-benchmarks/definitions/uc3-kstreams/uc3-load-generator-service.yaml b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-load-generator-service.yaml new file mode 100644 index 000000000..f8b26b3f6 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-load-generator-service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: titan-ccp-load-generator + labels: + app: titan-ccp-load-generator +spec: + type: ClusterIP + clusterIP: None + selector: + app: titan-ccp-load-generator + ports: + - name: coordination + port: 5701 + targetPort: 5701 + protocol: TCP diff --git a/theodolite-benchmarks/definitions/uc3-kstreams/uc3-service-monitor.yaml b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-service-monitor.yaml new file mode 100644 index 000000000..4e7e758ca --- /dev/null +++ b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-service-monitor.yaml @@ -0,0 +1,14 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + app: titan-ccp-aggregation + appScope: titan-ccp + name: titan-ccp-aggregation +spec: + selector: + matchLabels: + app: titan-ccp-aggregation + endpoints: + - port: metrics + interval: 10s diff --git a/theodolite-benchmarks/definitions/uc4-kstreams/uc4-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-benchmark-operator.yaml new file mode 100644 index 000000000..1e2b3db2e --- /dev/null +++ b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-benchmark-operator.yaml @@ -0,0 +1,32 @@ +apiVersion: theodolite.com/v1alpha1 +kind: benchmark +metadata: + name: uc4-kstreams +appResource: + - "uc4-kstreams-deployment.yaml" + - "uc4-kstreams-service.yaml" + - "uc4-jmx-configmap.yaml" + - "uc4-service-monitor.yaml" +loadGenResource: + - "uc4-load-generator-deployment.yaml" + - "uc4-load-generator-service.yaml" +resourceTypes: + - typeName: "Instances" + patchers: + - type: "ReplicaPatcher" + resource: "uc4-kstreams-deployment.yaml" +loadTypes: + - typeName: "NumSensors" + patchers: + - type: "EnvVarPatcher" + resource: "uc4-load-generator-deployment.yaml" + container: "workload-generator" + variableName: "NUM_SENSORS" + - type: NumSensorsLoadGeneratorReplicaPatcher + resource: "uc4-load-generator-deployment.yaml" +kafkaConfig: + bootstrapServer: "theodolite-cp-kafka:9092" + topics: + - name: "input" + numPartitions: 40 + replicationFactor: 1 \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc4-kstreams/uc4-benchmark-standalone.yaml b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-benchmark-standalone.yaml new file mode 100644 index 000000000..0769b7cdc --- /dev/null +++ b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-benchmark-standalone.yaml @@ -0,0 +1,29 @@ +name: "uc4-kstreams" +appResource: + - "uc4-kstreams-deployment.yaml" + - "uc4-kstreams-service.yaml" + - "uc4-jmx-configmap.yaml" + - "uc4-service-monitor.yaml" +loadGenResource: + - "uc4-load-generator-deployment.yaml" + - "uc4-load-generator-service.yaml" +resourceTypes: + - typeName: "Instances" + patchers: + - type: "ReplicaPatcher" + resource: "uc4-kstreams-deployment.yaml" +loadTypes: + - typeName: "NumSensors" + patchers: + - type: "EnvVarPatcher" + resource: "uc4-load-generator-deployment.yaml" + container: "workload-generator" + variableName: "NUM_NESTED_GROUPS" + - type: NumNestedGroupsLoadGeneratorReplicaPatcher + resource: "uc4-load-generator-deployment.yaml" +kafkaConfig: + bootstrapServer: "theodolite-cp-kafka:9092" + topics: + - name: "input" + numPartitions: 40 + replicationFactor: 1 \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc4-kstreams/uc4-jmx-configmap.yaml b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-jmx-configmap.yaml new file mode 100644 index 000000000..78496a86b --- /dev/null +++ b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-jmx-configmap.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: aggregation-jmx-configmap +data: + jmx-kafka-prometheus.yml: |+ + jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi + lowercaseOutputName: true + lowercaseOutputLabelNames: true + ssl: false diff --git a/theodolite-benchmarks/definitions/uc4-kstreams/uc4-kstreams-deployment.yaml b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-kstreams-deployment.yaml new file mode 100644 index 000000000..e3f63fae9 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-kstreams-deployment.yaml @@ -0,0 +1,55 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: titan-ccp-aggregation +spec: + selector: + matchLabels: + app: titan-ccp-aggregation + replicas: 1 + template: + metadata: + labels: + app: titan-ccp-aggregation + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: uc-application + image: ghcr.io/cau-se/theodolite-uc3-kstreams-app:latest + ports: + - containerPort: 5555 + name: jmx + env: + - name: KAFKA_BOOTSTRAP_SERVERS + value: "theodolite-cp-kafka:9092" + - name: SCHEMA_REGISTRY_URL + value: "http://theodolite-cp-schema-registry:8081" + - name: JAVA_OPTS + value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555" + - name: COMMIT_INTERVAL_MS # Set as default for the applications + value: "100" + resources: + limits: + memory: 4Gi + cpu: 1000m + - name: prometheus-jmx-exporter + image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143" + command: + - java + - -XX:+UnlockExperimentalVMOptions + - -XX:+UseCGroupMemoryLimitForHeap + - -XX:MaxRAMFraction=1 + - -XshowSettings:vm + - -jar + - jmx_prometheus_httpserver.jar + - "5556" + - /etc/jmx-aggregation/jmx-kafka-prometheus.yml + ports: + - containerPort: 5556 + volumeMounts: + - name: jmx-config + mountPath: /etc/jmx-aggregation + volumes: + - name: jmx-config + configMap: + name: aggregation-jmx-configmap \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc4-kstreams/uc4-kstreams-service.yaml b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-kstreams-service.yaml new file mode 100644 index 000000000..85432d04f --- /dev/null +++ b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-kstreams-service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: titan-ccp-aggregation + labels: + app: titan-ccp-aggregation +spec: + #type: NodePort + selector: + app: titan-ccp-aggregation + ports: + - name: http + port: 80 + targetPort: 80 + protocol: TCP + - name: metrics + port: 5556 diff --git a/theodolite-benchmarks/definitions/uc4-kstreams/uc4-load-generator-deployment.yaml b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-load-generator-deployment.yaml new file mode 100644 index 000000000..c1cad0b70 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-load-generator-deployment.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: titan-ccp-load-generator +spec: + selector: + matchLabels: + app: titan-ccp-load-generator + replicas: 1 + template: + metadata: + labels: + app: titan-ccp-load-generator + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: workload-generator + image: ghcr.io/cau-se/theodolite-uc3-workload-generator:latest + ports: + - containerPort: 5701 + name: coordination + env: + - name: KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: KUBERNETES_DNS_NAME + value: "titan-ccp-load-generator.$(KUBERNETES_NAMESPACE).svc.cluster.local" + - name: KAFKA_BOOTSTRAP_SERVERS + value: "theodolite-cp-kafka:9092" + - name: SCHEMA_REGISTRY_URL + value: "http://theodolite-cp-schema-registry:8081" diff --git a/theodolite-benchmarks/definitions/uc4-kstreams/uc4-load-generator-service.yaml b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-load-generator-service.yaml new file mode 100644 index 000000000..f8b26b3f6 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-load-generator-service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: titan-ccp-load-generator + labels: + app: titan-ccp-load-generator +spec: + type: ClusterIP + clusterIP: None + selector: + app: titan-ccp-load-generator + ports: + - name: coordination + port: 5701 + targetPort: 5701 + protocol: TCP diff --git a/theodolite-benchmarks/definitions/uc4-kstreams/uc4-service-monitor.yaml b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-service-monitor.yaml new file mode 100644 index 000000000..4e7e758ca --- /dev/null +++ b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-service-monitor.yaml @@ -0,0 +1,14 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + app: titan-ccp-aggregation + appScope: titan-ccp + name: titan-ccp-aggregation +spec: + selector: + matchLabels: + app: titan-ccp-aggregation + endpoints: + - port: metrics + interval: 10s -- GitLab