From 1cb447e04122933e82dbbd7948eb6e13e0b3a1aa Mon Sep 17 00:00:00 2001 From: lorenz <stu203404@mail.uni-kiel.de> Date: Wed, 9 Feb 2022 11:20:55 +0100 Subject: [PATCH] Add uc2-beam-samza benchmark resources --- .../infrastructure/kafka-client.yaml | 24 ++++++ .../resources/uc2-beam-samza-deployment.yaml | 40 ++++++++++ .../resources/uc2-beam-samza-service.yaml | 17 ++++ .../resources/uc2-service-monitor.yaml | 14 ++++ .../uc2-beam-samza-benchmark-operator.yaml | 78 +++++++++++++++++++ 5 files changed, 173 insertions(+) create mode 100644 theodolite-benchmarks/definitions/uc2-beam-samza/infrastructure/kafka-client.yaml create mode 100644 theodolite-benchmarks/definitions/uc2-beam-samza/resources/uc2-beam-samza-deployment.yaml create mode 100644 theodolite-benchmarks/definitions/uc2-beam-samza/resources/uc2-beam-samza-service.yaml create mode 100644 theodolite-benchmarks/definitions/uc2-beam-samza/resources/uc2-service-monitor.yaml create mode 100644 theodolite-benchmarks/definitions/uc2-beam-samza/uc2-beam-samza-benchmark-operator.yaml diff --git a/theodolite-benchmarks/definitions/uc2-beam-samza/infrastructure/kafka-client.yaml b/theodolite-benchmarks/definitions/uc2-beam-samza/infrastructure/kafka-client.yaml new file mode 100644 index 000000000..77ad4b3f3 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc2-beam-samza/infrastructure/kafka-client.yaml @@ -0,0 +1,24 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kafka-client + labels: + app: kafka-client +spec: + replicas: 3 + selector: + matchLabels: + app: kafka-client + template: + metadata: + labels: + app: kafka-client + spec: + containers: + - name: kafka-client + image: confluentinc/cp-enterprise-kafka:5.4.0 + command: + - sh + - -c + - "exec tail -f /dev/null" + restartPolicy: Never \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc2-beam-samza/resources/uc2-beam-samza-deployment.yaml b/theodolite-benchmarks/definitions/uc2-beam-samza/resources/uc2-beam-samza-deployment.yaml new file mode 100644 index 000000000..d2686b58d --- /dev/null +++ b/theodolite-benchmarks/definitions/uc2-beam-samza/resources/uc2-beam-samza-deployment.yaml @@ -0,0 +1,40 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: uc2-beam-samza +spec: + selector: + matchLabels: + app: uc2-beam-samza + replicas: 1 + template: + metadata: + labels: + app: uc2-beam-samza + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: uc2-beam-samza + image: ghcr.io/cau-se/theodolite-uc2-beam-samza + ports: + - containerPort: 5555 + name: jmx + env: + - name: SAMZA_JOB_COORDINATOR_ZK_CONNECT + value: "theodolite-cp-zookeeper:2181" + - name: SAMZA_SYSTEMS_KAFKA_PRODUCER_BOOTSTRAP_SERVERS + value: "theodolite-cp-kafka:9092" + - name: SAMZA_SYSTEMS_KAFKA_CONSUMER_BOOTSTRAP_SERVERS + value: "theodolite-cp-kafka:9092" + - name: MAX_SOURCE_PARALLELISM + value: "1024" + - name: KAFKA_BOOTSTRAP_SERVERS + value: "theodolite-cp-kafka:9092" + - name: SCHEMA_REGISTRY_URL + value: "http://theodolite-cp-schema-registry:8081" + - name: COMMIT_INTERVAL_MS # Set as default for the applications + value: "100" + resources: + limits: + memory: 4Gi + cpu: 1000m \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc2-beam-samza/resources/uc2-beam-samza-service.yaml b/theodolite-benchmarks/definitions/uc2-beam-samza/resources/uc2-beam-samza-service.yaml new file mode 100644 index 000000000..6317caf9f --- /dev/null +++ b/theodolite-benchmarks/definitions/uc2-beam-samza/resources/uc2-beam-samza-service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: titan-ccp-aggregation + labels: + app: titan-ccp-aggregation +spec: + #type: NodePort + selector: + app: titan-ccp-aggregation + ports: + - name: http + port: 80 + targetPort: 80 + protocol: TCP + - name: metrics + port: 5556 diff --git a/theodolite-benchmarks/definitions/uc2-beam-samza/resources/uc2-service-monitor.yaml b/theodolite-benchmarks/definitions/uc2-beam-samza/resources/uc2-service-monitor.yaml new file mode 100644 index 000000000..4e7e758ca --- /dev/null +++ b/theodolite-benchmarks/definitions/uc2-beam-samza/resources/uc2-service-monitor.yaml @@ -0,0 +1,14 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + app: titan-ccp-aggregation + appScope: titan-ccp + name: titan-ccp-aggregation +spec: + selector: + matchLabels: + app: titan-ccp-aggregation + endpoints: + - port: metrics + interval: 10s diff --git a/theodolite-benchmarks/definitions/uc2-beam-samza/uc2-beam-samza-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc2-beam-samza/uc2-beam-samza-benchmark-operator.yaml new file mode 100644 index 000000000..dc45a9014 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc2-beam-samza/uc2-beam-samza-benchmark-operator.yaml @@ -0,0 +1,78 @@ +apiVersion: theodolite.com/v1 +kind: benchmark +metadata: + name: uc2-beam-samza +spec: + sut: + resources: + - configMap: + name: "benchmark-resources-uc2-beam-samza" + files: + - "uc2-beam-samza-deployment.yaml" + - "uc2-beam-samza-service.yaml" + - "uc2-service-monitor.yaml" + # beforeActions: + # - selector: # delete zookeeper nodes to reset zookeeper + # pod: + # matchLabels: + # app: "cp-zookeeper" + # container: "cp-zookeeper-server" + # exec: + # command: ["/usr/bin/zookeeper-shell", "localhost:2181", "ls" , "/"] + # timeoutSeconds: 60 + # - selector: #Create input topic for samza with 10 partitions + # pod: + # matchLabels: + # app: "cp-zookeeper" + # container: "cp-zookeeper-server" + # exec: + # command: ["kafka-topics", "--zookeeper", "localhost:2181", + # "--topic", "input", "--partitions", "10", "--create"] + # timeoutSeconds: 90 + afterActions: + - selector: # delete zookeeper nodes to reset zookeeper + pod: + matchLabels: + app: "cp-zookeeper" + container: "cp-zookeeper-server" + exec: + command: ["zookeeper-shell", "localhost:2181", + "deleteall", "/app-theodolite-uc2-application-1"] + timeoutSeconds: 60 + loadGenerator: + resources: + - configMap: + name: "benchmark-resources-uc2-load-generator" + files: + - "uc2-load-generator-deployment.yaml" + - "uc2-load-generator-service.yaml" + resourceTypes: + - typeName: "Instances" + patchers: + - type: "ReplicaPatcher" + resource: "uc2-beam-samza-deployment.yaml" + loadTypes: + - typeName: "NumSensors" + patchers: + - type: "EnvVarPatcher" + resource: "uc2-load-generator-deployment.yaml" + properties: + container: "workload-generator" + variableName: "NUM_SENSORS" + - type: NumSensorsLoadGeneratorReplicaPatcher + resource: "uc2-load-generator-deployment.yaml" + properties: + loadGenMaxRecords: "150000" + kafkaConfig: + bootstrapServer: "theodolite-cp-kafka:9092" + topics: + - name: "input" + numPartitions: 40 + replicationFactor: 1 + - name: "output" + numPartitions: 40 + replicationFactor: 1 + - name: ".*samza.*" + removeOnly: true + - name: ".*uc2-application.*" + removeOnly: true \ No newline at end of file -- GitLab