From 3f6da3a1cd2ce0c370a2aec22c2cf4964e38dd01 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=B6ren=20Henning?= <post@soeren-henning.de>
Date: Tue, 5 May 2020 16:07:03 +0200
Subject: [PATCH] Allow for scaling up UC3 workload generation
---
execution/run_uc1-new.sh | 3 ---
execution/run_uc3-new.sh | 12 +++++++-----
execution/uc3-workload-generator/deployment.yaml | 13 ++++++++++---
3 files changed, 17 insertions(+), 11 deletions(-)
diff --git a/execution/run_uc1-new.sh b/execution/run_uc1-new.sh
index 932970596..e8f9e32b7 100755
--- a/execution/run_uc1-new.sh
+++ b/execution/run_uc1-new.sh
@@ -9,9 +9,6 @@ PARTITIONS=$4
#KAFKA_STREAMS_COMMIT_INTERVAL_MS=100
EXECUTION_MINUTES=5
-# Start up Kafka
-# TODO
-
# Create Topics
#PARTITIONS=40
#kubectl run temp-kafka --rm --attach --restart=Never --image=solsson/kafka --command -- bash -c "./bin/kafka-topics.sh --zookeeper my-confluent-cp-zookeeper:2181 --create --topic input --partitions $PARTITIONS --replication-factor 1; ./bin/kafka-topics.sh --zookeeper my-confluent-cp-zookeeper:2181 --create --topic configuration --partitions 1 --replication-factor 1; ./bin/kafka-topics.sh --zookeeper my-confluent-cp-zookeeper:2181 --create --topic output --partitions $PARTITIONS --replication-factor 1"
diff --git a/execution/run_uc3-new.sh b/execution/run_uc3-new.sh
index 7c1d10e69..4517041dc 100755
--- a/execution/run_uc3-new.sh
+++ b/execution/run_uc3-new.sh
@@ -9,8 +9,6 @@ PARTITIONS=$4
#KAFKA_STREAMS_COMMIT_INTERVAL_MS=100
EXECUTION_MINUTES=5
-# Maybe start up Kafka
-
# Create Topics
#PARTITIONS=40
#kubectl run temp-kafka --rm --attach --restart=Never --image=solsson/kafka --command -- bash -c "./bin/kafka-topics.sh --zookeeper my-confluent-cp-zookeeper:2181 --create --topic input --partitions $PARTITIONS --replication-factor 1; ./bin/kafka-topics.sh --zookeeper my-confluent-cp-zookeeper:2181 --create --topic configuration --partitions 1 --replication-factor 1; ./bin/kafka-topics.sh --zookeeper my-confluent-cp-zookeeper:2181 --create --topic output --partitions $PARTITIONS --replication-factor 1"
@@ -19,7 +17,9 @@ kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-z
# Start workload generator
NUM_SENSORS=$DIM_VALUE
-sed "s/{{NUM_SENSORS}}/$NUM_SENSORS/g" uc3-workload-generator/deployment.yaml | kubectl apply -f -
+WL_MAX_RECORDS=150000
+WL_INSTANCES=$(((NUM_SENSORS + (WL_MAX_RECORDS -1 ))/ WL_MAX_RECORDS))
+sed "s/{{NUM_SENSORS}}/$NUM_SENSORS/g; s/{{INSTANCES}}/$WL_INSTANCES/g" uc3-workload-generator/deployment.yaml | kubectl apply -f -
# Start application
REPLICAS=$INSTANCES
@@ -35,8 +35,10 @@ python lag_analysis.py $EXP_ID uc3 $DIM_VALUE $INSTANCES
deactivate
# Stop wl and app
-kubectl delete -f uc3-workload-generator/deployment.yaml
-kubectl delete -f uc3-application/aggregation-deployment.yaml
+#kubectl delete -f uc3-workload-generator/deployment.yaml
+#sed "s/{{INSTANCES}}/1/g" uc3-workload-generator/deployment.yaml | kubectl delete -f -
+sed "s/{{NUM_SENSORS}}/$NUM_SENSORS/g; s/{{INSTANCES}}/$WL_INSTANCES/g" uc3-workload-generator/deployment.yaml | kubectl delete -f -
+kubectl delete -f uc1-application/aggregation-deployment.yaml
# Delete topics instead of Kafka
diff --git a/execution/uc3-workload-generator/deployment.yaml b/execution/uc3-workload-generator/deployment.yaml
index b78653e23..9ecd2b67e 100644
--- a/execution/uc3-workload-generator/deployment.yaml
+++ b/execution/uc3-workload-generator/deployment.yaml
@@ -1,12 +1,13 @@
apiVersion: apps/v1
-kind: Deployment
+kind: StatefulSet
metadata:
name: titan-ccp-load-generator
spec:
selector:
matchLabels:
app: titan-ccp-load-generator
- replicas: 1
+ serviceName: titan-ccp-load-generator
+ replicas: {{INSTANCES}}
template:
metadata:
labels:
@@ -15,10 +16,16 @@ spec:
terminationGracePeriodSeconds: 0
containers:
- name: workload-generator
- image: benediktwetzel/uc3-wg:latest
+ image: soerenhenning/uc3-wg:latest
env:
- name: KAFKA_BOOTSTRAP_SERVERS
value: "my-confluent-cp-kafka:9092"
- name: NUM_SENSORS
value: "{{NUM_SENSORS}}"
+ - name: POD_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.name
+ - name: INSTANCES
+ value: "{{INSTANCES}}"
\ No newline at end of file
--
GitLab