From b5c80e48ad6d32dbd33c40802a71f09695fcffe7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Vonheiden?= <bjoern.vonheiden@hotmail.de>
Date: Sun, 7 Jun 2020 11:32:44 +0200
Subject: [PATCH] Use kustomize instead of sed approach in run uc

Create files for use case configurations in kubernetes and apply
them with kustomize.
---
 execution/run_uc1-new.sh                      | 59 ++++++++++++------
 execution/run_uc2-new.sh                      | 52 +++++++++++-----
 execution/run_uc3-new.sh                      | 62 ++++++++++++-------
 execution/run_uc4-new.sh                      | 56 +++++++++++------
 .../uc4-workload-generator/set_paramters.yaml |  2 -
 5 files changed, 157 insertions(+), 74 deletions(-)

diff --git a/execution/run_uc1-new.sh b/execution/run_uc1-new.sh
index 592986122..83e540bc4 100755
--- a/execution/run_uc1-new.sh
+++ b/execution/run_uc1-new.sh
@@ -29,18 +29,46 @@ NUM_SENSORS=$DIM_VALUE
 WL_MAX_RECORDS=150000
 WL_INSTANCES=$(((NUM_SENSORS + (WL_MAX_RECORDS -1 ))/ WL_MAX_RECORDS))
 
-WORKLOAD_GENERATOR_YAML=$(sed "s/{{NUM_SENSORS}}/$NUM_SENSORS/g; s/{{INSTANCES}}/$WL_INSTANCES/g" uc1-workload-generator/deployment.yaml)
-echo "$WORKLOAD_GENERATOR_YAML" | kubectl apply -f -
+cat <<EOF >uc1-workload-generator/set_paramters.yaml
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: titan-ccp-load-generator
+spec:
+  replicas: $WL_INSTANCES
+  template:
+    spec:
+      containers:
+      - name: workload-generator
+        env:
+        - name: NUM_SENSORS
+          value: $NUM_SENSORS
+        - name: INSTANCES
+          value: $WL_INSTANCES
+EOF
+kubectl apply -k uc1-workload-generator
 
 # Start application
 REPLICAS=$INSTANCES
-# When not using `sed` anymore, use `kubectl apply -f uc1-application`
-kubectl apply -f uc1-application/aggregation-service.yaml
-kubectl apply -f uc1-application/jmx-configmap.yaml
-kubectl apply -f uc1-application/service-monitor.yaml
-#kubectl apply -f uc1-application/aggregation-deployment.yaml
-APPLICATION_YAML=$(sed "s/{{CPU_LIMIT}}/$CPU_LIMIT/g; s/{{MEMORY_LIMIT}}/$MEMORY_LIMIT/g; s/{{KAFKA_STREAMS_COMMIT_INTERVAL_MS}}/$KAFKA_STREAMS_COMMIT_INTERVAL_MS/g" uc1-application/aggregation-deployment.yaml)
-echo "$APPLICATION_YAML" | kubectl apply -f -
+cat <<EOF >uc1-application/set_paramters.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: titan-ccp-aggregation
+spec:
+  template:
+    spec:
+      containers:
+      - name: uc1-application
+        env:
+        - name: COMMIT_INTERVAL_MS
+          value: $KAFKA_STREAMS_COMMIT_INTERVAL_MS
+        resources:
+          limits:
+            memory: $MEMORY_LIMIT
+            cpu: $CPU_LIMIT
+EOF
+kubectl apply -k uc1-application
 kubectl scale deployment titan-ccp-aggregation --replicas=$REPLICAS
 
 # Execute for certain time
@@ -51,16 +79,9 @@ source ../.venv/bin/activate
 python lag_analysis.py $EXP_ID uc1 $DIM_VALUE $INSTANCES $EXECUTION_MINUTES
 deactivate
 
-# Stop wl and app
-#kubectl delete -f uc1-workload-generator/deployment.yaml
-#sed "s/{{INSTANCES}}/1/g" uc1-workload-generator/deployment.yaml | kubectl delete -f -
-#sed "s/{{NUM_SENSORS}}/$NUM_SENSORS/g; s/{{INSTANCES}}/$WL_INSTANCES/g" uc1-workload-generator/deployment.yaml | kubectl delete -f -
-echo "$WORKLOAD_GENERATOR_YAML" | kubectl delete -f -
-kubectl delete -f uc1-application/aggregation-service.yaml
-kubectl delete -f uc1-application/jmx-configmap.yaml
-kubectl delete -f uc1-application/service-monitor.yaml
-#kubectl delete -f uc1-application/aggregation-deployment.yaml
-echo "$APPLICATION_YAML" | kubectl delete -f -
+# Stop workload generator and app
+kubectl delete -k uc1-workload-generator
+kubectl delete -k uc1-application
 
 
 # Delete topics instead of Kafka
diff --git a/execution/run_uc2-new.sh b/execution/run_uc2-new.sh
index e1571bf33..42ce812b8 100755
--- a/execution/run_uc2-new.sh
+++ b/execution/run_uc2-new.sh
@@ -26,17 +26,45 @@ kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-z
 
 # Start workload generator
 NUM_NESTED_GROUPS=$DIM_VALUE
-sed "s/{{NUM_NESTED_GROUPS}}/$NUM_NESTED_GROUPS/g" uc2-workload-generator/deployment.yaml | kubectl apply -f -
+cat <<EOF >uc2-workload-generator/set_paramters.yaml
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: titan-ccp-load-generator
+spec:
+  replicas: 1
+  template:
+    spec:
+      containers:
+      - name: workload-generator
+        env:
+        - name: NUM_NESTED_GROUPS
+          value: $NUM_NESTED_GROUPS
+EOF
+kubectl apply -k uc2-workload-generator
 
 # Start application
 REPLICAS=$INSTANCES
 # When not using `sed` anymore, use `kubectl apply -f uc2-application`
-kubectl apply -f uc2-application/aggregation-service.yaml
-kubectl apply -f uc2-application/jmx-configmap.yaml
-kubectl apply -f uc2-application/service-monitor.yaml
-#kubectl apply -f uc2-application/aggregation-deployment.yaml
-APPLICATION_YAML=$(sed "s/{{CPU_LIMIT}}/$CPU_LIMIT/g; s/{{MEMORY_LIMIT}}/$MEMORY_LIMIT/g; s/{{KAFKA_STREAMS_COMMIT_INTERVAL_MS}}/$KAFKA_STREAMS_COMMIT_INTERVAL_MS/g" uc2-application/aggregation-deployment.yaml)
-echo "$APPLICATION_YAML" | kubectl apply -f -
+cat <<EOF >uc2-application/set_paramters.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: titan-ccp-aggregation
+spec:
+  template:
+    spec:
+      containers:
+      - name: uc1-application
+        env:
+        - name: COMMIT_INTERVAL_MS
+          value: $KAFKA_STREAMS_COMMIT_INTERVAL_MS
+        resources:
+          limits:
+            memory: $MEMORY_LIMIT
+            cpu: $CPU_LIMIT
+EOF
+kubectl apply -k uc2-application
 kubectl scale deployment titan-ccp-aggregation --replicas=$REPLICAS
 
 # Execute for certain time
@@ -47,13 +75,9 @@ source ../.venv/bin/activate
 python lag_analysis.py $EXP_ID uc2 $DIM_VALUE $INSTANCES $EXECUTION_MINUTES
 deactivate
 
-# Stop wl and app
-kubectl delete -f uc2-workload-generator/deployment.yaml
-kubectl delete -f uc2-application/aggregation-service.yaml
-kubectl delete -f uc2-application/jmx-configmap.yaml
-kubectl delete -f uc2-application/service-monitor.yaml
-#kubectl delete -f uc2-application/aggregation-deployment.yaml
-echo "$APPLICATION_YAML" | kubectl delete -f -
+# Stop workload generator and app
+kubectl delete -k uc2-workload-generator
+kubectl delete -k uc2-application
 
 
 # Delete topics instead of Kafka
diff --git a/execution/run_uc3-new.sh b/execution/run_uc3-new.sh
index 99d005858..9af96fbcb 100755
--- a/execution/run_uc3-new.sh
+++ b/execution/run_uc3-new.sh
@@ -29,18 +29,47 @@ NUM_SENSORS=$DIM_VALUE
 WL_MAX_RECORDS=150000
 WL_INSTANCES=$(((NUM_SENSORS + (WL_MAX_RECORDS -1 ))/ WL_MAX_RECORDS))
 
-WORKLOAD_GENERATOR_YAML=$(sed "s/{{NUM_SENSORS}}/$NUM_SENSORS/g; s/{{INSTANCES}}/$WL_INSTANCES/g" uc3-workload-generator/deployment.yaml)
-echo "$WORKLOAD_GENERATOR_YAML" | kubectl apply -f -
+cat <<EOF >uc3-workload-generator/set_paramters.yaml
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: titan-ccp-load-generator
+spec:
+  replicas: $WL_INSTANCES
+  template:
+    spec:
+      containers:
+      - name: workload-generator
+        env:
+        - name: NUM_SENSORS
+          value: $NUM_SENSORS
+        - name: INSTANCES
+          value: $WL_INSTANCES
+EOF
+kubectl apply -k uc3-workload-generator
+
 
 # Start application
 REPLICAS=$INSTANCES
-# When not using `sed` anymore, use `kubectl apply -f uc3-application`
-kubectl apply -f uc3-application/aggregation-service.yaml
-kubectl apply -f uc3-application/jmx-configmap.yaml
-kubectl apply -f uc3-application/service-monitor.yaml
-#kubectl apply -f uc3-application/aggregation-deployment.yaml
-APPLICATION_YAML=$(sed "s/{{CPU_LIMIT}}/$CPU_LIMIT/g; s/{{MEMORY_LIMIT}}/$MEMORY_LIMIT/g; s/{{KAFKA_STREAMS_COMMIT_INTERVAL_MS}}/$KAFKA_STREAMS_COMMIT_INTERVAL_MS/g" uc3-application/aggregation-deployment.yaml)
-echo "$APPLICATION_YAML" | kubectl apply -f -
+cat <<EOF >uc3-application/set_paramters.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: titan-ccp-aggregation
+spec:
+  template:
+    spec:
+      containers:
+      - name: uc1-application
+        env:
+        - name: COMMIT_INTERVAL_MS
+          value: $KAFKA_STREAMS_COMMIT_INTERVAL_MS
+        resources:
+          limits:
+            memory: $MEMORY_LIMIT
+            cpu: $CPU_LIMIT
+EOF
+kubectl apply -k uc3-application
 kubectl scale deployment titan-ccp-aggregation --replicas=$REPLICAS
 
 # Execute for certain time
@@ -51,18 +80,9 @@ source ../.venv/bin/activate
 python lag_analysis.py $EXP_ID uc3 $DIM_VALUE $INSTANCES $EXECUTION_MINUTES
 deactivate
 
-# Stop wl and app
-#kubectl delete -f uc3-workload-generator/deployment.yaml
-#sed "s/{{INSTANCES}}/1/g" uc3-workload-generator/deployment.yaml | kubectl delete -f -
-echo "$WORKLOAD_GENERATOR_YAML" | kubectl delete -f -
-kubectl delete -f uc3-application/aggregation-service.yaml
-kubectl delete -f uc3-application/jmx-configmap.yaml
-kubectl delete -f uc3-application/service-monitor.yaml
-#kubectl delete -f uc3-application/aggregation-deployment.yaml
-#sed "s/{{CPU_LIMIT}}/1000m/g; s/{{MEMORY_LIMIT}}/4Gi/g; s/{{KAFKA_STREAMS_COMMIT_INTERVAL_MS}}/100/g" uc3-application/aggregation-deployment.yaml | kubectl delete -f -
-echo "$APPLICATION_YAML" | kubectl delete -f -
-
-
+# Stop workload generator and app
+kubectl delete -k uc1-workload-generator
+kubectl delete -k uc1-application
 
 # Delete topics instead of Kafka
 #kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --delete --topic 'input,output,configuration,titan-.*'"
diff --git a/execution/run_uc4-new.sh b/execution/run_uc4-new.sh
index 5864aec84..c22e49644 100755
--- a/execution/run_uc4-new.sh
+++ b/execution/run_uc4-new.sh
@@ -26,20 +26,44 @@ kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-z
 
 # Start workload generator
 NUM_SENSORS=$DIM_VALUE
-#NUM_SENSORS=xy
-sed "s/{{NUM_SENSORS}}/$NUM_SENSORS/g" uc4-workload-generator/deployment.yaml | kubectl apply -f -
+cat <<EOF >uc4-workload-generator/set_paramters.yaml
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: titan-ccp-load-generator
+spec:
+  replicas: 1
+  template:
+    spec:
+      containers:
+      - name: workload-generator
+        env:
+        - name: NUM_SENSORS
+          value: $NUM_SENSORS
+EOF
+kubectl apply -k uc4-workload-generator
 
 # Start application
 REPLICAS=$INSTANCES
-#AGGREGATION_DURATION_DAYS=$DIM_VALUE
-# When not using `sed` anymore, use `kubectl apply -f uc4-application`
-kubectl apply -f uc4-application/aggregation-service.yaml
-kubectl apply -f uc4-application/jmx-configmap.yaml
-kubectl apply -f uc4-application/service-monitor.yaml
-#kubectl apply -f uc4-application/aggregation-deployment.yaml
-#sed "s/{{AGGREGATION_DURATION_DAYS}}/$AGGREGATION_DURATION_DAYS/g" uc4-application/aggregation-deployment.yaml | kubectl apply -f -
-APPLICATION_YAML=$(sed "s/{{CPU_LIMIT}}/$CPU_LIMIT/g; s/{{MEMORY_LIMIT}}/$MEMORY_LIMIT/g; s/{{KAFKA_STREAMS_COMMIT_INTERVAL_MS}}/$KAFKA_STREAMS_COMMIT_INTERVAL_MS/g" uc4-application/aggregation-deployment.yaml)
-echo "$APPLICATION_YAML" | kubectl apply -f -
+cat <<EOF >uc1-application/set_paramters.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: titan-ccp-aggregation
+spec:
+  template:
+    spec:
+      containers:
+      - name: uc1-application
+        env:
+        - name: COMMIT_INTERVAL_MS
+          value: $KAFKA_STREAMS_COMMIT_INTERVAL_MS
+        resources:
+          limits:
+            memory: $MEMORY_LIMIT
+            cpu: $CPU_LIMIT
+EOF
+kubectl apply -k uc4-application
 kubectl scale deployment titan-ccp-aggregation --replicas=$REPLICAS
 
 # Execute for certain time
@@ -50,13 +74,9 @@ source ../.venv/bin/activate
 python lag_analysis.py $EXP_ID uc4 $DIM_VALUE $INSTANCES $EXECUTION_MINUTES
 deactivate
 
-# Stop wl and app
-kubectl delete -f uc4-workload-generator/deployment.yaml
-kubectl delete -f uc4-application/aggregation-service.yaml
-kubectl delete -f uc4-application/jmx-configmap.yaml
-kubectl delete -f uc4-application/service-monitor.yaml
-#kubectl delete -f uc4-application/aggregation-deployment.yaml
-echo "$APPLICATION_YAML" | kubectl delete -f -
+# Stop workload generator and app
+kubectl delete -k uc4-workload-generator
+kubectl delete -k uc4-application
 
 
 # Delete topics instead of Kafka
diff --git a/execution/uc4-workload-generator/set_paramters.yaml b/execution/uc4-workload-generator/set_paramters.yaml
index 9d512893f..140804368 100644
--- a/execution/uc4-workload-generator/set_paramters.yaml
+++ b/execution/uc4-workload-generator/set_paramters.yaml
@@ -11,5 +11,3 @@ spec:
         env:
         - name: NUM_SENSORS
           value: 25000
-        - name: INSTANCES
-          value: 1
-- 
GitLab