Use kustomize instead of sed approach in run uc

Create files for use case configurations in kubernetes and apply them with kustomize.

Use kustomize instead of sed approach in run uc
b5c80e48 · Björn Vonheiden · 0aabcffe · b5c80e48 · b5c80e48 · b5c80e48
Commit b5c80e48 authored Jun 7, 2020 by Björn Vonheiden
--- a/execution/run_uc1-new.sh
+++ b/execution/run_uc1-new.sh
@@ -29,18 +29,46 @@ NUM_SENSORS=$DIM_VALUE
 WL_MAX_RECORDS=150000
 WL_INSTANCES=$(((NUM_SENSORS + (WL_MAX_RECORDS -1 ))/ WL_MAX_RECORDS))

-WORKLOAD_GENERATOR_YAML=$(sed "s/{{NUM_SENSORS}}/$NUM_SENSORS/g; s/{{INSTANCES}}/$WL_INSTANCES/g" uc1-workload-generator/deployment.yaml)
-echo "$WORKLOAD_GENERATOR_YAML" | kubectl apply -f -
+cat <<EOF >uc1-workload-generator/set_paramters.yaml
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: titan-ccp-load-generator
+spec:
+  replicas: $WL_INSTANCES
+  template:
+    spec:
+      containers:
+      - name: workload-generator
+        env:
+        - name: NUM_SENSORS
+          value: $NUM_SENSORS
+        - name: INSTANCES
+          value: $WL_INSTANCES
+EOF
+kubectl apply -k uc1-workload-generator

 # Start application
 REPLICAS=$INSTANCES
-# When not using `sed` anymore, use `kubectl apply -f uc1-application`
-kubectl apply -f uc1-application/aggregation-service.yaml
-kubectl apply -f uc1-application/jmx-configmap.yaml
-kubectl apply -f uc1-application/service-monitor.yaml
-#kubectl apply -f uc1-application/aggregation-deployment.yaml
-APPLICATION_YAML=$(sed "s/{{CPU_LIMIT}}/$CPU_LIMIT/g; s/{{MEMORY_LIMIT}}/$MEMORY_LIMIT/g; s/{{KAFKA_STREAMS_COMMIT_INTERVAL_MS}}/$KAFKA_STREAMS_COMMIT_INTERVAL_MS/g" uc1-application/aggregation-deployment.yaml)
-echo "$APPLICATION_YAML" | kubectl apply -f -
+cat <<EOF >uc1-application/set_paramters.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: titan-ccp-aggregation
+spec:
+  template:
+    spec:
+      containers:
+      - name: uc1-application
+        env:
+        - name: COMMIT_INTERVAL_MS
+          value: $KAFKA_STREAMS_COMMIT_INTERVAL_MS
+        resources:
+          limits:
+            memory: $MEMORY_LIMIT
+            cpu: $CPU_LIMIT
+EOF
+kubectl apply -k uc1-application
 kubectl scale deployment titan-ccp-aggregation --replicas=$REPLICAS

 # Execute for certain time
@@ -51,16 +79,9 @@ source ../.venv/bin/activate
 python lag_analysis.py $EXP_ID uc1 $DIM_VALUE $INSTANCES $EXECUTION_MINUTES
 deactivate

-# Stop wl and app
-#kubectl delete -f uc1-workload-generator/deployment.yaml
-#sed "s/{{INSTANCES}}/1/g" uc1-workload-generator/deployment.yaml | kubectl delete -f -
-#sed "s/{{NUM_SENSORS}}/$NUM_SENSORS/g; s/{{INSTANCES}}/$WL_INSTANCES/g" uc1-workload-generator/deployment.yaml | kubectl delete -f -
-echo "$WORKLOAD_GENERATOR_YAML" | kubectl delete -f -
-kubectl delete -f uc1-application/aggregation-service.yaml
-kubectl delete -f uc1-application/jmx-configmap.yaml
-kubectl delete -f uc1-application/service-monitor.yaml
-#kubectl delete -f uc1-application/aggregation-deployment.yaml
-echo "$APPLICATION_YAML" | kubectl delete -f -
+# Stop workload generator and app
+kubectl delete -k uc1-workload-generator
+kubectl delete -k uc1-application


 # Delete topics instead of Kafka

--- a/execution/run_uc2-new.sh
+++ b/execution/run_uc2-new.sh
@@ -26,17 +26,45 @@ kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-z

 # Start workload generator
 NUM_NESTED_GROUPS=$DIM_VALUE
-sed "s/{{NUM_NESTED_GROUPS}}/$NUM_NESTED_GROUPS/g" uc2-workload-generator/deployment.yaml | kubectl apply -f -
+cat <<EOF >uc2-workload-generator/set_paramters.yaml
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: titan-ccp-load-generator
+spec:
+  replicas: 1
+  template:
+    spec:
+      containers:
+      - name: workload-generator
+        env:
+        - name: NUM_NESTED_GROUPS
+          value: $NUM_NESTED_GROUPS
+EOF
+kubectl apply -k uc2-workload-generator

 # Start application
 REPLICAS=$INSTANCES
 # When not using `sed` anymore, use `kubectl apply -f uc2-application`
-kubectl apply -f uc2-application/aggregation-service.yaml
-kubectl apply -f uc2-application/jmx-configmap.yaml
-kubectl apply -f uc2-application/service-monitor.yaml
-#kubectl apply -f uc2-application/aggregation-deployment.yaml
-APPLICATION_YAML=$(sed "s/{{CPU_LIMIT}}/$CPU_LIMIT/g; s/{{MEMORY_LIMIT}}/$MEMORY_LIMIT/g; s/{{KAFKA_STREAMS_COMMIT_INTERVAL_MS}}/$KAFKA_STREAMS_COMMIT_INTERVAL_MS/g" uc2-application/aggregation-deployment.yaml)
-echo "$APPLICATION_YAML" | kubectl apply -f -
+cat <<EOF >uc2-application/set_paramters.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: titan-ccp-aggregation
+spec:
+  template:
+    spec:
+      containers:
+      - name: uc1-application
+        env:
+        - name: COMMIT_INTERVAL_MS
+          value: $KAFKA_STREAMS_COMMIT_INTERVAL_MS
+        resources:
+          limits:
+            memory: $MEMORY_LIMIT
+            cpu: $CPU_LIMIT
+EOF
+kubectl apply -k uc2-application
 kubectl scale deployment titan-ccp-aggregation --replicas=$REPLICAS

 # Execute for certain time
@@ -47,13 +75,9 @@ source ../.venv/bin/activate
 python lag_analysis.py $EXP_ID uc2 $DIM_VALUE $INSTANCES $EXECUTION_MINUTES
 deactivate

-# Stop wl and app
-kubectl delete -f uc2-workload-generator/deployment.yaml
-kubectl delete -f uc2-application/aggregation-service.yaml
-kubectl delete -f uc2-application/jmx-configmap.yaml
-kubectl delete -f uc2-application/service-monitor.yaml
-#kubectl delete -f uc2-application/aggregation-deployment.yaml
-echo "$APPLICATION_YAML" | kubectl delete -f -
+# Stop workload generator and app
+kubectl delete -k uc2-workload-generator
+kubectl delete -k uc2-application


 # Delete topics instead of Kafka

--- a/execution/run_uc3-new.sh
+++ b/execution/run_uc3-new.sh
@@ -29,18 +29,47 @@ NUM_SENSORS=$DIM_VALUE
 WL_MAX_RECORDS=150000
 WL_INSTANCES=$(((NUM_SENSORS + (WL_MAX_RECORDS -1 ))/ WL_MAX_RECORDS))

-WORKLOAD_GENERATOR_YAML=$(sed "s/{{NUM_SENSORS}}/$NUM_SENSORS/g; s/{{INSTANCES}}/$WL_INSTANCES/g" uc3-workload-generator/deployment.yaml)
-echo "$WORKLOAD_GENERATOR_YAML" | kubectl apply -f -
+cat <<EOF >uc3-workload-generator/set_paramters.yaml
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: titan-ccp-load-generator
+spec:
+  replicas: $WL_INSTANCES
+  template:
+    spec:
+      containers:
+      - name: workload-generator
+        env:
+        - name: NUM_SENSORS
+          value: $NUM_SENSORS
+        - name: INSTANCES
+          value: $WL_INSTANCES
+EOF
+kubectl apply -k uc3-workload-generator
+

 # Start application
 REPLICAS=$INSTANCES
-# When not using `sed` anymore, use `kubectl apply -f uc3-application`
-kubectl apply -f uc3-application/aggregation-service.yaml
-kubectl apply -f uc3-application/jmx-configmap.yaml
-kubectl apply -f uc3-application/service-monitor.yaml
-#kubectl apply -f uc3-application/aggregation-deployment.yaml
-APPLICATION_YAML=$(sed "s/{{CPU_LIMIT}}/$CPU_LIMIT/g; s/{{MEMORY_LIMIT}}/$MEMORY_LIMIT/g; s/{{KAFKA_STREAMS_COMMIT_INTERVAL_MS}}/$KAFKA_STREAMS_COMMIT_INTERVAL_MS/g" uc3-application/aggregation-deployment.yaml)
-echo "$APPLICATION_YAML" | kubectl apply -f -
+cat <<EOF >uc3-application/set_paramters.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: titan-ccp-aggregation
+spec:
+  template:
+    spec:
+      containers:
+      - name: uc1-application
+        env:
+        - name: COMMIT_INTERVAL_MS
+          value: $KAFKA_STREAMS_COMMIT_INTERVAL_MS
+        resources:
+          limits:
+            memory: $MEMORY_LIMIT
+            cpu: $CPU_LIMIT
+EOF
+kubectl apply -k uc3-application
 kubectl scale deployment titan-ccp-aggregation --replicas=$REPLICAS

 # Execute for certain time
@@ -51,18 +80,9 @@ source ../.venv/bin/activate
 python lag_analysis.py $EXP_ID uc3 $DIM_VALUE $INSTANCES $EXECUTION_MINUTES
 deactivate

-# Stop wl and app
-#kubectl delete -f uc3-workload-generator/deployment.yaml
-#sed "s/{{INSTANCES}}/1/g" uc3-workload-generator/deployment.yaml | kubectl delete -f -
-echo "$WORKLOAD_GENERATOR_YAML" | kubectl delete -f -
-kubectl delete -f uc3-application/aggregation-service.yaml
-kubectl delete -f uc3-application/jmx-configmap.yaml
-kubectl delete -f uc3-application/service-monitor.yaml
-#kubectl delete -f uc3-application/aggregation-deployment.yaml
-#sed "s/{{CPU_LIMIT}}/1000m/g; s/{{MEMORY_LIMIT}}/4Gi/g; s/{{KAFKA_STREAMS_COMMIT_INTERVAL_MS}}/100/g" uc3-application/aggregation-deployment.yaml | kubectl delete -f -
-echo "$APPLICATION_YAML" | kubectl delete -f -
-
-
+# Stop workload generator and app
+kubectl delete -k uc1-workload-generator
+kubectl delete -k uc1-application

 # Delete topics instead of Kafka
 #kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --delete --topic 'input,output,configuration,titan-.*'"

--- a/execution/run_uc4-new.sh
+++ b/execution/run_uc4-new.sh
@@ -26,20 +26,44 @@ kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-z

 # Start workload generator
 NUM_SENSORS=$DIM_VALUE
-#NUM_SENSORS=xy
-sed "s/{{NUM_SENSORS}}/$NUM_SENSORS/g" uc4-workload-generator/deployment.yaml | kubectl apply -f -
+cat <<EOF >uc4-workload-generator/set_paramters.yaml
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: titan-ccp-load-generator
+spec:
+  replicas: 1
+  template:
+    spec:
+      containers:
+      - name: workload-generator
+        env:
+        - name: NUM_SENSORS
+          value: $NUM_SENSORS
+EOF
+kubectl apply -k uc4-workload-generator

 # Start application
 REPLICAS=$INSTANCES
-#AGGREGATION_DURATION_DAYS=$DIM_VALUE
-# When not using `sed` anymore, use `kubectl apply -f uc4-application`
-kubectl apply -f uc4-application/aggregation-service.yaml
-kubectl apply -f uc4-application/jmx-configmap.yaml
-kubectl apply -f uc4-application/service-monitor.yaml
-#kubectl apply -f uc4-application/aggregation-deployment.yaml
-#sed "s/{{AGGREGATION_DURATION_DAYS}}/$AGGREGATION_DURATION_DAYS/g" uc4-application/aggregation-deployment.yaml | kubectl apply -f -
-APPLICATION_YAML=$(sed "s/{{CPU_LIMIT}}/$CPU_LIMIT/g; s/{{MEMORY_LIMIT}}/$MEMORY_LIMIT/g; s/{{KAFKA_STREAMS_COMMIT_INTERVAL_MS}}/$KAFKA_STREAMS_COMMIT_INTERVAL_MS/g" uc4-application/aggregation-deployment.yaml)
-echo "$APPLICATION_YAML" | kubectl apply -f -
+cat <<EOF >uc1-application/set_paramters.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: titan-ccp-aggregation
+spec:
+  template:
+    spec:
+      containers:
+      - name: uc1-application
+        env:
+        - name: COMMIT_INTERVAL_MS
+          value: $KAFKA_STREAMS_COMMIT_INTERVAL_MS
+        resources:
+          limits:
+            memory: $MEMORY_LIMIT
+            cpu: $CPU_LIMIT
+EOF
+kubectl apply -k uc4-application
 kubectl scale deployment titan-ccp-aggregation --replicas=$REPLICAS

 # Execute for certain time
@@ -50,13 +74,9 @@ source ../.venv/bin/activate
 python lag_analysis.py $EXP_ID uc4 $DIM_VALUE $INSTANCES $EXECUTION_MINUTES
 deactivate

-# Stop wl and app
-kubectl delete -f uc4-workload-generator/deployment.yaml
-kubectl delete -f uc4-application/aggregation-service.yaml
-kubectl delete -f uc4-application/jmx-configmap.yaml
-kubectl delete -f uc4-application/service-monitor.yaml
-#kubectl delete -f uc4-application/aggregation-deployment.yaml
-echo "$APPLICATION_YAML" | kubectl delete -f -
+# Stop workload generator and app
+kubectl delete -k uc4-workload-generator
+kubectl delete -k uc4-application


 # Delete topics instead of Kafka

--- a/execution/uc4-workload-generator/set_paramters.yaml
+++ b/execution/uc4-workload-generator/set_paramters.yaml
@@ -11,5 +11,3 @@ spec:
        env:
        - name: NUM_SENSORS
          value: 25000
-        - name: INSTANCES
-          value: 1