Streamline workload generator kubernetes files using kustomize

Add the hierachie structure to the kubernetes workload generator
files.
Use the new structure in the run uc scripts.

execution/run_uc1-new.sh

@@ -29,7 +29,7 @@ NUM_SENSORS=$DIM_VALUE
 WL_MAX_RECORDS=150000
 WL_INSTANCES=$(((NUM_SENSORS + (WL_MAX_RECORDS -1 ))/ WL_MAX_RECORDS))
 
-cat <<EOF >uc1-workload-generator/set_paramters.yaml
+cat <<EOF >uc-workload-generator/overlay/uc1-workload-generator/set_paramters.yaml
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
@@ -46,7 +46,7 @@ spec:
         - name: INSTANCES
           value: "$WL_INSTANCES"
 EOF
-kubectl apply -k uc1-workload-generator
+kubectl apply -k uc-workload-generator/overlay/uc1-workload-generator
 
 # Start application
 REPLICAS=$INSTANCES
@@ -80,7 +80,7 @@ python lag_analysis.py $EXP_ID uc1 $DIM_VALUE $INSTANCES $EXECUTION_MINUTES
 deactivate
 
 # Stop workload generator and app
-kubectl delete -k uc1-workload-generator
+kubectl delete -k uc-workload-generator/overlay/uc1-workload-generator
 kubectl delete -k uc-application/overlay/uc1-application
 
 

execution/run_uc2-new.sh

@@ -26,7 +26,7 @@ kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-z
 
 # Start workload generator
 NUM_NESTED_GROUPS=$DIM_VALUE
-cat <<EOF >uc2-workload-generator/set_paramters.yaml
+cat <<EOF >uc-workload-generator/overlay/uc2-workload-generator/set_paramters.yaml
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
@@ -38,10 +38,14 @@ spec:
       containers:
       - name: workload-generator
         env:
+        - name: NUM_SENSORS
+          value: "4"
+        - name: HIERARCHY
+          value: "full"
         - name: NUM_NESTED_GROUPS
           value: "$NUM_NESTED_GROUPS"
 EOF
-kubectl apply -k uc2-workload-generator
+kubectl apply -k uc-workload-generator/overlay/uc2-workload-generator
 
 # Start application
 REPLICAS=$INSTANCES
@@ -75,7 +79,7 @@ python lag_analysis.py $EXP_ID uc2 $DIM_VALUE $INSTANCES $EXECUTION_MINUTES
 deactivate
 
 # Stop workload generator and app
-kubectl delete -k uc2-workload-generator
+kubectl delete -k uc-workload-generator/overlay/uc2-workload-generator
 kubectl delete -k uc-application/overlay/uc2-application
 
 

execution/run_uc3-new.sh

@@ -29,7 +29,7 @@ NUM_SENSORS=$DIM_VALUE
 WL_MAX_RECORDS=150000
 WL_INSTANCES=$(((NUM_SENSORS + (WL_MAX_RECORDS -1 ))/ WL_MAX_RECORDS))
 
-cat <<EOF >uc3-workload-generator/set_paramters.yaml
+cat <<EOF >uc-workload-generator/overlay/uc3-workload-generator/set_paramters.yaml
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
@@ -46,7 +46,7 @@ spec:
         - name: INSTANCES
           value: "$WL_INSTANCES"
 EOF
-kubectl apply -k uc3-workload-generator
+kubectl apply -k uc-workload-generator/overlay/uc3-workload-generator
 
 
 # Start application
@@ -81,7 +81,7 @@ python lag_analysis.py $EXP_ID uc3 $DIM_VALUE $INSTANCES $EXECUTION_MINUTES
 deactivate
 
 # Stop workload generator and app
-kubectl delete -k uc3-workload-generator
+kubectl delete -k uc-workload-generator/overlay/uc3-workload-generator
 kubectl delete -k uc-application/overlay/uc3-application
 
 # Delete topics instead of Kafka

execution/run_uc4-new.sh

@@ -26,7 +26,7 @@ kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-z
 
 # Start workload generator
 NUM_SENSORS=$DIM_VALUE
-cat <<EOF >uc4-workload-generator/set_paramters.yaml
+cat <<EOF >uuc-workload-generator/overlay/c4-workload-generator/set_paramters.yaml
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
@@ -41,7 +41,7 @@ spec:
         - name: NUM_SENSORS
           value: "$NUM_SENSORS"
 EOF
-kubectl apply -k uc4-workload-generator
+kubectl apply -k uc-workload-generator/overlay/uc4-workload-generator
 
 # Start application
 REPLICAS=$INSTANCES
@@ -75,7 +75,7 @@ python lag_analysis.py $EXP_ID uc4 $DIM_VALUE $INSTANCES $EXECUTION_MINUTES
 deactivate
 
 # Stop workload generator and app
-kubectl delete -k uc4-workload-generator
+kubectl delete -k uc-workload-generator/overlay/uc4-workload-generator
 kubectl delete -k uc-application/overlay/uc4-application
 
 

execution/uc-application/base/aggregation-deployment.yaml

@@ -15,7 +15,7 @@ spec:
       terminationGracePeriodSeconds: 0
       containers:
       - name: uc-application
-        image: "uc-app:latest"
+        image: uc-app:latest
         ports:
         - containerPort: 5555
           name: jmx

execution/uc1-workload-generator/deployment.yaml → execution/uc-workload-generator/base/deployment.yaml

@@ -16,7 +16,7 @@ spec:
       terminationGracePeriodSeconds: 0
       containers:
       - name: workload-generator
-        image: soerenhenning/uc1-wg:latest
+        image: workload-generator:latest
         env:
         - name: KAFKA_BOOTSTRAP_SERVERS
           value: "my-confluent-cp-kafka:9092"
@@ -27,4 +27,4 @@ spec:
             fieldRef:
               fieldPath: metadata.name
         - name: INSTANCES
-          value: 1
+          value: "1"

execution/uc3-workload-generator/kustomization.yaml → execution/uc-workload-generator/base/kustomization.yaml

 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
-# Use all resources to compose them into one file
 resources:
   - deployment.yaml
-
-patchesStrategicMerge:
-- set_paramters.yaml # Patch setting the parameters

execution/uc1-workload-generator/kustomization.yaml → execution/uc-workload-generator/overlay/uc1-workload-generator/kustomization.yaml

 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
-# Use all resources to compose them into one file
-resources:
-  - deployment.yaml
+namePrefix: uc1-
+
+images:
+  - name: workload-generator
+    newName: soerenhenning/uc1-wl
+    newTag: latest
+
+bases:
+- ../../base
 
 patchesStrategicMerge:
-- set_paramters.yaml # Patch setting the parameters
+- set_paramters.yaml # Patch setting the resource parameters

execution/uc4-workload-generator/kustomization.yaml → execution/uc-workload-generator/overlay/uc2-workload-generator/kustomization.yaml

 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
-# Use all resources to compose them into one file
-resources:
-  - deployment.yaml
+namePrefix: uc2-
+
+images:
+  - name: workload-generator
+    newName: soerenhenning/uc2-wl
+    newTag: latest
+
+bases:
+- ../../base
 
 patchesStrategicMerge:
-- set_paramters.yaml # Patch setting the parameters
+- set_paramters.yaml # Patch setting the resource parameters

execution/uc2-workload-generator/set_paramters.yaml → execution/uc-workload-generator/overlay/uc2-workload-generator/set_paramters.yaml

@@ -9,5 +9,10 @@ spec:
       containers:
       - name: workload-generator
         env:
+        - name: NUM_SENSORS
+          value: "4"
+        - name: HIERARCHY
+          value: "full"
         - name: NUM_NESTED_GROUPS
           value: "25000"
+

execution/uc2-workload-generator/kustomization.yaml → execution/uc-workload-generator/overlay/uc3-workload-generator/kustomization.yaml

 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
-# Use all resources to compose them into one file
-resources:
-  - deployment.yaml
+namePrefix: uc3-
+
+images:
+  - name: workload-generator
+    newName: soerenhenning/uc3-wl
+    newTag: latest
+
+bases:
+- ../../base
 
 patchesStrategicMerge:
-- set_paramters.yaml # Patch setting the parameters
+- set_paramters.yaml # Patch setting the resource parameters

execution/uc-workload-generator/overlay/uc4-workload-generator/kustomization.yaml

+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namePrefix: uc4-
+
+images:
+  - name: workload-generator
+    newName: soerenhenning/uc4-wl
+    newTag: latest
+
+bases:
+- ../../base
+
+patchesStrategicMerge:
+- set_paramters.yaml # Patch setting the resource parameters

execution/uc2-workload-generator/deployment.yaml

-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: titan-ccp-load-generator
-spec:
-  selector:
-    matchLabels:
-      app: titan-ccp-load-generator
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: titan-ccp-load-generator
-    spec:
-      terminationGracePeriodSeconds: 0
-      containers:
-      - name: workload-generator
-        image: benediktwetzel/uc2-wg:latest
-        env:
-        - name: KAFKA_BOOTSTRAP_SERVERS
-          value: "my-confluent-cp-kafka:9092"
-        - name: HIERARCHY
-          value: "full"
-        - name: NUM_SENSORS
-          value: "4"
-        - name: NUM_NESTED_GROUPS
-          value: 25000

execution/uc3-workload-generator/deployment.yaml

-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: titan-ccp-load-generator
-spec:
-  selector:
-    matchLabels:
-      app: titan-ccp-load-generator
-  serviceName: titan-ccp-load-generator
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: titan-ccp-load-generator
-    spec:
-      terminationGracePeriodSeconds: 0
-      containers:
-      - name: workload-generator
-        image: soerenhenning/uc3-wg:latest
-        env:
-        - name: KAFKA_BOOTSTRAP_SERVERS
-          value: "my-confluent-cp-kafka:9092"
-        - name: NUM_SENSORS
-          value: 25000
-        - name: POD_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-        - name: INSTANCES
-          value: 1

execution/uc4-workload-generator/deployment.yaml

-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: titan-ccp-load-generator
-spec:
-  selector:
-    matchLabels:
-      app: titan-ccp-load-generator
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: titan-ccp-load-generator
-    spec:
-      terminationGracePeriodSeconds: 0
-      containers:
-      - name: workload-generator
-        image: soerenhenning/uc4-wg:latest
-        env:
-        - name: KAFKA_BOOTSTRAP_SERVERS
-          value: "my-confluent-cp-kafka:9092"
-        - name: NUM_SENSORS
-          value: 25000