From b4871a47e17151920af33f3d69699a73b3c9db37 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Vonheiden?= <bjoern.vonheiden@hotmail.de>
Date: Mon, 8 Jun 2020 16:48:34 +0200
Subject: [PATCH] Streamline workload generator kubernetes files using
 kustomize

Add the hierachie structure to the kubernetes workload generator
files.
Use the new structure in the run uc scripts.
---
 execution/run_uc1-new.sh                      |  6 ++--
 execution/run_uc2-new.sh                      | 10 +++++--
 execution/run_uc3-new.sh                      |  6 ++--
 execution/run_uc4-new.sh                      |  6 ++--
 .../base/aggregation-deployment.yaml          |  2 +-
 .../base}/deployment.yaml                     |  4 +--
 .../base/kustomization.yaml                   |  5 ++++
 .../uc1-workload-generator/kustomization.yaml | 15 ++++++++++
 .../uc1-workload-generator/set_paramters.yaml |  0
 .../uc2-workload-generator/kustomization.yaml | 15 ++++++++++
 .../uc2-workload-generator/set_paramters.yaml |  5 ++++
 .../uc3-workload-generator/kustomization.yaml | 15 ++++++++++
 .../uc3-workload-generator/set_paramters.yaml |  0
 .../uc4-workload-generator/kustomization.yaml | 15 ++++++++++
 .../uc4-workload-generator/set_paramters.yaml |  0
 .../uc1-workload-generator/kustomization.yaml |  9 ------
 .../uc2-workload-generator/deployment.yaml    | 27 -----------------
 .../uc2-workload-generator/kustomization.yaml |  9 ------
 .../uc3-workload-generator/deployment.yaml    | 30 -------------------
 .../uc3-workload-generator/kustomization.yaml |  9 ------
 .../uc4-workload-generator/deployment.yaml    | 23 --------------
 .../uc4-workload-generator/kustomization.yaml |  9 ------
 22 files changed, 89 insertions(+), 131 deletions(-)
 rename execution/{uc1-workload-generator => uc-workload-generator/base}/deployment.yaml (91%)
 create mode 100644 execution/uc-workload-generator/base/kustomization.yaml
 create mode 100644 execution/uc-workload-generator/overlay/uc1-workload-generator/kustomization.yaml
 rename execution/{ => uc-workload-generator/overlay}/uc1-workload-generator/set_paramters.yaml (100%)
 create mode 100644 execution/uc-workload-generator/overlay/uc2-workload-generator/kustomization.yaml
 rename execution/{ => uc-workload-generator/overlay}/uc2-workload-generator/set_paramters.yaml (71%)
 create mode 100644 execution/uc-workload-generator/overlay/uc3-workload-generator/kustomization.yaml
 rename execution/{ => uc-workload-generator/overlay}/uc3-workload-generator/set_paramters.yaml (100%)
 create mode 100644 execution/uc-workload-generator/overlay/uc4-workload-generator/kustomization.yaml
 rename execution/{ => uc-workload-generator/overlay}/uc4-workload-generator/set_paramters.yaml (100%)
 delete mode 100644 execution/uc1-workload-generator/kustomization.yaml
 delete mode 100644 execution/uc2-workload-generator/deployment.yaml
 delete mode 100644 execution/uc2-workload-generator/kustomization.yaml
 delete mode 100644 execution/uc3-workload-generator/deployment.yaml
 delete mode 100644 execution/uc3-workload-generator/kustomization.yaml
 delete mode 100644 execution/uc4-workload-generator/deployment.yaml
 delete mode 100644 execution/uc4-workload-generator/kustomization.yaml

diff --git a/execution/run_uc1-new.sh b/execution/run_uc1-new.sh
index a598a9587..fa4455520 100755
--- a/execution/run_uc1-new.sh
+++ b/execution/run_uc1-new.sh
@@ -29,7 +29,7 @@ NUM_SENSORS=$DIM_VALUE
 WL_MAX_RECORDS=150000
 WL_INSTANCES=$(((NUM_SENSORS + (WL_MAX_RECORDS -1 ))/ WL_MAX_RECORDS))
 
-cat <<EOF >uc1-workload-generator/set_paramters.yaml
+cat <<EOF >uc-workload-generator/overlay/uc1-workload-generator/set_paramters.yaml
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
@@ -46,7 +46,7 @@ spec:
         - name: INSTANCES
           value: "$WL_INSTANCES"
 EOF
-kubectl apply -k uc1-workload-generator
+kubectl apply -k uc-workload-generator/overlay/uc1-workload-generator
 
 # Start application
 REPLICAS=$INSTANCES
@@ -80,7 +80,7 @@ python lag_analysis.py $EXP_ID uc1 $DIM_VALUE $INSTANCES $EXECUTION_MINUTES
 deactivate
 
 # Stop workload generator and app
-kubectl delete -k uc1-workload-generator
+kubectl delete -k uc-workload-generator/overlay/uc1-workload-generator
 kubectl delete -k uc-application/overlay/uc1-application
 
 
diff --git a/execution/run_uc2-new.sh b/execution/run_uc2-new.sh
index a52dd9c1d..3e3ddae13 100755
--- a/execution/run_uc2-new.sh
+++ b/execution/run_uc2-new.sh
@@ -26,7 +26,7 @@ kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-z
 
 # Start workload generator
 NUM_NESTED_GROUPS=$DIM_VALUE
-cat <<EOF >uc2-workload-generator/set_paramters.yaml
+cat <<EOF >uc-workload-generator/overlay/uc2-workload-generator/set_paramters.yaml
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
@@ -38,10 +38,14 @@ spec:
       containers:
       - name: workload-generator
         env:
+        - name: NUM_SENSORS
+          value: "4"
+        - name: HIERARCHY
+          value: "full"
         - name: NUM_NESTED_GROUPS
           value: "$NUM_NESTED_GROUPS"
 EOF
-kubectl apply -k uc2-workload-generator
+kubectl apply -k uc-workload-generator/overlay/uc2-workload-generator
 
 # Start application
 REPLICAS=$INSTANCES
@@ -75,7 +79,7 @@ python lag_analysis.py $EXP_ID uc2 $DIM_VALUE $INSTANCES $EXECUTION_MINUTES
 deactivate
 
 # Stop workload generator and app
-kubectl delete -k uc2-workload-generator
+kubectl delete -k uc-workload-generator/overlay/uc2-workload-generator
 kubectl delete -k uc-application/overlay/uc2-application
 
 
diff --git a/execution/run_uc3-new.sh b/execution/run_uc3-new.sh
index 33549d676..05507b647 100755
--- a/execution/run_uc3-new.sh
+++ b/execution/run_uc3-new.sh
@@ -29,7 +29,7 @@ NUM_SENSORS=$DIM_VALUE
 WL_MAX_RECORDS=150000
 WL_INSTANCES=$(((NUM_SENSORS + (WL_MAX_RECORDS -1 ))/ WL_MAX_RECORDS))
 
-cat <<EOF >uc3-workload-generator/set_paramters.yaml
+cat <<EOF >uc-workload-generator/overlay/uc3-workload-generator/set_paramters.yaml
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
@@ -46,7 +46,7 @@ spec:
         - name: INSTANCES
           value: "$WL_INSTANCES"
 EOF
-kubectl apply -k uc3-workload-generator
+kubectl apply -k uc-workload-generator/overlay/uc3-workload-generator
 
 
 # Start application
@@ -81,7 +81,7 @@ python lag_analysis.py $EXP_ID uc3 $DIM_VALUE $INSTANCES $EXECUTION_MINUTES
 deactivate
 
 # Stop workload generator and app
-kubectl delete -k uc3-workload-generator
+kubectl delete -k uc-workload-generator/overlay/uc3-workload-generator
 kubectl delete -k uc-application/overlay/uc3-application
 
 # Delete topics instead of Kafka
diff --git a/execution/run_uc4-new.sh b/execution/run_uc4-new.sh
index 13f814315..aecf2132b 100755
--- a/execution/run_uc4-new.sh
+++ b/execution/run_uc4-new.sh
@@ -26,7 +26,7 @@ kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-z
 
 # Start workload generator
 NUM_SENSORS=$DIM_VALUE
-cat <<EOF >uc4-workload-generator/set_paramters.yaml
+cat <<EOF >uuc-workload-generator/overlay/c4-workload-generator/set_paramters.yaml
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
@@ -41,7 +41,7 @@ spec:
         - name: NUM_SENSORS
           value: "$NUM_SENSORS"
 EOF
-kubectl apply -k uc4-workload-generator
+kubectl apply -k uc-workload-generator/overlay/uc4-workload-generator
 
 # Start application
 REPLICAS=$INSTANCES
@@ -75,7 +75,7 @@ python lag_analysis.py $EXP_ID uc4 $DIM_VALUE $INSTANCES $EXECUTION_MINUTES
 deactivate
 
 # Stop workload generator and app
-kubectl delete -k uc4-workload-generator
+kubectl delete -k uc-workload-generator/overlay/uc4-workload-generator
 kubectl delete -k uc-application/overlay/uc4-application
 
 
diff --git a/execution/uc-application/base/aggregation-deployment.yaml b/execution/uc-application/base/aggregation-deployment.yaml
index 129a9be64..55de21064 100644
--- a/execution/uc-application/base/aggregation-deployment.yaml
+++ b/execution/uc-application/base/aggregation-deployment.yaml
@@ -15,7 +15,7 @@ spec:
       terminationGracePeriodSeconds: 0
       containers:
       - name: uc-application
-        image: "uc-app:latest"
+        image: uc-app:latest
         ports:
         - containerPort: 5555
           name: jmx
diff --git a/execution/uc1-workload-generator/deployment.yaml b/execution/uc-workload-generator/base/deployment.yaml
similarity index 91%
rename from execution/uc1-workload-generator/deployment.yaml
rename to execution/uc-workload-generator/base/deployment.yaml
index 0db431d58..109e7c139 100644
--- a/execution/uc1-workload-generator/deployment.yaml
+++ b/execution/uc-workload-generator/base/deployment.yaml
@@ -16,7 +16,7 @@ spec:
       terminationGracePeriodSeconds: 0
       containers:
       - name: workload-generator
-        image: soerenhenning/uc1-wg:latest
+        image: workload-generator:latest
         env:
         - name: KAFKA_BOOTSTRAP_SERVERS
           value: "my-confluent-cp-kafka:9092"
@@ -27,4 +27,4 @@ spec:
             fieldRef:
               fieldPath: metadata.name
         - name: INSTANCES
-          value: 1
+          value: "1"
diff --git a/execution/uc-workload-generator/base/kustomization.yaml b/execution/uc-workload-generator/base/kustomization.yaml
new file mode 100644
index 000000000..88a04b541
--- /dev/null
+++ b/execution/uc-workload-generator/base/kustomization.yaml
@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - deployment.yaml
diff --git a/execution/uc-workload-generator/overlay/uc1-workload-generator/kustomization.yaml b/execution/uc-workload-generator/overlay/uc1-workload-generator/kustomization.yaml
new file mode 100644
index 000000000..ed5a89619
--- /dev/null
+++ b/execution/uc-workload-generator/overlay/uc1-workload-generator/kustomization.yaml
@@ -0,0 +1,15 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namePrefix: uc1-
+
+images:
+  - name: workload-generator
+    newName: soerenhenning/uc1-wl
+    newTag: latest
+
+bases:
+- ../../base
+
+patchesStrategicMerge:
+- set_paramters.yaml # Patch setting the resource parameters
diff --git a/execution/uc1-workload-generator/set_paramters.yaml b/execution/uc-workload-generator/overlay/uc1-workload-generator/set_paramters.yaml
similarity index 100%
rename from execution/uc1-workload-generator/set_paramters.yaml
rename to execution/uc-workload-generator/overlay/uc1-workload-generator/set_paramters.yaml
diff --git a/execution/uc-workload-generator/overlay/uc2-workload-generator/kustomization.yaml b/execution/uc-workload-generator/overlay/uc2-workload-generator/kustomization.yaml
new file mode 100644
index 000000000..f08be8612
--- /dev/null
+++ b/execution/uc-workload-generator/overlay/uc2-workload-generator/kustomization.yaml
@@ -0,0 +1,15 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namePrefix: uc2-
+
+images:
+  - name: workload-generator
+    newName: soerenhenning/uc2-wl
+    newTag: latest
+
+bases:
+- ../../base
+
+patchesStrategicMerge:
+- set_paramters.yaml # Patch setting the resource parameters
diff --git a/execution/uc2-workload-generator/set_paramters.yaml b/execution/uc-workload-generator/overlay/uc2-workload-generator/set_paramters.yaml
similarity index 71%
rename from execution/uc2-workload-generator/set_paramters.yaml
rename to execution/uc-workload-generator/overlay/uc2-workload-generator/set_paramters.yaml
index d585eb437..0f17f661e 100644
--- a/execution/uc2-workload-generator/set_paramters.yaml
+++ b/execution/uc-workload-generator/overlay/uc2-workload-generator/set_paramters.yaml
@@ -9,5 +9,10 @@ spec:
       containers:
       - name: workload-generator
         env:
+        - name: NUM_SENSORS
+          value: "4"
+        - name: HIERARCHY
+          value: "full"
         - name: NUM_NESTED_GROUPS
           value: "25000"
+
diff --git a/execution/uc-workload-generator/overlay/uc3-workload-generator/kustomization.yaml b/execution/uc-workload-generator/overlay/uc3-workload-generator/kustomization.yaml
new file mode 100644
index 000000000..8ccc61eaf
--- /dev/null
+++ b/execution/uc-workload-generator/overlay/uc3-workload-generator/kustomization.yaml
@@ -0,0 +1,15 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namePrefix: uc3-
+
+images:
+  - name: workload-generator
+    newName: soerenhenning/uc3-wl
+    newTag: latest
+
+bases:
+- ../../base
+
+patchesStrategicMerge:
+- set_paramters.yaml # Patch setting the resource parameters
diff --git a/execution/uc3-workload-generator/set_paramters.yaml b/execution/uc-workload-generator/overlay/uc3-workload-generator/set_paramters.yaml
similarity index 100%
rename from execution/uc3-workload-generator/set_paramters.yaml
rename to execution/uc-workload-generator/overlay/uc3-workload-generator/set_paramters.yaml
diff --git a/execution/uc-workload-generator/overlay/uc4-workload-generator/kustomization.yaml b/execution/uc-workload-generator/overlay/uc4-workload-generator/kustomization.yaml
new file mode 100644
index 000000000..3074b46cf
--- /dev/null
+++ b/execution/uc-workload-generator/overlay/uc4-workload-generator/kustomization.yaml
@@ -0,0 +1,15 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namePrefix: uc4-
+
+images:
+  - name: workload-generator
+    newName: soerenhenning/uc4-wl
+    newTag: latest
+
+bases:
+- ../../base
+
+patchesStrategicMerge:
+- set_paramters.yaml # Patch setting the resource parameters
diff --git a/execution/uc4-workload-generator/set_paramters.yaml b/execution/uc-workload-generator/overlay/uc4-workload-generator/set_paramters.yaml
similarity index 100%
rename from execution/uc4-workload-generator/set_paramters.yaml
rename to execution/uc-workload-generator/overlay/uc4-workload-generator/set_paramters.yaml
diff --git a/execution/uc1-workload-generator/kustomization.yaml b/execution/uc1-workload-generator/kustomization.yaml
deleted file mode 100644
index faf86aa9a..000000000
--- a/execution/uc1-workload-generator/kustomization.yaml
+++ /dev/null
@@ -1,9 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-# Use all resources to compose them into one file
-resources:
-  - deployment.yaml
-
-patchesStrategicMerge:
-- set_paramters.yaml # Patch setting the parameters
diff --git a/execution/uc2-workload-generator/deployment.yaml b/execution/uc2-workload-generator/deployment.yaml
deleted file mode 100644
index 277af4587..000000000
--- a/execution/uc2-workload-generator/deployment.yaml
+++ /dev/null
@@ -1,27 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: titan-ccp-load-generator
-spec:
-  selector:
-    matchLabels:
-      app: titan-ccp-load-generator
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: titan-ccp-load-generator
-    spec:
-      terminationGracePeriodSeconds: 0
-      containers:
-      - name: workload-generator
-        image: benediktwetzel/uc2-wg:latest
-        env:
-        - name: KAFKA_BOOTSTRAP_SERVERS
-          value: "my-confluent-cp-kafka:9092"
-        - name: HIERARCHY
-          value: "full"
-        - name: NUM_SENSORS
-          value: "4"
-        - name: NUM_NESTED_GROUPS
-          value: 25000
diff --git a/execution/uc2-workload-generator/kustomization.yaml b/execution/uc2-workload-generator/kustomization.yaml
deleted file mode 100644
index faf86aa9a..000000000
--- a/execution/uc2-workload-generator/kustomization.yaml
+++ /dev/null
@@ -1,9 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-# Use all resources to compose them into one file
-resources:
-  - deployment.yaml
-
-patchesStrategicMerge:
-- set_paramters.yaml # Patch setting the parameters
diff --git a/execution/uc3-workload-generator/deployment.yaml b/execution/uc3-workload-generator/deployment.yaml
deleted file mode 100644
index c9acb06a9..000000000
--- a/execution/uc3-workload-generator/deployment.yaml
+++ /dev/null
@@ -1,30 +0,0 @@
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: titan-ccp-load-generator
-spec:
-  selector:
-    matchLabels:
-      app: titan-ccp-load-generator
-  serviceName: titan-ccp-load-generator
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: titan-ccp-load-generator
-    spec:
-      terminationGracePeriodSeconds: 0
-      containers:
-      - name: workload-generator
-        image: soerenhenning/uc3-wg:latest
-        env:
-        - name: KAFKA_BOOTSTRAP_SERVERS
-          value: "my-confluent-cp-kafka:9092"
-        - name: NUM_SENSORS
-          value: 25000
-        - name: POD_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-        - name: INSTANCES
-          value: 1
diff --git a/execution/uc3-workload-generator/kustomization.yaml b/execution/uc3-workload-generator/kustomization.yaml
deleted file mode 100644
index faf86aa9a..000000000
--- a/execution/uc3-workload-generator/kustomization.yaml
+++ /dev/null
@@ -1,9 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-# Use all resources to compose them into one file
-resources:
-  - deployment.yaml
-
-patchesStrategicMerge:
-- set_paramters.yaml # Patch setting the parameters
diff --git a/execution/uc4-workload-generator/deployment.yaml b/execution/uc4-workload-generator/deployment.yaml
deleted file mode 100644
index 8241c734b..000000000
--- a/execution/uc4-workload-generator/deployment.yaml
+++ /dev/null
@@ -1,23 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: titan-ccp-load-generator
-spec:
-  selector:
-    matchLabels:
-      app: titan-ccp-load-generator
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: titan-ccp-load-generator
-    spec:
-      terminationGracePeriodSeconds: 0
-      containers:
-      - name: workload-generator
-        image: soerenhenning/uc4-wg:latest
-        env:
-        - name: KAFKA_BOOTSTRAP_SERVERS
-          value: "my-confluent-cp-kafka:9092"
-        - name: NUM_SENSORS
-          value: 25000
diff --git a/execution/uc4-workload-generator/kustomization.yaml b/execution/uc4-workload-generator/kustomization.yaml
deleted file mode 100644
index faf86aa9a..000000000
--- a/execution/uc4-workload-generator/kustomization.yaml
+++ /dev/null
@@ -1,9 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-# Use all resources to compose them into one file
-resources:
-  - deployment.yaml
-
-patchesStrategicMerge:
-- set_paramters.yaml # Patch setting the parameters
-- 
GitLab