Structure use case infrastructe better with kustomize

Reorder the structure of the use cases to use a base and customize
this accordingly to their needs.

execution/run_uc1-new.sh

@@ -42,15 +42,15 @@ spec:
       - name: workload-generator
         env:
         - name: NUM_SENSORS
-          value: $NUM_SENSORS
+          value: "$NUM_SENSORS"
         - name: INSTANCES
-          value: $WL_INSTANCES
+          value: "$WL_INSTANCES"
 EOF
 kubectl apply -k uc1-workload-generator
 
 # Start application
 REPLICAS=$INSTANCES
-cat <<EOF >uc1-application/set_paramters.yaml
+cat <<EOF >uc-application/overlay/uc1-application/set_paramters.yaml
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -59,17 +59,17 @@ spec:
   template:
     spec:
       containers:
-      - name: uc1-application
+      - name: uc-application
         env:
         - name: COMMIT_INTERVAL_MS
-          value: $KAFKA_STREAMS_COMMIT_INTERVAL_MS
+          value: "$KAFKA_STREAMS_COMMIT_INTERVAL_MS"
         resources:
           limits:
             memory: $MEMORY_LIMIT
             cpu: $CPU_LIMIT
 EOF
-kubectl apply -k uc1-application
-kubectl scale deployment titan-ccp-aggregation --replicas=$REPLICAS
+kubectl apply -k uc-application/overlay/uc1-application
+kubectl scale deployment uc1-titan-ccp-aggregation --replicas=$REPLICAS
 
 # Execute for certain time
 sleep ${EXECUTION_MINUTES}m
@@ -81,7 +81,7 @@ deactivate
 
 # Stop workload generator and app
 kubectl delete -k uc1-workload-generator
-kubectl delete -k uc1-application
+kubectl delete -k uc-application/overlay/uc1-application
 
 
 # Delete topics instead of Kafka

execution/run_uc2-new.sh

@@ -45,8 +45,7 @@ kubectl apply -k uc2-workload-generator
 
 # Start application
 REPLICAS=$INSTANCES
-# When not using `sed` anymore, use `kubectl apply -f uc2-application`
-cat <<EOF >uc2-application/set_paramters.yaml
+cat <<EOF >uc-application/overlay/uc1-application/set_paramters.yaml
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -55,17 +54,17 @@ spec:
   template:
     spec:
       containers:
-      - name: uc1-application
+      - name: uc-application
         env:
         - name: COMMIT_INTERVAL_MS
-          value: $KAFKA_STREAMS_COMMIT_INTERVAL_MS
+          value: "$KAFKA_STREAMS_COMMIT_INTERVAL_MS"
         resources:
           limits:
             memory: $MEMORY_LIMIT
             cpu: $CPU_LIMIT
 EOF
-kubectl apply -k uc2-application
-kubectl scale deployment titan-ccp-aggregation --replicas=$REPLICAS
+kubectl apply -k uc-application/overlay/uc2-application
+kubectl scale deployment uc2-titan-ccp-aggregation --replicas=$REPLICAS
 
 # Execute for certain time
 sleep ${EXECUTION_MINUTES}m
@@ -77,7 +76,7 @@ deactivate
 
 # Stop workload generator and app
 kubectl delete -k uc2-workload-generator
-kubectl delete -k uc2-application
+kubectl delete -k uc-application/overlay/uc2-application
 
 
 # Delete topics instead of Kafka

execution/run_uc3-new.sh

@@ -51,7 +51,7 @@ kubectl apply -k uc3-workload-generator
 
 # Start application
 REPLICAS=$INSTANCES
-cat <<EOF >uc3-application/set_paramters.yaml
+cat <<EOF >uc-application/overlay/uc3-application/set_paramters.yaml
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -60,17 +60,17 @@ spec:
   template:
     spec:
       containers:
-      - name: uc1-application
+      - name: uc-application
         env:
         - name: COMMIT_INTERVAL_MS
-          value: $KAFKA_STREAMS_COMMIT_INTERVAL_MS
+          value: "$KAFKA_STREAMS_COMMIT_INTERVAL_MS"
         resources:
           limits:
             memory: $MEMORY_LIMIT
             cpu: $CPU_LIMIT
 EOF
-kubectl apply -k uc3-application
-kubectl scale deployment titan-ccp-aggregation --replicas=$REPLICAS
+kubectl apply -k uc-application/overlay/uc3-application
+kubectl scale deployment uc3-titan-ccp-aggregation --replicas=$REPLICAS
 
 # Execute for certain time
 sleep ${EXECUTION_MINUTES}m
@@ -82,7 +82,7 @@ deactivate
 
 # Stop workload generator and app
 kubectl delete -k uc1-workload-generator
-kubectl delete -k uc1-application
+kubectl delete -k uc-application/overlay/uc3-application
 
 # Delete topics instead of Kafka
 #kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --delete --topic 'input,output,configuration,titan-.*'"

execution/run_uc4-new.sh

@@ -45,7 +45,7 @@ kubectl apply -k uc4-workload-generator
 
 # Start application
 REPLICAS=$INSTANCES
-cat <<EOF >uc1-application/set_paramters.yaml
+cat <<EOF >uc-application/overlay/uc1-application/set_paramters.yaml
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -54,17 +54,17 @@ spec:
   template:
     spec:
       containers:
-      - name: uc1-application
+      - name: uc-application
         env:
         - name: COMMIT_INTERVAL_MS
-          value: $KAFKA_STREAMS_COMMIT_INTERVAL_MS
+          value: "$KAFKA_STREAMS_COMMIT_INTERVAL_MS"
         resources:
           limits:
             memory: $MEMORY_LIMIT
             cpu: $CPU_LIMIT
 EOF
-kubectl apply -k uc4-application
-kubectl scale deployment titan-ccp-aggregation --replicas=$REPLICAS
+kubectl apply -k uc-application/overlay/uc4-application
+kubectl scale deployment uc4-titan-ccp-aggregation --replicas=$REPLICAS
 
 # Execute for certain time
 sleep ${EXECUTION_MINUTES}m
@@ -76,7 +76,7 @@ deactivate
 
 # Stop workload generator and app
 kubectl delete -k uc4-workload-generator
-kubectl delete -k uc4-application
+kubectl delete -k uc-application/overlay/uc4-application
 
 
 # Delete topics instead of Kafka

execution/uc1-application/aggregation-deployment.yaml → execution/uc-application/base/aggregation-deployment.yaml

@@ -14,8 +14,8 @@ spec:
     spec:
       terminationGracePeriodSeconds: 0
       containers:
-      - name: uc1-application
-        image: "soerenhenning/uc1-app:latest"
+      - name: uc-application
+        image: "uc-app:latest"
         ports:
         - containerPort: 5555
           name: jmx

execution/uc2-application/kustomization.yaml → execution/uc-application/base/kustomization.yaml

@@ -10,6 +10,3 @@ resources:
   - aggregation-service.yaml
   - service-monitor.yaml
   - jmx-configmap.yaml
-
-patchesStrategicMerge:
-- set_paramters.yaml # Patch setting the resource parameters

execution/uc1-application/kustomization.yaml → execution/uc-application/overlay/uc1-application/kustomization.yaml

 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
-commonLabels:
-  app: titan-ccp-aggregation
-
-# Use all resources to compose them into one file
-resources:
-  - aggregation-deployment.yaml
-  - aggregation-service.yaml
-  - service-monitor.yaml
-  - jmx-configmap.yaml
+namePrefix: uc1-
+
+images:
+  - name: uc-app
+    newName: soerenhenning/uc1-app
+    newTag: latest
+
+bases:
+- ../../base
 
 patchesStrategicMerge:
 - set_paramters.yaml # Patch setting the resource parameters

execution/uc2-application/set_paramters.yaml → execution/uc-application/overlay/uc1-application/set_paramters.yaml

@@ -6,10 +6,10 @@ spec:
   template:
     spec:
       containers:
-      - name: uc2-application
+      - name: uc-application
         env:
         - name: COMMIT_INTERVAL_MS
-          value: 100
+          value: "100"
         resources:
           limits:
             memory: 4Gi

execution/uc3-application/kustomization.yaml → execution/uc-application/overlay/uc2-application/kustomization.yaml

 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
-commonLabels:
-  app: titan-ccp-aggregation
-
-# Use all resources to compose them into one file
-resources:
-  - aggregation-deployment.yaml
-  - aggregation-service.yaml
-  - service-monitor.yaml
-  - jmx-configmap.yaml
+namePrefix: uc2-
+
+images:
+  - name: uc-app
+    newName: soerenhenning/uc2-app
+    newTag: latest
+
+bases:
+- ../../base
 
 patchesStrategicMerge:
 - set_paramters.yaml # Patch setting the resource parameters

execution/uc1-application/set_paramters.yaml → execution/uc-application/overlay/uc2-application/set_paramters.yaml

@@ -6,10 +6,10 @@ spec:
   template:
     spec:
       containers:
-      - name: uc1-application
+      - name: uc-application
         env:
         - name: COMMIT_INTERVAL_MS
-          value: 100
+          value: "100"
         resources:
           limits:
             memory: 4Gi

execution/uc4-application/kustomization.yaml → execution/uc-application/overlay/uc3-application/kustomization.yaml

 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
-commonLabels:
-  app: titan-ccp-aggregation
-
-# Use all resources to compose them into one file
-resources:
-  - aggregation-deployment.yaml
-  - aggregation-service.yaml
-  - service-monitor.yaml
-  - jmx-configmap.yaml
+namePrefix: uc3-
+
+images:
+  - name: uc-app
+    newName: soerenhenning/uc3-app
+    newTag: latest
+
+bases:
+- ../../base
 
 patchesStrategicMerge:
 - set_paramters.yaml # Patch setting the resource parameters

execution/uc3-application/set_paramters.yaml → execution/uc-application/overlay/uc3-application/set_paramters.yaml

@@ -6,7 +6,7 @@ spec:
   template:
     spec:
       containers:
-      - name: uc3-application
+      - name: uc-application
         env:
         - name: COMMIT_INTERVAL_MS
           value: 100

execution/uc-application/overlay/uc4-application/kustomization.yaml

+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namePrefix: uc4-
+
+images:
+  - name: uc-app
+    newName: soerenhenning/uc4-app
+    newTag: latest
+
+bases:
+- ../../base
+
+patchesStrategicMerge:
+- set_paramters.yaml # Patch setting the resource parameters

execution/uc4-application/set_paramters.yaml → execution/uc-application/overlay/uc4-application/set_paramters.yaml

@@ -6,10 +6,10 @@ spec:
   template:
     spec:
       containers:
-      - name: uc4-application
+      - name: uc-application
         env:
         - name: COMMIT_INTERVAL_MS
-          value: 100
+          value: "100"
         resources:
           limits:
             memory: 4Gi

execution/uc2-application/aggregation-deployment.yaml

-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: titan-ccp-aggregation
-spec:
-  selector:
-    matchLabels:
-      app: titan-ccp-aggregation
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: titan-ccp-aggregation
-    spec:
-      terminationGracePeriodSeconds: 0
-      containers:
-      - name: uc2-application
-        image: "benediktwetzel/uc2-app:latest"
-        ports:
-        - containerPort: 5555
-          name: jmx
-        env:
-        - name: KAFKA_BOOTSTRAP_SERVERS
-          value: "my-confluent-cp-kafka:9092"
-        - name: COMMIT_INTERVAL_MS
-          value: 100
-        - name: JAVA_OPTS
-          value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555"
-        resources:
-          limits:
-            memory: 4Gi
-            cpu: 1000m
-      - name: prometheus-jmx-exporter
-        image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143"
-        command:
-          - java
-          - -XX:+UnlockExperimentalVMOptions
-          - -XX:+UseCGroupMemoryLimitForHeap
-          - -XX:MaxRAMFraction=1
-          - -XshowSettings:vm
-          - -jar
-          - jmx_prometheus_httpserver.jar
-          - "5556"
-          - /etc/jmx-aggregation/jmx-kafka-prometheus.yml
-        ports:
-          - containerPort: 5556
-        volumeMounts:
-          - name: jmx-config
-            mountPath: /etc/jmx-aggregation
-      volumes:
-        - name: jmx-config
-          configMap:
-            name: aggregation-jmx-configmap

execution/uc2-application/aggregation-service.yaml

-apiVersion: v1
-kind: Service
-metadata:  
-  name: titan-ccp-aggregation
-  labels:
-    app: titan-ccp-aggregation
-spec:
-  #type: NodePort
-  selector:    
-    app: titan-ccp-aggregation
-  ports:  
-  - name: http
-    port: 80
-    targetPort: 80
-    protocol: TCP
-  - name: metrics
-    port: 5556

execution/uc2-application/jmx-configmap.yaml

-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: aggregation-jmx-configmap
-data:
-  jmx-kafka-prometheus.yml: |+
-    jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi
-    lowercaseOutputName: true
-    lowercaseOutputLabelNames: true
-    ssl: false