diff --git a/execution/run_uc1-new.sh b/execution/run_uc1-new.sh index 564e03a470723f2b4564ccf96d31b66fa7dd7d2f..136a395793d981b3a8f0a2582951a513f6157698 100755 --- a/execution/run_uc1-new.sh +++ b/execution/run_uc1-new.sh @@ -29,38 +29,59 @@ NUM_SENSORS=$DIM_VALUE WL_MAX_RECORDS=150000 WL_INSTANCES=$(((NUM_SENSORS + (WL_MAX_RECORDS -1 ))/ WL_MAX_RECORDS)) -WORKLOAD_GENERATOR_YAML=$(sed "s/{{NUM_SENSORS}}/$NUM_SENSORS/g; s/{{INSTANCES}}/$WL_INSTANCES/g" uc1-workload-generator/deployment.yaml) -echo "$WORKLOAD_GENERATOR_YAML" | kubectl apply -f - +cat <<EOF >uc-workload-generator/overlay/uc1-workload-generator/set_paramters.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: titan-ccp-load-generator +spec: + replicas: $WL_INSTANCES + template: + spec: + containers: + - name: workload-generator + env: + - name: NUM_SENSORS + value: "$NUM_SENSORS" + - name: INSTANCES + value: "$WL_INSTANCES" +EOF +kubectl apply -k uc-workload-generator/overlay/uc1-workload-generator # Start application REPLICAS=$INSTANCES -# When not using `sed` anymore, use `kubectl apply -f uc1-application` -kubectl apply -f uc1-application/aggregation-service.yaml -kubectl apply -f uc1-application/jmx-configmap.yaml -kubectl apply -f uc1-application/service-monitor.yaml -#kubectl apply -f uc1-application/aggregation-deployment.yaml -APPLICATION_YAML=$(sed "s/{{CPU_LIMIT}}/$CPU_LIMIT/g; s/{{MEMORY_LIMIT}}/$MEMORY_LIMIT/g; s/{{KAFKA_STREAMS_COMMIT_INTERVAL_MS}}/$KAFKA_STREAMS_COMMIT_INTERVAL_MS/g" uc1-application/aggregation-deployment.yaml) -echo "$APPLICATION_YAML" | kubectl apply -f - -kubectl scale deployment titan-ccp-aggregation --replicas=$REPLICAS +cat <<EOF >uc-application/overlay/uc1-application/set_paramters.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: titan-ccp-aggregation +spec: + template: + spec: + containers: + - name: uc-application + env: + - name: COMMIT_INTERVAL_MS + value: "$KAFKA_STREAMS_COMMIT_INTERVAL_MS" + resources: + limits: + memory: $MEMORY_LIMIT + cpu: $CPU_LIMIT +EOF +kubectl apply -k uc-application/overlay/uc1-application +kubectl scale deployment uc1-titan-ccp-aggregation --replicas=$REPLICAS # Execute for certain time -sleep ${EXECUTION_MINUTES}m +sleep $(($EXECUTION_MINUTES * 60)) # Run eval script source ../.venv/bin/activate python lag_analysis.py $EXP_ID uc1 $DIM_VALUE $INSTANCES $EXECUTION_MINUTES deactivate -# Stop wl and app -#kubectl delete -f uc1-workload-generator/deployment.yaml -#sed "s/{{INSTANCES}}/1/g" uc1-workload-generator/deployment.yaml | kubectl delete -f - -#sed "s/{{NUM_SENSORS}}/$NUM_SENSORS/g; s/{{INSTANCES}}/$WL_INSTANCES/g" uc1-workload-generator/deployment.yaml | kubectl delete -f - -echo "$WORKLOAD_GENERATOR_YAML" | kubectl delete -f - -kubectl delete -f uc1-application/aggregation-service.yaml -kubectl delete -f uc1-application/jmx-configmap.yaml -kubectl delete -f uc1-application/service-monitor.yaml -#kubectl delete -f uc1-application/aggregation-deployment.yaml -echo "$APPLICATION_YAML" | kubectl delete -f - +# Stop workload generator and app +kubectl delete -k uc-workload-generator/overlay/uc1-workload-generator +kubectl delete -k uc-application/overlay/uc1-application # Delete topics instead of Kafka diff --git a/execution/run_uc2-new.sh b/execution/run_uc2-new.sh index aca65894b5d791eb20fd97b9bc9ab279f693eda7..88db87087e2386744c26366171addc46aaf73443 100755 --- a/execution/run_uc2-new.sh +++ b/execution/run_uc2-new.sh @@ -26,34 +26,61 @@ kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-z # Start workload generator NUM_NESTED_GROUPS=$DIM_VALUE -sed "s/{{NUM_NESTED_GROUPS}}/$NUM_NESTED_GROUPS/g" uc2-workload-generator/deployment.yaml | kubectl apply -f - +cat <<EOF >uc-workload-generator/overlay/uc2-workload-generator/set_paramters.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: titan-ccp-load-generator +spec: + replicas: 1 + template: + spec: + containers: + - name: workload-generator + env: + - name: NUM_SENSORS + value: "4" + - name: HIERARCHY + value: "full" + - name: NUM_NESTED_GROUPS + value: "$NUM_NESTED_GROUPS" +EOF +kubectl apply -k uc-workload-generator/overlay/uc2-workload-generator # Start application REPLICAS=$INSTANCES -# When not using `sed` anymore, use `kubectl apply -f uc2-application` -kubectl apply -f uc2-application/aggregation-service.yaml -kubectl apply -f uc2-application/jmx-configmap.yaml -kubectl apply -f uc2-application/service-monitor.yaml -#kubectl apply -f uc2-application/aggregation-deployment.yaml -APPLICATION_YAML=$(sed "s/{{CPU_LIMIT}}/$CPU_LIMIT/g; s/{{MEMORY_LIMIT}}/$MEMORY_LIMIT/g; s/{{KAFKA_STREAMS_COMMIT_INTERVAL_MS}}/$KAFKA_STREAMS_COMMIT_INTERVAL_MS/g" uc2-application/aggregation-deployment.yaml) -echo "$APPLICATION_YAML" | kubectl apply -f - -kubectl scale deployment titan-ccp-aggregation --replicas=$REPLICAS +cat <<EOF >uc-application/overlay/uc2-application/set_paramters.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: titan-ccp-aggregation +spec: + template: + spec: + containers: + - name: uc-application + env: + - name: COMMIT_INTERVAL_MS + value: "$KAFKA_STREAMS_COMMIT_INTERVAL_MS" + resources: + limits: + memory: $MEMORY_LIMIT + cpu: $CPU_LIMIT +EOF +kubectl apply -k uc-application/overlay/uc2-application +kubectl scale deployment uc2-titan-ccp-aggregation --replicas=$REPLICAS # Execute for certain time -sleep ${EXECUTION_MINUTES}m +sleep $(($EXECUTION_MINUTES * 60)) # Run eval script source ../.venv/bin/activate python lag_analysis.py $EXP_ID uc2 $DIM_VALUE $INSTANCES $EXECUTION_MINUTES deactivate -# Stop wl and app -kubectl delete -f uc2-workload-generator/deployment.yaml -kubectl delete -f uc2-application/aggregation-service.yaml -kubectl delete -f uc2-application/jmx-configmap.yaml -kubectl delete -f uc2-application/service-monitor.yaml -#kubectl delete -f uc2-application/aggregation-deployment.yaml -echo "$APPLICATION_YAML" | kubectl delete -f - +# Stop workload generator and app +kubectl delete -k uc-workload-generator/overlay/uc2-workload-generator +kubectl delete -k uc-application/overlay/uc2-application # Delete topics instead of Kafka diff --git a/execution/run_uc3-new.sh b/execution/run_uc3-new.sh index 79500eb508e39d9460c965494a4b7d0b34b6585a..46084c7a889a29019de0b5edbe9c9bbf61fe329a 100755 --- a/execution/run_uc3-new.sh +++ b/execution/run_uc3-new.sh @@ -29,40 +29,60 @@ NUM_SENSORS=$DIM_VALUE WL_MAX_RECORDS=150000 WL_INSTANCES=$(((NUM_SENSORS + (WL_MAX_RECORDS -1 ))/ WL_MAX_RECORDS)) -WORKLOAD_GENERATOR_YAML=$(sed "s/{{NUM_SENSORS}}/$NUM_SENSORS/g; s/{{INSTANCES}}/$WL_INSTANCES/g" uc3-workload-generator/deployment.yaml) -echo "$WORKLOAD_GENERATOR_YAML" | kubectl apply -f - +cat <<EOF >uc-workload-generator/overlay/uc3-workload-generator/set_paramters.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: titan-ccp-load-generator +spec: + replicas: $WL_INSTANCES + template: + spec: + containers: + - name: workload-generator + env: + - name: NUM_SENSORS + value: "$NUM_SENSORS" + - name: INSTANCES + value: "$WL_INSTANCES" +EOF +kubectl apply -k uc-workload-generator/overlay/uc3-workload-generator + # Start application REPLICAS=$INSTANCES -# When not using `sed` anymore, use `kubectl apply -f uc3-application` -kubectl apply -f uc3-application/aggregation-service.yaml -kubectl apply -f uc3-application/jmx-configmap.yaml -kubectl apply -f uc3-application/service-monitor.yaml -#kubectl apply -f uc3-application/aggregation-deployment.yaml -APPLICATION_YAML=$(sed "s/{{CPU_LIMIT}}/$CPU_LIMIT/g; s/{{MEMORY_LIMIT}}/$MEMORY_LIMIT/g; s/{{KAFKA_STREAMS_COMMIT_INTERVAL_MS}}/$KAFKA_STREAMS_COMMIT_INTERVAL_MS/g" uc3-application/aggregation-deployment.yaml) -echo "$APPLICATION_YAML" | kubectl apply -f - -kubectl scale deployment titan-ccp-aggregation --replicas=$REPLICAS +cat <<EOF >uc-application/overlay/uc3-application/set_paramters.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: titan-ccp-aggregation +spec: + template: + spec: + containers: + - name: uc-application + env: + - name: COMMIT_INTERVAL_MS + value: "$KAFKA_STREAMS_COMMIT_INTERVAL_MS" + resources: + limits: + memory: $MEMORY_LIMIT + cpu: $CPU_LIMIT +EOF +kubectl apply -k uc-application/overlay/uc3-application +kubectl scale deployment uc3-titan-ccp-aggregation --replicas=$REPLICAS # Execute for certain time -sleep ${EXECUTION_MINUTES}m +sleep $(($EXECUTION_MINUTES * 60)) # Run eval script source ../.venv/bin/activate python lag_analysis.py $EXP_ID uc3 $DIM_VALUE $INSTANCES $EXECUTION_MINUTES deactivate -# Stop wl and app -#kubectl delete -f uc3-workload-generator/deployment.yaml -#sed "s/{{INSTANCES}}/1/g" uc3-workload-generator/deployment.yaml | kubectl delete -f - -echo "$WORKLOAD_GENERATOR_YAML" | kubectl delete -f - -kubectl delete -f uc3-application/aggregation-service.yaml -kubectl delete -f uc3-application/jmx-configmap.yaml -kubectl delete -f uc3-application/service-monitor.yaml -#kubectl delete -f uc3-application/aggregation-deployment.yaml -#sed "s/{{CPU_LIMIT}}/1000m/g; s/{{MEMORY_LIMIT}}/4Gi/g; s/{{KAFKA_STREAMS_COMMIT_INTERVAL_MS}}/100/g" uc3-application/aggregation-deployment.yaml | kubectl delete -f - -echo "$APPLICATION_YAML" | kubectl delete -f - - - +# Stop workload generator and app +kubectl delete -k uc-workload-generator/overlay/uc3-workload-generator +kubectl delete -k uc-application/overlay/uc3-application # Delete topics instead of Kafka #kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --delete --topic 'input,output,configuration,titan-.*'" diff --git a/execution/run_uc4-new.sh b/execution/run_uc4-new.sh index 664d866f88d894eda37a30a72875151f1d545e98..8c7eef3a762a90b3b63b9569f6d956ec5603c71e 100755 --- a/execution/run_uc4-new.sh +++ b/execution/run_uc4-new.sh @@ -26,37 +26,57 @@ kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-z # Start workload generator NUM_SENSORS=$DIM_VALUE -#NUM_SENSORS=xy -sed "s/{{NUM_SENSORS}}/$NUM_SENSORS/g" uc4-workload-generator/deployment.yaml | kubectl apply -f - +cat <<EOF >uuc-workload-generator/overlay/c4-workload-generator/set_paramters.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: titan-ccp-load-generator +spec: + replicas: 1 + template: + spec: + containers: + - name: workload-generator + env: + - name: NUM_SENSORS + value: "$NUM_SENSORS" +EOF +kubectl apply -k uc-workload-generator/overlay/uc4-workload-generator # Start application REPLICAS=$INSTANCES -#AGGREGATION_DURATION_DAYS=$DIM_VALUE -# When not using `sed` anymore, use `kubectl apply -f uc4-application` -kubectl apply -f uc4-application/aggregation-service.yaml -kubectl apply -f uc4-application/jmx-configmap.yaml -kubectl apply -f uc4-application/service-monitor.yaml -#kubectl apply -f uc4-application/aggregation-deployment.yaml -#sed "s/{{AGGREGATION_DURATION_DAYS}}/$AGGREGATION_DURATION_DAYS/g" uc4-application/aggregation-deployment.yaml | kubectl apply -f - -APPLICATION_YAML=$(sed "s/{{CPU_LIMIT}}/$CPU_LIMIT/g; s/{{MEMORY_LIMIT}}/$MEMORY_LIMIT/g; s/{{KAFKA_STREAMS_COMMIT_INTERVAL_MS}}/$KAFKA_STREAMS_COMMIT_INTERVAL_MS/g" uc4-application/aggregation-deployment.yaml) -echo "$APPLICATION_YAML" | kubectl apply -f - -kubectl scale deployment titan-ccp-aggregation --replicas=$REPLICAS +cat <<EOF >uc-application/overlay/uc4-application/set_paramters.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: titan-ccp-aggregation +spec: + template: + spec: + containers: + - name: uc-application + env: + - name: COMMIT_INTERVAL_MS + value: "$KAFKA_STREAMS_COMMIT_INTERVAL_MS" + resources: + limits: + memory: $MEMORY_LIMIT + cpu: $CPU_LIMIT +EOF +kubectl apply -k uc-application/overlay/uc4-application +kubectl scale deployment uc4-titan-ccp-aggregation --replicas=$REPLICAS # Execute for certain time -sleep ${EXECUTION_MINUTES}m +sleep $(($EXECUTION_MINUTES * 60)) # Run eval script source ../.venv/bin/activate python lag_analysis.py $EXP_ID uc4 $DIM_VALUE $INSTANCES $EXECUTION_MINUTES deactivate -# Stop wl and app -kubectl delete -f uc4-workload-generator/deployment.yaml -kubectl delete -f uc4-application/aggregation-service.yaml -kubectl delete -f uc4-application/jmx-configmap.yaml -kubectl delete -f uc4-application/service-monitor.yaml -#kubectl delete -f uc4-application/aggregation-deployment.yaml -echo "$APPLICATION_YAML" | kubectl delete -f - +# Stop workload generator and app +kubectl delete -k uc-workload-generator/overlay/uc4-workload-generator +kubectl delete -k uc-application/overlay/uc4-application # Delete topics instead of Kafka diff --git a/execution/uc1-application/aggregation-deployment.yaml b/execution/uc-application/base/aggregation-deployment.yaml similarity index 85% rename from execution/uc1-application/aggregation-deployment.yaml rename to execution/uc-application/base/aggregation-deployment.yaml index d5bccca4a72f6a47a855ed8a7ca47fac4a8a19ca..55de21064c2aa47898554af35d33f025c3c83cec 100644 --- a/execution/uc1-application/aggregation-deployment.yaml +++ b/execution/uc-application/base/aggregation-deployment.yaml @@ -14,8 +14,8 @@ spec: spec: terminationGracePeriodSeconds: 0 containers: - - name: uc1-application - image: "soerenhenning/uc1-app:latest" + - name: uc-application + image: uc-app:latest ports: - containerPort: 5555 name: jmx @@ -23,13 +23,13 @@ spec: - name: KAFKA_BOOTSTRAP_SERVERS value: "my-confluent-cp-kafka:9092" - name: COMMIT_INTERVAL_MS - value: "{{KAFKA_STREAMS_COMMIT_INTERVAL_MS}}" + value: 100 - name: JAVA_OPTS value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555" resources: limits: - memory: "{{MEMORY_LIMIT}}" - cpu: "{{CPU_LIMIT}}" + memory: 4Gi + cpu: 1000m - name: prometheus-jmx-exporter image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143" command: @@ -50,4 +50,4 @@ spec: volumes: - name: jmx-config configMap: - name: aggregation-jmx-configmap \ No newline at end of file + name: aggregation-jmx-configmap diff --git a/execution/uc2-application/aggregation-service.yaml b/execution/uc-application/base/aggregation-service.yaml similarity index 86% rename from execution/uc2-application/aggregation-service.yaml rename to execution/uc-application/base/aggregation-service.yaml index 85432d04f225c30469f3232153ef6bd72bd02bdf..6317caf9fe624e42449b8f630d040a068709cda3 100644 --- a/execution/uc2-application/aggregation-service.yaml +++ b/execution/uc-application/base/aggregation-service.yaml @@ -1,14 +1,14 @@ apiVersion: v1 kind: Service -metadata: +metadata: name: titan-ccp-aggregation labels: app: titan-ccp-aggregation spec: #type: NodePort - selector: + selector: app: titan-ccp-aggregation - ports: + ports: - name: http port: 80 targetPort: 80 diff --git a/execution/uc1-application/jmx-configmap.yaml b/execution/uc-application/base/jmx-configmap.yaml similarity index 100% rename from execution/uc1-application/jmx-configmap.yaml rename to execution/uc-application/base/jmx-configmap.yaml diff --git a/execution/uc-application/base/kustomization.yaml b/execution/uc-application/base/kustomization.yaml new file mode 100644 index 0000000000000000000000000000000000000000..24c89cfdafb17cdc91f65198b9faf3665bfc6822 --- /dev/null +++ b/execution/uc-application/base/kustomization.yaml @@ -0,0 +1,12 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +commonLabels: + app: titan-ccp-aggregation + +# Use all resources to compose them into one file +resources: + - aggregation-deployment.yaml + - aggregation-service.yaml + - service-monitor.yaml + - jmx-configmap.yaml diff --git a/execution/uc1-application/service-monitor.yaml b/execution/uc-application/base/service-monitor.yaml similarity index 100% rename from execution/uc1-application/service-monitor.yaml rename to execution/uc-application/base/service-monitor.yaml diff --git a/execution/uc-application/overlay/uc1-application/kustomization.yaml b/execution/uc-application/overlay/uc1-application/kustomization.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5dc746cc26ab31e96f925717b2145a3e3b89aee3 --- /dev/null +++ b/execution/uc-application/overlay/uc1-application/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namePrefix: uc1- + +images: + - name: uc-app + newName: soerenhenning/uc1-app + newTag: latest + +bases: +- ../../base + +patchesStrategicMerge: +- set_paramters.yaml # Patch setting the resource parameters diff --git a/execution/uc-application/overlay/uc1-application/set_paramters.yaml b/execution/uc-application/overlay/uc1-application/set_paramters.yaml new file mode 100644 index 0000000000000000000000000000000000000000..57611fabd1a89a8655ce03b9cb8a059e1f695e77 --- /dev/null +++ b/execution/uc-application/overlay/uc1-application/set_paramters.yaml @@ -0,0 +1,16 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: titan-ccp-aggregation +spec: + template: + spec: + containers: + - name: uc-application + env: + - name: COMMIT_INTERVAL_MS + value: "100" + resources: + limits: + memory: 4Gi + cpu: 1000m diff --git a/execution/uc-application/overlay/uc2-application/kustomization.yaml b/execution/uc-application/overlay/uc2-application/kustomization.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8c62daebd716c79451ce065c3e9d73bc96e8ff25 --- /dev/null +++ b/execution/uc-application/overlay/uc2-application/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namePrefix: uc2- + +images: + - name: uc-app + newName: soerenhenning/uc2-app + newTag: latest + +bases: +- ../../base + +patchesStrategicMerge: +- set_paramters.yaml # Patch setting the resource parameters diff --git a/execution/uc-application/overlay/uc2-application/set_paramters.yaml b/execution/uc-application/overlay/uc2-application/set_paramters.yaml new file mode 100644 index 0000000000000000000000000000000000000000..57611fabd1a89a8655ce03b9cb8a059e1f695e77 --- /dev/null +++ b/execution/uc-application/overlay/uc2-application/set_paramters.yaml @@ -0,0 +1,16 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: titan-ccp-aggregation +spec: + template: + spec: + containers: + - name: uc-application + env: + - name: COMMIT_INTERVAL_MS + value: "100" + resources: + limits: + memory: 4Gi + cpu: 1000m diff --git a/execution/uc-application/overlay/uc3-application/kustomization.yaml b/execution/uc-application/overlay/uc3-application/kustomization.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0a898a9ca0ecd98e1ea411ce9997aed020351428 --- /dev/null +++ b/execution/uc-application/overlay/uc3-application/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namePrefix: uc3- + +images: + - name: uc-app + newName: soerenhenning/uc3-app + newTag: latest + +bases: +- ../../base + +patchesStrategicMerge: +- set_paramters.yaml # Patch setting the resource parameters diff --git a/execution/uc-application/overlay/uc3-application/set_paramters.yaml b/execution/uc-application/overlay/uc3-application/set_paramters.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d25865a7f39e29aaa3f6f22d19b682cc78179de0 --- /dev/null +++ b/execution/uc-application/overlay/uc3-application/set_paramters.yaml @@ -0,0 +1,16 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: titan-ccp-aggregation +spec: + template: + spec: + containers: + - name: uc-application + env: + - name: COMMIT_INTERVAL_MS + value: 100 + resources: + limits: + memory: 4Gi + cpu: 1000m diff --git a/execution/uc-application/overlay/uc4-application/kustomization.yaml b/execution/uc-application/overlay/uc4-application/kustomization.yaml new file mode 100644 index 0000000000000000000000000000000000000000..00ad540e66b0ccfd1f4322c6f04951faf344b3bd --- /dev/null +++ b/execution/uc-application/overlay/uc4-application/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namePrefix: uc4- + +images: + - name: uc-app + newName: soerenhenning/uc4-app + newTag: latest + +bases: +- ../../base + +patchesStrategicMerge: +- set_paramters.yaml # Patch setting the resource parameters diff --git a/execution/uc-application/overlay/uc4-application/set_paramters.yaml b/execution/uc-application/overlay/uc4-application/set_paramters.yaml new file mode 100644 index 0000000000000000000000000000000000000000..57611fabd1a89a8655ce03b9cb8a059e1f695e77 --- /dev/null +++ b/execution/uc-application/overlay/uc4-application/set_paramters.yaml @@ -0,0 +1,16 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: titan-ccp-aggregation +spec: + template: + spec: + containers: + - name: uc-application + env: + - name: COMMIT_INTERVAL_MS + value: "100" + resources: + limits: + memory: 4Gi + cpu: 1000m diff --git a/execution/uc3-workload-generator/deployment.yaml b/execution/uc-workload-generator/base/deployment.yaml similarity index 80% rename from execution/uc3-workload-generator/deployment.yaml rename to execution/uc-workload-generator/base/deployment.yaml index 9ecd2b67e757c94221e36edcfcfd43c22782270a..109e7c139bdc7867d154cfa07f562148b1763d79 100644 --- a/execution/uc3-workload-generator/deployment.yaml +++ b/execution/uc-workload-generator/base/deployment.yaml @@ -7,7 +7,7 @@ spec: matchLabels: app: titan-ccp-load-generator serviceName: titan-ccp-load-generator - replicas: {{INSTANCES}} + replicas: 1 template: metadata: labels: @@ -16,16 +16,15 @@ spec: terminationGracePeriodSeconds: 0 containers: - name: workload-generator - image: soerenhenning/uc3-wg:latest + image: workload-generator:latest env: - name: KAFKA_BOOTSTRAP_SERVERS value: "my-confluent-cp-kafka:9092" - name: NUM_SENSORS - value: "{{NUM_SENSORS}}" + value: "25000" - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name - name: INSTANCES - value: "{{INSTANCES}}" - \ No newline at end of file + value: "1" diff --git a/execution/uc-workload-generator/base/kustomization.yaml b/execution/uc-workload-generator/base/kustomization.yaml new file mode 100644 index 0000000000000000000000000000000000000000..88a04b54175c0d7a1f31ecbc0358be8092e74b77 --- /dev/null +++ b/execution/uc-workload-generator/base/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - deployment.yaml diff --git a/execution/uc-workload-generator/overlay/uc1-workload-generator/kustomization.yaml b/execution/uc-workload-generator/overlay/uc1-workload-generator/kustomization.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ed5a89619cf921a0c8864948eae1a6a9f3e463b3 --- /dev/null +++ b/execution/uc-workload-generator/overlay/uc1-workload-generator/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namePrefix: uc1- + +images: + - name: workload-generator + newName: soerenhenning/uc1-wl + newTag: latest + +bases: +- ../../base + +patchesStrategicMerge: +- set_paramters.yaml # Patch setting the resource parameters diff --git a/execution/uc-workload-generator/overlay/uc1-workload-generator/set_paramters.yaml b/execution/uc-workload-generator/overlay/uc1-workload-generator/set_paramters.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a63afa46fe0ecc4602ace583cc4fa0a7e4943366 --- /dev/null +++ b/execution/uc-workload-generator/overlay/uc1-workload-generator/set_paramters.yaml @@ -0,0 +1,15 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: titan-ccp-load-generator +spec: + replicas: 1 + template: + spec: + containers: + - name: workload-generator + env: + - name: NUM_SENSORS + value: "25000" + - name: INSTANCES + value: "1" diff --git a/execution/uc-workload-generator/overlay/uc2-workload-generator/kustomization.yaml b/execution/uc-workload-generator/overlay/uc2-workload-generator/kustomization.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f08be86125f02513ec22bd2545e24b6d3b0efc70 --- /dev/null +++ b/execution/uc-workload-generator/overlay/uc2-workload-generator/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namePrefix: uc2- + +images: + - name: workload-generator + newName: soerenhenning/uc2-wl + newTag: latest + +bases: +- ../../base + +patchesStrategicMerge: +- set_paramters.yaml # Patch setting the resource parameters diff --git a/execution/uc-workload-generator/overlay/uc2-workload-generator/set_paramters.yaml b/execution/uc-workload-generator/overlay/uc2-workload-generator/set_paramters.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0f17f661eccbe1412d7b5a9d5d299c2f5167db52 --- /dev/null +++ b/execution/uc-workload-generator/overlay/uc2-workload-generator/set_paramters.yaml @@ -0,0 +1,18 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: titan-ccp-load-generator +spec: + replicas: 1 + template: + spec: + containers: + - name: workload-generator + env: + - name: NUM_SENSORS + value: "4" + - name: HIERARCHY + value: "full" + - name: NUM_NESTED_GROUPS + value: "25000" + diff --git a/execution/uc-workload-generator/overlay/uc3-workload-generator/kustomization.yaml b/execution/uc-workload-generator/overlay/uc3-workload-generator/kustomization.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ccc61eafe9eb33769d361c0760d2a4f764d2184 --- /dev/null +++ b/execution/uc-workload-generator/overlay/uc3-workload-generator/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namePrefix: uc3- + +images: + - name: workload-generator + newName: soerenhenning/uc3-wl + newTag: latest + +bases: +- ../../base + +patchesStrategicMerge: +- set_paramters.yaml # Patch setting the resource parameters diff --git a/execution/uc-workload-generator/overlay/uc3-workload-generator/set_paramters.yaml b/execution/uc-workload-generator/overlay/uc3-workload-generator/set_paramters.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a63afa46fe0ecc4602ace583cc4fa0a7e4943366 --- /dev/null +++ b/execution/uc-workload-generator/overlay/uc3-workload-generator/set_paramters.yaml @@ -0,0 +1,15 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: titan-ccp-load-generator +spec: + replicas: 1 + template: + spec: + containers: + - name: workload-generator + env: + - name: NUM_SENSORS + value: "25000" + - name: INSTANCES + value: "1" diff --git a/execution/uc-workload-generator/overlay/uc4-workload-generator/kustomization.yaml b/execution/uc-workload-generator/overlay/uc4-workload-generator/kustomization.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3074b46cfc568dd31c089b8927d58de71825efaa --- /dev/null +++ b/execution/uc-workload-generator/overlay/uc4-workload-generator/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namePrefix: uc4- + +images: + - name: workload-generator + newName: soerenhenning/uc4-wl + newTag: latest + +bases: +- ../../base + +patchesStrategicMerge: +- set_paramters.yaml # Patch setting the resource parameters diff --git a/execution/uc-workload-generator/overlay/uc4-workload-generator/set_paramters.yaml b/execution/uc-workload-generator/overlay/uc4-workload-generator/set_paramters.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50ca424fbac14502ed51ed441d4a1787fc9b1063 --- /dev/null +++ b/execution/uc-workload-generator/overlay/uc4-workload-generator/set_paramters.yaml @@ -0,0 +1,13 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: titan-ccp-load-generator +spec: + replicas: 1 + template: + spec: + containers: + - name: workload-generator + env: + - name: NUM_SENSORS + value: "25000" diff --git a/execution/uc1-application/aggregation-service.yaml b/execution/uc1-application/aggregation-service.yaml deleted file mode 100644 index 85432d04f225c30469f3232153ef6bd72bd02bdf..0000000000000000000000000000000000000000 --- a/execution/uc1-application/aggregation-service.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: titan-ccp-aggregation - labels: - app: titan-ccp-aggregation -spec: - #type: NodePort - selector: - app: titan-ccp-aggregation - ports: - - name: http - port: 80 - targetPort: 80 - protocol: TCP - - name: metrics - port: 5556 diff --git a/execution/uc1-workload-generator/deployment.yaml b/execution/uc1-workload-generator/deployment.yaml deleted file mode 100644 index a0fde4bbf9765b2bb56bd36acde430d97169f34b..0000000000000000000000000000000000000000 --- a/execution/uc1-workload-generator/deployment.yaml +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: titan-ccp-load-generator -spec: - selector: - matchLabels: - app: titan-ccp-load-generator - serviceName: titan-ccp-load-generator - replicas: {{INSTANCES}} - template: - metadata: - labels: - app: titan-ccp-load-generator - spec: - terminationGracePeriodSeconds: 0 - containers: - - name: workload-generator - image: soerenhenning/uc1-wg:latest - env: - - name: KAFKA_BOOTSTRAP_SERVERS - value: "my-confluent-cp-kafka:9092" - - name: NUM_SENSORS - value: "{{NUM_SENSORS}}" - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: INSTANCES - value: "{{INSTANCES}}" - \ No newline at end of file diff --git a/execution/uc2-application/aggregation-deployment.yaml b/execution/uc2-application/aggregation-deployment.yaml deleted file mode 100644 index ce52421731ea5fc044c435ad10adb311e7e7e878..0000000000000000000000000000000000000000 --- a/execution/uc2-application/aggregation-deployment.yaml +++ /dev/null @@ -1,53 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: titan-ccp-aggregation -spec: - selector: - matchLabels: - app: titan-ccp-aggregation - replicas: 1 - template: - metadata: - labels: - app: titan-ccp-aggregation - spec: - terminationGracePeriodSeconds: 0 - containers: - - name: uc2-application - image: "benediktwetzel/uc2-app:latest" - ports: - - containerPort: 5555 - name: jmx - env: - - name: KAFKA_BOOTSTRAP_SERVERS - value: "my-confluent-cp-kafka:9092" - - name: COMMIT_INTERVAL_MS - value: "{{KAFKA_STREAMS_COMMIT_INTERVAL_MS}}" - - name: JAVA_OPTS - value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555" - resources: - limits: - memory: "{{MEMORY_LIMIT}}" - cpu: "{{CPU_LIMIT}}" - - name: prometheus-jmx-exporter - image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143" - command: - - java - - -XX:+UnlockExperimentalVMOptions - - -XX:+UseCGroupMemoryLimitForHeap - - -XX:MaxRAMFraction=1 - - -XshowSettings:vm - - -jar - - jmx_prometheus_httpserver.jar - - "5556" - - /etc/jmx-aggregation/jmx-kafka-prometheus.yml - ports: - - containerPort: 5556 - volumeMounts: - - name: jmx-config - mountPath: /etc/jmx-aggregation - volumes: - - name: jmx-config - configMap: - name: aggregation-jmx-configmap \ No newline at end of file diff --git a/execution/uc2-application/jmx-configmap.yaml b/execution/uc2-application/jmx-configmap.yaml deleted file mode 100644 index 78496a86b1242a89b9e844ead3e700fd0b9a9667..0000000000000000000000000000000000000000 --- a/execution/uc2-application/jmx-configmap.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: aggregation-jmx-configmap -data: - jmx-kafka-prometheus.yml: |+ - jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi - lowercaseOutputName: true - lowercaseOutputLabelNames: true - ssl: false diff --git a/execution/uc2-application/service-monitor.yaml b/execution/uc2-application/service-monitor.yaml deleted file mode 100644 index 4e7e758cacb5086305efa26292ddef2afc958096..0000000000000000000000000000000000000000 --- a/execution/uc2-application/service-monitor.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - labels: - app: titan-ccp-aggregation - appScope: titan-ccp - name: titan-ccp-aggregation -spec: - selector: - matchLabels: - app: titan-ccp-aggregation - endpoints: - - port: metrics - interval: 10s diff --git a/execution/uc2-workload-generator/deployment.yaml b/execution/uc2-workload-generator/deployment.yaml deleted file mode 100644 index 52592626f2a6bf93415c29f5bb4f020b527a5899..0000000000000000000000000000000000000000 --- a/execution/uc2-workload-generator/deployment.yaml +++ /dev/null @@ -1,28 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: titan-ccp-load-generator -spec: - selector: - matchLabels: - app: titan-ccp-load-generator - replicas: 1 - template: - metadata: - labels: - app: titan-ccp-load-generator - spec: - terminationGracePeriodSeconds: 0 - containers: - - name: workload-generator - image: benediktwetzel/uc2-wg:latest - env: - - name: KAFKA_BOOTSTRAP_SERVERS - value: "my-confluent-cp-kafka:9092" - - name: HIERARCHY - value: "full" - - name: NUM_SENSORS - value: "4" - - name: NUM_NESTED_GROUPS - value: "{{NUM_NESTED_GROUPS}}" - \ No newline at end of file diff --git a/execution/uc3-application/aggregation-deployment.yaml b/execution/uc3-application/aggregation-deployment.yaml deleted file mode 100644 index 0f3327af3119df125e3431574e3e406183abc132..0000000000000000000000000000000000000000 --- a/execution/uc3-application/aggregation-deployment.yaml +++ /dev/null @@ -1,55 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: titan-ccp-aggregation -spec: - selector: - matchLabels: - app: titan-ccp-aggregation - replicas: 1 - template: - metadata: - labels: - app: titan-ccp-aggregation - spec: - terminationGracePeriodSeconds: 0 - containers: - - name: uc3-application - image: "soerenhenning/uc3-app:latest" - ports: - - containerPort: 5555 - name: jmx - env: - - name: KAFKA_BOOTSTRAP_SERVERS - value: "my-confluent-cp-kafka:9092" - - name: KAFKA_WINDOW_DURATION_MINUTES - value: "1" - - name: COMMIT_INTERVAL_MS - value: "{{KAFKA_STREAMS_COMMIT_INTERVAL_MS}}" - - name: JAVA_OPTS - value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555" - resources: - limits: - memory: "{{MEMORY_LIMIT}}" - cpu: "{{CPU_LIMIT}}" - - name: prometheus-jmx-exporter - image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143" - command: - - java - - -XX:+UnlockExperimentalVMOptions - - -XX:+UseCGroupMemoryLimitForHeap - - -XX:MaxRAMFraction=1 - - -XshowSettings:vm - - -jar - - jmx_prometheus_httpserver.jar - - "5556" - - /etc/jmx-aggregation/jmx-kafka-prometheus.yml - ports: - - containerPort: 5556 - volumeMounts: - - name: jmx-config - mountPath: /etc/jmx-aggregation - volumes: - - name: jmx-config - configMap: - name: aggregation-jmx-configmap \ No newline at end of file diff --git a/execution/uc3-application/aggregation-service.yaml b/execution/uc3-application/aggregation-service.yaml deleted file mode 100644 index 85432d04f225c30469f3232153ef6bd72bd02bdf..0000000000000000000000000000000000000000 --- a/execution/uc3-application/aggregation-service.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: titan-ccp-aggregation - labels: - app: titan-ccp-aggregation -spec: - #type: NodePort - selector: - app: titan-ccp-aggregation - ports: - - name: http - port: 80 - targetPort: 80 - protocol: TCP - - name: metrics - port: 5556 diff --git a/execution/uc3-application/jmx-configmap.yaml b/execution/uc3-application/jmx-configmap.yaml deleted file mode 100644 index 78496a86b1242a89b9e844ead3e700fd0b9a9667..0000000000000000000000000000000000000000 --- a/execution/uc3-application/jmx-configmap.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: aggregation-jmx-configmap -data: - jmx-kafka-prometheus.yml: |+ - jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi - lowercaseOutputName: true - lowercaseOutputLabelNames: true - ssl: false diff --git a/execution/uc3-application/service-monitor.yaml b/execution/uc3-application/service-monitor.yaml deleted file mode 100644 index 4e7e758cacb5086305efa26292ddef2afc958096..0000000000000000000000000000000000000000 --- a/execution/uc3-application/service-monitor.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - labels: - app: titan-ccp-aggregation - appScope: titan-ccp - name: titan-ccp-aggregation -spec: - selector: - matchLabels: - app: titan-ccp-aggregation - endpoints: - - port: metrics - interval: 10s diff --git a/execution/uc4-application/aggregation-deployment.yaml b/execution/uc4-application/aggregation-deployment.yaml deleted file mode 100644 index f7a750c790b6a9eab8453fa91e05176de665104e..0000000000000000000000000000000000000000 --- a/execution/uc4-application/aggregation-deployment.yaml +++ /dev/null @@ -1,57 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: titan-ccp-aggregation -spec: - selector: - matchLabels: - app: titan-ccp-aggregation - replicas: 1 - template: - metadata: - labels: - app: titan-ccp-aggregation - spec: - terminationGracePeriodSeconds: 0 - containers: - - name: uc4-application - image: "soerenhenning/uc4-app:latest" - ports: - - containerPort: 5555 - name: jmx - env: - - name: KAFKA_BOOTSTRAP_SERVERS - value: "my-confluent-cp-kafka:9092" - - name: AGGREGATION_DURATION_DAYS - value: "3" #AGGREGATION_DURATION_DAYS - - name: AGGREGATION_DURATION_ADVANCE - value: "1" - - name: COMMIT_INTERVAL_MS - value: "{{KAFKA_STREAMS_COMMIT_INTERVAL_MS}}" - - name: JAVA_OPTS - value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555" - resources: - limits: - memory: "{{MEMORY_LIMIT}}" - cpu: "{{CPU_LIMIT}}" - - name: prometheus-jmx-exporter - image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143" - command: - - java - - -XX:+UnlockExperimentalVMOptions - - -XX:+UseCGroupMemoryLimitForHeap - - -XX:MaxRAMFraction=1 - - -XshowSettings:vm - - -jar - - jmx_prometheus_httpserver.jar - - "5556" - - /etc/jmx-aggregation/jmx-kafka-prometheus.yml - ports: - - containerPort: 5556 - volumeMounts: - - name: jmx-config - mountPath: /etc/jmx-aggregation - volumes: - - name: jmx-config - configMap: - name: aggregation-jmx-configmap \ No newline at end of file diff --git a/execution/uc4-application/aggregation-service.yaml b/execution/uc4-application/aggregation-service.yaml deleted file mode 100644 index 85432d04f225c30469f3232153ef6bd72bd02bdf..0000000000000000000000000000000000000000 --- a/execution/uc4-application/aggregation-service.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: titan-ccp-aggregation - labels: - app: titan-ccp-aggregation -spec: - #type: NodePort - selector: - app: titan-ccp-aggregation - ports: - - name: http - port: 80 - targetPort: 80 - protocol: TCP - - name: metrics - port: 5556 diff --git a/execution/uc4-application/jmx-configmap.yaml b/execution/uc4-application/jmx-configmap.yaml deleted file mode 100644 index 78496a86b1242a89b9e844ead3e700fd0b9a9667..0000000000000000000000000000000000000000 --- a/execution/uc4-application/jmx-configmap.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: aggregation-jmx-configmap -data: - jmx-kafka-prometheus.yml: |+ - jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi - lowercaseOutputName: true - lowercaseOutputLabelNames: true - ssl: false diff --git a/execution/uc4-application/service-monitor.yaml b/execution/uc4-application/service-monitor.yaml deleted file mode 100644 index 4e7e758cacb5086305efa26292ddef2afc958096..0000000000000000000000000000000000000000 --- a/execution/uc4-application/service-monitor.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - labels: - app: titan-ccp-aggregation - appScope: titan-ccp - name: titan-ccp-aggregation -spec: - selector: - matchLabels: - app: titan-ccp-aggregation - endpoints: - - port: metrics - interval: 10s diff --git a/execution/uc4-workload-generator/deployment.yaml b/execution/uc4-workload-generator/deployment.yaml deleted file mode 100644 index 6400abc345dcfb902364d3225bc6eb174380eb8b..0000000000000000000000000000000000000000 --- a/execution/uc4-workload-generator/deployment.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: titan-ccp-load-generator -spec: - selector: - matchLabels: - app: titan-ccp-load-generator - replicas: 1 - template: - metadata: - labels: - app: titan-ccp-load-generator - spec: - terminationGracePeriodSeconds: 0 - containers: - - name: workload-generator - image: soerenhenning/uc4-wg:latest - env: - - name: KAFKA_BOOTSTRAP_SERVERS - value: "my-confluent-cp-kafka:9092" - - name: NUM_SENSORS - value: "{{NUM_SENSORS}}" - \ No newline at end of file