From 6a451f643211ce924f28fe82716ff92197dde3ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Henning?= <soeren.henning@email.uni-kiel.de> Date: Thu, 11 Feb 2021 18:02:07 +0100 Subject: [PATCH] Remove unsued Kustomize and run_uc.sh files Fix #123 --- execution/run_uc.py | 10 +- execution/run_uc1.sh | 124 ----------------- execution/run_uc2.sh | 129 ------------------ execution/run_uc3.sh | 125 ----------------- execution/run_uc4.sh | 124 ----------------- .../{base => }/aggregation-deployment.yaml | 0 .../{base => }/aggregation-service.yaml | 0 .../{base => }/jmx-configmap.yaml | 0 .../{base => }/kustomization.yaml | 0 .../uc1-application/kustomization.yaml | 15 -- .../uc1-application/set_paramters.yaml | 17 --- .../uc2-application/kustomization.yaml | 15 -- .../uc2-application/set_paramters.yaml | 17 --- .../uc3-application/kustomization.yaml | 15 -- .../uc3-application/set_paramters.yaml | 17 --- .../uc4-application/kustomization.yaml | 15 -- .../uc4-application/set_paramters.yaml | 17 --- .../{base => }/service-monitor.yaml | 0 .../{base => }/kustomization.yaml | 0 .../uc1-workload-generator/kustomization.yaml | 15 -- .../uc1-workload-generator/set_paramters.yaml | 15 -- .../uc2-workload-generator/kustomization.yaml | 15 -- .../uc2-workload-generator/set_paramters.yaml | 19 --- .../uc3-workload-generator/kustomization.yaml | 15 -- .../uc3-workload-generator/set_paramters.yaml | 15 -- .../uc4-workload-generator/kustomization.yaml | 15 -- .../uc4-workload-generator/set_paramters.yaml | 15 -- .../{base => }/workloadGenerator.yaml | 0 28 files changed, 5 insertions(+), 759 deletions(-) delete mode 100755 execution/run_uc1.sh delete mode 100755 execution/run_uc2.sh delete mode 100755 execution/run_uc3.sh delete mode 100755 execution/run_uc4.sh rename execution/uc-application/{base => }/aggregation-deployment.yaml (100%) rename execution/uc-application/{base => }/aggregation-service.yaml (100%) rename execution/uc-application/{base => }/jmx-configmap.yaml (100%) rename execution/uc-application/{base => }/kustomization.yaml (100%) delete mode 100644 execution/uc-application/overlay/uc1-application/kustomization.yaml delete mode 100644 execution/uc-application/overlay/uc1-application/set_paramters.yaml delete mode 100644 execution/uc-application/overlay/uc2-application/kustomization.yaml delete mode 100644 execution/uc-application/overlay/uc2-application/set_paramters.yaml delete mode 100644 execution/uc-application/overlay/uc3-application/kustomization.yaml delete mode 100644 execution/uc-application/overlay/uc3-application/set_paramters.yaml delete mode 100644 execution/uc-application/overlay/uc4-application/kustomization.yaml delete mode 100644 execution/uc-application/overlay/uc4-application/set_paramters.yaml rename execution/uc-application/{base => }/service-monitor.yaml (100%) rename execution/uc-workload-generator/{base => }/kustomization.yaml (100%) delete mode 100644 execution/uc-workload-generator/overlay/uc1-workload-generator/kustomization.yaml delete mode 100644 execution/uc-workload-generator/overlay/uc1-workload-generator/set_paramters.yaml delete mode 100644 execution/uc-workload-generator/overlay/uc2-workload-generator/kustomization.yaml delete mode 100644 execution/uc-workload-generator/overlay/uc2-workload-generator/set_paramters.yaml delete mode 100644 execution/uc-workload-generator/overlay/uc3-workload-generator/kustomization.yaml delete mode 100644 execution/uc-workload-generator/overlay/uc3-workload-generator/set_paramters.yaml delete mode 100644 execution/uc-workload-generator/overlay/uc4-workload-generator/kustomization.yaml delete mode 100644 execution/uc-workload-generator/overlay/uc4-workload-generator/set_paramters.yaml rename execution/uc-workload-generator/{base => }/workloadGenerator.yaml (100%) diff --git a/execution/run_uc.py b/execution/run_uc.py index a0fcdbb6d..9bbb28764 100644 --- a/execution/run_uc.py +++ b/execution/run_uc.py @@ -94,11 +94,11 @@ def load_yaml_files(): :return: wg, app_svc, app_svc_monitor ,app_jmx, app_deploy """ print('Load kubernetes yaml files') - wg = load_yaml('uc-workload-generator/base/workloadGenerator.yaml') - app_svc = load_yaml('uc-application/base/aggregation-service.yaml') - app_svc_monitor = load_yaml('uc-application/base/service-monitor.yaml') - app_jmx = load_yaml('uc-application/base/jmx-configmap.yaml') - app_deploy = load_yaml('uc-application/base/aggregation-deployment.yaml') + wg = load_yaml('uc-workload-generator/workloadGenerator.yaml') + app_svc = load_yaml('uc-application/aggregation-service.yaml') + app_svc_monitor = load_yaml('uc-application/service-monitor.yaml') + app_jmx = load_yaml('uc-application/jmx-configmap.yaml') + app_deploy = load_yaml('uc-application/aggregation-deployment.yaml') print('Kubernetes yaml files loaded') return wg, app_svc, app_svc_monitor, app_jmx, app_deploy diff --git a/execution/run_uc1.sh b/execution/run_uc1.sh deleted file mode 100755 index 02c46d883..000000000 --- a/execution/run_uc1.sh +++ /dev/null @@ -1,124 +0,0 @@ -#!/bin/bash - -EXP_ID=$1 -DIM_VALUE=$2 -INSTANCES=$3 -PARTITIONS=${4:-40} -CPU_LIMIT=${5:-1000m} -MEMORY_LIMIT=${6:-4Gi} -KAFKA_STREAMS_COMMIT_INTERVAL_MS=${7:-100} -EXECUTION_MINUTES=${8:-5} - -echo "EXP_ID: $EXP_ID" -echo "DIM_VALUE: $DIM_VALUE" -echo "INSTANCES: $INSTANCES" -echo "PARTITIONS: $PARTITIONS" -echo "CPU_LIMIT: $CPU_LIMIT" -echo "MEMORY_LIMIT: $MEMORY_LIMIT" -echo "KAFKA_STREAMS_COMMIT_INTERVAL_MS: $KAFKA_STREAMS_COMMIT_INTERVAL_MS" -echo "EXECUTION_MINUTES: $EXECUTION_MINUTES" - -# Create Topics -#PARTITIONS=40 -#kubectl run temp-kafka --rm --attach --restart=Never --image=solsson/kafka --command -- bash -c "./bin/kafka-topics.sh --zookeeper my-confluent-cp-zookeeper:2181 --create --topic input --partitions $PARTITIONS --replication-factor 1; ./bin/kafka-topics.sh --zookeeper my-confluent-cp-zookeeper:2181 --create --topic configuration --partitions 1 --replication-factor 1; ./bin/kafka-topics.sh --zookeeper my-confluent-cp-zookeeper:2181 --create --topic output --partitions $PARTITIONS --replication-factor 1" -PARTITIONS=$PARTITIONS -kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --create --topic input --partitions $PARTITIONS --replication-factor 1; kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --create --topic configuration --partitions 1 --replication-factor 1; kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --create --topic output --partitions $PARTITIONS --replication-factor 1" - -# Start workload generator -NUM_SENSORS=$DIM_VALUE -WL_MAX_RECORDS=150000 -WL_INSTANCES=$(((NUM_SENSORS + (WL_MAX_RECORDS -1 ))/ WL_MAX_RECORDS)) - -cat <<EOF >uc-workload-generator/overlay/uc1-workload-generator/set_paramters.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: titan-ccp-load-generator -spec: - replicas: $WL_INSTANCES - template: - spec: - containers: - - name: workload-generator - env: - - name: NUM_SENSORS - value: "$NUM_SENSORS" - - name: INSTANCES - value: "$WL_INSTANCES" -EOF -kubectl apply -k uc-workload-generator/overlay/uc1-workload-generator - -# Start application -REPLICAS=$INSTANCES -cat <<EOF >uc-application/overlay/uc1-application/set_paramters.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: titan-ccp-aggregation -spec: - replicas: $REPLICAS - template: - spec: - containers: - - name: uc-application - env: - - name: COMMIT_INTERVAL_MS - value: "$KAFKA_STREAMS_COMMIT_INTERVAL_MS" - resources: - limits: - memory: $MEMORY_LIMIT - cpu: $CPU_LIMIT -EOF -kubectl apply -k uc-application/overlay/uc1-application - -# Execute for certain time -sleep $(($EXECUTION_MINUTES * 60)) - -# Run eval script -source ../.venv/bin/activate -python lag_analysis.py $EXP_ID uc1 $DIM_VALUE $INSTANCES $EXECUTION_MINUTES -deactivate - -# Stop workload generator and app -kubectl delete -k uc-workload-generator/overlay/uc1-workload-generator -kubectl delete -k uc-application/overlay/uc1-application - - -# Delete topics instead of Kafka -#kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --delete --topic 'input,output,configuration,titan-.*'" -# kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --delete --topic '.*' -#sleep 30s # TODO check -#kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --list" | sed -n '/^titan-.*/p;/^input$/p;/^output$/p;/^configuration$/p' -#kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --list" | sed -n '/^titan-.*/p;/^input$/p;/^output$/p;/^configuration$/p' | wc -l -#kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --list" - -#kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --delete --topic 'input,output,configuration,titan-.*'" -echo "Finished execution, print topics:" -#kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --list" | sed -n -E '/^(titan-.*|input|output|configuration)( - marked for deletion)?$/p' -while test $(kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --list" | sed -n -E '/^(theodolite-.*|input|output|configuration)( - marked for deletion)?$/p' | wc -l) -gt 0 -do - kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --delete --topic 'input|output|configuration|theodolite-.*' --if-exists" - echo "Wait for topic deletion" - sleep 5s - #echo "Finished waiting, print topics:" - #kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --list" | sed -n -E '/^(titan-.*|input|output|configuration)( - marked for deletion)?$/p' - # Sometimes a second deletion seems to be required -done -echo "Finish topic deletion, print topics:" -#kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --list" | sed -n -E '/^(titan-.*|input|output|configuration)( - marked for deletion)?$/p' - -# delete zookeeper nodes used for workload generation -echo "Delete ZooKeeper configurations used for workload generation" -kubectl exec zookeeper-client -- bash -c "zookeeper-shell my-confluent-cp-zookeeper:2181 deleteall /workload-generation" -echo "Waiting for deletion" -while kubectl exec zookeeper-client -- bash -c "zookeeper-shell my-confluent-cp-zookeeper:2181 get /workload-generation" -do - echo "Wait for ZooKeeper state deletion." - sleep 5s -done -echo "Deletion finished" - -echo "Exiting script" - -KAFKA_LAG_EXPORTER_POD=$(kubectl get pod -l app.kubernetes.io/name=kafka-lag-exporter -o jsonpath="{.items[0].metadata.name}") -kubectl delete pod $KAFKA_LAG_EXPORTER_POD diff --git a/execution/run_uc2.sh b/execution/run_uc2.sh deleted file mode 100755 index 4544d3609..000000000 --- a/execution/run_uc2.sh +++ /dev/null @@ -1,129 +0,0 @@ -#!/bin/bash - -EXP_ID=$1 -DIM_VALUE=$2 -INSTANCES=$3 -PARTITIONS=${4:-40} -CPU_LIMIT=${5:-1000m} -MEMORY_LIMIT=${6:-4Gi} -KAFKA_STREAMS_COMMIT_INTERVAL_MS=${7:-100} -EXECUTION_MINUTES=${8:-5} - -echo "EXP_ID: $EXP_ID" -echo "DIM_VALUE: $DIM_VALUE" -echo "INSTANCES: $INSTANCES" -echo "PARTITIONS: $PARTITIONS" -echo "CPU_LIMIT: $CPU_LIMIT" -echo "MEMORY_LIMIT: $MEMORY_LIMIT" -echo "KAFKA_STREAMS_COMMIT_INTERVAL_MS: $KAFKA_STREAMS_COMMIT_INTERVAL_MS" -echo "EXECUTION_MINUTES: $EXECUTION_MINUTES" - -# Create Topics -#PARTITIONS=40 -#kubectl run temp-kafka --rm --attach --restart=Never --image=solsson/kafka --command -- bash -c "./bin/kafka-topics.sh --zookeeper my-confluent-cp-zookeeper:2181 --create --topic input --partitions $PARTITIONS --replication-factor 1; ./bin/kafka-topics.sh --zookeeper my-confluent-cp-zookeeper:2181 --create --topic configuration --partitions 1 --replication-factor 1; ./bin/kafka-topics.sh --zookeeper my-confluent-cp-zookeeper:2181 --create --topic output --partitions $PARTITIONS --replication-factor 1" -PARTITIONS=$PARTITIONS -kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --create --topic input --partitions $PARTITIONS --replication-factor 1; kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --create --topic aggregation-feedback --partitions $PARTITIONS --replication-factor 1; kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --create --topic configuration --partitions 1 --replication-factor 1; kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --create --topic output --partitions $PARTITIONS --replication-factor 1" - -# Start workload generator -NUM_NESTED_GROUPS=$DIM_VALUE -WL_MAX_RECORDS=150000 -APPROX_NUM_SENSORS=$((4**NUM_NESTED_GROUPS)) -WL_INSTANCES=$(((APPROX_NUM_SENSORS + (WL_MAX_RECORDS -1 ))/ WL_MAX_RECORDS)) - -cat <<EOF >uc-workload-generator/overlay/uc2-workload-generator/set_paramters.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: titan-ccp-load-generator -spec: - replicas: $WL_INSTANCES - template: - spec: - containers: - - name: workload-generator - env: - - name: NUM_SENSORS - value: "4" - - name: HIERARCHY - value: "full" - - name: NUM_NESTED_GROUPS - value: "$NUM_NESTED_GROUPS" - - name: INSTANCES - value: "$WL_INSTANCES" -EOF -kubectl apply -k uc-workload-generator/overlay/uc2-workload-generator - -# Start application -REPLICAS=$INSTANCES -cat <<EOF >uc-application/overlay/uc2-application/set_paramters.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: titan-ccp-aggregation -spec: - replicas: $REPLICAS - template: - spec: - containers: - - name: uc-application - env: - - name: COMMIT_INTERVAL_MS - value: "$KAFKA_STREAMS_COMMIT_INTERVAL_MS" - resources: - limits: - memory: $MEMORY_LIMIT - cpu: $CPU_LIMIT -EOF -kubectl apply -k uc-application/overlay/uc2-application - -# Execute for certain time -sleep $(($EXECUTION_MINUTES * 60)) - -# Run eval script -source ../.venv/bin/activate -python lag_analysis.py $EXP_ID uc2 $DIM_VALUE $INSTANCES $EXECUTION_MINUTES -deactivate - -# Stop workload generator and app -kubectl delete -k uc-workload-generator/overlay/uc2-workload-generator -kubectl delete -k uc-application/overlay/uc2-application - - -# Delete topics instead of Kafka -#kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --delete --topic 'input,output,configuration,titan-.*'" -# kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --delete --topic '.*' -#sleep 30s # TODO check -#kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --list" | sed -n '/^titan-.*/p;/^input$/p;/^output$/p;/^configuration$/p' -#kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --list" | sed -n '/^titan-.*/p;/^input$/p;/^output$/p;/^configuration$/p' | wc -l -#kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --list" - -#kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --delete --topic 'input,output,configuration,titan-.*'" -echo "Finished execution, print topics:" -#kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --list" | sed -n -E '/^(titan-.*|input|output|configuration)( - marked for deletion)?$/p' -while test $(kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --list" | sed -n -E '/^(theodolite-.*|input|aggregation-feedback|output|configuration)( - marked for deletion)?$/p' | wc -l) -gt 0 -do - kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --delete --topic 'input|aggregation-feedback|output|configuration|theodolite-.*' --if-exists" - echo "Wait for topic deletion" - sleep 5s - #echo "Finished waiting, print topics:" - #kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --list" | sed -n -E '/^(titan-.*|input|output|configuration)( - marked for deletion)?$/p' - # Sometimes a second deletion seems to be required -done -echo "Finish topic deletion, print topics:" -#kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --list" | sed -n -E '/^(titan-.*|input|output|configuration)( - marked for deletion)?$/p' - -# delete zookeeper nodes used for workload generation -echo "Delete ZooKeeper configurations used for workload generation" -kubectl exec zookeeper-client -- bash -c "zookeeper-shell my-confluent-cp-zookeeper:2181 deleteall /workload-generation" -echo "Waiting for deletion" -while kubectl exec zookeeper-client -- bash -c "zookeeper-shell my-confluent-cp-zookeeper:2181 get /workload-generation" -do - echo "Wait for ZooKeeper state deletion." - sleep 5s -done -echo "Deletion finished" - -echo "Exiting script" - -KAFKA_LAG_EXPORTER_POD=$(kubectl get pod -l app.kubernetes.io/name=kafka-lag-exporter -o jsonpath="{.items[0].metadata.name}") -kubectl delete pod $KAFKA_LAG_EXPORTER_POD diff --git a/execution/run_uc3.sh b/execution/run_uc3.sh deleted file mode 100755 index 4f2323f93..000000000 --- a/execution/run_uc3.sh +++ /dev/null @@ -1,125 +0,0 @@ -#!/bin/bash - -EXP_ID=$1 -DIM_VALUE=$2 -INSTANCES=$3 -PARTITIONS=${4:-40} -CPU_LIMIT=${5:-1000m} -MEMORY_LIMIT=${6:-4Gi} -KAFKA_STREAMS_COMMIT_INTERVAL_MS=${7:-100} -EXECUTION_MINUTES=${8:-5} - -echo "EXP_ID: $EXP_ID" -echo "DIM_VALUE: $DIM_VALUE" -echo "INSTANCES: $INSTANCES" -echo "PARTITIONS: $PARTITIONS" -echo "CPU_LIMIT: $CPU_LIMIT" -echo "MEMORY_LIMIT: $MEMORY_LIMIT" -echo "KAFKA_STREAMS_COMMIT_INTERVAL_MS: $KAFKA_STREAMS_COMMIT_INTERVAL_MS" -echo "EXECUTION_MINUTES: $EXECUTION_MINUTES" - -# Create Topics -#PARTITIONS=40 -#kubectl run temp-kafka --rm --attach --restart=Never --image=solsson/kafka --command -- bash -c "./bin/kafka-topics.sh --zookeeper my-confluent-cp-zookeeper:2181 --create --topic input --partitions $PARTITIONS --replication-factor 1; ./bin/kafka-topics.sh --zookeeper my-confluent-cp-zookeeper:2181 --create --topic configuration --partitions 1 --replication-factor 1; ./bin/kafka-topics.sh --zookeeper my-confluent-cp-zookeeper:2181 --create --topic output --partitions $PARTITIONS --replication-factor 1" -PARTITIONS=$PARTITIONS -kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --create --topic input --partitions $PARTITIONS --replication-factor 1; kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --create --topic configuration --partitions 1 --replication-factor 1; kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --create --topic output --partitions $PARTITIONS --replication-factor 1" - -# Start workload generator -NUM_SENSORS=$DIM_VALUE -WL_MAX_RECORDS=150000 -WL_INSTANCES=$(((NUM_SENSORS + (WL_MAX_RECORDS -1 ))/ WL_MAX_RECORDS)) - -cat <<EOF >uc-workload-generator/overlay/uc3-workload-generator/set_paramters.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: titan-ccp-load-generator -spec: - replicas: $WL_INSTANCES - template: - spec: - containers: - - name: workload-generator - env: - - name: NUM_SENSORS - value: "$NUM_SENSORS" - - name: INSTANCES - value: "$WL_INSTANCES" -EOF -kubectl apply -k uc-workload-generator/overlay/uc3-workload-generator - - -# Start application -REPLICAS=$INSTANCES -cat <<EOF >uc-application/overlay/uc3-application/set_paramters.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: titan-ccp-aggregation -spec: - replicas: $REPLICAS - template: - spec: - containers: - - name: uc-application - env: - - name: COMMIT_INTERVAL_MS - value: "$KAFKA_STREAMS_COMMIT_INTERVAL_MS" - resources: - limits: - memory: $MEMORY_LIMIT - cpu: $CPU_LIMIT -EOF -kubectl apply -k uc-application/overlay/uc3-application -kubectl scale deployment uc3-titan-ccp-aggregation --replicas=$REPLICAS - -# Execute for certain time -sleep $(($EXECUTION_MINUTES * 60)) - -# Run eval script -source ../.venv/bin/activate -python lag_analysis.py $EXP_ID uc3 $DIM_VALUE $INSTANCES $EXECUTION_MINUTES -deactivate - -# Stop workload generator and app -kubectl delete -k uc-workload-generator/overlay/uc3-workload-generator -kubectl delete -k uc-application/overlay/uc3-application - -# Delete topics instead of Kafka -#kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --delete --topic 'input,output,configuration,titan-.*'" -# kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --delete --topic '.*' -#sleep 30s # TODO check -#kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --list" | sed -n '/^titan-.*/p;/^input$/p;/^output$/p;/^configuration$/p' -#kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --list" | sed -n '/^titan-.*/p;/^input$/p;/^output$/p;/^configuration$/p' | wc -l -#kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --list" - -#kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --delete --topic 'input,output,configuration,titan-.*'" -echo "Finished execution, print topics:" -#kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --list" | sed -n -E '/^(titan-.*|input|output|configuration)( - marked for deletion)?$/p' -while test $(kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --list" | sed -n -E '/^(theodolite-.*|input|output|configuration)( - marked for deletion)?$/p' | wc -l) -gt 0 -do - kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --delete --topic 'input|output|configuration|theodolite-.*' --if-exists" - echo "Wait for topic deletion" - sleep 5s - #echo "Finished waiting, print topics:" - #kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --list" | sed -n -E '/^(titan-.*|input|output|configuration)( - marked for deletion)?$/p' - # Sometimes a second deletion seems to be required -done -echo "Finish topic deletion, print topics:" -#kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --list" | sed -n -E '/^(titan-.*|input|output|configuration)( - marked for deletion)?$/p' - -# delete zookeeper nodes used for workload generation -echo "Delete ZooKeeper configurations used for workload generation" -kubectl exec zookeeper-client -- bash -c "zookeeper-shell my-confluent-cp-zookeeper:2181 deleteall /workload-generation" -echo "Waiting for deletion" -while kubectl exec zookeeper-client -- bash -c "zookeeper-shell my-confluent-cp-zookeeper:2181 get /workload-generation" -do - echo "Wait for ZooKeeper state deletion." - sleep 5s -done -echo "Deletion finished" - -echo "Exiting script" - -KAFKA_LAG_EXPORTER_POD=$(kubectl get pod -l app.kubernetes.io/name=kafka-lag-exporter -o jsonpath="{.items[0].metadata.name}") -kubectl delete pod $KAFKA_LAG_EXPORTER_POD diff --git a/execution/run_uc4.sh b/execution/run_uc4.sh deleted file mode 100755 index 08a384988..000000000 --- a/execution/run_uc4.sh +++ /dev/null @@ -1,124 +0,0 @@ -#!/bin/bash - -EXP_ID=$1 -DIM_VALUE=$2 -INSTANCES=$3 -PARTITIONS=${4:-40} -CPU_LIMIT=${5:-1000m} -MEMORY_LIMIT=${6:-4Gi} -KAFKA_STREAMS_COMMIT_INTERVAL_MS=${7:-100} -EXECUTION_MINUTES=${8:-5} - -echo "EXP_ID: $EXP_ID" -echo "DIM_VALUE: $DIM_VALUE" -echo "INSTANCES: $INSTANCES" -echo "PARTITIONS: $PARTITIONS" -echo "CPU_LIMIT: $CPU_LIMIT" -echo "MEMORY_LIMIT: $MEMORY_LIMIT" -echo "KAFKA_STREAMS_COMMIT_INTERVAL_MS: $KAFKA_STREAMS_COMMIT_INTERVAL_MS" -echo "EXECUTION_MINUTES: $EXECUTION_MINUTES" - -# Create Topics -#PARTITIONS=40 -#kubectl run temp-kafka --rm --attach --restart=Never --image=solsson/kafka --command -- bash -c "./bin/kafka-topics.sh --zookeeper my-confluent-cp-zookeeper:2181 --create --topic input --partitions $PARTITIONS --replication-factor 1; ./bin/kafka-topics.sh --zookeeper my-confluent-cp-zookeeper:2181 --create --topic configuration --partitions 1 --replication-factor 1; ./bin/kafka-topics.sh --zookeeper my-confluent-cp-zookeeper:2181 --create --topic output --partitions $PARTITIONS --replication-factor 1" -PARTITIONS=$PARTITIONS -kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --create --topic input --partitions $PARTITIONS --replication-factor 1; kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --create --topic configuration --partitions 1 --replication-factor 1; kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --create --topic output --partitions $PARTITIONS --replication-factor 1" - -# Start workload generator -NUM_SENSORS=$DIM_VALUE -WL_MAX_RECORDS=150000 -WL_INSTANCES=$(((NUM_SENSORS + (WL_MAX_RECORDS -1 ))/ WL_MAX_RECORDS)) - -cat <<EOF >uuc-workload-generator/overlay/uc4-workload-generator/set_paramters.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: titan-ccp-load-generator -spec: - replicas: $WL_INSTANCES - template: - spec: - containers: - - name: workload-generator - env: - - name: NUM_SENSORS - value: "$NUM_SENSORS" - - name: INSTANCES - value: "$WL_INSTANCES" -EOF -kubectl apply -k uc-workload-generator/overlay/uc4-workload-generator - -# Start application -REPLICAS=$INSTANCES -cat <<EOF >uc-application/overlay/uc4-application/set_paramters.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: titan-ccp-aggregation -spec: - replicas: $REPLICAS - template: - spec: - containers: - - name: uc-application - env: - - name: COMMIT_INTERVAL_MS - value: "$KAFKA_STREAMS_COMMIT_INTERVAL_MS" - resources: - limits: - memory: $MEMORY_LIMIT - cpu: $CPU_LIMIT -EOF -kubectl apply -k uc-application/overlay/uc4-application -kubectl scale deployment uc4-titan-ccp-aggregation --replicas=$REPLICAS - -# Execute for certain time -sleep $(($EXECUTION_MINUTES * 60)) - -# Run eval script -source ../.venv/bin/activate -python lag_analysis.py $EXP_ID uc4 $DIM_VALUE $INSTANCES $EXECUTION_MINUTES -deactivate - -# Stop workload generator and app -kubectl delete -k uc-workload-generator/overlay/uc4-workload-generator -kubectl delete -k uc-application/overlay/uc4-application - -# Delete topics instead of Kafka -#kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --delete --topic 'input,output,configuration,titan-.*'" -# kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --delete --topic '.*' -#sleep 30s # TODO check -#kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --list" | sed -n '/^titan-.*/p;/^input$/p;/^output$/p;/^configuration$/p' -#kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --list" | sed -n '/^titan-.*/p;/^input$/p;/^output$/p;/^configuration$/p' | wc -l -#kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --list" - -#kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --delete --topic 'input,output,configuration,titan-.*'" -echo "Finished execution, print topics:" -#kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --list" | sed -n -E '/^(titan-.*|input|output|configuration)( - marked for deletion)?$/p' -while test $(kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --list" | sed -n -E '/^(theodolite-.*|input|output|configuration)( - marked for deletion)?$/p' | wc -l) -gt 0 -do - kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --delete --topic 'input|output|configuration|theodolite-.*' --if-exists" - echo "Wait for topic deletion" - sleep 5s - #echo "Finished waiting, print topics:" - #kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --list" | sed -n -E '/^(titan-.*|input|output|configuration)( - marked for deletion)?$/p' - # Sometimes a second deletion seems to be required -done -echo "Finish topic deletion, print topics:" -#kubectl exec kafka-client -- bash -c "kafka-topics --zookeeper my-confluent-cp-zookeeper:2181 --list" | sed -n -E '/^(titan-.*|input|output|configuration)( - marked for deletion)?$/p' - -# delete zookeeper nodes used for workload generation -echo "Delete ZooKeeper configurations used for workload generation" -kubectl exec zookeeper-client -- bash -c "zookeeper-shell my-confluent-cp-zookeeper:2181 deleteall /workload-generation" -echo "Waiting for deletion" -while kubectl exec zookeeper-client -- bash -c "zookeeper-shell my-confluent-cp-zookeeper:2181 get /workload-generation" -do - echo "Wait for ZooKeeper state deletion." - sleep 5s -done -echo "Deletion finished" - -echo "Exiting script" - -KAFKA_LAG_EXPORTER_POD=$(kubectl get pod -l app.kubernetes.io/name=kafka-lag-exporter -o jsonpath="{.items[0].metadata.name}") -kubectl delete pod $KAFKA_LAG_EXPORTER_POD diff --git a/execution/uc-application/base/aggregation-deployment.yaml b/execution/uc-application/aggregation-deployment.yaml similarity index 100% rename from execution/uc-application/base/aggregation-deployment.yaml rename to execution/uc-application/aggregation-deployment.yaml diff --git a/execution/uc-application/base/aggregation-service.yaml b/execution/uc-application/aggregation-service.yaml similarity index 100% rename from execution/uc-application/base/aggregation-service.yaml rename to execution/uc-application/aggregation-service.yaml diff --git a/execution/uc-application/base/jmx-configmap.yaml b/execution/uc-application/jmx-configmap.yaml similarity index 100% rename from execution/uc-application/base/jmx-configmap.yaml rename to execution/uc-application/jmx-configmap.yaml diff --git a/execution/uc-application/base/kustomization.yaml b/execution/uc-application/kustomization.yaml similarity index 100% rename from execution/uc-application/base/kustomization.yaml rename to execution/uc-application/kustomization.yaml diff --git a/execution/uc-application/overlay/uc1-application/kustomization.yaml b/execution/uc-application/overlay/uc1-application/kustomization.yaml deleted file mode 100644 index 0d3820fe3..000000000 --- a/execution/uc-application/overlay/uc1-application/kustomization.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -namePrefix: uc1- - -images: - - name: uc-app - newName: theodolite/theodolite-uc1-kstreams-app - newTag: latest - -bases: -- ../../base - -patchesStrategicMerge: -- set_paramters.yaml # Patch setting the resource parameters diff --git a/execution/uc-application/overlay/uc1-application/set_paramters.yaml b/execution/uc-application/overlay/uc1-application/set_paramters.yaml deleted file mode 100644 index cb8504812..000000000 --- a/execution/uc-application/overlay/uc1-application/set_paramters.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: titan-ccp-aggregation -spec: - replicas: 1 - template: - spec: - containers: - - name: uc-application - env: - - name: COMMIT_INTERVAL_MS - value: "100" - resources: - limits: - memory: 4Gi - cpu: 1000m diff --git a/execution/uc-application/overlay/uc2-application/kustomization.yaml b/execution/uc-application/overlay/uc2-application/kustomization.yaml deleted file mode 100644 index cd32cabf7..000000000 --- a/execution/uc-application/overlay/uc2-application/kustomization.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -namePrefix: uc2- - -images: - - name: uc-app - newName: theodolite/theodolite-uc2-kstreams-app - newTag: latest - -bases: -- ../../base - -patchesStrategicMerge: -- set_paramters.yaml # Patch setting the resource parameters diff --git a/execution/uc-application/overlay/uc2-application/set_paramters.yaml b/execution/uc-application/overlay/uc2-application/set_paramters.yaml deleted file mode 100644 index cb8504812..000000000 --- a/execution/uc-application/overlay/uc2-application/set_paramters.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: titan-ccp-aggregation -spec: - replicas: 1 - template: - spec: - containers: - - name: uc-application - env: - - name: COMMIT_INTERVAL_MS - value: "100" - resources: - limits: - memory: 4Gi - cpu: 1000m diff --git a/execution/uc-application/overlay/uc3-application/kustomization.yaml b/execution/uc-application/overlay/uc3-application/kustomization.yaml deleted file mode 100644 index 5722cbca8..000000000 --- a/execution/uc-application/overlay/uc3-application/kustomization.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -namePrefix: uc3- - -images: - - name: uc-app - newName: theodolite/theodolite-uc3-kstreams-app - newTag: latest - -bases: -- ../../base - -patchesStrategicMerge: -- set_paramters.yaml # Patch setting the resource parameters diff --git a/execution/uc-application/overlay/uc3-application/set_paramters.yaml b/execution/uc-application/overlay/uc3-application/set_paramters.yaml deleted file mode 100644 index cb8504812..000000000 --- a/execution/uc-application/overlay/uc3-application/set_paramters.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: titan-ccp-aggregation -spec: - replicas: 1 - template: - spec: - containers: - - name: uc-application - env: - - name: COMMIT_INTERVAL_MS - value: "100" - resources: - limits: - memory: 4Gi - cpu: 1000m diff --git a/execution/uc-application/overlay/uc4-application/kustomization.yaml b/execution/uc-application/overlay/uc4-application/kustomization.yaml deleted file mode 100644 index b44a9bb64..000000000 --- a/execution/uc-application/overlay/uc4-application/kustomization.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -namePrefix: uc4- - -images: - - name: uc-app - newName: theodolite/theodolite-uc4-kstreams-app - newTag: latest - -bases: -- ../../base - -patchesStrategicMerge: -- set_paramters.yaml # Patch setting the resource parameters diff --git a/execution/uc-application/overlay/uc4-application/set_paramters.yaml b/execution/uc-application/overlay/uc4-application/set_paramters.yaml deleted file mode 100644 index cb8504812..000000000 --- a/execution/uc-application/overlay/uc4-application/set_paramters.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: titan-ccp-aggregation -spec: - replicas: 1 - template: - spec: - containers: - - name: uc-application - env: - - name: COMMIT_INTERVAL_MS - value: "100" - resources: - limits: - memory: 4Gi - cpu: 1000m diff --git a/execution/uc-application/base/service-monitor.yaml b/execution/uc-application/service-monitor.yaml similarity index 100% rename from execution/uc-application/base/service-monitor.yaml rename to execution/uc-application/service-monitor.yaml diff --git a/execution/uc-workload-generator/base/kustomization.yaml b/execution/uc-workload-generator/kustomization.yaml similarity index 100% rename from execution/uc-workload-generator/base/kustomization.yaml rename to execution/uc-workload-generator/kustomization.yaml diff --git a/execution/uc-workload-generator/overlay/uc1-workload-generator/kustomization.yaml b/execution/uc-workload-generator/overlay/uc1-workload-generator/kustomization.yaml deleted file mode 100644 index 553b769a3..000000000 --- a/execution/uc-workload-generator/overlay/uc1-workload-generator/kustomization.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -namePrefix: uc1- - -images: - - name: workload-generator - newName: theodolite/theodolite-uc1-workload-generator - newTag: latest - -bases: -- ../../base - -patchesStrategicMerge: -- set_paramters.yaml # Patch setting the resource parameters diff --git a/execution/uc-workload-generator/overlay/uc1-workload-generator/set_paramters.yaml b/execution/uc-workload-generator/overlay/uc1-workload-generator/set_paramters.yaml deleted file mode 100644 index b275607c2..000000000 --- a/execution/uc-workload-generator/overlay/uc1-workload-generator/set_paramters.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: titan-ccp-load-generator -spec: - replicas: 1 - template: - spec: - containers: - - name: workload-generator - env: - - name: NUM_SENSORS - value: "25000" - - name: INSTANCES - value: "1" diff --git a/execution/uc-workload-generator/overlay/uc2-workload-generator/kustomization.yaml b/execution/uc-workload-generator/overlay/uc2-workload-generator/kustomization.yaml deleted file mode 100644 index ff6874335..000000000 --- a/execution/uc-workload-generator/overlay/uc2-workload-generator/kustomization.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -namePrefix: uc2- - -images: - - name: workload-generator - newName: theodolite/theodolite-uc2-workload-generator - newTag: latest - -bases: -- ../../base - -patchesStrategicMerge: -- set_paramters.yaml # Patch setting the resource parameters diff --git a/execution/uc-workload-generator/overlay/uc2-workload-generator/set_paramters.yaml b/execution/uc-workload-generator/overlay/uc2-workload-generator/set_paramters.yaml deleted file mode 100644 index 187cb4717..000000000 --- a/execution/uc-workload-generator/overlay/uc2-workload-generator/set_paramters.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: titan-ccp-load-generator -spec: - replicas: 1 - template: - spec: - containers: - - name: workload-generator - env: - - name: NUM_SENSORS - value: "4" - - name: HIERARCHY - value: "full" - - name: NUM_NESTED_GROUPS - value: "5" - - name: INSTANCES - value: "1" diff --git a/execution/uc-workload-generator/overlay/uc3-workload-generator/kustomization.yaml b/execution/uc-workload-generator/overlay/uc3-workload-generator/kustomization.yaml deleted file mode 100644 index a7022480f..000000000 --- a/execution/uc-workload-generator/overlay/uc3-workload-generator/kustomization.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -namePrefix: uc3- - -images: - - name: workload-generator - newName: theodolite/theodolite-uc3-workload-generator - newTag: latest - -bases: -- ../../base - -patchesStrategicMerge: -- set_paramters.yaml # Patch setting the resource parameters diff --git a/execution/uc-workload-generator/overlay/uc3-workload-generator/set_paramters.yaml b/execution/uc-workload-generator/overlay/uc3-workload-generator/set_paramters.yaml deleted file mode 100644 index b275607c2..000000000 --- a/execution/uc-workload-generator/overlay/uc3-workload-generator/set_paramters.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: titan-ccp-load-generator -spec: - replicas: 1 - template: - spec: - containers: - - name: workload-generator - env: - - name: NUM_SENSORS - value: "25000" - - name: INSTANCES - value: "1" diff --git a/execution/uc-workload-generator/overlay/uc4-workload-generator/kustomization.yaml b/execution/uc-workload-generator/overlay/uc4-workload-generator/kustomization.yaml deleted file mode 100644 index 5efb0eb25..000000000 --- a/execution/uc-workload-generator/overlay/uc4-workload-generator/kustomization.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -namePrefix: uc4- - -images: - - name: workload-generator - newName: theodolite/theodolite-uc4-workload-generator - newTag: latest - -bases: -- ../../base - -patchesStrategicMerge: -- set_paramters.yaml # Patch setting the resource parameters diff --git a/execution/uc-workload-generator/overlay/uc4-workload-generator/set_paramters.yaml b/execution/uc-workload-generator/overlay/uc4-workload-generator/set_paramters.yaml deleted file mode 100644 index b275607c2..000000000 --- a/execution/uc-workload-generator/overlay/uc4-workload-generator/set_paramters.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: titan-ccp-load-generator -spec: - replicas: 1 - template: - spec: - containers: - - name: workload-generator - env: - - name: NUM_SENSORS - value: "25000" - - name: INSTANCES - value: "1" diff --git a/execution/uc-workload-generator/base/workloadGenerator.yaml b/execution/uc-workload-generator/workloadGenerator.yaml similarity index 100% rename from execution/uc-workload-generator/base/workloadGenerator.yaml rename to execution/uc-workload-generator/workloadGenerator.yaml -- GitLab