Merge

execution/experiments.txt

-# Test Partition count of 100
-./run_loop.sh 1 "10000 50000 100000 200000" "1, 2, 4, 8, 12, 16, 20" 100
\ No newline at end of file

execution/infrastructure/kafka-lag-exporter/service-monitor.yaml

-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
-  labels:
-    app: kafka-lag-exporter
-    appScope: titan-ccp
-  name: kafka-lag-exporter
-spec:
-  selector:
-    matchLabels:
-      #app: cp-kafka
-      jobLabel: kafka-lag-exporter
-  endpoints:
-    - port: http
-      interval: 5s
\ No newline at end of file

execution/infrastructure/kafka-lag-exporter/values.yaml

@@ -7,8 +7,10 @@ pollIntervalSeconds: 15
 
 prometheus:
   serviceMonitor:
-    enabled: false
-    interval: "30s"
+    enabled: true
+    interval: "5s"
+    additionalLabels:
+      appScope: titan-ccp
     # service monitor label selectors: https://github.com/helm/charts/blob/f5a751f174263971fafd21eee4e35416d6612a3d/stable/prometheus-operator/templates/prometheus/prometheus.yaml#L74
     # additionalLabels:
-    #   prometheus: k8s
+    #   prometheus: k8s
\ No newline at end of file

execution/infrastructure/kafka/kafka-client.yaml

+apiVersion: v1
+kind: Pod
+metadata:
+  name: kafka-client-2
+spec:
+  containers:
+  - name: kafka-client
+    image: confluentinc/cp-enterprise-kafka:5.4.0
+    command:
+      - sh
+      - -c
+      - "exec tail -f /dev/null"
\ No newline at end of file

execution/lag_analysis.py

@@ -11,7 +11,7 @@ exp_id =  sys.argv[1]
 benchmark = sys.argv[2]
 dim_value = sys.argv[3]
 instances = sys.argv[4]
-execution_minutes = 5
+execution_minutes = int(sys.argv[5])
 time_diff_ms = int(os.getenv('CLOCK_DIFF_MS', 0))
 
 #http://localhost:9090/api/v1/query_range?query=sum%20by(job,topic)(kafka_consumer_consumer_fetch_manager_metrics_records_lag)&start=2015-07-01T20:10:30.781Z&end=2020-07-01T20:11:00.781Z&step=15s

execution/requirements.txt

-attrs==19.3.0
-backcall==0.1.0
-bleach==3.1.1
-certifi==2019.11.28
-chardet==3.0.4
-cycler==0.10.0
-decorator==4.4.2
-defusedxml==0.6.0
-entrypoints==0.3
-idna==2.9
-importlib-metadata==1.5.0
-ipykernel==5.1.4
-ipython==7.13.0
-ipython-genutils==0.2.0
-ipywidgets==7.5.1
-jedi==0.16.0
-Jinja2==2.11.1
-joblib==0.14.1
-jsonschema==3.2.0
-jupyter==1.0.0
-jupyter-client==6.0.0
-jupyter-console==6.1.0
-jupyter-core==4.6.3
-kiwisolver==1.1.0
-MarkupSafe==1.1.1
 matplotlib==3.2.0
-mistune==0.8.4
-nbconvert==5.6.1
-nbformat==5.0.4
-notebook==6.0.3
-numpy==1.18.1
 pandas==1.0.1
-pandocfilters==1.4.2
-parso==0.6.2
-pexpect==4.8.0
-pickleshare==0.7.5
-prometheus-client==0.7.1
-prompt-toolkit==3.0.4
-ptyprocess==0.6.0
-Pygments==2.6.1
-pyparsing==2.4.6
-pyrsistent==0.15.7
-python-dateutil==2.8.1
-pytz==2019.3
-pyzmq==19.0.0
-qtconsole==4.7.1
-QtPy==1.9.0
 requests==2.23.0
-scikit-learn==0.22.2.post1
-scipy==1.4.1
-Send2Trash==1.5.0
-six==1.14.0
-sklearn==0.0
-terminado==0.8.3
-testpath==0.4.4
-tornado==6.0.4
-traitlets==4.3.3
-urllib3==1.25.8
-wcwidth==0.1.8
-webencodings==0.5.1
-widgetsnbextension==3.5.1
-zipp==3.1.0
+scikit-learn==0.22.2.post1
\ No newline at end of file

execution/run_loop.sh

@@ -10,7 +10,7 @@ KAFKA_STREAMS_COMMIT_INTERVAL_MS=${7:-100}
 EXECUTION_MINUTES=${8:-5}
 
 # Get and increment counter
-EXP_ID=$(cat exp_counter.txt)
+EXP_ID=$(cat exp_counter.txt 2>/dev/null || echo "0")
 echo $((EXP_ID+1)) > exp_counter.txt
 
 # Store meta information

execution/run_uc1-new.sh

@@ -34,7 +34,11 @@ echo "$WORKLOAD_GENERATOR_YAML" | kubectl apply -f -
 
 # Start application
 REPLICAS=$INSTANCES
-#kubectl apply -f uc3-application/aggregation-deployment.yaml
+# When not using `sed` anymore, use `kubectl apply -f uc1-application`
+kubectl apply -f uc1-application/aggregation-service.yaml
+kubectl apply -f uc1-application/jmx-configmap.yaml
+kubectl apply -f uc1-application/service-monitor.yaml
+#kubectl apply -f uc1-application/aggregation-deployment.yaml
 APPLICATION_YAML=$(sed "s/{{CPU_LIMIT}}/$CPU_LIMIT/g; s/{{MEMORY_LIMIT}}/$MEMORY_LIMIT/g; s/{{KAFKA_STREAMS_COMMIT_INTERVAL_MS}}/$KAFKA_STREAMS_COMMIT_INTERVAL_MS/g" uc1-application/aggregation-deployment.yaml)
 echo "$APPLICATION_YAML" | kubectl apply -f -
 kubectl scale deployment titan-ccp-aggregation --replicas=$REPLICAS
@@ -44,7 +48,7 @@ sleep ${EXECUTION_MINUTES}m
 
 # Run eval script
 source ../.venv/bin/activate
-python lag_analysis.py $EXP_ID uc1 $DIM_VALUE $INSTANCES
+python lag_analysis.py $EXP_ID uc1 $DIM_VALUE $INSTANCES $EXECUTION_MINUTES
 deactivate
 
 # Stop wl and app
@@ -52,6 +56,9 @@ deactivate
 #sed "s/{{INSTANCES}}/1/g" uc1-workload-generator/deployment.yaml | kubectl delete -f -
 #sed "s/{{NUM_SENSORS}}/$NUM_SENSORS/g; s/{{INSTANCES}}/$WL_INSTANCES/g" uc1-workload-generator/deployment.yaml | kubectl delete -f -
 echo "$WORKLOAD_GENERATOR_YAML" | kubectl delete -f -
+kubectl delete -f uc1-application/aggregation-service.yaml
+kubectl delete -f uc1-application/jmx-configmap.yaml
+kubectl delete -f uc1-application/service-monitor.yaml
 #kubectl delete -f uc1-application/aggregation-deployment.yaml
 echo "$APPLICATION_YAML" | kubectl delete -f -
 

execution/run_uc2-new.sh

@@ -30,6 +30,10 @@ sed "s/{{NUM_NESTED_GROUPS}}/$NUM_NESTED_GROUPS/g" uc2-workload-generator/deploy
 
 # Start application
 REPLICAS=$INSTANCES
+# When not using `sed` anymore, use `kubectl apply -f uc2-application`
+kubectl apply -f uc2-application/aggregation-service.yaml
+kubectl apply -f uc2-application/jmx-configmap.yaml
+kubectl apply -f uc2-application/service-monitor.yaml
 #kubectl apply -f uc2-application/aggregation-deployment.yaml
 APPLICATION_YAML=$(sed "s/{{CPU_LIMIT}}/$CPU_LIMIT/g; s/{{MEMORY_LIMIT}}/$MEMORY_LIMIT/g; s/{{KAFKA_STREAMS_COMMIT_INTERVAL_MS}}/$KAFKA_STREAMS_COMMIT_INTERVAL_MS/g" uc2-application/aggregation-deployment.yaml)
 echo "$APPLICATION_YAML" | kubectl apply -f -
@@ -40,11 +44,14 @@ sleep ${EXECUTION_MINUTES}m
 
 # Run eval script
 source ../.venv/bin/activate
-python lag_analysis.py $EXP_ID uc2 $DIM_VALUE $INSTANCES
+python lag_analysis.py $EXP_ID uc2 $DIM_VALUE $INSTANCES $EXECUTION_MINUTES
 deactivate
 
 # Stop wl and app
 kubectl delete -f uc2-workload-generator/deployment.yaml
+kubectl delete -f uc2-application/aggregation-service.yaml
+kubectl delete -f uc2-application/jmx-configmap.yaml
+kubectl delete -f uc2-application/service-monitor.yaml
 #kubectl delete -f uc2-application/aggregation-deployment.yaml
 echo "$APPLICATION_YAML" | kubectl delete -f -
 

execution/run_uc3-new.sh

@@ -34,6 +34,10 @@ echo "$WORKLOAD_GENERATOR_YAML" | kubectl apply -f -
 
 # Start application
 REPLICAS=$INSTANCES
+# When not using `sed` anymore, use `kubectl apply -f uc3-application`
+kubectl apply -f uc3-application/aggregation-service.yaml
+kubectl apply -f uc3-application/jmx-configmap.yaml
+kubectl apply -f uc3-application/service-monitor.yaml
 #kubectl apply -f uc3-application/aggregation-deployment.yaml
 APPLICATION_YAML=$(sed "s/{{CPU_LIMIT}}/$CPU_LIMIT/g; s/{{MEMORY_LIMIT}}/$MEMORY_LIMIT/g; s/{{KAFKA_STREAMS_COMMIT_INTERVAL_MS}}/$KAFKA_STREAMS_COMMIT_INTERVAL_MS/g" uc3-application/aggregation-deployment.yaml)
 echo "$APPLICATION_YAML" | kubectl apply -f -
@@ -44,14 +48,17 @@ sleep ${EXECUTION_MINUTES}m
 
 # Run eval script
 source ../.venv/bin/activate
-python lag_analysis.py $EXP_ID uc3 $DIM_VALUE $INSTANCES
+python lag_analysis.py $EXP_ID uc3 $DIM_VALUE $INSTANCES $EXECUTION_MINUTES
 deactivate
 
 # Stop wl and app
 #kubectl delete -f uc3-workload-generator/deployment.yaml
 #sed "s/{{INSTANCES}}/1/g" uc3-workload-generator/deployment.yaml | kubectl delete -f -
 echo "$WORKLOAD_GENERATOR_YAML" | kubectl delete -f -
-#kubectl delete -f uc1-application/aggregation-deployment.yaml
+kubectl delete -f uc3-application/aggregation-service.yaml
+kubectl delete -f uc3-application/jmx-configmap.yaml
+kubectl delete -f uc3-application/service-monitor.yaml
+#kubectl delete -f uc3-application/aggregation-deployment.yaml
 #sed "s/{{CPU_LIMIT}}/1000m/g; s/{{MEMORY_LIMIT}}/4Gi/g; s/{{KAFKA_STREAMS_COMMIT_INTERVAL_MS}}/100/g" uc3-application/aggregation-deployment.yaml | kubectl delete -f -
 echo "$APPLICATION_YAML" | kubectl delete -f -
 

execution/run_uc4-new.sh

@@ -32,6 +32,10 @@ sed "s/{{NUM_SENSORS}}/$NUM_SENSORS/g" uc4-workload-generator/deployment.yaml |
 # Start application
 REPLICAS=$INSTANCES
 #AGGREGATION_DURATION_DAYS=$DIM_VALUE
+# When not using `sed` anymore, use `kubectl apply -f uc4-application`
+kubectl apply -f uc4-application/aggregation-service.yaml
+kubectl apply -f uc4-application/jmx-configmap.yaml
+kubectl apply -f uc4-application/service-monitor.yaml
 #kubectl apply -f uc4-application/aggregation-deployment.yaml
 #sed "s/{{AGGREGATION_DURATION_DAYS}}/$AGGREGATION_DURATION_DAYS/g" uc4-application/aggregation-deployment.yaml | kubectl apply -f -
 APPLICATION_YAML=$(sed "s/{{CPU_LIMIT}}/$CPU_LIMIT/g; s/{{MEMORY_LIMIT}}/$MEMORY_LIMIT/g; s/{{KAFKA_STREAMS_COMMIT_INTERVAL_MS}}/$KAFKA_STREAMS_COMMIT_INTERVAL_MS/g" uc4-application/aggregation-deployment.yaml)
@@ -43,11 +47,14 @@ sleep ${EXECUTION_MINUTES}m
 
 # Run eval script
 source ../.venv/bin/activate
-python lag_analysis.py $EXP_ID uc4 $DIM_VALUE $INSTANCES
+python lag_analysis.py $EXP_ID uc4 $DIM_VALUE $INSTANCES $EXECUTION_MINUTES
 deactivate
 
 # Stop wl and app
 kubectl delete -f uc4-workload-generator/deployment.yaml
+kubectl delete -f uc4-application/aggregation-service.yaml
+kubectl delete -f uc4-application/jmx-configmap.yaml
+kubectl delete -f uc4-application/service-monitor.yaml
 #kubectl delete -f uc4-application/aggregation-deployment.yaml
 echo "$APPLICATION_YAML" | kubectl delete -f -
 

execution/uc1-workload-generator/jmx-configmap.yaml

-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: load-generator-jmx-configmap
-data:
-  jmx-kafka-prometheus.yml: |+
-    jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi
-    lowercaseOutputName: true
-    lowercaseOutputLabelNames: true
-    ssl: false

execution/uc2-workload-generator/jmx-configmap.yaml

-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: load-generator-jmx-configmap
-data:
-  jmx-kafka-prometheus.yml: |+
-    jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi
-    lowercaseOutputName: true
-    lowercaseOutputLabelNames: true
-    ssl: false

execution/uc3-workload-generator/jmx-configmap.yaml

-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: load-generator-jmx-configmap
-data:
-  jmx-kafka-prometheus.yml: |+
-    jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi
-    lowercaseOutputName: true
-    lowercaseOutputLabelNames: true
-    ssl: false

execution/uc4-workload-generator/jmx-configmap.yaml

-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: load-generator-jmx-configmap
-data:
-  jmx-kafka-prometheus.yml: |+
-    jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi
-    lowercaseOutputName: true
-    lowercaseOutputLabelNames: true
-    ssl: false

settings.gradle

 rootProject.name = 'scalability-benchmarking'
 
-include 'workload-generator-common'
+include 'workload-generator-commons'
+include 'application-kafkastreams-commons'
 
 include 'uc1-workload-generator'
 include 'uc1-application'

uc1-application/build.gradle

-mainClassName = "spesb.uc1.application.HistoryService"
+mainClassName = "theodolite.uc1.application.HistoryService"

uc1-application/src/main/java/theodolite/uc1/application/ConfigurationKeys.java

+package theodolite.uc1.application;
+
+/**
+ * Keys to access configuration parameters.
+ */
+public final class ConfigurationKeys {
+
+  public static final String APPLICATION_NAME = "application.name";
+
+  public static final String APPLICATION_VERSION = "application.version";
+
+  public static final String NUM_THREADS = "num.threads";
+
+  public static final String COMMIT_INTERVAL_MS = "commit.interval.ms";
+
+  public static final String CACHE_MAX_BYTES_BUFFERING = "cache.max.bytes.buffering";
+
+  public static final String KAFKA_BOOTSTRAP_SERVERS = "kafka.bootstrap.servers";
+
+  public static final String KAFKA_INPUT_TOPIC = "kafka.input.topic";
+
+
+  private ConfigurationKeys() {}
+
+}

uc1-application/src/main/java/spesb/uc1/application/HistoryService.java → uc1-application/src/main/java/theodolite/uc1/application/HistoryService.java

-package spesb.uc1.application;
+package theodolite.uc1.application;
 
 import java.util.concurrent.CompletableFuture;
 import org.apache.commons.configuration2.Configuration;
 import org.apache.kafka.streams.KafkaStreams;
-import spesb.uc1.streamprocessing.KafkaStreamsBuilder;
+import theodolite.uc1.streamprocessing.Uc1KafkaStreamsBuilder;
 import titan.ccp.common.configuration.Configurations;
 
 /**
@@ -30,14 +30,20 @@ public class HistoryService {
    */
   private void createKafkaStreamsApplication() {
 
-    final KafkaStreams kafkaStreams = new KafkaStreamsBuilder()
-        .bootstrapServers(this.config.getString(ConfigurationKeys.KAFKA_BOOTSTRAP_SERVERS))
-        .inputTopic(this.config.getString(ConfigurationKeys.KAFKA_INPUT_TOPIC))
+    final Uc1KafkaStreamsBuilder uc1KafkaStreamsBuilder = new Uc1KafkaStreamsBuilder();
+    uc1KafkaStreamsBuilder.inputTopic(this.config.getString(ConfigurationKeys.KAFKA_INPUT_TOPIC));
+
+    final KafkaStreams kafkaStreams = uc1KafkaStreamsBuilder
+        .applicationName(this.config.getString(ConfigurationKeys.APPLICATION_NAME))
+        .applicationVersion(this.config.getString(ConfigurationKeys.APPLICATION_VERSION))
         .numThreads(this.config.getInt(ConfigurationKeys.NUM_THREADS))
         .commitIntervalMs(this.config.getInt(ConfigurationKeys.COMMIT_INTERVAL_MS))
         .cacheMaxBytesBuffering(this.config.getInt(ConfigurationKeys.CACHE_MAX_BYTES_BUFFERING))
+        .bootstrapServers(this.config.getString(ConfigurationKeys.KAFKA_BOOTSTRAP_SERVERS))
         .build();
+
     this.stopEvent.thenRun(kafkaStreams::close);
+
     kafkaStreams.start();
   }