Merge branch 'feature/147-add-beam-implementations' of...

Merge branch 'feature/147-add-beam-implementations' of git.se.informatik.uni-kiel.de:stu203404/theodolite into feature/147-add-beam-implementations

theodolite-benchmarks/README.md

+# Theodolite Benchmarks
+
+Theodolite comes with a set of 4 benchmarks for event-driven microservices, which are implemented with Kafka Streams
+and Apache Flink. The benchmarks are based on typical use cases for stream processing and named: UC1, UC2, UC3 and UC4.
+
+## Building and packaging the benchmarks
+
+All benchmarks can be built with:
+
+```sh
+./gradlew build
+```
+
+This produces `.tar` files in `<benchmark-impl>/build/distribution`, where `<benchmark-impl>` is for example
+`uc1-kstreams`.
+
+## Building Docker images of the benchmarks
+
+Each benchmark implementation directory contains a Dockerfile. To build an image (in this case of `uc1-kstreams`), run:
+
+```sh
+docker build -t theodolite-uc1-kstreams ./uc1-kstreams
+```

theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.flink.gradle

@@ -6,9 +6,6 @@ plugins {
 
 applicationDefaultJvmArgs = ["-Dlog4j.configuration=log4j.properties"]
 
-
-run.classpath = sourceSets.main.runtimeClasspath
-
 jar {
     manifest {
         attributes 'Built-By': System.getProperty('user.name'),
@@ -17,7 +14,6 @@ jar {
 }
 
 shadowJar {
-    configurations = [project.configurations.compile]
     zip64 true
 }
 
@@ -43,24 +39,22 @@ dependencies {
     implementation('org.industrial-devops:titan-ccp-common:0.1.0-flink-ready-SNAPSHOT') { changing = true }
     implementation('org.industrial-devops:titan-ccp-common-kafka:0.1.0-SNAPSHOT') { changing = true }
 
-    // These dependencies are used internally, and not exposed to consumers on their own compile classpath.
     implementation 'org.apache.kafka:kafka-clients:2.2.0'
-    implementation 'com.google.guava:guava:24.1-jre'
+    implementation 'com.google.guava:guava:30.1-jre'
     implementation 'com.google.code.gson:gson:2.8.2'
     implementation 'org.slf4j:slf4j-simple:1.6.1'
     implementation project(':flink-commons')
 
-    //compile group: 'org.apache.kafka', name: 'kafka-clients', version: "2.2.0"
     implementation "org.apache.flink:flink-java:${flinkVersion}"
     implementation "org.apache.flink:flink-streaming-java_${scalaBinaryVersion}:${flinkVersion}"
-    implementation "org.apache.flink:flink-table-api-java-bridge_${scalaBinaryVersion}:${flinkVersion}"
-    implementation "org.apache.flink:flink-table-planner-blink_${scalaBinaryVersion}:${flinkVersion}"
     implementation "org.apache.flink:flink-connector-kafka_${scalaBinaryVersion}:${flinkVersion}"
     implementation "org.apache.flink:flink-avro:${flinkVersion}"
     implementation "org.apache.flink:flink-avro-confluent-registry:${flinkVersion}"
-    implementation "org.apache.flink:flink-runtime-web_${scalaBinaryVersion}:${flinkVersion}" // For debugging
     implementation "org.apache.flink:flink-statebackend-rocksdb_${scalaBinaryVersion}:${flinkVersion}"
     implementation "org.apache.flink:flink-metrics-prometheus_${scalaBinaryVersion}:${flinkVersion}"
+    implementation "org.apache.flink:flink-runtime-web_${scalaBinaryVersion}:${flinkVersion}" // For debugging
+    
+    
 
     // Use JUnit test framework
     testImplementation 'junit:junit:4.12'

theodolite-benchmarks/docker-test/uc1-beam-flink/docker-compose.yml

@@ -18,6 +18,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1
@@ -44,8 +45,8 @@ services:
       NUM_SENSORS: 10
   benchmark-jobmanager:
       image: uc1-beam-flink:latest
-      ports:
-        - "8080:8081"
+      #ports:
+      #  - "8080:8081"
       command: >
         standalone-job --job-classname application.Uc1BeamFlink
         --streaming

theodolite-benchmarks/docker-test/uc1-beam-samza/docker-compose.yml

@@ -20,6 +20,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1

theodolite-benchmarks/docker-test/uc1-flink-docker-compose/docker-compose.yml

@@ -18,6 +18,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1
@@ -44,8 +45,8 @@ services:
       NUM_SENSORS: 10
   benchmark-jobmanager:
     image: ghcr.io/cau-se/theodolite-uc1-flink:latest
-    ports:
-      - "8080:8081"
+    #ports:
+    #  - "8080:8081"
     command: standalone-job --job-classname theodolite.uc1.application.HistoryServiceFlinkJob
     environment:
       - KAFKA_BOOTSTRAP_SERVERS=kafka:9092

theodolite-benchmarks/docker-test/uc1-kstreams-docker-compose/docker-compose.yml

@@ -18,6 +18,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1

theodolite-benchmarks/docker-test/uc2-beam-flink/docker-compose.yml

@@ -18,6 +18,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1
@@ -44,8 +45,8 @@ services:
       NUM_SENSORS: 10
   benchmark-jobmanager:
       image: uc2-beam-flink:latest
-      ports:
-        - "8080:8081"
+      #ports:
+      #  - "8080:8081"
       command: >
         standalone-job --job-classname application.Uc2BeamFlink
         --streaming

theodolite-benchmarks/docker-test/uc2-beam-samza/docker-compose.yml

@@ -20,6 +20,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1

theodolite-benchmarks/docker-test/uc2-flink-docker-compose/docker-compose.yml

@@ -18,6 +18,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1
@@ -44,8 +45,8 @@ services:
       NUM_SENSORS: 10  
   benchmark-jobmanager:
     image: ghcr.io/cau-se/theodolite-uc2-flink:latest
-    ports:
-      - "8080:8081"
+    #ports:
+    #  - "8080:8081"
     command: standalone-job --job-classname theodolite.uc2.application.HistoryServiceFlinkJob
     environment:
       - KAFKA_BOOTSTRAP_SERVERS=kafka:9092

theodolite-benchmarks/docker-test/uc2-kstreams-docker-compose/docker-compose.yml

@@ -18,6 +18,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1

theodolite-benchmarks/docker-test/uc3-beam-flink/docker-compose.yml

@@ -18,6 +18,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1
@@ -44,8 +45,8 @@ services:
       NUM_SENSORS: 10
   benchmark-jobmanager:
       image: uc3-beam-flink:latest
-      ports:
-        - "8080:8081"
+      #ports:
+      #  - "8080:8081"
       command: >
         standalone-job --job-classname application.Uc3BeamFlink
         --streaming

theodolite-benchmarks/docker-test/uc3-beam-samza/docker-compose.yml

@@ -20,6 +20,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1

theodolite-benchmarks/docker-test/uc3-flink-docker-compose/docker-compose.yml

@@ -18,6 +18,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1
@@ -44,8 +45,8 @@ services:
       NUM_SENSORS: 10
   benchmark-jobmanager:
     image: ghcr.io/cau-se/theodolite-uc3-flink:latest
-    ports:
-      - "8080:8081"
+    #ports:
+    #  - "8080:8081"
     command: standalone-job --job-classname theodolite.uc3.application.HistoryServiceFlinkJob
     environment:
       - KAFKA_BOOTSTRAP_SERVERS=kafka:9092

theodolite-benchmarks/docker-test/uc3-kstreams-docker-compose/docker-compose.yml

@@ -18,6 +18,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1

theodolite-benchmarks/docker-test/uc4-beam-flink/docker-compose.yml

@@ -18,6 +18,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1
@@ -45,8 +46,8 @@ services:
       NUM_NESTED_GROUPS: 4
   benchmark-jobmanager:
       image: uc4-beam-flink:latest
-      ports:
-        - "8080:8081"
+      #ports:
+      #  - "8080:8081"
       command: >
         standalone-job
         --job-classname application.Uc4BeamFlink

theodolite-benchmarks/docker-test/uc4-beam-samza/docker-compose.yml

@@ -20,6 +20,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1

theodolite-benchmarks/docker-test/uc4-flink-docker-compose/docker-compose.yml

@@ -18,6 +18,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1
@@ -45,8 +46,8 @@ services:
       NUM_NESTED_GROUPS: 4
   benchmark-jobmanager:
     image: ghcr.io/cau-se/theodolite-uc4-flink:latest
-    ports:
-      - "8080:8081"
+    #ports:
+    #  - "8080:8081"
     command: standalone-job --job-classname theodolite.uc4.application.AggregationServiceFlinkJob
     environment:
       - KAFKA_BOOTSTRAP_SERVERS=kafka:9092

theodolite-benchmarks/docker-test/uc4-kstreams-docker-compose/docker-compose.yml

@@ -18,6 +18,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1

theodolite-benchmarks/flink-commons/build.gradle

@@ -20,17 +20,13 @@ repositories {
 dependencies {
     // Special version required because of https://issues.apache.org/jira/browse/FLINK-13703
     implementation('org.industrial-devops:titan-ccp-common:0.1.0-flink-ready-SNAPSHOT') { changing = true }
-    implementation('org.industrial-devops:titan-ccp-common-kafka:0.1.0-SNAPSHOT') { changing = true }
     implementation 'com.google.guava:guava:30.1-jre'
 
-    implementation "org.apache.flink:flink-java:${flinkVersion}"    
+    implementation "org.apache.flink:flink-streaming-java_${scalaBinaryVersion}:${flinkVersion}"
     implementation "org.apache.flink:flink-connector-kafka_${scalaBinaryVersion}:${flinkVersion}"
-    implementation "org.apache.flink:flink-avro:${flinkVersion}"
     implementation "org.apache.flink:flink-avro-confluent-registry:${flinkVersion}"
-    implementation "org.apache.flink:flink-runtime-web_${scalaBinaryVersion}:${flinkVersion}" // For debugging
     implementation "org.apache.flink:flink-statebackend-rocksdb_${scalaBinaryVersion}:${flinkVersion}"
-    implementation "org.apache.flink:flink-metrics-prometheus_${scalaBinaryVersion}:${flinkVersion}"
-    
+
     // Use JUnit test framework
     testImplementation 'junit:junit:4.12'
   }

theodolite-benchmarks/uc2-flink/build.gradle

@@ -2,16 +2,4 @@ plugins {
   id 'theodolite.flink'
 }
 
-allprojects {
-	repositories {
-    	maven {
-    		url 'https://packages.confluent.io/maven/'
-    	}
-	}
-}
-
-dependencies {
-    compile('org.industrial-devops:titan-ccp-common-kafka:0.1.0-SNAPSHOT')
-}
-
 mainClassName = "theodolite.uc2.application.HistoryServiceFlinkJob"