Merge branch 'feature/147-add-beam-implementations' into...

Merge branch 'feature/147-add-beam-implementations' into feature/281-Add-Beam-Kubernetes-Benchmark-Definitions

theodolite-benchmarks/README.md

+# Theodolite Benchmarks
+
+Theodolite comes with a set of 4 benchmarks for event-driven microservices, which are implemented with Kafka Streams
+and Apache Flink. The benchmarks are based on typical use cases for stream processing and named: UC1, UC2, UC3 and UC4.
+
+## Building and packaging the benchmarks
+
+All benchmarks can be built with:
+
+```sh
+./gradlew build
+```
+
+This produces `.tar` files in `<benchmark-impl>/build/distribution`, where `<benchmark-impl>` is for example
+`uc1-kstreams`.
+
+## Building Docker images of the benchmarks
+
+Each benchmark implementation directory contains a Dockerfile. To build an image (in this case of `uc1-kstreams`), run:
+
+```sh
+docker build -t theodolite-uc1-kstreams ./uc1-kstreams
+```

theodolite-benchmarks/beam-commons/src/main/java/theodolite/commons/beam/AbstractBeamService.java

@@ -8,7 +8,7 @@ import org.slf4j.LoggerFactory;
 import titan.ccp.common.configuration.ServiceConfigurations;
 
 /**
- * Abstraction of a beam microservice.
+ * Abstraction of a Beam microservice.
  * Encapsulates the corresponding {@link PipelineOptions} and the beam Runner.
  */
 public class AbstractBeamService {

theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.beam.flink.gradle

@@ -3,6 +3,6 @@ plugins {
 }
 
 dependencies {
-    compile group: 'org.apache.beam', name: 'beam-runners-flink-1.12', version: '2.27.0'
-    compile group: 'org.apache.flink', name: 'flink-statebackend-rocksdb_2.11', version: '1.12.0'
+    implementation group: 'org.apache.beam', name: 'beam-runners-flink-1.12', version: '2.27.0'
+    implementation group: 'org.apache.flink', name: 'flink-statebackend-rocksdb_2.11', version: '1.12.0'
 }
\ No newline at end of file

theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.beam.gradle

@@ -36,9 +36,9 @@ dependencies {
     }
     implementation group: 'org.slf4j', name: 'slf4j-simple', version: '1.7.30'
 
-    runtime "org.apache.beam:beam-runners-direct-java:${apacheBeamVersion}"
-    runtime 'org.slf4j:slf4j-api:1.7.32'
-    runtime 'org.slf4j:slf4j-jdk14:1.7.32'
+    runtimeOnly "org.apache.beam:beam-runners-direct-java:${apacheBeamVersion}"
+    runtimeOnly 'org.slf4j:slf4j-api:1.7.32'
+    runtimeOnly 'org.slf4j:slf4j-jdk14:1.7.32'
 
     // Use JUnit test framework
     testImplementation 'junit:junit:4.12'

theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.beam.samza.gradle

@@ -3,5 +3,7 @@ plugins {
 }
 
 dependencies {
-    compile group: 'org.apache.beam', name: 'beam-runners-samza', version: '2.22.0'
+    implementation('org.apache.beam:beam-runners-samza:2.22.0') {
+        exclude group: 'org.apache.samza', module: 'samza-yarn_2.11'
+    }
 }
\ No newline at end of file

theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.flink.gradle

@@ -6,9 +6,6 @@ plugins {
 
 applicationDefaultJvmArgs = ["-Dlog4j.configuration=log4j.properties"]
 
-
-run.classpath = sourceSets.main.runtimeClasspath
-
 jar {
     manifest {
         attributes 'Built-By': System.getProperty('user.name'),
@@ -17,7 +14,6 @@ jar {
 }
 
 shadowJar {
-    configurations = [project.configurations.compile]
     zip64 true
 }
 
@@ -43,24 +39,22 @@ dependencies {
     implementation('org.industrial-devops:titan-ccp-common:0.1.0-flink-ready-SNAPSHOT') { changing = true }
     implementation('org.industrial-devops:titan-ccp-common-kafka:0.1.0-SNAPSHOT') { changing = true }
 
-    // These dependencies are used internally, and not exposed to consumers on their own compile classpath.
     implementation 'org.apache.kafka:kafka-clients:2.2.0'
-    implementation 'com.google.guava:guava:24.1-jre'
+    implementation 'com.google.guava:guava:30.1-jre'
     implementation 'com.google.code.gson:gson:2.8.2'
     implementation 'org.slf4j:slf4j-simple:1.6.1'
     implementation project(':flink-commons')
 
-    //compile group: 'org.apache.kafka', name: 'kafka-clients', version: "2.2.0"
     implementation "org.apache.flink:flink-java:${flinkVersion}"
     implementation "org.apache.flink:flink-streaming-java_${scalaBinaryVersion}:${flinkVersion}"
-    implementation "org.apache.flink:flink-table-api-java-bridge_${scalaBinaryVersion}:${flinkVersion}"
-    implementation "org.apache.flink:flink-table-planner-blink_${scalaBinaryVersion}:${flinkVersion}"
     implementation "org.apache.flink:flink-connector-kafka_${scalaBinaryVersion}:${flinkVersion}"
     implementation "org.apache.flink:flink-avro:${flinkVersion}"
     implementation "org.apache.flink:flink-avro-confluent-registry:${flinkVersion}"
-    implementation "org.apache.flink:flink-runtime-web_${scalaBinaryVersion}:${flinkVersion}" // For debugging
     implementation "org.apache.flink:flink-statebackend-rocksdb_${scalaBinaryVersion}:${flinkVersion}"
     implementation "org.apache.flink:flink-metrics-prometheus_${scalaBinaryVersion}:${flinkVersion}"
+    implementation "org.apache.flink:flink-runtime-web_${scalaBinaryVersion}:${flinkVersion}" // For debugging
+    
+    
 
     // Use JUnit test framework
     testImplementation 'junit:junit:4.12'

theodolite-benchmarks/docker-test/uc1-beam-flink/docker-compose.yml

@@ -18,6 +18,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1
@@ -44,8 +45,8 @@ services:
       NUM_SENSORS: 10
   benchmark-jobmanager:
       image: uc1-beam-flink:latest
-      ports:
-        - "8080:8081"
+      #ports:
+      #  - "8080:8081"
       command: >
         standalone-job --job-classname application.Uc1BeamFlink
         --streaming

theodolite-benchmarks/docker-test/uc1-beam-samza/docker-compose.yml

@@ -4,6 +4,8 @@ services:
     image: confluentinc/cp-zookeeper
     expose:
       - "2181"
+    #ports:
+    #  - 2181:2181
     environment:
       ZOOKEEPER_CLIENT_PORT: 2181
   kafka:
@@ -18,6 +20,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1
@@ -38,6 +41,9 @@ services:
       - schema-registry
       - kafka
     environment:
+      SAMZA_JOB_COORDINATOR_ZK_CONNECT: zookeeper:2181
+      SAMZA_SYSTEMS_KAFKA_PRODUCER_BOOTSTRAP_SERVERS: kafka:9092
+      SAMZA_SYSTEMS_KAFKA_CONSUMER_BOOTSTRAP_SERVERS: kafka:9092
       MAX_SOURCE_PARALLELISM: 1024
       KAFKA_BOOTSTRAP_SERVERS: kafka:9092
       SCHEMA_REGISTRY_URL: http://schema-registry:8081

theodolite-benchmarks/docker-test/uc1-flink-docker-compose/docker-compose.yml

@@ -18,6 +18,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1
@@ -44,8 +45,8 @@ services:
       NUM_SENSORS: 10
   benchmark-jobmanager:
     image: ghcr.io/cau-se/theodolite-uc1-flink:latest
-    ports:
-      - "8080:8081"
+    #ports:
+    #  - "8080:8081"
     command: standalone-job --job-classname theodolite.uc1.application.HistoryServiceFlinkJob
     environment:
       - KAFKA_BOOTSTRAP_SERVERS=kafka:9092

theodolite-benchmarks/docker-test/uc1-kstreams-docker-compose/docker-compose.yml

@@ -18,6 +18,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1

theodolite-benchmarks/docker-test/uc2-beam-flink/docker-compose.yml

@@ -18,6 +18,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1
@@ -44,8 +45,8 @@ services:
       NUM_SENSORS: 10
   benchmark-jobmanager:
       image: uc2-beam-flink:latest
-      ports:
-        - "8080:8081"
+      #ports:
+      #  - "8080:8081"
       command: >
         standalone-job --job-classname application.Uc2BeamFlink
         --streaming

theodolite-benchmarks/docker-test/uc2-beam-samza/docker-compose.yml

@@ -4,6 +4,8 @@ services:
     image: confluentinc/cp-zookeeper
     expose:
       - "2181"
+    #ports:
+    #  - 2181:2181
     environment:
       ZOOKEEPER_CLIENT_PORT: 2181
   kafka:
@@ -18,6 +20,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1
@@ -38,6 +41,9 @@ services:
       - schema-registry
       - kafka
     environment:
+      SAMZA_JOB_COORDINATOR_ZK_CONNECT: zookeeper:2181
+      SAMZA_SYSTEMS_KAFKA_PRODUCER_BOOTSTRAP_SERVERS: kafka:9092
+      SAMZA_SYSTEMS_KAFKA_CONSUMER_BOOTSTRAP_SERVERS: kafka:9092
       MAX_SOURCE_PARALLELISM: 1024
       KAFKA_BOOTSTRAP_SERVERS: kafka:9092
       SCHEMA_REGISTRY_URL: http://schema-registry:8081

theodolite-benchmarks/docker-test/uc2-flink-docker-compose/docker-compose.yml

@@ -18,6 +18,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1
@@ -44,8 +45,8 @@ services:
       NUM_SENSORS: 10  
   benchmark-jobmanager:
     image: ghcr.io/cau-se/theodolite-uc2-flink:latest
-    ports:
-      - "8080:8081"
+    #ports:
+    #  - "8080:8081"
     command: standalone-job --job-classname theodolite.uc2.application.HistoryServiceFlinkJob
     environment:
       - KAFKA_BOOTSTRAP_SERVERS=kafka:9092

theodolite-benchmarks/docker-test/uc2-kstreams-docker-compose/docker-compose.yml

@@ -18,6 +18,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1

theodolite-benchmarks/docker-test/uc3-beam-flink/docker-compose.yml

@@ -18,6 +18,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1
@@ -44,8 +45,8 @@ services:
       NUM_SENSORS: 10
   benchmark-jobmanager:
       image: uc3-beam-flink:latest
-      ports:
-        - "8080:8081"
+      #ports:
+      #  - "8080:8081"
       command: >
         standalone-job --job-classname application.Uc3BeamFlink
         --streaming

theodolite-benchmarks/docker-test/uc3-beam-samza/docker-compose.yml

@@ -4,6 +4,8 @@ services:
     image: confluentinc/cp-zookeeper
     expose:
       - "2181"
+    #ports:
+    #  - 2181:2181
     environment:
       ZOOKEEPER_CLIENT_PORT: 2181
   kafka:
@@ -18,6 +20,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1
@@ -38,6 +41,9 @@ services:
       - schema-registry
       - kafka
     environment:
+      SAMZA_JOB_COORDINATOR_ZK_CONNECT: zookeeper:2181
+      SAMZA_SYSTEMS_KAFKA_PRODUCER_BOOTSTRAP_SERVERS: kafka:9092
+      SAMZA_SYSTEMS_KAFKA_CONSUMER_BOOTSTRAP_SERVERS: kafka:9092
       MAX_SOURCE_PARALLELISM: 1024
       KAFKA_BOOTSTRAP_SERVERS: kafka:9092
       SCHEMA_REGISTRY_URL: http://schema-registry:8081

theodolite-benchmarks/docker-test/uc3-flink-docker-compose/docker-compose.yml

@@ -18,6 +18,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1
@@ -44,8 +45,8 @@ services:
       NUM_SENSORS: 10
   benchmark-jobmanager:
     image: ghcr.io/cau-se/theodolite-uc3-flink:latest
-    ports:
-      - "8080:8081"
+    #ports:
+    #  - "8080:8081"
     command: standalone-job --job-classname theodolite.uc3.application.HistoryServiceFlinkJob
     environment:
       - KAFKA_BOOTSTRAP_SERVERS=kafka:9092

theodolite-benchmarks/docker-test/uc3-kstreams-docker-compose/docker-compose.yml

@@ -18,6 +18,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1

theodolite-benchmarks/docker-test/uc4-beam-flink/docker-compose.yml

@@ -18,6 +18,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1
@@ -45,8 +46,8 @@ services:
       NUM_NESTED_GROUPS: 4
   benchmark-jobmanager:
       image: uc4-beam-flink:latest
-      ports:
-        - "8080:8081"
+      #ports:
+      #  - "8080:8081"
       command: >
         standalone-job
         --job-classname application.Uc4BeamFlink

theodolite-benchmarks/docker-test/uc4-beam-samza/docker-compose.yml

@@ -4,6 +4,8 @@ services:
     image: confluentinc/cp-zookeeper
     expose:
       - "2181"
+    #ports:
+    #  - 2181:2181
     environment:
       ZOOKEEPER_CLIENT_PORT: 2181
   kafka:
@@ -18,6 +20,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1
@@ -38,6 +41,9 @@ services:
       - schema-registry
       - kafka
     environment:
+      SAMZA_JOB_COORDINATOR_ZK_CONNECT: zookeeper:2181
+      SAMZA_SYSTEMS_KAFKA_PRODUCER_BOOTSTRAP_SERVERS: kafka:9092
+      SAMZA_SYSTEMS_KAFKA_CONSUMER_BOOTSTRAP_SERVERS: kafka:9092
       MAX_SOURCE_PARALLELISM: 1024
       KAFKA_BOOTSTRAP_SERVERS: kafka:9092
       SCHEMA_REGISTRY_URL: http://schema-registry:8081