Merge branch 'master' into hazelcastjet-she

.gitlab-ci.yml

@@ -235,7 +235,7 @@ deploy-uc4-load-generator:
 
 .theodolite:
   image:
-    name: ghcr.io/graalvm/native-image:java11-21.1.0
+    name: ghcr.io/graalvm/native-image:java11-21.3.0
     entrypoint: [""]
   variables:
     GRADLE_OPTS: "-Dorg.gradle.daemon=false"

helm/values.yaml

@@ -66,7 +66,7 @@ cp-helm-charts:
   cp-zookeeper:
     enabled: true
     nodeSelector: {}
-    servers: 3 # default: 3 
+    servers: 3
     image: confluentinc/cp-zookeeper
     imageTag: 5.4.0
     ## Optionally specify an array of imagePullSecrets. Secrets must be manually created in the namespace.
@@ -83,7 +83,7 @@ cp-helm-charts:
   cp-kafka:
     enabled: true
     nodeSelector: {}
-    brokers: 10 # default: 10
+    brokers: 3
     image: confluentinc/cp-enterprise-kafka
     imageTag: 5.4.0
     ## Optionally specify an array of imagePullSecrets. Secrets must be manually created in the namespace.

theodolite-benchmarks/README.md

+# Theodolite Benchmarks
+
+Theodolite comes with a set of 4 benchmarks for event-driven microservices, which are implemented with Kafka Streams
+and Apache Flink. The benchmarks are based on typical use cases for stream processing and named: UC1, UC2, UC3 and UC4.
+
+## Building and packaging the benchmarks
+
+All benchmarks can be built with:
+
+```sh
+./gradlew build
+```
+
+This produces `.tar` files in `<benchmark-impl>/build/distribution`, where `<benchmark-impl>` is for example
+`uc1-kstreams`.
+
+## Building Docker images of the benchmarks
+
+Each benchmark implementation directory contains a Dockerfile. To build an image (in this case of `uc1-kstreams`), run:
+
+```sh
+docker build -t theodolite-uc1-kstreams ./uc1-kstreams
+```

theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.flink.gradle

@@ -6,9 +6,6 @@ plugins {
 
 applicationDefaultJvmArgs = ["-Dlog4j.configuration=log4j.properties"]
 
-
-run.classpath = sourceSets.main.runtimeClasspath
-
 jar {
     manifest {
         attributes 'Built-By': System.getProperty('user.name'),
@@ -17,7 +14,6 @@ jar {
 }
 
 shadowJar {
-    configurations = [project.configurations.compile]
     zip64 true
 }
 
@@ -43,24 +39,22 @@ dependencies {
     implementation('org.industrial-devops:titan-ccp-common:0.1.0-flink-ready-SNAPSHOT') { changing = true }
     implementation('org.industrial-devops:titan-ccp-common-kafka:0.1.0-SNAPSHOT') { changing = true }
 
-    // These dependencies are used internally, and not exposed to consumers on their own compile classpath.
     implementation 'org.apache.kafka:kafka-clients:2.2.0'
-    implementation 'com.google.guava:guava:24.1-jre'
+    implementation 'com.google.guava:guava:30.1-jre'
     implementation 'com.google.code.gson:gson:2.8.2'
     implementation 'org.slf4j:slf4j-simple:1.6.1'
     implementation project(':flink-commons')
 
-    //compile group: 'org.apache.kafka', name: 'kafka-clients', version: "2.2.0"
     implementation "org.apache.flink:flink-java:${flinkVersion}"
     implementation "org.apache.flink:flink-streaming-java_${scalaBinaryVersion}:${flinkVersion}"
-    implementation "org.apache.flink:flink-table-api-java-bridge_${scalaBinaryVersion}:${flinkVersion}"
-    implementation "org.apache.flink:flink-table-planner-blink_${scalaBinaryVersion}:${flinkVersion}"
     implementation "org.apache.flink:flink-connector-kafka_${scalaBinaryVersion}:${flinkVersion}"
     implementation "org.apache.flink:flink-avro:${flinkVersion}"
     implementation "org.apache.flink:flink-avro-confluent-registry:${flinkVersion}"
-    implementation "org.apache.flink:flink-runtime-web_${scalaBinaryVersion}:${flinkVersion}" // For debugging
     implementation "org.apache.flink:flink-statebackend-rocksdb_${scalaBinaryVersion}:${flinkVersion}"
     implementation "org.apache.flink:flink-metrics-prometheus_${scalaBinaryVersion}:${flinkVersion}"
+    implementation "org.apache.flink:flink-runtime-web_${scalaBinaryVersion}:${flinkVersion}" // For debugging
+    
+    
 
     // Use JUnit test framework
     testImplementation 'junit:junit:4.12'

theodolite-benchmarks/docker-test/uc1-flink-docker-compose/docker-compose.yml

@@ -18,6 +18,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1
@@ -44,8 +45,8 @@ services:
       NUM_SENSORS: 10
   benchmark-jobmanager:
     image: ghcr.io/cau-se/theodolite-uc1-flink:latest
-    ports:
-      - "8080:8081"
+    #ports:
+    #  - "8080:8081"
     command: standalone-job --job-classname theodolite.uc1.application.HistoryServiceFlinkJob
     environment:
       - KAFKA_BOOTSTRAP_SERVERS=kafka:9092

theodolite-benchmarks/docker-test/uc1-kstreams-docker-compose/docker-compose.yml

@@ -18,6 +18,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1

theodolite-benchmarks/docker-test/uc2-flink-docker-compose/docker-compose.yml

@@ -18,6 +18,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1
@@ -44,8 +45,8 @@ services:
       NUM_SENSORS: 10  
   benchmark-jobmanager:
     image: ghcr.io/cau-se/theodolite-uc2-flink:latest
-    ports:
-      - "8080:8081"
+    #ports:
+    #  - "8080:8081"
     command: standalone-job --job-classname theodolite.uc2.application.HistoryServiceFlinkJob
     environment:
       - KAFKA_BOOTSTRAP_SERVERS=kafka:9092

theodolite-benchmarks/docker-test/uc2-kstreams-docker-compose/docker-compose.yml

@@ -18,6 +18,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1

theodolite-benchmarks/docker-test/uc3-flink-docker-compose/docker-compose.yml

@@ -18,6 +18,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1
@@ -44,8 +45,8 @@ services:
       NUM_SENSORS: 10
   benchmark-jobmanager:
     image: ghcr.io/cau-se/theodolite-uc3-flink:latest
-    ports:
-      - "8080:8081"
+    #ports:
+    #  - "8080:8081"
     command: standalone-job --job-classname theodolite.uc3.application.HistoryServiceFlinkJob
     environment:
       - KAFKA_BOOTSTRAP_SERVERS=kafka:9092

theodolite-benchmarks/docker-test/uc3-kstreams-docker-compose/docker-compose.yml

@@ -18,6 +18,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1

theodolite-benchmarks/docker-test/uc4-flink-docker-compose/docker-compose.yml

@@ -18,6 +18,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1
@@ -45,8 +46,8 @@ services:
       NUM_NESTED_GROUPS: 4
   benchmark-jobmanager:
     image: ghcr.io/cau-se/theodolite-uc4-flink:latest
-    ports:
-      - "8080:8081"
+    #ports:
+    #  - "8080:8081"
     command: standalone-job --job-classname theodolite.uc4.application.AggregationServiceFlinkJob
     environment:
       - KAFKA_BOOTSTRAP_SERVERS=kafka:9092

theodolite-benchmarks/docker-test/uc4-kstreams-docker-compose/docker-compose.yml

@@ -18,6 +18,7 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
   schema-registry:
     image: confluentinc/cp-schema-registry:5.3.1

theodolite-benchmarks/flink-commons/build.gradle

@@ -20,16 +20,12 @@ repositories {
 dependencies {
     // Special version required because of https://issues.apache.org/jira/browse/FLINK-13703
     implementation('org.industrial-devops:titan-ccp-common:0.1.0-flink-ready-SNAPSHOT') { changing = true }
-    implementation('org.industrial-devops:titan-ccp-common-kafka:0.1.0-SNAPSHOT') { changing = true }
     implementation 'com.google.guava:guava:30.1-jre'
 
-    implementation "org.apache.flink:flink-java:${flinkVersion}"    
+    implementation "org.apache.flink:flink-streaming-java_${scalaBinaryVersion}:${flinkVersion}"
     implementation "org.apache.flink:flink-connector-kafka_${scalaBinaryVersion}:${flinkVersion}"
-    implementation "org.apache.flink:flink-avro:${flinkVersion}"
     implementation "org.apache.flink:flink-avro-confluent-registry:${flinkVersion}"
-    implementation "org.apache.flink:flink-runtime-web_${scalaBinaryVersion}:${flinkVersion}" // For debugging
     implementation "org.apache.flink:flink-statebackend-rocksdb_${scalaBinaryVersion}:${flinkVersion}"
-    implementation "org.apache.flink:flink-metrics-prometheus_${scalaBinaryVersion}:${flinkVersion}"
 
     // Use JUnit test framework
     testImplementation 'junit:junit:4.12'

theodolite-benchmarks/uc2-flink/build.gradle

@@ -2,16 +2,4 @@ plugins {
   id 'theodolite.flink'
 }
 
-allprojects {
-	repositories {
-    	maven {
-    		url 'https://packages.confluent.io/maven/'
-    	}
-	}
-}
-
-dependencies {
-    compile('org.industrial-devops:titan-ccp-common-kafka:0.1.0-SNAPSHOT')
-}
-
 mainClassName = "theodolite.uc2.application.HistoryServiceFlinkJob"

theodolite-benchmarks/uc3-flink/build.gradle

@@ -2,16 +2,4 @@ plugins {
   id 'theodolite.flink'
 }
 
-allprojects {
-	repositories {
-    	maven {
-    		url 'https://packages.confluent.io/maven/'
-    	}
-	}
-}
-
-dependencies {
-    compile('org.industrial-devops:titan-ccp-common-kafka:0.1.0-SNAPSHOT')
-}
-
 mainClassName = "theodolite.uc3.application.HistoryServiceFlinkJob"

theodolite/README.md

@@ -29,7 +29,7 @@ The application can be packaged using:
 ./gradlew build
 ```
 
-It produces the `theodolite-1.0.0-SNAPSHOT-runner.jar` file in the `/build` directory. Be aware that it’s not
+It produces the `theodolite-0.6.0-SNAPSHOT-runner.jar` file in the `/build` directory. Be aware that it’s not
 an _über-jar_ as the dependencies are copied into the `build/lib` directory.
 
 If you want to build an _über-jar_, execute the following command:
@@ -38,7 +38,7 @@ If you want to build an _über-jar_, execute the following command:
 ./gradlew build -Dquarkus.package.type=uber-jar
 ```
 
-The application is now runnable using `java -jar build/theodolite-1.0.0-SNAPSHOT-runner.jar`.
+The application is now runnable using `java -jar build/theodolite-0.6.0-SNAPSHOT-runner.jar`.
 
 ## Creating a native executable
 
@@ -57,7 +57,7 @@ Or, if you don't have GraalVM installed, you can run the native executable build
 ```
 
 You can then execute your native executable with:
-```./build/theodolite-1.0.0-SNAPSHOT-runner```
+```./build/theodolite-0.6.0-SNAPSHOT-runner```
 
 If you want to learn more about building native executables, please consult <https://quarkus.io/guides/gradle-tooling>.
 

theodolite/build.gradle

@@ -36,7 +36,7 @@ dependencies {
 }
 
 group 'theodolite'
-version '0.5.0-SNAPSHOT'
+version '0.6.0-SNAPSHOT'
 
 java {
     sourceCompatibility = JavaVersion.VERSION_11