diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 87a448be0a59ca30a3218837d3efe5384027d23c..49630fb74a5d89e497d9008ab6f334979c773f92 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -235,7 +235,7 @@ deploy-uc4-load-generator: .theodolite: image: - name: ghcr.io/graalvm/native-image:java11-21.1.0 + name: ghcr.io/graalvm/native-image:java11-21.3.0 entrypoint: [""] variables: GRADLE_OPTS: "-Dorg.gradle.daemon=false" diff --git a/helm/values.yaml b/helm/values.yaml index 9de0155cf26d26bc20975d8ec6524607ac9a1d43..1e57b42c485eb20a5525f25cfc0ef616e65a325c 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -66,7 +66,7 @@ cp-helm-charts: cp-zookeeper: enabled: true nodeSelector: {} - servers: 3 # default: 3 + servers: 3 image: confluentinc/cp-zookeeper imageTag: 5.4.0 ## Optionally specify an array of imagePullSecrets. Secrets must be manually created in the namespace. @@ -83,7 +83,7 @@ cp-helm-charts: cp-kafka: enabled: true nodeSelector: {} - brokers: 10 # default: 10 + brokers: 3 image: confluentinc/cp-enterprise-kafka imageTag: 5.4.0 ## Optionally specify an array of imagePullSecrets. Secrets must be manually created in the namespace. diff --git a/theodolite-benchmarks/README.md b/theodolite-benchmarks/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f794d201859b815fc26a4cbc7bd3be891c964ab4 --- /dev/null +++ b/theodolite-benchmarks/README.md @@ -0,0 +1,23 @@ +# Theodolite Benchmarks + +Theodolite comes with a set of 4 benchmarks for event-driven microservices, which are implemented with Kafka Streams +and Apache Flink. The benchmarks are based on typical use cases for stream processing and named: UC1, UC2, UC3 and UC4. + +## Building and packaging the benchmarks + +All benchmarks can be built with: + +```sh +./gradlew build +``` + +This produces `.tar` files in `<benchmark-impl>/build/distribution`, where `<benchmark-impl>` is for example +`uc1-kstreams`. + +## Building Docker images of the benchmarks + +Each benchmark implementation directory contains a Dockerfile. To build an image (in this case of `uc1-kstreams`), run: + +```sh +docker build -t theodolite-uc1-kstreams ./uc1-kstreams +``` diff --git a/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.flink.gradle b/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.flink.gradle index 333a87bf55bcd0051be05ca91dfe8dc9a2e9e8fa..26a827b6049d09e422d48609590614f383f6cae8 100644 --- a/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.flink.gradle +++ b/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.flink.gradle @@ -6,9 +6,6 @@ plugins { applicationDefaultJvmArgs = ["-Dlog4j.configuration=log4j.properties"] - -run.classpath = sourceSets.main.runtimeClasspath - jar { manifest { attributes 'Built-By': System.getProperty('user.name'), @@ -17,7 +14,6 @@ jar { } shadowJar { - configurations = [project.configurations.compile] zip64 true } @@ -43,24 +39,22 @@ dependencies { implementation('org.industrial-devops:titan-ccp-common:0.1.0-flink-ready-SNAPSHOT') { changing = true } implementation('org.industrial-devops:titan-ccp-common-kafka:0.1.0-SNAPSHOT') { changing = true } - // These dependencies are used internally, and not exposed to consumers on their own compile classpath. implementation 'org.apache.kafka:kafka-clients:2.2.0' - implementation 'com.google.guava:guava:24.1-jre' + implementation 'com.google.guava:guava:30.1-jre' implementation 'com.google.code.gson:gson:2.8.2' implementation 'org.slf4j:slf4j-simple:1.6.1' implementation project(':flink-commons') - //compile group: 'org.apache.kafka', name: 'kafka-clients', version: "2.2.0" implementation "org.apache.flink:flink-java:${flinkVersion}" implementation "org.apache.flink:flink-streaming-java_${scalaBinaryVersion}:${flinkVersion}" - implementation "org.apache.flink:flink-table-api-java-bridge_${scalaBinaryVersion}:${flinkVersion}" - implementation "org.apache.flink:flink-table-planner-blink_${scalaBinaryVersion}:${flinkVersion}" implementation "org.apache.flink:flink-connector-kafka_${scalaBinaryVersion}:${flinkVersion}" implementation "org.apache.flink:flink-avro:${flinkVersion}" implementation "org.apache.flink:flink-avro-confluent-registry:${flinkVersion}" - implementation "org.apache.flink:flink-runtime-web_${scalaBinaryVersion}:${flinkVersion}" // For debugging implementation "org.apache.flink:flink-statebackend-rocksdb_${scalaBinaryVersion}:${flinkVersion}" implementation "org.apache.flink:flink-metrics-prometheus_${scalaBinaryVersion}:${flinkVersion}" + implementation "org.apache.flink:flink-runtime-web_${scalaBinaryVersion}:${flinkVersion}" // For debugging + + // Use JUnit test framework testImplementation 'junit:junit:4.12' diff --git a/theodolite-benchmarks/docker-test/uc1-flink-docker-compose/docker-compose.yml b/theodolite-benchmarks/docker-test/uc1-flink-docker-compose/docker-compose.yml index 419c9cfb741578cccd91845c8164d4e5554d2ab6..5a252f07e23205cf20390230ec956240ad2dc7a6 100755 --- a/theodolite-benchmarks/docker-test/uc1-flink-docker-compose/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc1-flink-docker-compose/docker-compose.yml @@ -18,6 +18,7 @@ services: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1" schema-registry: image: confluentinc/cp-schema-registry:5.3.1 @@ -44,8 +45,8 @@ services: NUM_SENSORS: 10 benchmark-jobmanager: image: ghcr.io/cau-se/theodolite-uc1-flink:latest - ports: - - "8080:8081" + #ports: + # - "8080:8081" command: standalone-job --job-classname theodolite.uc1.application.HistoryServiceFlinkJob environment: - KAFKA_BOOTSTRAP_SERVERS=kafka:9092 diff --git a/theodolite-benchmarks/docker-test/uc1-kstreams-docker-compose/docker-compose.yml b/theodolite-benchmarks/docker-test/uc1-kstreams-docker-compose/docker-compose.yml index cebf3676e92f8ececa5b6707df156f9f22f3be38..88ffadfcf3ce7e372fad1e3cbf28cc3aa847756d 100755 --- a/theodolite-benchmarks/docker-test/uc1-kstreams-docker-compose/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc1-kstreams-docker-compose/docker-compose.yml @@ -18,6 +18,7 @@ services: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1" schema-registry: image: confluentinc/cp-schema-registry:5.3.1 diff --git a/theodolite-benchmarks/docker-test/uc2-flink-docker-compose/docker-compose.yml b/theodolite-benchmarks/docker-test/uc2-flink-docker-compose/docker-compose.yml index c4265702b6f2b833e7b3792a787e3d8a67486ac7..f7047a7af7d0b613dd128b9d8d4d9fffd22b4692 100755 --- a/theodolite-benchmarks/docker-test/uc2-flink-docker-compose/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc2-flink-docker-compose/docker-compose.yml @@ -18,6 +18,7 @@ services: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1" schema-registry: image: confluentinc/cp-schema-registry:5.3.1 @@ -44,8 +45,8 @@ services: NUM_SENSORS: 10 benchmark-jobmanager: image: ghcr.io/cau-se/theodolite-uc2-flink:latest - ports: - - "8080:8081" + #ports: + # - "8080:8081" command: standalone-job --job-classname theodolite.uc2.application.HistoryServiceFlinkJob environment: - KAFKA_BOOTSTRAP_SERVERS=kafka:9092 diff --git a/theodolite-benchmarks/docker-test/uc2-kstreams-docker-compose/docker-compose.yml b/theodolite-benchmarks/docker-test/uc2-kstreams-docker-compose/docker-compose.yml index b520611e4855f6e942fab62b02d27d5f360860d1..89f2633b390b08a3a18128e98f261cc264e2b41d 100755 --- a/theodolite-benchmarks/docker-test/uc2-kstreams-docker-compose/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc2-kstreams-docker-compose/docker-compose.yml @@ -18,6 +18,7 @@ services: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1" schema-registry: image: confluentinc/cp-schema-registry:5.3.1 diff --git a/theodolite-benchmarks/docker-test/uc3-flink-docker-compose/docker-compose.yml b/theodolite-benchmarks/docker-test/uc3-flink-docker-compose/docker-compose.yml index 2c69a659c6ed1e83c149e699484ec148196806c5..c2b8d7ad436301138acdf8dfae1654e2feb9b9bb 100755 --- a/theodolite-benchmarks/docker-test/uc3-flink-docker-compose/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc3-flink-docker-compose/docker-compose.yml @@ -18,6 +18,7 @@ services: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1" schema-registry: image: confluentinc/cp-schema-registry:5.3.1 @@ -44,8 +45,8 @@ services: NUM_SENSORS: 10 benchmark-jobmanager: image: ghcr.io/cau-se/theodolite-uc3-flink:latest - ports: - - "8080:8081" + #ports: + # - "8080:8081" command: standalone-job --job-classname theodolite.uc3.application.HistoryServiceFlinkJob environment: - KAFKA_BOOTSTRAP_SERVERS=kafka:9092 diff --git a/theodolite-benchmarks/docker-test/uc3-kstreams-docker-compose/docker-compose.yml b/theodolite-benchmarks/docker-test/uc3-kstreams-docker-compose/docker-compose.yml index 5ed8e7a673afd825b2e1426fa018db3e00848296..65b0a3467e123a84d0e719d8702749ed33773aea 100755 --- a/theodolite-benchmarks/docker-test/uc3-kstreams-docker-compose/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc3-kstreams-docker-compose/docker-compose.yml @@ -18,6 +18,7 @@ services: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1" schema-registry: image: confluentinc/cp-schema-registry:5.3.1 diff --git a/theodolite-benchmarks/docker-test/uc4-flink-docker-compose/docker-compose.yml b/theodolite-benchmarks/docker-test/uc4-flink-docker-compose/docker-compose.yml index b6bb905e2a950e23970392f256f16935a7777fed..0f7e4e656dede1aad3342fb79816e3ebf88e84d8 100755 --- a/theodolite-benchmarks/docker-test/uc4-flink-docker-compose/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc4-flink-docker-compose/docker-compose.yml @@ -18,6 +18,7 @@ services: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1" schema-registry: image: confluentinc/cp-schema-registry:5.3.1 @@ -45,8 +46,8 @@ services: NUM_NESTED_GROUPS: 4 benchmark-jobmanager: image: ghcr.io/cau-se/theodolite-uc4-flink:latest - ports: - - "8080:8081" + #ports: + # - "8080:8081" command: standalone-job --job-classname theodolite.uc4.application.AggregationServiceFlinkJob environment: - KAFKA_BOOTSTRAP_SERVERS=kafka:9092 diff --git a/theodolite-benchmarks/docker-test/uc4-kstreams-docker-compose/docker-compose.yml b/theodolite-benchmarks/docker-test/uc4-kstreams-docker-compose/docker-compose.yml index 68264b244c16f1a1be7b370bb4e78052d3a8518f..5fca44708006d1fae3ae2f9f46b5c42f6431fc3a 100755 --- a/theodolite-benchmarks/docker-test/uc4-kstreams-docker-compose/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc4-kstreams-docker-compose/docker-compose.yml @@ -18,6 +18,7 @@ services: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1" schema-registry: image: confluentinc/cp-schema-registry:5.3.1 diff --git a/theodolite-benchmarks/flink-commons/build.gradle b/theodolite-benchmarks/flink-commons/build.gradle index 0da7c6f93f4e77e1376f5f2d006ec0bf0f398ec8..edd48c914b8c909ff196bb98e9bbc8b9d99865b9 100644 --- a/theodolite-benchmarks/flink-commons/build.gradle +++ b/theodolite-benchmarks/flink-commons/build.gradle @@ -20,17 +20,13 @@ repositories { dependencies { // Special version required because of https://issues.apache.org/jira/browse/FLINK-13703 implementation('org.industrial-devops:titan-ccp-common:0.1.0-flink-ready-SNAPSHOT') { changing = true } - implementation('org.industrial-devops:titan-ccp-common-kafka:0.1.0-SNAPSHOT') { changing = true } implementation 'com.google.guava:guava:30.1-jre' - implementation "org.apache.flink:flink-java:${flinkVersion}" + implementation "org.apache.flink:flink-streaming-java_${scalaBinaryVersion}:${flinkVersion}" implementation "org.apache.flink:flink-connector-kafka_${scalaBinaryVersion}:${flinkVersion}" - implementation "org.apache.flink:flink-avro:${flinkVersion}" implementation "org.apache.flink:flink-avro-confluent-registry:${flinkVersion}" - implementation "org.apache.flink:flink-runtime-web_${scalaBinaryVersion}:${flinkVersion}" // For debugging implementation "org.apache.flink:flink-statebackend-rocksdb_${scalaBinaryVersion}:${flinkVersion}" - implementation "org.apache.flink:flink-metrics-prometheus_${scalaBinaryVersion}:${flinkVersion}" - + // Use JUnit test framework testImplementation 'junit:junit:4.12' } diff --git a/theodolite-benchmarks/uc2-flink/build.gradle b/theodolite-benchmarks/uc2-flink/build.gradle index 680b802ce527d538062658874146ce1f0bd3b1e8..0be6f3773f508dc84cad6a62b125b6a889edb383 100644 --- a/theodolite-benchmarks/uc2-flink/build.gradle +++ b/theodolite-benchmarks/uc2-flink/build.gradle @@ -2,16 +2,4 @@ plugins { id 'theodolite.flink' } -allprojects { - repositories { - maven { - url 'https://packages.confluent.io/maven/' - } - } -} - -dependencies { - compile('org.industrial-devops:titan-ccp-common-kafka:0.1.0-SNAPSHOT') -} - mainClassName = "theodolite.uc2.application.HistoryServiceFlinkJob" diff --git a/theodolite-benchmarks/uc3-flink/build.gradle b/theodolite-benchmarks/uc3-flink/build.gradle index 7f74b4b6f3d75f213d2fae868775423381076641..b8bb79bc1d41627f57c3d9f3b897cbc4c260d620 100644 --- a/theodolite-benchmarks/uc3-flink/build.gradle +++ b/theodolite-benchmarks/uc3-flink/build.gradle @@ -2,16 +2,4 @@ plugins { id 'theodolite.flink' } -allprojects { - repositories { - maven { - url 'https://packages.confluent.io/maven/' - } - } -} - -dependencies { - compile('org.industrial-devops:titan-ccp-common-kafka:0.1.0-SNAPSHOT') -} - mainClassName = "theodolite.uc3.application.HistoryServiceFlinkJob" diff --git a/theodolite/README.md b/theodolite/README.md index 60bd56d933d8955217120465c47e70b4b34585e1..521f6aeba12e5d6bc17fb80ef0f2a525bbe20675 100644 --- a/theodolite/README.md +++ b/theodolite/README.md @@ -29,7 +29,7 @@ The application can be packaged using: ./gradlew build ``` -It produces the `theodolite-1.0.0-SNAPSHOT-runner.jar` file in the `/build` directory. Be aware that it’s not +It produces the `theodolite-0.6.0-SNAPSHOT-runner.jar` file in the `/build` directory. Be aware that it’s not an _über-jar_ as the dependencies are copied into the `build/lib` directory. If you want to build an _über-jar_, execute the following command: @@ -38,7 +38,7 @@ If you want to build an _über-jar_, execute the following command: ./gradlew build -Dquarkus.package.type=uber-jar ``` -The application is now runnable using `java -jar build/theodolite-1.0.0-SNAPSHOT-runner.jar`. +The application is now runnable using `java -jar build/theodolite-0.6.0-SNAPSHOT-runner.jar`. ## Creating a native executable @@ -57,7 +57,7 @@ Or, if you don't have GraalVM installed, you can run the native executable build ``` You can then execute your native executable with: -```./build/theodolite-1.0.0-SNAPSHOT-runner``` +```./build/theodolite-0.6.0-SNAPSHOT-runner``` If you want to learn more about building native executables, please consult <https://quarkus.io/guides/gradle-tooling>. diff --git a/theodolite/build.gradle b/theodolite/build.gradle index 3082deaf12fc48c6aca97ffd00b9c74cd7e6c143..39cc62804a382888f8d0019e639c5ff4febe8d59 100644 --- a/theodolite/build.gradle +++ b/theodolite/build.gradle @@ -36,7 +36,7 @@ dependencies { } group 'theodolite' -version '0.5.0-SNAPSHOT' +version '0.6.0-SNAPSHOT' java { sourceCompatibility = JavaVersion.VERSION_11