Skip to content
Snippets Groups Projects
Commit e03a14ad authored by Sören Henning's avatar Sören Henning
Browse files

Merge branch 'master' into hazelcastjet-she

parents 3939a93a e6bd76b5
No related branches found
No related tags found
1 merge request!208Add benchmark implementations for Hazelcast Jet
Hide whitespace changes
Inline Side-by-side
Showing
with 52 additions and 55 deletions
.gitlab-ci.yml
+ 1
− 1
View file @ e03a14ad
......@@ -235,7 +235,7 @@ deploy-uc4-load-generator:
.theodolite:
image:
name: ghcr.io/graalvm/native-image:java11-21.1.0
name: ghcr.io/graalvm/native-image:java11-21.3.0
entrypoint: [""]
variables:
GRADLE_OPTS: "-Dorg.gradle.daemon=false"
......
helm/values.yaml
+ 2
− 2
View file @ e03a14ad
......@@ -66,7 +66,7 @@ cp-helm-charts:
cp-zookeeper:
enabled: true
nodeSelector: {}
servers: 3 # default: 3
servers: 3
image: confluentinc/cp-zookeeper
imageTag: 5.4.0
## Optionally specify an array of imagePullSecrets. Secrets must be manually created in the namespace.
......@@ -83,7 +83,7 @@ cp-helm-charts:
cp-kafka:
enabled: true
nodeSelector: {}
brokers: 10 # default: 10
brokers: 3
image: confluentinc/cp-enterprise-kafka
imageTag: 5.4.0
## Optionally specify an array of imagePullSecrets. Secrets must be manually created in the namespace.
......
theodolite-benchmarks/README.md 0 → 100644
+ 23
− 0
View file @ e03a14ad
# Theodolite Benchmarks
Theodolite comes with a set of 4 benchmarks for event-driven microservices, which are implemented with Kafka Streams
and Apache Flink. The benchmarks are based on typical use cases for stream processing and named: UC1, UC2, UC3 and UC4.
## Building and packaging the benchmarks
All benchmarks can be built with:
```sh
./gradlew build
```
This produces `.tar` files in `<benchmark-impl>/build/distribution`, where `<benchmark-impl>` is for example
`uc1-kstreams`.
## Building Docker images of the benchmarks
Each benchmark implementation directory contains a Dockerfile. To build an image (in this case of `uc1-kstreams`), run:
```sh
docker build -t theodolite-uc1-kstreams ./uc1-kstreams
```
theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.flink.gradle
+ 4
− 10
View file @ e03a14ad
......@@ -6,9 +6,6 @@ plugins {
applicationDefaultJvmArgs = ["-Dlog4j.configuration=log4j.properties"]
run.classpath = sourceSets.main.runtimeClasspath
jar {
manifest {
attributes 'Built-By': System.getProperty('user.name'),
......@@ -17,7 +14,6 @@ jar {
}
shadowJar {
configurations = [project.configurations.compile]
zip64 true
}
......@@ -43,24 +39,22 @@ dependencies {
implementation('org.industrial-devops:titan-ccp-common:0.1.0-flink-ready-SNAPSHOT') { changing = true }
implementation('org.industrial-devops:titan-ccp-common-kafka:0.1.0-SNAPSHOT') { changing = true }
// These dependencies are used internally, and not exposed to consumers on their own compile classpath.
implementation 'org.apache.kafka:kafka-clients:2.2.0'
implementation 'com.google.guava:guava:24.1-jre'
implementation 'com.google.guava:guava:30.1-jre'
implementation 'com.google.code.gson:gson:2.8.2'
implementation 'org.slf4j:slf4j-simple:1.6.1'
implementation project(':flink-commons')
//compile group: 'org.apache.kafka', name: 'kafka-clients', version: "2.2.0"
implementation "org.apache.flink:flink-java:${flinkVersion}"
implementation "org.apache.flink:flink-streaming-java_${scalaBinaryVersion}:${flinkVersion}"
implementation "org.apache.flink:flink-table-api-java-bridge_${scalaBinaryVersion}:${flinkVersion}"
implementation "org.apache.flink:flink-table-planner-blink_${scalaBinaryVersion}:${flinkVersion}"
implementation "org.apache.flink:flink-connector-kafka_${scalaBinaryVersion}:${flinkVersion}"
implementation "org.apache.flink:flink-avro:${flinkVersion}"
implementation "org.apache.flink:flink-avro-confluent-registry:${flinkVersion}"
implementation "org.apache.flink:flink-runtime-web_${scalaBinaryVersion}:${flinkVersion}" // For debugging
implementation "org.apache.flink:flink-statebackend-rocksdb_${scalaBinaryVersion}:${flinkVersion}"
implementation "org.apache.flink:flink-metrics-prometheus_${scalaBinaryVersion}:${flinkVersion}"
implementation "org.apache.flink:flink-runtime-web_${scalaBinaryVersion}:${flinkVersion}" // For debugging
// Use JUnit test framework
testImplementation 'junit:junit:4.12'
......
theodolite-benchmarks/docker-test/uc1-flink-docker-compose/docker-compose.yml
+ 3
− 2
View file @ e03a14ad
......@@ -18,6 +18,7 @@ services:
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
schema-registry:
image: confluentinc/cp-schema-registry:5.3.1
......@@ -44,8 +45,8 @@ services:
NUM_SENSORS: 10
benchmark-jobmanager:
image: ghcr.io/cau-se/theodolite-uc1-flink:latest
ports:
- "8080:8081"
#ports:
# - "8080:8081"
command: standalone-job --job-classname theodolite.uc1.application.HistoryServiceFlinkJob
environment:
- KAFKA_BOOTSTRAP_SERVERS=kafka:9092
......
theodolite-benchmarks/docker-test/uc1-kstreams-docker-compose/docker-compose.yml
+ 1
− 0
View file @ e03a14ad
......@@ -18,6 +18,7 @@ services:
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
schema-registry:
image: confluentinc/cp-schema-registry:5.3.1
......
theodolite-benchmarks/docker-test/uc2-flink-docker-compose/docker-compose.yml
+ 3
− 2
View file @ e03a14ad
......@@ -18,6 +18,7 @@ services:
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
schema-registry:
image: confluentinc/cp-schema-registry:5.3.1
......@@ -44,8 +45,8 @@ services:
NUM_SENSORS: 10
benchmark-jobmanager:
image: ghcr.io/cau-se/theodolite-uc2-flink:latest
ports:
- "8080:8081"
#ports:
# - "8080:8081"
command: standalone-job --job-classname theodolite.uc2.application.HistoryServiceFlinkJob
environment:
- KAFKA_BOOTSTRAP_SERVERS=kafka:9092
......
theodolite-benchmarks/docker-test/uc2-kstreams-docker-compose/docker-compose.yml
+ 1
− 0
View file @ e03a14ad
......@@ -18,6 +18,7 @@ services:
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
schema-registry:
image: confluentinc/cp-schema-registry:5.3.1
......
theodolite-benchmarks/docker-test/uc3-flink-docker-compose/docker-compose.yml
+ 3
− 2
View file @ e03a14ad
......@@ -18,6 +18,7 @@ services:
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
schema-registry:
image: confluentinc/cp-schema-registry:5.3.1
......@@ -44,8 +45,8 @@ services:
NUM_SENSORS: 10
benchmark-jobmanager:
image: ghcr.io/cau-se/theodolite-uc3-flink:latest
ports:
- "8080:8081"
#ports:
# - "8080:8081"
command: standalone-job --job-classname theodolite.uc3.application.HistoryServiceFlinkJob
environment:
- KAFKA_BOOTSTRAP_SERVERS=kafka:9092
......
theodolite-benchmarks/docker-test/uc3-kstreams-docker-compose/docker-compose.yml
+ 1
− 0
View file @ e03a14ad
......@@ -18,6 +18,7 @@ services:
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
schema-registry:
image: confluentinc/cp-schema-registry:5.3.1
......
theodolite-benchmarks/docker-test/uc4-flink-docker-compose/docker-compose.yml
+ 3
− 2
View file @ e03a14ad
......@@ -18,6 +18,7 @@ services:
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
schema-registry:
image: confluentinc/cp-schema-registry:5.3.1
......@@ -45,8 +46,8 @@ services:
NUM_NESTED_GROUPS: 4
benchmark-jobmanager:
image: ghcr.io/cau-se/theodolite-uc4-flink:latest
ports:
- "8080:8081"
#ports:
# - "8080:8081"
command: standalone-job --job-classname theodolite.uc4.application.AggregationServiceFlinkJob
environment:
- KAFKA_BOOTSTRAP_SERVERS=kafka:9092
......
theodolite-benchmarks/docker-test/uc4-kstreams-docker-compose/docker-compose.yml
+ 1
− 0
View file @ e03a14ad
......@@ -18,6 +18,7 @@ services:
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
schema-registry:
image: confluentinc/cp-schema-registry:5.3.1
......
theodolite-benchmarks/flink-commons/build.gradle
+ 2
− 6
View file @ e03a14ad
......@@ -20,17 +20,13 @@ repositories {
dependencies {
// Special version required because of https://issues.apache.org/jira/browse/FLINK-13703
implementation('org.industrial-devops:titan-ccp-common:0.1.0-flink-ready-SNAPSHOT') { changing = true }
implementation('org.industrial-devops:titan-ccp-common-kafka:0.1.0-SNAPSHOT') { changing = true }
implementation 'com.google.guava:guava:30.1-jre'
implementation "org.apache.flink:flink-java:${flinkVersion}"
implementation "org.apache.flink:flink-streaming-java_${scalaBinaryVersion}:${flinkVersion}"
implementation "org.apache.flink:flink-connector-kafka_${scalaBinaryVersion}:${flinkVersion}"
implementation "org.apache.flink:flink-avro:${flinkVersion}"
implementation "org.apache.flink:flink-avro-confluent-registry:${flinkVersion}"
implementation "org.apache.flink:flink-runtime-web_${scalaBinaryVersion}:${flinkVersion}" // For debugging
implementation "org.apache.flink:flink-statebackend-rocksdb_${scalaBinaryVersion}:${flinkVersion}"
implementation "org.apache.flink:flink-metrics-prometheus_${scalaBinaryVersion}:${flinkVersion}"
// Use JUnit test framework
testImplementation 'junit:junit:4.12'
}
theodolite-benchmarks/uc2-flink/build.gradle
+ 0
− 12
View file @ e03a14ad
......@@ -2,16 +2,4 @@ plugins {
id 'theodolite.flink'
}
allprojects {
repositories {
maven {
url 'https://packages.confluent.io/maven/'
}
}
}
dependencies {
compile('org.industrial-devops:titan-ccp-common-kafka:0.1.0-SNAPSHOT')
}
mainClassName = "theodolite.uc2.application.HistoryServiceFlinkJob"
theodolite-benchmarks/uc3-flink/build.gradle
+ 0
− 12
View file @ e03a14ad
......@@ -2,16 +2,4 @@ plugins {
id 'theodolite.flink'
}
allprojects {
repositories {
maven {
url 'https://packages.confluent.io/maven/'
}
}
}
dependencies {
compile('org.industrial-devops:titan-ccp-common-kafka:0.1.0-SNAPSHOT')
}
mainClassName = "theodolite.uc3.application.HistoryServiceFlinkJob"
theodolite/README.md
+ 3
− 3
View file @ e03a14ad
......@@ -29,7 +29,7 @@ The application can be packaged using:
./gradlew build
```
It produces the `theodolite-1.0.0-SNAPSHOT-runner.jar` file in the `/build` directory. Be aware that it’s not
It produces the `theodolite-0.6.0-SNAPSHOT-runner.jar` file in the `/build` directory. Be aware that it’s not
an _über-jar_ as the dependencies are copied into the `build/lib` directory.
If you want to build an _über-jar_, execute the following command:
......@@ -38,7 +38,7 @@ If you want to build an _über-jar_, execute the following command:
./gradlew build -Dquarkus.package.type=uber-jar
```
The application is now runnable using `java -jar build/theodolite-1.0.0-SNAPSHOT-runner.jar`.
The application is now runnable using `java -jar build/theodolite-0.6.0-SNAPSHOT-runner.jar`.
## Creating a native executable
......@@ -57,7 +57,7 @@ Or, if you don't have GraalVM installed, you can run the native executable build
```
You can then execute your native executable with:
```./build/theodolite-1.0.0-SNAPSHOT-runner```
```./build/theodolite-0.6.0-SNAPSHOT-runner```
If you want to learn more about building native executables, please consult <https://quarkus.io/guides/gradle-tooling>.
......
theodolite/build.gradle
+ 1
− 1
View file @ e03a14ad
......@@ -36,7 +36,7 @@ dependencies {
}
group 'theodolite'
version '0.5.0-SNAPSHOT'
version '0.6.0-SNAPSHOT'
java {
sourceCompatibility = JavaVersion.VERSION_11
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment