Merge branch 'master' into feature/281-Add-Beam-Kubernetes-Benchmark-Definitions

theodolite-benchmarks/README.md

+# Theodolite Benchmarks
+
+Theodolite comes with a set of 4 benchmarks for event-driven microservices, which are implemented with Kafka Streams
+and Apache Flink. The benchmarks are based on typical use cases for stream processing and named: UC1, UC2, UC3 and UC4.
+
+## Building and packaging the benchmarks
+
+All benchmarks can be built with:
+
+```sh
+./gradlew build
+```
+
+This produces `.tar` files in `<benchmark-impl>/build/distribution`, where `<benchmark-impl>` is for example
+`uc1-kstreams`.
+
+## Building Docker images of the benchmarks
+
+Each benchmark implementation directory contains a Dockerfile. To build an image (in this case of `uc1-kstreams`), run:
+
+```sh
+docker build -t theodolite-uc1-kstreams ./uc1-kstreams
+```

theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.flink.gradle

@@ -6,9 +6,6 @@ plugins {
 
 applicationDefaultJvmArgs = ["-Dlog4j.configuration=log4j.properties"]
 
-
-run.classpath = sourceSets.main.runtimeClasspath
-
 jar {
     manifest {
         attributes 'Built-By': System.getProperty('user.name'),
@@ -17,7 +14,6 @@ jar {
 }
 
 shadowJar {
-    configurations = [project.configurations.compile]
     zip64 true
 }
 
@@ -43,24 +39,22 @@ dependencies {
     implementation('org.industrial-devops:titan-ccp-common:0.1.0-flink-ready-SNAPSHOT') { changing = true }
     implementation('org.industrial-devops:titan-ccp-common-kafka:0.1.0-SNAPSHOT') { changing = true }
 
-    // These dependencies are used internally, and not exposed to consumers on their own compile classpath.
     implementation 'org.apache.kafka:kafka-clients:2.2.0'
-    implementation 'com.google.guava:guava:24.1-jre'
+    implementation 'com.google.guava:guava:30.1-jre'
     implementation 'com.google.code.gson:gson:2.8.2'
     implementation 'org.slf4j:slf4j-simple:1.6.1'
     implementation project(':flink-commons')
 
-    //compile group: 'org.apache.kafka', name: 'kafka-clients', version: "2.2.0"
     implementation "org.apache.flink:flink-java:${flinkVersion}"
     implementation "org.apache.flink:flink-streaming-java_${scalaBinaryVersion}:${flinkVersion}"
-    implementation "org.apache.flink:flink-table-api-java-bridge_${scalaBinaryVersion}:${flinkVersion}"
-    implementation "org.apache.flink:flink-table-planner-blink_${scalaBinaryVersion}:${flinkVersion}"
     implementation "org.apache.flink:flink-connector-kafka_${scalaBinaryVersion}:${flinkVersion}"
     implementation "org.apache.flink:flink-avro:${flinkVersion}"
     implementation "org.apache.flink:flink-avro-confluent-registry:${flinkVersion}"
-    implementation "org.apache.flink:flink-runtime-web_${scalaBinaryVersion}:${flinkVersion}" // For debugging
     implementation "org.apache.flink:flink-statebackend-rocksdb_${scalaBinaryVersion}:${flinkVersion}"
     implementation "org.apache.flink:flink-metrics-prometheus_${scalaBinaryVersion}:${flinkVersion}"
+    implementation "org.apache.flink:flink-runtime-web_${scalaBinaryVersion}:${flinkVersion}" // For debugging
+    
+    
 
     // Use JUnit test framework
     testImplementation 'junit:junit:4.12'

theodolite-benchmarks/flink-commons/build.gradle

@@ -20,17 +20,13 @@ repositories {
 dependencies {
     // Special version required because of https://issues.apache.org/jira/browse/FLINK-13703
     implementation('org.industrial-devops:titan-ccp-common:0.1.0-flink-ready-SNAPSHOT') { changing = true }
-    implementation('org.industrial-devops:titan-ccp-common-kafka:0.1.0-SNAPSHOT') { changing = true }
     implementation 'com.google.guava:guava:30.1-jre'
 
-    implementation "org.apache.flink:flink-java:${flinkVersion}"    
+    implementation "org.apache.flink:flink-streaming-java_${scalaBinaryVersion}:${flinkVersion}"
     implementation "org.apache.flink:flink-connector-kafka_${scalaBinaryVersion}:${flinkVersion}"
-    implementation "org.apache.flink:flink-avro:${flinkVersion}"
     implementation "org.apache.flink:flink-avro-confluent-registry:${flinkVersion}"
-    implementation "org.apache.flink:flink-runtime-web_${scalaBinaryVersion}:${flinkVersion}" // For debugging
     implementation "org.apache.flink:flink-statebackend-rocksdb_${scalaBinaryVersion}:${flinkVersion}"
-    implementation "org.apache.flink:flink-metrics-prometheus_${scalaBinaryVersion}:${flinkVersion}"
-    
+
     // Use JUnit test framework
     testImplementation 'junit:junit:4.12'
   }

theodolite-benchmarks/uc2-flink/build.gradle

@@ -2,16 +2,4 @@ plugins {
   id 'theodolite.flink'
 }
 
-allprojects {
-	repositories {
-    	maven {
-    		url 'https://packages.confluent.io/maven/'
-    	}
-	}
-}
-
-dependencies {
-    compile('org.industrial-devops:titan-ccp-common-kafka:0.1.0-SNAPSHOT')
-}
-
 mainClassName = "theodolite.uc2.application.HistoryServiceFlinkJob"

theodolite-benchmarks/uc3-flink/build.gradle

@@ -2,16 +2,4 @@ plugins {
   id 'theodolite.flink'
 }
 
-allprojects {
-	repositories {
-    	maven {
-    		url 'https://packages.confluent.io/maven/'
-    	}
-	}
-}
-
-dependencies {
-    compile('org.industrial-devops:titan-ccp-common-kafka:0.1.0-SNAPSHOT')
-}
-
 mainClassName = "theodolite.uc3.application.HistoryServiceFlinkJob"