From 661c834d2f0de280fedff2d76d4ba5d941448b76 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Vonheiden?= <bjoern.vonheiden@hotmail.de>
Date: Fri, 12 Mar 2021 14:52:47 +0100
Subject: [PATCH] Extract streams, flink and commons dependencies to convention
 plugin

Extract the kafka streams and flink use case and load generator
and commons dependencies to a convention plugin to reduce the
complexety of the main build gradle file.
---
 .../build.gradle                              |   7 ++
 theodolite-benchmarks/build.gradle            | 118 +-----------------
 theodolite-benchmarks/buildSrc/build.gradle   |   1 +
 .../src/main/groovy/theodolite.java-uc.gradle |   7 --
 .../main/groovy/theodolite.uc-flink.gradle    |  63 ++++++++++
 .../groovy/theodolite.uc-kafka-streams.gradle |  31 +++++
 .../main/groovy/theodolite.uc-workload.gradle |  30 +++++
 .../flink-commons/build.gradle                |   7 ++
 .../uc1-application-flink/build.gradle        |   2 +-
 .../uc1-application/build.gradle              |   2 +-
 .../uc1-workload-generator/build.gradle       |   2 +-
 .../uc2-application-flink/build.gradle        |   2 +-
 .../uc2-application/build.gradle              |   2 +-
 .../uc2-workload-generator/build.gradle       |   2 +-
 .../uc3-application-flink/build.gradle        |   2 +-
 .../uc3-application/build.gradle              |   2 +-
 .../uc3-workload-generator/build.gradle       |   2 +-
 .../uc4-application-flink/build.gradle        |   2 +-
 .../uc4-application/build.gradle              |   2 +-
 .../uc4-workload-generator/build.gradle       |   2 +-
 .../workload-generator-commons/build.gradle   |  10 ++
 21 files changed, 162 insertions(+), 136 deletions(-)
 delete mode 100644 theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.java-uc.gradle
 create mode 100644 theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.uc-flink.gradle
 create mode 100644 theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.uc-kafka-streams.gradle
 create mode 100644 theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.uc-workload.gradle

diff --git a/theodolite-benchmarks/application-kafkastreams-commons/build.gradle b/theodolite-benchmarks/application-kafkastreams-commons/build.gradle
index 01daacba9..e177aa5c4 100644
--- a/theodolite-benchmarks/application-kafkastreams-commons/build.gradle
+++ b/theodolite-benchmarks/application-kafkastreams-commons/build.gradle
@@ -2,6 +2,13 @@ plugins {
     id 'theodolite.java-commons'
 }
 
+repositories {
+  jcenter()
+  maven {
+    url "https://oss.sonatype.org/content/repositories/snapshots/"
+  }
+}
+
 dependencies {
   // These dependencies are used internally, and not exposed to consumers on their own compile classpath.
   // implementation 'org.slf4j:slf4j-simple:1.7.25'
diff --git a/theodolite-benchmarks/build.gradle b/theodolite-benchmarks/build.gradle
index 3395dcf80..3d980c333 100644
--- a/theodolite-benchmarks/build.gradle
+++ b/theodolite-benchmarks/build.gradle
@@ -1,4 +1,5 @@
 // Inherited to all subprojects
+// Make shadow plugin available, used for flink
 buildscript {
   repositories {
     maven {
@@ -10,133 +11,16 @@ buildscript {
   }
 }
 
-// Variables used to distinct different subprojects
-def useCaseProjects = subprojects.findAll {it -> it.name.matches('uc(.)*')}
-def useCaseApplications = subprojects.findAll {it -> it.name.matches('uc[0-9]+-application')}
-def useCaseApplicationsFlink = subprojects.findAll {it -> it.name.matches('uc[0-9]+-application-flink')}
-def useCaseGenerators = subprojects.findAll {it -> it.name.matches('uc[0-9]+-workload-generator*')}
-
 // Plugins
 allprojects {
   apply plugin: 'eclipse'
 }
 
-subprojects {
-  apply plugin: 'theodolite.java-conventions'
-}
-
-configure(useCaseProjects){
-    apply plugin: 'application'
-}
-
-configure(useCaseApplicationsFlink){
-    apply plugin: 'com.github.johnrengelman.shadow'
-    applicationDefaultJvmArgs = ["-Dlog4j.configuration=log4j.properties"]
-}
-
 // Check for updates every build
 configurations.all {
     resolutionStrategy.cacheChangingModulesFor 0, 'seconds'
 }
 
-// Repositories for all projects
-allprojects {
-	repositories {
-	    jcenter()
-	    maven {
-	    	url "https://oss.sonatype.org/content/repositories/snapshots/"
-	    }
-	    maven {
-	        url 'https://packages.confluent.io/maven/'
-	    }
-    	maven {
-        	url 'https://repository.apache.org/content/repositories/snapshots/'
-    	} // TODO required?
-    }
-}
-
-// Dependencies for all Kafka Streams benchmarks (use case applications)
-configure(useCaseApplications) {
-  dependencies {
-      // These dependencies are used internally, and not exposed to consumers on their own compile classpath.
-      implementation('org.industrial-devops:titan-ccp-common:0.1.0-SNAPSHOT') { changing = true }
-      implementation('org.industrial-devops:titan-ccp-common-kafka:0.1.0-SNAPSHOT') { changing = true }
-      implementation 'org.apache.kafka:kafka-streams:2.6.0' // enable TransformerSuppliers
-      implementation 'com.google.code.gson:gson:2.8.2'
-      implementation 'com.google.guava:guava:24.1-jre'
-      implementation 'org.slf4j:slf4j-simple:1.7.25'
-      implementation project(':application-kafkastreams-commons')
-
-      // Use JUnit test framework
-      testImplementation 'junit:junit:4.12'
-  }
-}
-
-// Dependencies for all Flink benchmarks (use case applications)
-configure(useCaseApplicationsFlink) {
-  ext {
-    flinkVersion = '1.12.2'
-    scalaBinaryVersion = '2.12'
-  }
-
-  dependencies {
-      // Special version required because of https://issues.apache.org/jira/browse/FLINK-13703
-      implementation('org.industrial-devops:titan-ccp-common:0.1.0-flink-ready-SNAPSHOT') { changing = true }
-      implementation('org.industrial-devops:titan-ccp-common-kafka:0.1.0-SNAPSHOT') { changing = true }
-
-      // These dependencies are used internally, and not exposed to consumers on their own compile classpath.
-      implementation 'org.apache.kafka:kafka-clients:2.2.0'
-      implementation 'com.google.guava:guava:24.1-jre'
-      implementation 'org.slf4j:slf4j-simple:1.6.1'
-      compile project(':flink-commons')
-
-      //compile group: 'org.apache.kafka', name: 'kafka-clients', version: "2.2.0"
-      compile group: 'org.apache.flink', name: 'flink-java', version: "${flinkVersion}"
-      compile group: 'org.apache.flink', name: "flink-streaming-java_${scalaBinaryVersion}", version:"${flinkVersion}"
-      compile group: 'org.apache.flink', name: "flink-table-api-java-bridge_${scalaBinaryVersion}", version: "${flinkVersion}"
-      compile group: 'org.apache.flink', name: "flink-table-planner-blink_${scalaBinaryVersion}", version: "${flinkVersion}"
-      compile group: 'org.apache.flink', name: "flink-connector-kafka_${scalaBinaryVersion}", version: "${flinkVersion}"
-      implementation "org.apache.flink:flink-avro:${flinkVersion}"
-      implementation "org.apache.flink:flink-avro-confluent-registry:${flinkVersion}"
-      compile group: 'org.apache.flink', name: "flink-runtime-web_${scalaBinaryVersion}", version: "${flinkVersion}" // TODO: remove after development
-      compile group: 'org.apache.flink', name: "flink-statebackend-rocksdb_${scalaBinaryVersion}", version: "${flinkVersion}"
-      compile group: 'org.apache.flink', name: "flink-metrics-prometheus_${scalaBinaryVersion}", version: "${flinkVersion}"
-
-      // Use JUnit test framework
-      testImplementation 'junit:junit:4.12'
-  }
-
-  run.classpath = sourceSets.main.runtimeClasspath
-
-  jar {
-      manifest {
-          attributes 'Built-By': System.getProperty('user.name'),
-                     'Build-Jdk': System.getProperty('java.version')
-      }
-  }
-
-  shadowJar {
-      configurations = [project.configurations.compile]
-      zip64 true
-  }
-}
-
-// Dependencies for all load generators
-configure(useCaseGenerators) {
-  dependencies {
-      // These dependencies are used internally, and not exposed to consumers on their own compile classpath.
-      implementation('org.industrial-devops:titan-ccp-common:0.1.0-SNAPSHOT') { changing = true }
-      implementation('org.industrial-devops:titan-ccp-common-kafka:0.1.0-SNAPSHOT') { changing = true }
-      implementation 'org.slf4j:slf4j-simple:1.7.25'
-
-      // These dependencies are used for the workload-generator-commmon
-      implementation project(':workload-generator-commons')
-
-      // Use JUnit test framework
-      testImplementation 'junit:junit:4.12'
-  }
-}
-
 allprojects {
   eclipse {
       classpath {
diff --git a/theodolite-benchmarks/buildSrc/build.gradle b/theodolite-benchmarks/buildSrc/build.gradle
index 7abe60a90..f6acd3b4c 100644
--- a/theodolite-benchmarks/buildSrc/build.gradle
+++ b/theodolite-benchmarks/buildSrc/build.gradle
@@ -1,3 +1,4 @@
+// to discover the precompiled script plugins
 plugins {
     id 'groovy-gradle-plugin'
 }
diff --git a/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.java-uc.gradle b/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.java-uc.gradle
deleted file mode 100644
index b182c9d7f..000000000
--- a/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.java-uc.gradle
+++ /dev/null
@@ -1,7 +0,0 @@
-plugins {
-  // common java conventions
-  id 'theodolite.java-conventions'
-
-  // make executable
-  id 'application'
-}
diff --git a/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.uc-flink.gradle b/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.uc-flink.gradle
new file mode 100644
index 000000000..c19f930d8
--- /dev/null
+++ b/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.uc-flink.gradle
@@ -0,0 +1,63 @@
+plugins {
+  id 'theodolite.java-conventions'
+  id 'application' // executable
+  id 'com.github.johnrengelman.shadow' // create fat jar
+}
+
+applicationDefaultJvmArgs = ["-Dlog4j.configuration=log4j.properties"]
+
+ext {
+  flinkVersion = '1.12.2'
+  scalaBinaryVersion = '2.12'
+}
+
+repositories {
+  jcenter()
+  maven {
+    url "https://oss.sonatype.org/content/repositories/snapshots/"
+  }
+  maven {
+      url 'https://packages.confluent.io/maven/'
+  }
+}
+
+dependencies {
+    // Special version required because of https://issues.apache.org/jira/browse/FLINK-13703
+    implementation('org.industrial-devops:titan-ccp-common:0.1.0-flink-ready-SNAPSHOT') { changing = true }
+    implementation('org.industrial-devops:titan-ccp-common-kafka:0.1.0-SNAPSHOT') { changing = true }
+
+    // These dependencies are used internally, and not exposed to consumers on their own compile classpath.
+    implementation 'org.apache.kafka:kafka-clients:2.2.0'
+    implementation 'com.google.guava:guava:24.1-jre'
+    implementation 'org.slf4j:slf4j-simple:1.6.1'
+    compile project(':flink-commons')
+
+    //compile group: 'org.apache.kafka', name: 'kafka-clients', version: "2.2.0"
+    compile group: 'org.apache.flink', name: 'flink-java', version: "${flinkVersion}"
+    compile group: 'org.apache.flink', name: "flink-streaming-java_${scalaBinaryVersion}", version:"${flinkVersion}"
+    compile group: 'org.apache.flink', name: "flink-table-api-java-bridge_${scalaBinaryVersion}", version: "${flinkVersion}"
+    compile group: 'org.apache.flink', name: "flink-table-planner-blink_${scalaBinaryVersion}", version: "${flinkVersion}"
+    compile group: 'org.apache.flink', name: "flink-connector-kafka_${scalaBinaryVersion}", version: "${flinkVersion}"
+    implementation "org.apache.flink:flink-avro:${flinkVersion}"
+    implementation "org.apache.flink:flink-avro-confluent-registry:${flinkVersion}"
+    compile group: 'org.apache.flink', name: "flink-runtime-web_${scalaBinaryVersion}", version: "${flinkVersion}" // TODO: remove after development
+    compile group: 'org.apache.flink', name: "flink-statebackend-rocksdb_${scalaBinaryVersion}", version: "${flinkVersion}"
+    compile group: 'org.apache.flink', name: "flink-metrics-prometheus_${scalaBinaryVersion}", version: "${flinkVersion}"
+
+    // Use JUnit test framework
+    testImplementation 'junit:junit:4.12'
+}
+
+run.classpath = sourceSets.main.runtimeClasspath
+
+jar {
+    manifest {
+        attributes 'Built-By': System.getProperty('user.name'),
+                   'Build-Jdk': System.getProperty('java.version')
+    }
+}
+
+shadowJar {
+    configurations = [project.configurations.compile]
+    zip64 true
+}
diff --git a/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.uc-kafka-streams.gradle b/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.uc-kafka-streams.gradle
new file mode 100644
index 000000000..c664038cf
--- /dev/null
+++ b/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.uc-kafka-streams.gradle
@@ -0,0 +1,31 @@
+plugins {
+  // common java conventions
+  id 'theodolite.java-conventions'
+
+  // make executable
+  id 'application'
+}
+
+repositories {
+  jcenter()
+  maven {
+    url "https://oss.sonatype.org/content/repositories/snapshots/"
+  }
+  maven {
+      url 'https://packages.confluent.io/maven/'
+  }
+}
+
+dependencies {
+    // These dependencies are used internally, and not exposed to consumers on their own compile classpath.
+    implementation('org.industrial-devops:titan-ccp-common:0.1.0-SNAPSHOT') { changing = true }
+    implementation('org.industrial-devops:titan-ccp-common-kafka:0.1.0-SNAPSHOT') { changing = true }
+    implementation 'org.apache.kafka:kafka-streams:2.6.0' // enable TransformerSuppliers
+    implementation 'com.google.code.gson:gson:2.8.2'
+    implementation 'com.google.guava:guava:24.1-jre'
+    implementation 'org.slf4j:slf4j-simple:1.7.25'
+    implementation project(':application-kafkastreams-commons')
+
+    // Use JUnit test framework
+    testImplementation 'junit:junit:4.12'
+}
diff --git a/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.uc-workload.gradle b/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.uc-workload.gradle
new file mode 100644
index 000000000..2c1f9509a
--- /dev/null
+++ b/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.uc-workload.gradle
@@ -0,0 +1,30 @@
+plugins {
+  // common java conventions
+  id 'theodolite.java-conventions'
+
+  // make executable
+  id 'application'
+}
+
+repositories {
+  jcenter()
+  maven {
+    url "https://oss.sonatype.org/content/repositories/snapshots/"
+  }
+  maven {
+      url 'https://packages.confluent.io/maven/'
+  }
+}
+
+dependencies {
+  // These dependencies are used internally, and not exposed to consumers on their own compile classpath.
+  implementation('org.industrial-devops:titan-ccp-common:0.1.0-SNAPSHOT') { changing = true }
+  implementation('org.industrial-devops:titan-ccp-common-kafka:0.1.0-SNAPSHOT') { changing = true }
+  implementation 'org.slf4j:slf4j-simple:1.7.25'
+
+  // These dependencies are used for the workload-generator-commmon
+  implementation project(':workload-generator-commons')
+
+  // Use JUnit test framework
+  testImplementation 'junit:junit:4.12'
+}
diff --git a/theodolite-benchmarks/flink-commons/build.gradle b/theodolite-benchmarks/flink-commons/build.gradle
index a1f048b67..cff49b845 100644
--- a/theodolite-benchmarks/flink-commons/build.gradle
+++ b/theodolite-benchmarks/flink-commons/build.gradle
@@ -7,6 +7,13 @@ ext {
     scalaBinaryVersion = '2.12'
 }
 
+repositories {
+  jcenter()
+  maven {
+    url "https://oss.sonatype.org/content/repositories/snapshots/"
+  }
+}
+
 dependencies {
     // Special version required because of https://issues.apache.org/jira/browse/FLINK-13703
     implementation('org.industrial-devops:titan-ccp-common:0.1.0-flink-ready-SNAPSHOT') { changing = true }
diff --git a/theodolite-benchmarks/uc1-application-flink/build.gradle b/theodolite-benchmarks/uc1-application-flink/build.gradle
index 5626314b0..8a57c61ba 100644
--- a/theodolite-benchmarks/uc1-application-flink/build.gradle
+++ b/theodolite-benchmarks/uc1-application-flink/build.gradle
@@ -1,5 +1,5 @@
 plugins {
-  id 'theodolite.java-uc'
+  id 'theodolite.uc-flink'
 }
 
 mainClassName = "theodolite.uc1.application.HistoryServiceFlinkJob"
diff --git a/theodolite-benchmarks/uc1-application/build.gradle b/theodolite-benchmarks/uc1-application/build.gradle
index c13b89627..3c4f6cda2 100644
--- a/theodolite-benchmarks/uc1-application/build.gradle
+++ b/theodolite-benchmarks/uc1-application/build.gradle
@@ -1,5 +1,5 @@
 plugins {
-  id 'theodolite.java-uc'
+  id 'theodolite.uc-kafka-streams'
 }
 
 mainClassName = "theodolite.uc1.application.HistoryService"
diff --git a/theodolite-benchmarks/uc1-workload-generator/build.gradle b/theodolite-benchmarks/uc1-workload-generator/build.gradle
index 490dbb663..cec494c8e 100644
--- a/theodolite-benchmarks/uc1-workload-generator/build.gradle
+++ b/theodolite-benchmarks/uc1-workload-generator/build.gradle
@@ -1,5 +1,5 @@
 plugins {
-  id 'theodolite.java-uc'
+  id 'theodolite.uc-workload'
 }
 
 mainClassName = "theodolite.uc1.workloadgenerator.LoadGenerator"
diff --git a/theodolite-benchmarks/uc2-application-flink/build.gradle b/theodolite-benchmarks/uc2-application-flink/build.gradle
index 173c3ceac..ee30302ac 100644
--- a/theodolite-benchmarks/uc2-application-flink/build.gradle
+++ b/theodolite-benchmarks/uc2-application-flink/build.gradle
@@ -1,5 +1,5 @@
 plugins {
-  id 'theodolite.java-uc'
+  id 'theodolite.uc-flink'
 }
 
 allprojects {
diff --git a/theodolite-benchmarks/uc2-application/build.gradle b/theodolite-benchmarks/uc2-application/build.gradle
index 919555031..da9ea85fa 100644
--- a/theodolite-benchmarks/uc2-application/build.gradle
+++ b/theodolite-benchmarks/uc2-application/build.gradle
@@ -1,5 +1,5 @@
 plugins {
-  id 'theodolite.java-uc'
+  id 'theodolite.uc-kafka-streams'
 }
 
 mainClassName = "theodolite.uc2.application.HistoryService"
diff --git a/theodolite-benchmarks/uc2-workload-generator/build.gradle b/theodolite-benchmarks/uc2-workload-generator/build.gradle
index 498e05fc8..e949b625b 100644
--- a/theodolite-benchmarks/uc2-workload-generator/build.gradle
+++ b/theodolite-benchmarks/uc2-workload-generator/build.gradle
@@ -1,5 +1,5 @@
 plugins {
-  id 'theodolite.java-uc'
+  id 'theodolite.uc-workload'
 }
 
 mainClassName = "theodolite.uc2.workloadgenerator.LoadGenerator"
diff --git a/theodolite-benchmarks/uc3-application-flink/build.gradle b/theodolite-benchmarks/uc3-application-flink/build.gradle
index a1ed6ba07..20801fa5f 100644
--- a/theodolite-benchmarks/uc3-application-flink/build.gradle
+++ b/theodolite-benchmarks/uc3-application-flink/build.gradle
@@ -1,5 +1,5 @@
 plugins {
-  id 'theodolite.java-uc'
+  id 'theodolite.uc-flink'
 }
 
 allprojects {
diff --git a/theodolite-benchmarks/uc3-application/build.gradle b/theodolite-benchmarks/uc3-application/build.gradle
index ee4917278..ac6bd76b5 100644
--- a/theodolite-benchmarks/uc3-application/build.gradle
+++ b/theodolite-benchmarks/uc3-application/build.gradle
@@ -1,5 +1,5 @@
 plugins {
-  id 'theodolite.java-uc'
+  id 'theodolite.uc-kafka-streams'
 }
 
 mainClassName = "theodolite.uc3.application.HistoryService"
diff --git a/theodolite-benchmarks/uc3-workload-generator/build.gradle b/theodolite-benchmarks/uc3-workload-generator/build.gradle
index 22103aaf2..d536e6e5a 100644
--- a/theodolite-benchmarks/uc3-workload-generator/build.gradle
+++ b/theodolite-benchmarks/uc3-workload-generator/build.gradle
@@ -1,5 +1,5 @@
 plugins {
-  id 'theodolite.java-uc'
+  id 'theodolite.uc-workload'
 }
 
 mainClassName = "theodolite.uc3.workloadgenerator.LoadGenerator"
diff --git a/theodolite-benchmarks/uc4-application-flink/build.gradle b/theodolite-benchmarks/uc4-application-flink/build.gradle
index 9ef66cb39..e5466514f 100644
--- a/theodolite-benchmarks/uc4-application-flink/build.gradle
+++ b/theodolite-benchmarks/uc4-application-flink/build.gradle
@@ -1,5 +1,5 @@
 plugins {
-  id 'theodolite.java-uc'
+  id 'theodolite.uc-flink'
 }
 
 mainClassName = "theodolite.uc4.application.AggregationServiceFlinkJob"
diff --git a/theodolite-benchmarks/uc4-application/build.gradle b/theodolite-benchmarks/uc4-application/build.gradle
index 061650e31..8486e9302 100644
--- a/theodolite-benchmarks/uc4-application/build.gradle
+++ b/theodolite-benchmarks/uc4-application/build.gradle
@@ -1,5 +1,5 @@
 plugins {
-  id 'theodolite.java-uc'
+  id 'theodolite.uc-kafka-streams'
 }
 
 mainClassName = "theodolite.uc4.application.AggregationService"
diff --git a/theodolite-benchmarks/uc4-workload-generator/build.gradle b/theodolite-benchmarks/uc4-workload-generator/build.gradle
index 5d419e67c..14b39ccd5 100644
--- a/theodolite-benchmarks/uc4-workload-generator/build.gradle
+++ b/theodolite-benchmarks/uc4-workload-generator/build.gradle
@@ -1,5 +1,5 @@
 plugins {
-  id 'theodolite.java-uc'
+  id 'theodolite.uc-workload'
 }
 
 mainClassName = "theodolite.uc4.workloadgenerator.LoadGenerator"
diff --git a/theodolite-benchmarks/workload-generator-commons/build.gradle b/theodolite-benchmarks/workload-generator-commons/build.gradle
index 06dd5b2b8..118f3e648 100644
--- a/theodolite-benchmarks/workload-generator-commons/build.gradle
+++ b/theodolite-benchmarks/workload-generator-commons/build.gradle
@@ -2,6 +2,16 @@ plugins {
     id 'theodolite.java-commons'
 }
 
+repositories {
+  jcenter()
+  maven {
+    url "https://oss.sonatype.org/content/repositories/snapshots/"
+  }
+  maven {
+      url 'https://packages.confluent.io/maven/'
+  }
+}
+
 dependencies {
   implementation 'com.google.guava:guava:30.1-jre'
   implementation 'com.hazelcast:hazelcast:4.1.1'
-- 
GitLab