Merge branch 'upgrade-beam' into 'master'

Upgrade Beam version

Closes #304

See merge request !223

theodolite-benchmarks/beam-commons/build.gradle

@@ -16,14 +16,14 @@ dependencies {
   implementation('org.industrial-devops:titan-ccp-common:0.1.0-SNAPSHOT') { changing = true }
   implementation('org.industrial-devops:titan-ccp-common-kafka:0.1.0-SNAPSHOT') { changing = true }
 
-  implementation group: 'org.apache.beam', name: 'beam-sdks-java-core', version: '2.22.0'
-  implementation('org.apache.beam:beam-sdks-java-io-kafka:2.22.0'){
+  implementation group: 'org.apache.beam', name: 'beam-sdks-java-core', version: '2.35.0'
+  implementation('org.apache.beam:beam-sdks-java-io-kafka:2.35.0'){
     exclude group: 'org.apache.kafka', module: 'kafka-clients'
   }
   implementation ('io.confluent:kafka-streams-avro-serde:5.3.2') 
   
   implementation group: 'org.slf4j', name: 'slf4j-simple', version: '1.7.30'
-  
+
   runtimeOnly 'org.slf4j:slf4j-api:1.7.32'
   runtimeOnly 'org.slf4j:slf4j-jdk14:1.7.32'
 

theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.beam.flink.gradle

@@ -3,6 +3,6 @@ plugins {
 }
 
 dependencies {
-    implementation group: 'org.apache.beam', name: 'beam-runners-flink-1.12', version: '2.27.0'
-    implementation group: 'org.apache.flink', name: 'flink-statebackend-rocksdb_2.11', version: '1.12.0'
+    implementation group: 'org.apache.beam', name: 'beam-runners-flink-1.13', version: '2.35.0'
+    implementation group: 'org.apache.flink', name: 'flink-statebackend-rocksdb_2.11', version: '1.13.0'
 }
\ No newline at end of file

theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.beam.gradle

@@ -18,7 +18,7 @@ repositories {
     }
 }
 
-def apacheBeamVersion =  '2.22.0' //'2.27.0' // '2.34.0'
+def apacheBeamVersion =  '2.35.0'
 
 dependencies {
     // These dependencies are used internally, and not exposed to consumers on their own compile classpath.

theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.beam.samza.gradle

@@ -3,7 +3,8 @@ plugins {
 }
 
 dependencies {
-    implementation('org.apache.beam:beam-runners-samza:2.22.0') {
+    implementation('org.apache.beam:beam-runners-samza:2.35.0') {
         exclude group: 'org.apache.samza', module: 'samza-yarn_2.11'
     }
+    implementation 'org.apache.samza:samza-kafka_2.11:1.5.0'
 }
\ No newline at end of file

theodolite-benchmarks/config/spotbugs-exclude-filter.xml

@@ -5,4 +5,7 @@
     <Rank value="16" />
   </Match>
 
+  <!-- Temporary disabled due to potential false positive reported in https://github.com/spotbugs/spotbugs/issues/1923. -->
+  <Bug code="NP" />
+
 </FindBugsFilter>
\ No newline at end of file

theodolite-benchmarks/uc1-beam-samza/Dockerfile

@@ -5,5 +5,5 @@ ENV MAX_SOURCE_PARALLELISM=1024
 ADD build/distributions/uc1-beam-samza.tar /
 ADD samza-standalone.properties /
 
-CMD /uc1-beam-samza/bin/uc1-beam-samza --configFactory=org.apache.samza.config.factories.PropertiesConfigFactory --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --maxSourceParallelism=$MAX_SOURCE_PARALLELISM --enableMetrics=false --configOverride="{\"job.coordinator.zk.connect\":\"$SAMZA_JOB_COORDINATOR_ZK_CONNECT\"}"
+CMD /uc1-beam-samza/bin/uc1-beam-samza --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --maxSourceParallelism=$MAX_SOURCE_PARALLELISM --enableMetrics=false --configOverride="{\"job.coordinator.zk.connect\":\"$SAMZA_JOB_COORDINATOR_ZK_CONNECT\"}"
 

theodolite-benchmarks/uc2-beam-samza/Dockerfile

@@ -5,4 +5,4 @@ ENV MAX_SOURCE_PARALLELISM=1024
 ADD build/distributions/uc2-beam-samza.tar /
 ADD samza-standalone.properties /
 
-CMD /uc2-beam-samza/bin/uc2-beam-samza --configFactory=org.apache.samza.config.factories.PropertiesConfigFactory --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --maxSourceParallelism=$MAX_SOURCE_PARALLELISM --enableMetrics=false --configOverride="{\"job.coordinator.zk.connect\":\"$SAMZA_JOB_COORDINATOR_ZK_CONNECT\"}"
+CMD /uc2-beam-samza/bin/uc2-beam-samza --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --maxSourceParallelism=$MAX_SOURCE_PARALLELISM --enableMetrics=false --configOverride="{\"job.coordinator.zk.connect\":\"$SAMZA_JOB_COORDINATOR_ZK_CONNECT\"}"

theodolite-benchmarks/uc3-beam-samza/Dockerfile

@@ -5,4 +5,4 @@ ENV MAX_SOURCE_PARALLELISM=1024
 ADD build/distributions/uc3-beam-samza.tar /
 ADD samza-standalone.properties /
 
-CMD /uc3-beam-samza/bin/uc3-beam-samza --configFactory=org.apache.samza.config.factories.PropertiesConfigFactory --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --maxSourceParallelism=$MAX_SOURCE_PARALLELISM --enableMetrics=false --configOverride="{\"job.coordinator.zk.connect\":\"$SAMZA_JOB_COORDINATOR_ZK_CONNECT\"}"
+CMD /uc3-beam-samza/bin/uc3-beam-samza --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --maxSourceParallelism=$MAX_SOURCE_PARALLELISM --enableMetrics=false --configOverride="{\"job.coordinator.zk.connect\":\"$SAMZA_JOB_COORDINATOR_ZK_CONNECT\"}"

theodolite-benchmarks/uc4-beam-samza/Dockerfile

@@ -5,4 +5,4 @@ ENV MAX_SOURCE_PARALLELISM=1024
 ADD build/distributions/uc4-beam-samza.tar /
 ADD samza-standalone.properties /
 
-CMD /uc4-beam-samza/bin/uc4-beam-samza --configFactory=org.apache.samza.config.factories.PropertiesConfigFactory --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --maxSourceParallelism=$MAX_SOURCE_PARALLELISM --enableMetrics=false --configOverride="{\"job.coordinator.zk.connect\":\"$SAMZA_JOB_COORDINATOR_ZK_CONNECT\"}"
+CMD /uc4-beam-samza/bin/uc4-beam-samza --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --maxSourceParallelism=$MAX_SOURCE_PARALLELISM --enableMetrics=false --configOverride="{\"job.coordinator.zk.connect\":\"$SAMZA_JOB_COORDINATOR_ZK_CONNECT\"}"