Migrate Beam benchmark implementation

Co-authored-by: Jan Bensien stu128012@mail.uni-kiel.de

This merge request will fix #147 (closed). It is based on the existing implementation in the https://git.se.informatik.uni-kiel.de/stu128012/theodolite/-/tree/beam_current_theodolite fork but has to tackle the following requirements:

• Migrate the individual benchmarks with their new enumeration
  • Migrate UC1
  • Migrate UC2 (f.k.a. UC3)
  • Migrate UC3 (f.k.a. UC4)
  • Migrate UC4 (f.k.a. UC2)
• Migrate to Titan Control Center Commons 0.1.0 (Avro records and Schema Registry are already used)
• Fix code quality issues if any
• Support Docker builds (already available in the base branch)
• Build and push Docker images in CI pipeline
• Update Docker Compose files in docker-test
• Share code among Beam runners (either create subprojects or dedicated Gradle tasks or dedicated Dockerfiles or environment variables or ...)
  • Current approach is to use different subprojects. Other approaches would increase the complexity and is currently not possible because of:
    • https://issues.apache.org/jira/browse/BEAM-9830
    • https://stackoverflow.com/questions/61441333/conflict-with-runner-dependencies-in-beam
    • build beam commons project (for builing configurations, common PT transformations)
    • outsource uc1-4 pipelines into single projects
    • outsource uc1-4 beam samza into single projects
    • outsource uc1-4 beam flink into single projects
• Clean up Gradle build files
  • Build Gradle plugins:
    • Build beam commons plugin
    • Build beam flink plugin
    • Build beam samza plugin
  • Check if same versions of dependecies are used (slf4, etc)
• Provide corresponding Kubernetes benchmark definitions -> Moved to #281 (closed)
• Solve #278 (closed)
• Fix #293 (closed)
• Solve #291 (not part of this MR)
• Make samza configurable through environment variables
• Resolve ./gradlew build --warning-mode all
• @she Image names in Docker Compose files must be replaced when merging and pushing images

Depends on !188 (merged)

.gitlab-ci.yml

@@ -129,172 +129,152 @@ spotbugs-benchmarks:
     - changes:
       - theodolite-benchmarks/*
       - theodolite-benchmarks/$JAVA_PROJECT_NAME/**/*
-      - theodolite-benchmarks/kstreams-commons/**/*
-      - theodolite-benchmarks/flink-commons/**/*
-      - theodolite-benchmarks/beam-commons/**/*
-      - theodolite-benchmarks/load-generator-commons/**/*
-      if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW && $IMAGE_NAME && $JAVA_PROJECT_NAME"
+      - theodolite-benchmarks/{$JAVA_PROJECT_DEPS}/**/*
+      if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW && $IMAGE_NAME && $JAVA_PROJECT_NAME && $JAVA_PROJECT_DEPS"
       when: always
     - if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW && $IMAGE_NAME && $JAVA_PROJECT_NAME"
       when: manual
       allow_failure: true
 
-# ---------------------- Kstreams:
 deploy-uc1-kstreams:
   extends: .deploy-benchmarks
   variables:
     IMAGE_NAME: "theodolite-uc1-kstreams-app"
     JAVA_PROJECT_NAME: "uc1-kstreams"
+    JAVA_PROJECT_DEPS: "kstreams-commons"
 
 deploy-uc2-kstreams:
   extends: .deploy-benchmarks
   variables:
     IMAGE_NAME: "theodolite-uc2-kstreams-app"
     JAVA_PROJECT_NAME: "uc2-kstreams"
+    JAVA_PROJECT_DEPS: "kstreams-commons"
 
 deploy-uc3-kstreams:
   extends: .deploy-benchmarks
   variables:
     IMAGE_NAME: "theodolite-uc3-kstreams-app"
     JAVA_PROJECT_NAME: "uc3-kstreams"
+    JAVA_PROJECT_DEPS: "kstreams-commons"
 
 deploy-uc4-kstreams:
   extends: .deploy-benchmarks
   variables:
     IMAGE_NAME: "theodolite-uc4-kstreams-app"
     JAVA_PROJECT_NAME: "uc4-kstreams"
+    JAVA_PROJECT_DEPS: "kstreams-commons"
 
-# ---------------------- Flink:
 deploy-uc1-flink:
   extends: .deploy-benchmarks
   variables:
     IMAGE_NAME: "theodolite-uc1-flink"
     JAVA_PROJECT_NAME: "uc1-flink"
+    JAVA_PROJECT_DEPS: "flink-commons"
 
 deploy-uc2-flink:
   extends: .deploy-benchmarks
   variables:
     IMAGE_NAME: "theodolite-uc2-flink"
     JAVA_PROJECT_NAME: "uc2-flink"
+    JAVA_PROJECT_DEPS: "flink-commons"
 
 deploy-uc3-flink:
   extends: .deploy-benchmarks
   variables:
     IMAGE_NAME: "theodolite-uc3-flink"
     JAVA_PROJECT_NAME: "uc3-flink"
+    JAVA_PROJECT_DEPS: "flink-commons"
 
 deploy-uc4-flink:
   extends: .deploy-benchmarks
   variables:
     IMAGE_NAME: "theodolite-uc4-flink"
     JAVA_PROJECT_NAME: "uc4-flink"
+    JAVA_PROJECT_DEPS: "flink-commons"
 
-# ---------------------- Beam - Flink :
 deploy-uc1-beam-flink:
   extends: .deploy-benchmarks
-  rules:
-    - changes:
-        - theodolite-benchmarks/uc1-beam/**/*
-      when: always
   variables:
     IMAGE_NAME: "theodolite-uc1-beam-flink"
     JAVA_PROJECT_NAME: "uc1-beam-flink"
+    JAVA_PROJECT_DEPS: "beam-commons,uc1-beam"
 
 deploy-uc2-beam-flink:
   extends: .deploy-benchmarks
-  rules:
-    - changes:
-        - theodolite-benchmarks/uc2-beam/**/*
-      when: always
   variables:
     IMAGE_NAME: "theodolite-uc2-beam-flink"
     JAVA_PROJECT_NAME: "uc2-beam-flink"
+    JAVA_PROJECT_DEPS: "beam-commons,uc2-beam"
 
 deploy-uc3-beam-flink:
   extends: .deploy-benchmarks
-  rules:
-    - changes:
-        - theodolite-benchmarks/uc3-beam/**/*
-      when: always
   variables:
     IMAGE_NAME: "theodolite-uc3-beam-flink"
     JAVA_PROJECT_NAME: "uc3-beam-flink"
+    JAVA_PROJECT_DEPS: "beam-commons,uc3-beam"
 
 deploy-uc4-beam-flink:
   extends: .deploy-benchmarks
-  rules:
-    - changes:
-      -  theodolite-benchmarks/uc4-beam/**/*
-      when: always
   variables:
     IMAGE_NAME: "theodolite-uc4-beam-flink"
     JAVA_PROJECT_NAME: "uc4-beam-flink"
+    JAVA_PROJECT_DEPS: "beam-commons,uc4-beam"
 
-# ---------------------- Beam - Samza:
 deploy-uc1-beam-samza:
   extends: .deploy-benchmarks
-  rules:
-    - changes:
-      - theodolite-benchmarks/uc1-beam/**/*
-      when: always
   variables:
     IMAGE_NAME: "theodolite-uc1-beam-samza"
     JAVA_PROJECT_NAME: "uc1-beam-samza"
+    JAVA_PROJECT_DEPS: "beam-commons,uc1-beam"
 
 deploy-uc2-beam-samza:
   extends: .deploy-benchmarks
-  rules:
-    - changes:
-      - theodolite-benchmarks/uc2-beam/**/*
-      when: always
   variables:
     IMAGE_NAME: "theodolite-uc2-beam-samza"
     JAVA_PROJECT_NAME: "uc2-beam-samza"
+    JAVA_PROJECT_DEPS: "beam-commons,uc2-beam"
 
 deploy-uc3-beam-samza:
   extends: .deploy-benchmarks
-  rules:
-    - changes:
-      - theodolite-benchmarks/uc3-beam/**/*
-      when: always
   variables:
     IMAGE_NAME: "theodolite-uc3-beam-samza"
     JAVA_PROJECT_NAME: "uc3-beam-samza"
+    JAVA_PROJECT_DEPS: "beam-commons,uc3-beam"
 
 deploy-uc4-beam-samza:
   extends: .deploy-benchmarks
-  rules:
-    - changes:
-      - theodolite-benchmarks/uc4-beam/**/*
-      when: always
   variables:
     IMAGE_NAME: "theodolite-uc4-beam-samza"
     JAVA_PROJECT_NAME: "uc4-beam-samza"
+    JAVA_PROJECT_DEPS: "beam-commons,uc4-beam"
 
-# ---------------------- Loadgenerator:
 deploy-uc1-load-generator:
   extends: .deploy-benchmarks
   variables:
     IMAGE_NAME: "theodolite-uc1-workload-generator"
     JAVA_PROJECT_NAME: "uc1-load-generator"
+    JAVA_PROJECT_DEPS: "load-generator-commons"
 
 deploy-uc2-load-generator:
   extends: .deploy-benchmarks
   variables:
     IMAGE_NAME: "theodolite-uc2-workload-generator"
     JAVA_PROJECT_NAME: "uc2-load-generator"
+    JAVA_PROJECT_DEPS: "load-generator-commons"
 
 deploy-uc3-load-generator:
   extends: .deploy-benchmarks
   variables:
     IMAGE_NAME: "theodolite-uc3-workload-generator"
     JAVA_PROJECT_NAME: "uc3-load-generator"
+    JAVA_PROJECT_DEPS: "load-generator-commons"
 
 deploy-uc4-load-generator:
   extends: .deploy-benchmarks
   variables:
     IMAGE_NAME: "theodolite-uc4-workload-generator"
     JAVA_PROJECT_NAME: "uc4-load-generator"
+    JAVA_PROJECT_DEPS: "load-generator-commons"
       
 
 # Theodolite Framework