.gitlab-ci.yml

 workflow:
   rules:
     - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
-    - if: '$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push"'
+    - if: '$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_REF_PROTECTED != "true"'
       when: never
     - when: always
 
@@ -10,6 +10,7 @@ stages:
   - test
   - check
   - deploy
+  - smoketest
 
 default:
   tags:
@@ -20,9 +21,9 @@ default:
     - exec-dind
   # see https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#tls-enabled
   # for image usage and settings for building with TLS and docker in docker
-  image: docker:19.03.1
+  image: docker:20.10.12
   services:
-    - docker:19.03.1-dind
+    - docker:20.10.12-dind
   variables:
     DOCKER_TLS_CERTDIR: "/certs"
 
@@ -45,6 +46,10 @@ default:
       fi
     - "[ $DOCKERFILE ] && KANIKO_DOCKERFILE=\"--dockerfile $DOCKERFILE\""
     - /kaniko/executor --context `pwd`/$CONTEXT $KANIKO_DOCKERFILE $KANIKO_D
+    - echo "PUBLISHED_IMAGE_TAG=${CI_COMMIT_TAG-$CI_COMMIT_SHORT_SHA}" >> build.env
+  artifacts:
+    reports:
+      dotenv: build.env
 
 
 # Theodolite Docs
@@ -372,7 +377,167 @@ deploy-uc4-load-generator:
     IMAGE_NAME: "theodolite-uc4-workload-generator"
     JAVA_PROJECT_NAME: "uc4-load-generator"
     JAVA_PROJECT_DEPS: "load-generator-commons"
-      
+
+.smoketest-benchmarks:
+  stage: smoketest
+  extends:
+    - .dind
+  image: ghcr.io/cau-se/theodolite-build-docker-compose-jq:20.10.12
+  before_script:
+    - cd theodolite-benchmarks/docker-test
+  # variables:
+  #   TEST_LOG_FILE: "test.log"
+  script:
+    - export THEODOLITE_TAG=$PUBLISHED_IMAGE_TAG
+    - ./smoketest-runner.sh ./$DOCKER_COMPOSE_DIR
+    # - cat test.log
+  after_script:
+    - cd ./$DOCKER_COMPOSE_DIR
+    - docker-compose down
+  rules:
+    - changes:
+      - theodolite-benchmarks/*
+      - theodolite-benchmarks/{$JAVA_PROJECT_DEPS}/**/*
+      if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW && $DOCKER_COMPOSE_DIR && $JAVA_PROJECT_DEPS"
+    - if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW && $DOCKER_COMPOSE_DIR && $JAVA_PROJECT_DEPS"
+      when: manual
+      allow_failure: true
+
+smoketest-uc1-kstreams:
+  extends: .smoketest-benchmarks
+  needs:
+    - deploy-uc1-kstreams
+    - deploy-uc1-load-generator
+  variables:
+    DOCKER_COMPOSE_DIR: "uc1-kstreams"
+    JAVA_PROJECT_DEPS: "uc1-kstreams,kstreams-commons,uc1-load-generator,load-generator-commons"
+
+smoketest-uc1-flink:
+  extends: .smoketest-benchmarks
+  needs:
+    - deploy-uc1-flink
+    - deploy-uc1-load-generator
+  variables:
+    DOCKER_COMPOSE_DIR: "uc1-flink"
+    JAVA_PROJECT_DEPS: "uc1-flink,flink-commons,uc1-load-generator,load-generator-commons"
+
+smoketest-uc1-beam-flink:
+  extends: .smoketest-benchmarks
+  needs:
+    - deploy-uc1-beam-flink
+    - deploy-uc1-load-generator
+  variables:
+    DOCKER_COMPOSE_DIR: "uc1-beam-flink"
+    JAVA_PROJECT_DEPS: "uc1-beam-flink,uc1-beam,beam-commons,uc1-load-generator,load-generator-commons"
+
+smoketest-uc1-beam-samza:
+  extends: .smoketest-benchmarks
+  needs:
+    - deploy-uc1-beam-samza
+    - deploy-uc1-load-generator
+  variables:
+    DOCKER_COMPOSE_DIR: "uc1-beam-samza"
+    JAVA_PROJECT_DEPS: "uc1-beam-samza,uc1-beam,beam-commons,uc1-load-generator,load-generator-commons"
+
+smoketest-uc2-kstreams:
+  extends: .smoketest-benchmarks
+  needs:
+    - deploy-uc2-kstreams
+    - deploy-uc2-load-generator
+  variables:
+    DOCKER_COMPOSE_DIR: "uc2-kstreams"
+    JAVA_PROJECT_DEPS: "uc2-kstreams,kstreams-commons,uc2-load-generator,load-generator-commons"
+
+smoketest-uc2-flink:
+  extends: .smoketest-benchmarks
+  needs:
+    - deploy-uc2-flink
+    - deploy-uc2-load-generator
+  variables:
+    DOCKER_COMPOSE_DIR: "uc2-flink"
+    JAVA_PROJECT_DEPS: "uc2-flink,flink-commons,uc2-load-generator,load-generator-commons"
+
+smoketest-uc2-beam-flink:
+  extends: .smoketest-benchmarks
+  needs:
+    - deploy-uc2-beam-flink
+    - deploy-uc2-load-generator
+  variables:
+    DOCKER_COMPOSE_DIR: "uc2-beam-flink"
+    JAVA_PROJECT_DEPS: "uc2-beam-flink,uc2-beam,beam-commons,uc2-load-generator,load-generator-commons"
+
+smoketest-uc2-beam-samza:
+  extends: .smoketest-benchmarks
+  needs:
+    - deploy-uc2-beam-samza
+    - deploy-uc2-load-generator
+  variables:
+    DOCKER_COMPOSE_DIR: "uc2-beam-samza"
+    JAVA_PROJECT_DEPS: "uc2-beam-samza,uc2-beam,beam-commons,uc2-load-generator,load-generator-commons"
+
+smoketest-uc3-kstreams:
+  extends: .smoketest-benchmarks
+  needs:
+    - deploy-uc3-kstreams
+    - deploy-uc3-load-generator
+  variables:
+    DOCKER_COMPOSE_DIR: "uc3-kstreams"
+    JAVA_PROJECT_DEPS: "uc3-kstreams,kstreams-commons,uc3-load-generator,load-generator-commons"
+
+smoketest-uc3-beam-flink:
+  extends: .smoketest-benchmarks
+  needs:
+    - deploy-uc3-beam-flink
+    - deploy-uc3-load-generator
+  variables:
+    DOCKER_COMPOSE_DIR: "uc3-beam-flink"
+    JAVA_PROJECT_DEPS: "uc3-beam-flink,uc3-beam,beam-commons,uc3-load-generator,load-generator-commons"
+
+smoketest-uc3-beam-samza:
+  extends: .smoketest-benchmarks
+  needs:
+    - deploy-uc3-beam-samza
+    - deploy-uc3-load-generator
+  variables:
+    DOCKER_COMPOSE_DIR: "uc3-beam-samza"
+    JAVA_PROJECT_DEPS: "uc3-beam-samza,uc3-beam,beam-commons,uc3-load-generator,load-generator-commons"
+
+smoketest-uc4-kstreams:
+  extends: .smoketest-benchmarks
+  needs:
+    - deploy-uc4-kstreams
+    - deploy-uc4-load-generator
+  variables:
+    DOCKER_COMPOSE_DIR: "uc4-kstreams"
+    JAVA_PROJECT_DEPS: "uc4-kstreams,kstreams-commons,uc4-load-generator,load-generator-commons"
+
+smoketest-uc4-flink:
+  extends: .smoketest-benchmarks
+  needs:
+    - deploy-uc4-flink
+    - deploy-uc4-load-generator
+  variables:
+    DOCKER_COMPOSE_DIR: "uc4-flink"
+    JAVA_PROJECT_DEPS: "uc4-flink,flink-commons,uc4-load-generator,load-generator-commons"
+
+smoketest-uc4-beam-flink:
+  extends: .smoketest-benchmarks
+  needs:
+    - deploy-uc4-beam-flink
+    - deploy-uc4-load-generator
+  variables:
+    DOCKER_COMPOSE_DIR: "uc4-beam-flink"
+    JAVA_PROJECT_DEPS: "uc4-beam-flink,uc4-beam,beam-commons,uc4-load-generator,load-generator-commons"
+
+smoketest-uc4-beam-samza:
+  extends: .smoketest-benchmarks
+  needs:
+    - deploy-uc4-beam-samza
+    - deploy-uc4-load-generator
+  variables:
+    DOCKER_COMPOSE_DIR: "uc4-beam-samza"
+    JAVA_PROJECT_DEPS: "uc4-beam-samza,uc4-beam,beam-commons,uc4-load-generator,load-generator-commons"
+
 
 # Theodolite Framework
 

theodolite-benchmarks/beam-commons/build.gradle

@@ -16,14 +16,14 @@ dependencies {
   implementation('org.industrial-devops:titan-ccp-common:0.1.0-SNAPSHOT') { changing = true }
   implementation('org.industrial-devops:titan-ccp-common-kafka:0.1.0-SNAPSHOT') { changing = true }
 
-  implementation group: 'org.apache.beam', name: 'beam-sdks-java-core', version: '2.22.0'
-  implementation('org.apache.beam:beam-sdks-java-io-kafka:2.22.0'){
+  implementation group: 'org.apache.beam', name: 'beam-sdks-java-core', version: '2.35.0'
+  implementation('org.apache.beam:beam-sdks-java-io-kafka:2.35.0'){
     exclude group: 'org.apache.kafka', module: 'kafka-clients'
   }
   implementation ('io.confluent:kafka-streams-avro-serde:5.3.2') 
   
   implementation group: 'org.slf4j', name: 'slf4j-simple', version: '1.7.30'
-  
+
   runtimeOnly 'org.slf4j:slf4j-api:1.7.32'
   runtimeOnly 'org.slf4j:slf4j-jdk14:1.7.32'
 

theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.beam.flink.gradle

@@ -3,6 +3,6 @@ plugins {
 }
 
 dependencies {
-    implementation group: 'org.apache.beam', name: 'beam-runners-flink-1.12', version: '2.27.0'
-    implementation group: 'org.apache.flink', name: 'flink-statebackend-rocksdb_2.11', version: '1.12.0'
+    implementation group: 'org.apache.beam', name: 'beam-runners-flink-1.13', version: '2.35.0'
+    implementation group: 'org.apache.flink', name: 'flink-statebackend-rocksdb_2.11', version: '1.13.0'
 }
\ No newline at end of file

theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.beam.gradle

@@ -18,7 +18,7 @@ repositories {
     }
 }
 
-def apacheBeamVersion =  '2.22.0' //'2.27.0' // '2.34.0'
+def apacheBeamVersion =  '2.35.0'
 
 dependencies {
     // These dependencies are used internally, and not exposed to consumers on their own compile classpath.

theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.beam.samza.gradle

@@ -3,7 +3,8 @@ plugins {
 }
 
 dependencies {
-    implementation('org.apache.beam:beam-runners-samza:2.22.0') {
+    implementation('org.apache.beam:beam-runners-samza:2.35.0') {
         exclude group: 'org.apache.samza', module: 'samza-yarn_2.11'
     }
+    implementation 'org.apache.samza:samza-kafka_2.11:1.5.0'
 }
\ No newline at end of file

theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.flink.gradle

@@ -20,7 +20,7 @@ shadowJar {
 tasks.distZip.enabled = false
 
 ext {
-  flinkVersion = '1.12.2'
+  flinkVersion = '1.13.5'
   scalaBinaryVersion = '2.12'
 }
 

theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.kstreams.gradle

@@ -22,7 +22,7 @@ dependencies {
     // These dependencies are used internally, and not exposed to consumers on their own compile classpath.
     implementation('org.industrial-devops:titan-ccp-common:0.1.0-SNAPSHOT') { changing = true }
     implementation('org.industrial-devops:titan-ccp-common-kafka:0.1.0-SNAPSHOT') { changing = true }
-    implementation 'org.apache.kafka:kafka-streams:2.6.0' // enable TransformerSuppliers
+    implementation 'org.apache.kafka:kafka-streams:3.1.0'
     implementation 'com.google.code.gson:gson:2.8.2'
     implementation 'com.google.guava:guava:24.1-jre'
     implementation 'org.slf4j:slf4j-simple:1.7.25'

theodolite-benchmarks/config/spotbugs-exclude-filter.xml

@@ -5,4 +5,7 @@
     <Rank value="16" />
   </Match>
 
+  <!-- Temporary disabled due to potential false positive reported in https://github.com/spotbugs/spotbugs/issues/1923. -->
+  <Bug code="NP" />
+
 </FindBugsFilter>
\ No newline at end of file

theodolite-benchmarks/docker-test/README.md

@@ -36,3 +36,19 @@ the host, for example, from the IDE or Gradle. In such cases, the following adju
 
 You can now connect to Kafka from your host system with bootstrap server `localhost:19092` and contact the Schema
 Registry via `localhost:8081`. **Pay attention to the Kafka port, which is *19092* instead of the default one *9092*.**
+
+## Running Smoke Tests
+
+The `smoketest-runner.sh` script can be used to run a simple test for a specific Docker Compose file. You can call it with
+
+```sh
+./smoketest-runner.sh <docker-compose-dir>
+```
+
+where `<docker-compose-dir>` is the directory of a Docker-Compose file, for example, `uc2-beam-samza`. The script exists with a zero exit code in case of success and a non-zero exit code otherwise.
+
+You can also run the set of all smoke test with:
+
+```sh
+./smoketest-runner-all.sh
+```

theodolite-benchmarks/docker-test/smoketest-runner-all.sh

+#!/bin/sh
+
+find . -name 'test.sh' -type f -exec dirname {} \; |
+    sort |
+    xargs -I %s sh -c "./smoketest-runner.sh %s 1>&2; echo $?" |
+    sort |
+    awk 'BEGIN {count[0]=0; count[1]=0} {count[$1!=0]++} END {print count[0] " tests successful, " count[1] " test failed."; exit count[1]}'

theodolite-benchmarks/docker-test/smoketest-runner.sh

+#!/bin/sh
+
+COMPOSE_FILE_PATH=$1
+echo "Run test for '$COMPOSE_FILE_PATH'."
+
+cd $COMPOSE_FILE_PATH
+docker-compose pull -q
+docker-compose up -d kafka zookeeper schema-registry
+sleep 30s
+docker-compose up -d
+sleep 5s
+docker-compose ps
+
+if test -f "./test.sh"; then
+    #timeout --foreground 3m ./test.sh
+    ./test.sh
+    RETURN=$?
+else
+    RETURN=$?
+    echo "test.sh does not exists for '$COMPOSE_FILE_PATH'." 
+fi
+if [ $RETURN -eq 0 ]; then
+    echo "Test for '$COMPOSE_FILE_PATH' has passed."
+else
+    echo "Test for '$COMPOSE_FILE_PATH' has failed."
+fi
+
+docker-compose down
+exit $RETURN

theodolite-benchmarks/docker-test/uc1-beam-flink/docker-compose.yml

@@ -20,18 +20,23 @@ services:
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1"
+  kcat:
+    image: edenhill/kcat:1.7.1
+    entrypoint: "sh"
+    tty: true
   schema-registry:
-    image: confluentinc/cp-schema-registry:5.3.1
+    image: confluentinc/cp-schema-registry:7.0.1
     depends_on:
       - zookeeper
       - kafka
+    restart: "on-failure"
     expose:
       - "8081"
     #ports:
     #  - 8081:8081
     environment:
       SCHEMA_REGISTRY_HOST_NAME: schema-registry
-      SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
+      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092
   load-generator: 
     image: ghcr.io/cau-se/theodolite-uc1-workload-generator:${THEODOLITE_TAG:-latest}
     depends_on:

theodolite-benchmarks/docker-test/uc1-beam-flink/test.sh

+#!/bin/sh
+
+sleep 55s # to let the benchmark and produce some output
+docker-compose logs --tail 100 benchmark-taskmanager |
+    sed -n "s/^.*Key:\s\(\S*\), Value:\s\(\S*\).*$/\2/p" |
+    tee /dev/stderr |
+    jq .identifier |
+    sort |
+    uniq |
+    wc -l |
+    grep "\b10\b"

theodolite-benchmarks/docker-test/uc1-beam-samza/docker-compose.yml

@@ -22,18 +22,23 @@ services:
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1"
+  kcat:
+    image: edenhill/kcat:1.7.1
+    entrypoint: "sh"
+    tty: true
   schema-registry:
-    image: confluentinc/cp-schema-registry:5.3.1
+    image: confluentinc/cp-schema-registry:7.0.1
     depends_on:
       - zookeeper
       - kafka
+    restart: "on-failure"
     expose:
       - "8081"
     #ports:
     #  - 8081:8081
     environment:
       SCHEMA_REGISTRY_HOST_NAME: schema-registry
-      SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
+      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092
   benchmark:
     image: ghcr.io/cau-se/theodolite-uc1-beam-samza:${THEODOLITE_TAG:-latest}
     scale: 1

theodolite-benchmarks/docker-test/uc1-beam-samza/test.sh

+#!/bin/sh
+
+sleep 55s # to let the benchmark and produce some output
+docker-compose logs --tail 100 benchmark |
+    sed -n "s/^.*Key:\s\(\S*\), Value:\s\(\S*\).*$/\2/p" |
+    tee /dev/stderr |
+    jq .identifier |
+    sort |
+    uniq |
+    wc -l |
+    grep "\b10\b"
+
+

theodolite-benchmarks/docker-test/uc1-flink-docker-compose/docker-compose.yml → theodolite-benchmarks/docker-test/uc1-flink/docker-compose.yml

-version: '2'
+version: '2.2'
 services:
   zookeeper:
     image: confluentinc/cp-zookeeper
@@ -20,18 +20,23 @@ services:
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1"
+  kcat:
+    image: edenhill/kcat:1.7.1
+    entrypoint: "sh"
+    tty: true
   schema-registry:
-    image: confluentinc/cp-schema-registry:5.3.1
+    image: confluentinc/cp-schema-registry:7.0.1
     depends_on:
       - zookeeper
       - kafka
+    restart: "on-failure"
     expose:
       - "8081"
     #ports:
     #  - 8081:8081
     environment:
       SCHEMA_REGISTRY_HOST_NAME: schema-registry
-      SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
+      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092
   load-generator: 
     image: ghcr.io/cau-se/theodolite-uc1-workload-generator:${THEODOLITE_TAG:-latest}
     depends_on:

theodolite-benchmarks/docker-test/uc1-flink/test.sh

+#!/bin/sh
+
+sleep 55s # to let the benchmark and produce some output
+docker-compose logs --tail 100 benchmark-taskmanager |
+    sed -n "s/^.*Record:\s\(\S*\)$/\1/p" |
+    tee /dev/stderr |
+    jq .identifier |
+    sort |
+    uniq |
+    wc -l |
+    grep "\b10\b"

theodolite-benchmarks/docker-test/uc1-kstreams-docker-compose/docker-compose.yml → theodolite-benchmarks/docker-test/uc1-kstreams/docker-compose.yml

-version: '2'
+version: '2.2'
 services:
   zookeeper:
     image: confluentinc/cp-zookeeper
@@ -20,20 +20,25 @@ services:
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1"
+  kcat:
+    image: edenhill/kcat:1.7.1
+    entrypoint: "sh"
+    tty: true
   schema-registry:
-    image: confluentinc/cp-schema-registry:5.3.1
+    image: confluentinc/cp-schema-registry:7.0.1
     depends_on:
       - zookeeper
       - kafka
+    restart: "on-failure"
     expose:
       - "8081"
     #ports:
     #  - 8081:8081
     environment:
       SCHEMA_REGISTRY_HOST_NAME: schema-registry
-      SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
+      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092
   benchmark:
-    image: ghcr.io/cau-se/theodolite-uc1-kstreams-app:${THEODOLITE_TAG:-latest}
+    image: ghcr.io/cau-se/theodolite-uc1-kstreams-app:latest
     depends_on:
       - schema-registry
       - kafka

theodolite-benchmarks/docker-test/uc1-kstreams/test.sh

+#!/bin/sh
+
+sleep 55s # to let the benchmark and produce some output
+docker-compose logs --tail 100 benchmark |
+    sed -n "s/^.*Record:\s\(\S*\)$/\1/p" |
+    tee /dev/stderr |
+    jq .identifier |
+    sort |
+    uniq |
+    wc -l |
+    grep "\b10\b"

theodolite-benchmarks/docker-test/uc2-beam-flink/docker-compose.yml

@@ -20,18 +20,23 @@ services:
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
       KAFKA_CREATE_TOPICS: "input:3:1,output:3:1"
+  kcat:
+    image: edenhill/kcat:1.7.1
+    entrypoint: "sh"
+    tty: true
   schema-registry:
-    image: confluentinc/cp-schema-registry:5.3.1
+    image: confluentinc/cp-schema-registry:7.0.1
     depends_on:
       - zookeeper
       - kafka
+    restart: "on-failure"
     expose:
       - "8081"
     #ports:
     #  - 8081:8081
     environment:
       SCHEMA_REGISTRY_HOST_NAME: schema-registry
-      SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
+      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092
   load-generator: 
     image: ghcr.io/cau-se/theodolite-uc2-workload-generator:${THEODOLITE_TAG:-latest}
     depends_on: