diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9634c635c37dd93bfb0bb0f6ef9641cbf0432cb2..cce4be0f92d4abd6344f61541335d9e8f7b97da9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,7 @@ workflow: rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - - if: '$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push"' + - if: '$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_REF_PROTECTED != "true"' when: never - when: always @@ -10,6 +10,7 @@ stages: - test - check - deploy + - smoketest default: tags: @@ -20,9 +21,9 @@ default: - exec-dind # see https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#tls-enabled # for image usage and settings for building with TLS and docker in docker - image: docker:19.03.1 + image: docker:20.10.12 services: - - docker:19.03.1-dind + - docker:20.10.12-dind variables: DOCKER_TLS_CERTDIR: "/certs" @@ -45,6 +46,10 @@ default: fi - "[ $DOCKERFILE ] && KANIKO_DOCKERFILE=\"--dockerfile $DOCKERFILE\"" - /kaniko/executor --context `pwd`/$CONTEXT $KANIKO_DOCKERFILE $KANIKO_D + - echo "PUBLISHED_IMAGE_TAG=${CI_COMMIT_TAG-$CI_COMMIT_SHORT_SHA}" >> build.env + artifacts: + reports: + dotenv: build.env # Theodolite Docs @@ -372,7 +377,167 @@ deploy-uc4-load-generator: IMAGE_NAME: "theodolite-uc4-workload-generator" JAVA_PROJECT_NAME: "uc4-load-generator" JAVA_PROJECT_DEPS: "load-generator-commons" - + +.smoketest-benchmarks: + stage: smoketest + extends: + - .dind + image: ghcr.io/cau-se/theodolite-build-docker-compose-jq:20.10.12 + before_script: + - cd theodolite-benchmarks/docker-test + # variables: + # TEST_LOG_FILE: "test.log" + script: + - export THEODOLITE_TAG=$PUBLISHED_IMAGE_TAG + - ./smoketest-runner.sh ./$DOCKER_COMPOSE_DIR + # - cat test.log + after_script: + - cd ./$DOCKER_COMPOSE_DIR + - docker-compose down + rules: + - changes: + - theodolite-benchmarks/* + - theodolite-benchmarks/{$JAVA_PROJECT_DEPS}/**/* + if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW && $DOCKER_COMPOSE_DIR && $JAVA_PROJECT_DEPS" + - if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW && $DOCKER_COMPOSE_DIR && $JAVA_PROJECT_DEPS" + when: manual + allow_failure: true + +smoketest-uc1-kstreams: + extends: .smoketest-benchmarks + needs: + - deploy-uc1-kstreams + - deploy-uc1-load-generator + variables: + DOCKER_COMPOSE_DIR: "uc1-kstreams" + JAVA_PROJECT_DEPS: "uc1-kstreams,kstreams-commons,uc1-load-generator,load-generator-commons" + +smoketest-uc1-flink: + extends: .smoketest-benchmarks + needs: + - deploy-uc1-flink + - deploy-uc1-load-generator + variables: + DOCKER_COMPOSE_DIR: "uc1-flink" + JAVA_PROJECT_DEPS: "uc1-flink,flink-commons,uc1-load-generator,load-generator-commons" + +smoketest-uc1-beam-flink: + extends: .smoketest-benchmarks + needs: + - deploy-uc1-beam-flink + - deploy-uc1-load-generator + variables: + DOCKER_COMPOSE_DIR: "uc1-beam-flink" + JAVA_PROJECT_DEPS: "uc1-beam-flink,uc1-beam,beam-commons,uc1-load-generator,load-generator-commons" + +smoketest-uc1-beam-samza: + extends: .smoketest-benchmarks + needs: + - deploy-uc1-beam-samza + - deploy-uc1-load-generator + variables: + DOCKER_COMPOSE_DIR: "uc1-beam-samza" + JAVA_PROJECT_DEPS: "uc1-beam-samza,uc1-beam,beam-commons,uc1-load-generator,load-generator-commons" + +smoketest-uc2-kstreams: + extends: .smoketest-benchmarks + needs: + - deploy-uc2-kstreams + - deploy-uc2-load-generator + variables: + DOCKER_COMPOSE_DIR: "uc2-kstreams" + JAVA_PROJECT_DEPS: "uc2-kstreams,kstreams-commons,uc2-load-generator,load-generator-commons" + +smoketest-uc2-flink: + extends: .smoketest-benchmarks + needs: + - deploy-uc2-flink + - deploy-uc2-load-generator + variables: + DOCKER_COMPOSE_DIR: "uc2-flink" + JAVA_PROJECT_DEPS: "uc2-flink,flink-commons,uc2-load-generator,load-generator-commons" + +smoketest-uc2-beam-flink: + extends: .smoketest-benchmarks + needs: + - deploy-uc2-beam-flink + - deploy-uc2-load-generator + variables: + DOCKER_COMPOSE_DIR: "uc2-beam-flink" + JAVA_PROJECT_DEPS: "uc2-beam-flink,uc2-beam,beam-commons,uc2-load-generator,load-generator-commons" + +smoketest-uc2-beam-samza: + extends: .smoketest-benchmarks + needs: + - deploy-uc2-beam-samza + - deploy-uc2-load-generator + variables: + DOCKER_COMPOSE_DIR: "uc2-beam-samza" + JAVA_PROJECT_DEPS: "uc2-beam-samza,uc2-beam,beam-commons,uc2-load-generator,load-generator-commons" + +smoketest-uc3-kstreams: + extends: .smoketest-benchmarks + needs: + - deploy-uc3-kstreams + - deploy-uc3-load-generator + variables: + DOCKER_COMPOSE_DIR: "uc3-kstreams" + JAVA_PROJECT_DEPS: "uc3-kstreams,kstreams-commons,uc3-load-generator,load-generator-commons" + +smoketest-uc3-beam-flink: + extends: .smoketest-benchmarks + needs: + - deploy-uc3-beam-flink + - deploy-uc3-load-generator + variables: + DOCKER_COMPOSE_DIR: "uc3-beam-flink" + JAVA_PROJECT_DEPS: "uc3-beam-flink,uc3-beam,beam-commons,uc3-load-generator,load-generator-commons" + +smoketest-uc3-beam-samza: + extends: .smoketest-benchmarks + needs: + - deploy-uc3-beam-samza + - deploy-uc3-load-generator + variables: + DOCKER_COMPOSE_DIR: "uc3-beam-samza" + JAVA_PROJECT_DEPS: "uc3-beam-samza,uc3-beam,beam-commons,uc3-load-generator,load-generator-commons" + +smoketest-uc4-kstreams: + extends: .smoketest-benchmarks + needs: + - deploy-uc4-kstreams + - deploy-uc4-load-generator + variables: + DOCKER_COMPOSE_DIR: "uc4-kstreams" + JAVA_PROJECT_DEPS: "uc4-kstreams,kstreams-commons,uc4-load-generator,load-generator-commons" + +smoketest-uc4-flink: + extends: .smoketest-benchmarks + needs: + - deploy-uc4-flink + - deploy-uc4-load-generator + variables: + DOCKER_COMPOSE_DIR: "uc4-flink" + JAVA_PROJECT_DEPS: "uc4-flink,flink-commons,uc4-load-generator,load-generator-commons" + +smoketest-uc4-beam-flink: + extends: .smoketest-benchmarks + needs: + - deploy-uc4-beam-flink + - deploy-uc4-load-generator + variables: + DOCKER_COMPOSE_DIR: "uc4-beam-flink" + JAVA_PROJECT_DEPS: "uc4-beam-flink,uc4-beam,beam-commons,uc4-load-generator,load-generator-commons" + +smoketest-uc4-beam-samza: + extends: .smoketest-benchmarks + needs: + - deploy-uc4-beam-samza + - deploy-uc4-load-generator + variables: + DOCKER_COMPOSE_DIR: "uc4-beam-samza" + JAVA_PROJECT_DEPS: "uc4-beam-samza,uc4-beam,beam-commons,uc4-load-generator,load-generator-commons" + # Theodolite Framework diff --git a/docs/theodolite-benchmarks/load-generator.md b/docs/theodolite-benchmarks/load-generator.md index 6d42ea06d9cb008a9aeddcc8145a2868c8d916b1..17845c42d47e94a5b696dee1d774890de8d6fff1 100644 --- a/docs/theodolite-benchmarks/load-generator.md +++ b/docs/theodolite-benchmarks/load-generator.md @@ -17,7 +17,7 @@ For each benchmark, we provide a [load generator as OCI container image](https:/ You can simply run a load generator container, for example, for benchmark UC1 with: ```sh -docker run ghcr.io/cau-se/theodolite-uc1-workload-generator +docker run -it ghcr.io/cau-se/theodolite-uc1-workload-generator ``` ### Message format diff --git a/theodolite-benchmarks/docker-test/README.md b/theodolite-benchmarks/docker-test/README.md index fd1e9bf4730f897273be45a022ad2adeae1b7e6e..5d7ca3f4ac470202579f154fe8f066a246c84d23 100644 --- a/theodolite-benchmarks/docker-test/README.md +++ b/theodolite-benchmarks/docker-test/README.md @@ -36,3 +36,19 @@ the host, for example, from the IDE or Gradle. In such cases, the following adju You can now connect to Kafka from your host system with bootstrap server `localhost:19092` and contact the Schema Registry via `localhost:8081`. **Pay attention to the Kafka port, which is *19092* instead of the default one *9092*.** + +## Running Smoke Tests + +The `smoketest-runner.sh` script can be used to run a simple test for a specific Docker Compose file. You can call it with + +```sh +./smoketest-runner.sh <docker-compose-dir> +``` + +where `<docker-compose-dir>` is the directory of a Docker-Compose file, for example, `uc2-beam-samza`. The script exists with a zero exit code in case of success and a non-zero exit code otherwise. + +You can also run the set of all smoke test with: + +```sh +./smoketest-runner-all.sh +``` diff --git a/theodolite-benchmarks/docker-test/smoketest-runner-all.sh b/theodolite-benchmarks/docker-test/smoketest-runner-all.sh new file mode 100755 index 0000000000000000000000000000000000000000..0129a485d98a90d453b284408b755986f64208de --- /dev/null +++ b/theodolite-benchmarks/docker-test/smoketest-runner-all.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +find . -name 'test.sh' -type f -exec dirname {} \; | + sort | + xargs -I %s sh -c "./smoketest-runner.sh %s 1>&2; echo $?" | + sort | + awk 'BEGIN {count[0]=0; count[1]=0} {count[$1!=0]++} END {print count[0] " tests successful, " count[1] " test failed."; exit count[1]}' diff --git a/theodolite-benchmarks/docker-test/smoketest-runner.sh b/theodolite-benchmarks/docker-test/smoketest-runner.sh new file mode 100755 index 0000000000000000000000000000000000000000..3167c90a567eed3cc2678c80c722dbd0b8684f1e --- /dev/null +++ b/theodolite-benchmarks/docker-test/smoketest-runner.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +COMPOSE_FILE_PATH=$1 +echo "Run test for '$COMPOSE_FILE_PATH'." + +cd $COMPOSE_FILE_PATH +docker-compose pull -q +docker-compose up -d kafka zookeeper schema-registry +sleep 30s +docker-compose up -d +sleep 5s +docker-compose ps + +if test -f "./test.sh"; then + #timeout --foreground 3m ./test.sh + ./test.sh + RETURN=$? +else + RETURN=$? + echo "test.sh does not exists for '$COMPOSE_FILE_PATH'." +fi +if [ $RETURN -eq 0 ]; then + echo "Test for '$COMPOSE_FILE_PATH' has passed." +else + echo "Test for '$COMPOSE_FILE_PATH' has failed." +fi + +docker-compose down +exit $RETURN diff --git a/theodolite-benchmarks/docker-test/uc1-beam-flink/docker-compose.yml b/theodolite-benchmarks/docker-test/uc1-beam-flink/docker-compose.yml index 1b683b4ca65a2582aa6a4d68444c4bbef7895b73..69de6049aca50a6a8e10a9e709b38d322bc8cf61 100644 --- a/theodolite-benchmarks/docker-test/uc1-beam-flink/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc1-beam-flink/docker-compose.yml @@ -20,18 +20,23 @@ services: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1" + kcat: + image: edenhill/kcat:1.7.1 + entrypoint: "sh" + tty: true schema-registry: - image: confluentinc/cp-schema-registry:5.3.1 + image: confluentinc/cp-schema-registry:7.0.1 depends_on: - zookeeper - kafka + restart: "on-failure" expose: - "8081" #ports: # - 8081:8081 environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 load-generator: image: ghcr.io/cau-se/theodolite-uc1-workload-generator:${THEODOLITE_TAG:-latest} depends_on: diff --git a/theodolite-benchmarks/docker-test/uc1-beam-flink/test.sh b/theodolite-benchmarks/docker-test/uc1-beam-flink/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..ebbecd1c5336c5dd907db11b8c8c45924e5924a8 --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc1-beam-flink/test.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +sleep 55s # to let the benchmark and produce some output +docker-compose logs --tail 100 benchmark-taskmanager | + sed -n "s/^.*Key:\s\(\S*\), Value:\s\(\S*\).*$/\2/p" | + tee /dev/stderr | + jq .identifier | + sort | + uniq | + wc -l | + grep "\b10\b" diff --git a/theodolite-benchmarks/docker-test/uc1-beam-samza/docker-compose.yml b/theodolite-benchmarks/docker-test/uc1-beam-samza/docker-compose.yml index f5213799daa2d51eea53e794becdffc151a4da56..2212c3b539045114f31760d605ad928e237ed924 100644 --- a/theodolite-benchmarks/docker-test/uc1-beam-samza/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc1-beam-samza/docker-compose.yml @@ -22,18 +22,23 @@ services: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1" + kcat: + image: edenhill/kcat:1.7.1 + entrypoint: "sh" + tty: true schema-registry: - image: confluentinc/cp-schema-registry:5.3.1 + image: confluentinc/cp-schema-registry:7.0.1 depends_on: - zookeeper - kafka + restart: "on-failure" expose: - "8081" #ports: # - 8081:8081 environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 benchmark: image: ghcr.io/cau-se/theodolite-uc1-beam-samza:${THEODOLITE_TAG:-latest} scale: 1 diff --git a/theodolite-benchmarks/docker-test/uc1-beam-samza/test.sh b/theodolite-benchmarks/docker-test/uc1-beam-samza/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..ed17db3a44d5c4a8dacfbc956c2f36dd47503508 --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc1-beam-samza/test.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +sleep 55s # to let the benchmark and produce some output +docker-compose logs --tail 100 benchmark | + sed -n "s/^.*Key:\s\(\S*\), Value:\s\(\S*\).*$/\2/p" | + tee /dev/stderr | + jq .identifier | + sort | + uniq | + wc -l | + grep "\b10\b" + + diff --git a/theodolite-benchmarks/docker-test/uc1-flink-docker-compose/docker-compose.yml b/theodolite-benchmarks/docker-test/uc1-flink/docker-compose.yml similarity index 90% rename from theodolite-benchmarks/docker-test/uc1-flink-docker-compose/docker-compose.yml rename to theodolite-benchmarks/docker-test/uc1-flink/docker-compose.yml index 6c661bb49cb4173357acc89c06783fe5e0a2ce49..55e359665de56cb03a5049c299761cb07690df30 100755 --- a/theodolite-benchmarks/docker-test/uc1-flink-docker-compose/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc1-flink/docker-compose.yml @@ -1,4 +1,4 @@ -version: '2' +version: '2.2' services: zookeeper: image: confluentinc/cp-zookeeper @@ -20,18 +20,23 @@ services: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1" + kcat: + image: edenhill/kcat:1.7.1 + entrypoint: "sh" + tty: true schema-registry: - image: confluentinc/cp-schema-registry:5.3.1 + image: confluentinc/cp-schema-registry:7.0.1 depends_on: - zookeeper - kafka + restart: "on-failure" expose: - "8081" #ports: # - 8081:8081 environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 load-generator: image: ghcr.io/cau-se/theodolite-uc1-workload-generator:${THEODOLITE_TAG:-latest} depends_on: diff --git a/theodolite-benchmarks/docker-test/uc1-flink/test.sh b/theodolite-benchmarks/docker-test/uc1-flink/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..7c7f11a94f42d56d91d383f27d58ad9a09a918e5 --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc1-flink/test.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +sleep 55s # to let the benchmark and produce some output +docker-compose logs --tail 100 benchmark-taskmanager | + sed -n "s/^.*Record:\s\(\S*\)$/\1/p" | + tee /dev/stderr | + jq .identifier | + sort | + uniq | + wc -l | + grep "\b10\b" diff --git a/theodolite-benchmarks/docker-test/uc1-kstreams-docker-compose/docker-compose.yml b/theodolite-benchmarks/docker-test/uc1-kstreams/docker-compose.yml similarity index 83% rename from theodolite-benchmarks/docker-test/uc1-kstreams-docker-compose/docker-compose.yml rename to theodolite-benchmarks/docker-test/uc1-kstreams/docker-compose.yml index 25c19e35e5dae1807ef46fb8ade4e888dff0c2d8..c85ce305c2f1383a77d4c405d52089ec1d2b02a6 100755 --- a/theodolite-benchmarks/docker-test/uc1-kstreams-docker-compose/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc1-kstreams/docker-compose.yml @@ -1,4 +1,4 @@ -version: '2' +version: '2.2' services: zookeeper: image: confluentinc/cp-zookeeper @@ -20,20 +20,25 @@ services: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1" + kcat: + image: edenhill/kcat:1.7.1 + entrypoint: "sh" + tty: true schema-registry: - image: confluentinc/cp-schema-registry:5.3.1 + image: confluentinc/cp-schema-registry:7.0.1 depends_on: - zookeeper - kafka + restart: "on-failure" expose: - "8081" #ports: # - 8081:8081 environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 benchmark: - image: ghcr.io/cau-se/theodolite-uc1-kstreams-app:${THEODOLITE_TAG:-latest} + image: ghcr.io/cau-se/theodolite-uc1-kstreams-app:latest depends_on: - schema-registry - kafka diff --git a/theodolite-benchmarks/docker-test/uc1-kstreams/test.sh b/theodolite-benchmarks/docker-test/uc1-kstreams/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..da711e3fac1b1d664b1c8487687ceacdddce6efa --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc1-kstreams/test.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +sleep 55s # to let the benchmark and produce some output +docker-compose logs --tail 100 benchmark | + sed -n "s/^.*Record:\s\(\S*\)$/\1/p" | + tee /dev/stderr | + jq .identifier | + sort | + uniq | + wc -l | + grep "\b10\b" diff --git a/theodolite-benchmarks/docker-test/uc2-beam-flink/docker-compose.yml b/theodolite-benchmarks/docker-test/uc2-beam-flink/docker-compose.yml index 8427161e43faa920e011973d76f32f9cc9d62f8c..79f6f040144d3c4da56d469095ad3ddce90026a2 100644 --- a/theodolite-benchmarks/docker-test/uc2-beam-flink/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc2-beam-flink/docker-compose.yml @@ -20,18 +20,23 @@ services: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1,output:3:1" + kcat: + image: edenhill/kcat:1.7.1 + entrypoint: "sh" + tty: true schema-registry: - image: confluentinc/cp-schema-registry:5.3.1 + image: confluentinc/cp-schema-registry:7.0.1 depends_on: - zookeeper - kafka + restart: "on-failure" expose: - "8081" #ports: # - 8081:8081 environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 load-generator: image: ghcr.io/cau-se/theodolite-uc2-workload-generator:${THEODOLITE_TAG:-latest} depends_on: diff --git a/theodolite-benchmarks/docker-test/uc2-beam-flink/test.sh b/theodolite-benchmarks/docker-test/uc2-beam-flink/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..3c33834b0b21bc5dbe4e6a7c3ff947121bb2ce71 --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc2-beam-flink/test.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +until docker-compose exec -T kcat kcat -L -b kafka:9092 -t output -J | jq -r '.topics[0].partitions | length' | grep "\b3\b"; do sleep 5s; done + +docker-compose exec -T kcat kcat -C -b kafka:9092 -t output -s key=s -s value=s -r http://schema-registry:8081 -f '%k:%s\n' -c 20 | + tee /dev/stderr | + awk -F ':' '!/^%/ {print $1}' | + sort | + uniq | + wc -l | + grep "\b10\b" diff --git a/theodolite-benchmarks/docker-test/uc2-beam-samza/docker-compose.yml b/theodolite-benchmarks/docker-test/uc2-beam-samza/docker-compose.yml index 820bb25dd370cf6c7410b20fbdbdb1d4281f47d3..cc6bc7a7112c35f11ce9cfd27d09aebe401c8c51 100644 --- a/theodolite-benchmarks/docker-test/uc2-beam-samza/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc2-beam-samza/docker-compose.yml @@ -22,18 +22,23 @@ services: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1,output:3:1" + kcat: + image: edenhill/kcat:1.7.1 + entrypoint: "sh" + tty: true schema-registry: - image: confluentinc/cp-schema-registry:5.3.1 + image: confluentinc/cp-schema-registry:7.0.1 depends_on: - zookeeper - kafka + restart: "on-failure" expose: - "8081" #ports: # - 8081:8081 environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 benchmark: image: ghcr.io/cau-se/theodolite-uc2-beam-samza:${THEODOLITE_TAG:-latest} scale: 1 diff --git a/theodolite-benchmarks/docker-test/uc2-beam-samza/test.sh b/theodolite-benchmarks/docker-test/uc2-beam-samza/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..3c33834b0b21bc5dbe4e6a7c3ff947121bb2ce71 --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc2-beam-samza/test.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +until docker-compose exec -T kcat kcat -L -b kafka:9092 -t output -J | jq -r '.topics[0].partitions | length' | grep "\b3\b"; do sleep 5s; done + +docker-compose exec -T kcat kcat -C -b kafka:9092 -t output -s key=s -s value=s -r http://schema-registry:8081 -f '%k:%s\n' -c 20 | + tee /dev/stderr | + awk -F ':' '!/^%/ {print $1}' | + sort | + uniq | + wc -l | + grep "\b10\b" diff --git a/theodolite-benchmarks/docker-test/uc2-flink-docker-compose/docker-compose.yml b/theodolite-benchmarks/docker-test/uc2-flink/docker-compose.yml similarity index 89% rename from theodolite-benchmarks/docker-test/uc2-flink-docker-compose/docker-compose.yml rename to theodolite-benchmarks/docker-test/uc2-flink/docker-compose.yml index e51c544bfd64c6f540f5a84a5f617de745a1a6b6..1397c8ab0e7bb8d7ee67b7fa9c18143c404fd9ff 100755 --- a/theodolite-benchmarks/docker-test/uc2-flink-docker-compose/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc2-flink/docker-compose.yml @@ -1,4 +1,4 @@ -version: '2' +version: '2.2' services: zookeeper: image: confluentinc/cp-zookeeper @@ -20,18 +20,23 @@ services: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1,output:3:1" + kcat: + image: edenhill/kcat:1.7.1 + entrypoint: "sh" + tty: true schema-registry: - image: confluentinc/cp-schema-registry:5.3.1 + image: confluentinc/cp-schema-registry:7.0.1 depends_on: - zookeeper - kafka - #ports: - # - "8081:8081" + restart: "on-failure" expose: - "8081" + #ports: + # - 8081:8081 environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 load-generator: image: ghcr.io/cau-se/theodolite-uc2-workload-generator:${THEODOLITE_TAG:-latest} depends_on: diff --git a/theodolite-benchmarks/docker-test/uc2-flink/test.sh b/theodolite-benchmarks/docker-test/uc2-flink/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..3c33834b0b21bc5dbe4e6a7c3ff947121bb2ce71 --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc2-flink/test.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +until docker-compose exec -T kcat kcat -L -b kafka:9092 -t output -J | jq -r '.topics[0].partitions | length' | grep "\b3\b"; do sleep 5s; done + +docker-compose exec -T kcat kcat -C -b kafka:9092 -t output -s key=s -s value=s -r http://schema-registry:8081 -f '%k:%s\n' -c 20 | + tee /dev/stderr | + awk -F ':' '!/^%/ {print $1}' | + sort | + uniq | + wc -l | + grep "\b10\b" diff --git a/theodolite-benchmarks/docker-test/uc2-kstreams-docker-compose/docker-compose.yml b/theodolite-benchmarks/docker-test/uc2-kstreams/docker-compose.yml similarity index 86% rename from theodolite-benchmarks/docker-test/uc2-kstreams-docker-compose/docker-compose.yml rename to theodolite-benchmarks/docker-test/uc2-kstreams/docker-compose.yml index 4cc4978d844a8b361af4061ab41d1adc4e7a1813..efdba90bef634bab76012316f67b0f9be9f79c77 100755 --- a/theodolite-benchmarks/docker-test/uc2-kstreams-docker-compose/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc2-kstreams/docker-compose.yml @@ -1,4 +1,4 @@ -version: '2' +version: '2.2' services: zookeeper: image: confluentinc/cp-zookeeper @@ -20,18 +20,23 @@ services: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1,output:3:1" + kcat: + image: edenhill/kcat:1.7.1 + entrypoint: "sh" + tty: true schema-registry: - image: confluentinc/cp-schema-registry:5.3.1 + image: confluentinc/cp-schema-registry:7.0.1 depends_on: - zookeeper - kafka - #ports: - # - "8081:8081" + restart: "on-failure" expose: - "8081" + #ports: + # - 8081:8081 environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 benchmark: image: ghcr.io/cau-se/theodolite-uc2-kstreams-app:${THEODOLITE_TAG:-latest} depends_on: diff --git a/theodolite-benchmarks/docker-test/uc2-kstreams/test.sh b/theodolite-benchmarks/docker-test/uc2-kstreams/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..3c33834b0b21bc5dbe4e6a7c3ff947121bb2ce71 --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc2-kstreams/test.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +until docker-compose exec -T kcat kcat -L -b kafka:9092 -t output -J | jq -r '.topics[0].partitions | length' | grep "\b3\b"; do sleep 5s; done + +docker-compose exec -T kcat kcat -C -b kafka:9092 -t output -s key=s -s value=s -r http://schema-registry:8081 -f '%k:%s\n' -c 20 | + tee /dev/stderr | + awk -F ':' '!/^%/ {print $1}' | + sort | + uniq | + wc -l | + grep "\b10\b" diff --git a/theodolite-benchmarks/docker-test/uc3-beam-flink/docker-compose.yml b/theodolite-benchmarks/docker-test/uc3-beam-flink/docker-compose.yml index b9e934b61f9742ad4e83601ff13707be0641c875..e68dc94d4185d789272279e8fa2e5d178ffdd14b 100644 --- a/theodolite-benchmarks/docker-test/uc3-beam-flink/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc3-beam-flink/docker-compose.yml @@ -20,18 +20,23 @@ services: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1,output:3:1" + kcat: + image: edenhill/kcat:1.7.1 + entrypoint: "sh" + tty: true schema-registry: - image: confluentinc/cp-schema-registry:5.3.1 + image: confluentinc/cp-schema-registry:7.0.1 depends_on: - zookeeper - kafka + restart: "on-failure" expose: - "8081" #ports: # - 8081:8081 environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 load-generator: image: ghcr.io/cau-se/theodolite-uc3-workload-generator:${THEODOLITE_TAG:-latest} depends_on: diff --git a/theodolite-benchmarks/docker-test/uc3-beam-flink/test.sh b/theodolite-benchmarks/docker-test/uc3-beam-flink/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..0612838a3973a302c9acb3bbfa6b8d59ea2596c5 --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc3-beam-flink/test.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +until docker-compose exec -T kcat kcat -L -b kafka:9092 -t output -J | jq -r '.topics[0].partitions | length' | grep "\b3\b"; do sleep 5s; done + +docker-compose exec -T kcat kcat -C -b kafka:9092 -t output -s key=s -s value=s -r http://schema-registry:8081 -f '%k:%s\n' -c 600 | + tee /dev/stderr | + awk -F ':' '!/^%/ {print $1}' | + sort | + uniq | + wc -l | + grep "\b10\b" diff --git a/theodolite-benchmarks/docker-test/uc3-beam-samza/docker-compose.yml b/theodolite-benchmarks/docker-test/uc3-beam-samza/docker-compose.yml index 3ae3a2faa7c0849b76ea838b09ba0254bdfea936..bf120f31dbfda2384b314ba4a90a25362f37b6c2 100644 --- a/theodolite-benchmarks/docker-test/uc3-beam-samza/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc3-beam-samza/docker-compose.yml @@ -22,18 +22,23 @@ services: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1,output:3:1" + kcat: + image: edenhill/kcat:1.7.1 + entrypoint: "sh" + tty: true schema-registry: - image: confluentinc/cp-schema-registry:5.3.1 + image: confluentinc/cp-schema-registry:7.0.1 depends_on: - zookeeper - kafka + restart: "on-failure" expose: - "8081" #ports: # - 8081:8081 environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 benchmark: image: ghcr.io/cau-se/theodolite-uc3-beam-samza:${THEODOLITE_TAG:-latest} scale: 1 diff --git a/theodolite-benchmarks/docker-test/uc3-beam-samza/test.sh b/theodolite-benchmarks/docker-test/uc3-beam-samza/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..0612838a3973a302c9acb3bbfa6b8d59ea2596c5 --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc3-beam-samza/test.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +until docker-compose exec -T kcat kcat -L -b kafka:9092 -t output -J | jq -r '.topics[0].partitions | length' | grep "\b3\b"; do sleep 5s; done + +docker-compose exec -T kcat kcat -C -b kafka:9092 -t output -s key=s -s value=s -r http://schema-registry:8081 -f '%k:%s\n' -c 600 | + tee /dev/stderr | + awk -F ':' '!/^%/ {print $1}' | + sort | + uniq | + wc -l | + grep "\b10\b" diff --git a/theodolite-benchmarks/docker-test/uc3-flink-docker-compose/docker-compose.yml b/theodolite-benchmarks/docker-test/uc3-flink/docker-compose.yml similarity index 89% rename from theodolite-benchmarks/docker-test/uc3-flink-docker-compose/docker-compose.yml rename to theodolite-benchmarks/docker-test/uc3-flink/docker-compose.yml index 42c55950fbea022f00019699ae72678fbe88b8cb..0c83d8a25d19d4989f982d251893635ea1222cb0 100755 --- a/theodolite-benchmarks/docker-test/uc3-flink-docker-compose/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc3-flink/docker-compose.yml @@ -1,4 +1,4 @@ -version: '2' +version: '2.2' services: zookeeper: image: confluentinc/cp-zookeeper @@ -20,18 +20,23 @@ services: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1,output:3:1" + kcat: + image: edenhill/kcat:1.7.1 + entrypoint: "sh" + tty: true schema-registry: - image: confluentinc/cp-schema-registry:5.3.1 + image: confluentinc/cp-schema-registry:7.0.1 depends_on: - zookeeper - kafka - #ports: - # - "8081:8081" + restart: "on-failure" expose: - "8081" + #ports: + # - 8081:8081 environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 load-generator: image: ghcr.io/cau-se/theodolite-uc3-workload-generator:${THEODOLITE_TAG:-latest} depends_on: diff --git a/theodolite-benchmarks/docker-test/uc3-kstreams-docker-compose/docker-compose.yml b/theodolite-benchmarks/docker-test/uc3-kstreams/docker-compose.yml similarity index 85% rename from theodolite-benchmarks/docker-test/uc3-kstreams-docker-compose/docker-compose.yml rename to theodolite-benchmarks/docker-test/uc3-kstreams/docker-compose.yml index f943de372b75bf64df0028679d56950fedfaec48..7da1fa3b420e2dfa0d6d357723583bc3a256823e 100755 --- a/theodolite-benchmarks/docker-test/uc3-kstreams-docker-compose/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc3-kstreams/docker-compose.yml @@ -1,4 +1,4 @@ -version: '2' +version: '2.2' services: zookeeper: image: confluentinc/cp-zookeeper @@ -20,18 +20,23 @@ services: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1,output:3:1" + kcat: + image: edenhill/kcat:1.7.1 + entrypoint: "sh" + tty: true schema-registry: - image: confluentinc/cp-schema-registry:5.3.1 + image: confluentinc/cp-schema-registry:7.0.1 depends_on: - zookeeper - kafka - #ports: - # - "8081:8081" + restart: "on-failure" expose: - "8081" + #ports: + # - 8081:8081 environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 benchmark: image: ghcr.io/cau-se/theodolite-uc3-kstreams-app:${THEODOLITE_TAG:-latest} depends_on: diff --git a/theodolite-benchmarks/docker-test/uc3-kstreams/test.sh b/theodolite-benchmarks/docker-test/uc3-kstreams/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..0612838a3973a302c9acb3bbfa6b8d59ea2596c5 --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc3-kstreams/test.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +until docker-compose exec -T kcat kcat -L -b kafka:9092 -t output -J | jq -r '.topics[0].partitions | length' | grep "\b3\b"; do sleep 5s; done + +docker-compose exec -T kcat kcat -C -b kafka:9092 -t output -s key=s -s value=s -r http://schema-registry:8081 -f '%k:%s\n' -c 600 | + tee /dev/stderr | + awk -F ':' '!/^%/ {print $1}' | + sort | + uniq | + wc -l | + grep "\b10\b" diff --git a/theodolite-benchmarks/docker-test/uc4-beam-flink/docker-compose.yml b/theodolite-benchmarks/docker-test/uc4-beam-flink/docker-compose.yml index b0d18dfe34c23847b3cc0b04d788fdc4b3552e68..2968b718eb06ade4d42bbe104e214d573881ac14 100644 --- a/theodolite-benchmarks/docker-test/uc4-beam-flink/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc4-beam-flink/docker-compose.yml @@ -20,18 +20,23 @@ services: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1" + kcat: + image: edenhill/kcat:1.7.1 + entrypoint: "sh" + tty: true schema-registry: - image: confluentinc/cp-schema-registry:5.3.1 + image: confluentinc/cp-schema-registry:7.0.1 depends_on: - zookeeper - kafka + restart: "on-failure" expose: - "8081" #ports: # - 8081:8081 environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 load-generator: image: ghcr.io/cau-se/theodolite-uc4-workload-generator:${THEODOLITE_TAG:-latest} depends_on: @@ -43,7 +48,7 @@ services: KAFKA_BOOTSTRAP_SERVERS: kafka:9092 SCHEMA_REGISTRY_URL: http://schema-registry:8081 NUM_SENSORS: 4 - NUM_NESTED_GROUPS: 4 + NUM_NESTED_GROUPS: 3 benchmark-jobmanager: image: ghcr.io/cau-se/theodolite-uc4-beam-flink:${THEODOLITE_TAG:-latest} #ports: diff --git a/theodolite-benchmarks/docker-test/uc4-beam-flink/test.sh b/theodolite-benchmarks/docker-test/uc4-beam-flink/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..6a4c6dbf4f583e7598baefae8f48136bb2113630 --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc4-beam-flink/test.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +until docker-compose exec -T kcat kcat -L -b kafka:9092 -t output -J | jq -r '.topics[0].partitions | length' | grep "\b3\b"; do sleep 5s; done + +docker-compose exec -T kcat kcat -C -b kafka:9092 -t output -s key=s -s value=avro -r http://schema-registry:8081 -f '%k:%s\n' -c 2000 | + tee /dev/stderr | + awk -F ':' '!/^%/ {print $1}' | + sort | + uniq | + wc -l | + grep "\b21\b" diff --git a/theodolite-benchmarks/docker-test/uc4-beam-samza/docker-compose.yml b/theodolite-benchmarks/docker-test/uc4-beam-samza/docker-compose.yml index b9e4142cb0be2701f7772cff5725b524aaffcbc7..d236af7d284ebb085c78110feb6001cb28d18290 100644 --- a/theodolite-benchmarks/docker-test/uc4-beam-samza/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc4-beam-samza/docker-compose.yml @@ -22,18 +22,23 @@ services: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1" + kcat: + image: edenhill/kcat:1.7.1 + entrypoint: "sh" + tty: true schema-registry: - image: confluentinc/cp-schema-registry:5.3.1 + image: confluentinc/cp-schema-registry:7.0.1 depends_on: - zookeeper - kafka + restart: "on-failure" expose: - "8081" #ports: # - 8081:8081 environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 benchmark: image: ghcr.io/cau-se/theodolite-uc4-beam-samza:${THEODOLITE_TAG:-latest} scale: 1 @@ -57,4 +62,4 @@ services: KAFKA_BOOTSTRAP_SERVERS: kafka:9092 SCHEMA_REGISTRY_URL: http://schema-registry:8081 NUM_SENSORS: 4 - NUM_NESTED_GROUPS: 4 + NUM_NESTED_GROUPS: 3 diff --git a/theodolite-benchmarks/docker-test/uc4-beam-samza/test.sh b/theodolite-benchmarks/docker-test/uc4-beam-samza/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..6a4c6dbf4f583e7598baefae8f48136bb2113630 --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc4-beam-samza/test.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +until docker-compose exec -T kcat kcat -L -b kafka:9092 -t output -J | jq -r '.topics[0].partitions | length' | grep "\b3\b"; do sleep 5s; done + +docker-compose exec -T kcat kcat -C -b kafka:9092 -t output -s key=s -s value=avro -r http://schema-registry:8081 -f '%k:%s\n' -c 2000 | + tee /dev/stderr | + awk -F ':' '!/^%/ {print $1}' | + sort | + uniq | + wc -l | + grep "\b21\b" diff --git a/theodolite-benchmarks/docker-test/uc4-flink-docker-compose/docker-compose.yml b/theodolite-benchmarks/docker-test/uc4-flink/docker-compose.yml similarity index 89% rename from theodolite-benchmarks/docker-test/uc4-flink-docker-compose/docker-compose.yml rename to theodolite-benchmarks/docker-test/uc4-flink/docker-compose.yml index 5a5a5924ef13cc9691b267ce6169bdb85a1e267b..24e5acdf94d1ba3d5c2807d172ba054309a2ef4a 100755 --- a/theodolite-benchmarks/docker-test/uc4-flink-docker-compose/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc4-flink/docker-compose.yml @@ -1,4 +1,4 @@ -version: '2' +version: '2.2' services: zookeeper: image: confluentinc/cp-zookeeper @@ -20,18 +20,23 @@ services: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1" + kcat: + image: edenhill/kcat:1.7.1 + entrypoint: "sh" + tty: true schema-registry: - image: confluentinc/cp-schema-registry:5.3.1 + image: confluentinc/cp-schema-registry:7.0.1 depends_on: - zookeeper - kafka + restart: "on-failure" expose: - "8081" #ports: # - 8081:8081 environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 load-generator: image: ghcr.io/cau-se/theodolite-uc4-workload-generator:${THEODOLITE_TAG:-latest} depends_on: @@ -43,7 +48,7 @@ services: KAFKA_BOOTSTRAP_SERVERS: kafka:9092 SCHEMA_REGISTRY_URL: http://schema-registry:8081 NUM_SENSORS: 4 - NUM_NESTED_GROUPS: 4 + NUM_NESTED_GROUPS: 3 benchmark-jobmanager: image: ghcr.io/cau-se/theodolite-uc4-flink:${THEODOLITE_TAG:-latest} #ports: diff --git a/theodolite-benchmarks/docker-test/uc4-flink/test.sh b/theodolite-benchmarks/docker-test/uc4-flink/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..0a478cf6641a1333f65281ae43cb525e32cb2510 --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc4-flink/test.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +until docker-compose exec -T kcat kcat -L -b kafka:9092 -t output -J | jq -r '.topics[0].partitions | length' | grep "\b3\b"; do sleep 5s; done + +docker-compose exec -T kcat kcat -C -b kafka:9092 -t output -s key=s -s value=avro -r http://schema-registry:8081 -f '%k:%s\n' -c 500 | + tee /dev/stderr | + awk -F ':' '!/^%/ {print $1}' | + sort | + uniq | + wc -l | + grep "\b21\b" diff --git a/theodolite-benchmarks/docker-test/uc4-kstreams-docker-compose/docker-compose.yml b/theodolite-benchmarks/docker-test/uc4-kstreams/docker-compose.yml similarity index 86% rename from theodolite-benchmarks/docker-test/uc4-kstreams-docker-compose/docker-compose.yml rename to theodolite-benchmarks/docker-test/uc4-kstreams/docker-compose.yml index 1818505787f662d57f1a96c8c79b03d61cf9da2a..6aaa02990841547edb6059e4e2fbf3b28b50985c 100755 --- a/theodolite-benchmarks/docker-test/uc4-kstreams-docker-compose/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc4-kstreams/docker-compose.yml @@ -1,4 +1,4 @@ -version: '2' +version: '2.2' services: zookeeper: image: confluentinc/cp-zookeeper @@ -20,18 +20,23 @@ services: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1" + kcat: + image: edenhill/kcat:1.7.1 + entrypoint: "sh" + tty: true schema-registry: - image: confluentinc/cp-schema-registry:5.3.1 + image: confluentinc/cp-schema-registry:7.0.1 depends_on: - zookeeper - kafka + restart: "on-failure" expose: - "8081" #ports: # - 8081:8081 environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 benchmark: image: ghcr.io/cau-se/theodolite-uc4-kstreams-app:${THEODOLITE_TAG:-latest} depends_on: @@ -51,4 +56,4 @@ services: KAFKA_BOOTSTRAP_SERVERS: kafka:9092 SCHEMA_REGISTRY_URL: http://schema-registry:8081 NUM_SENSORS: 4 - NUM_NESTED_GROUPS: 4 \ No newline at end of file + NUM_NESTED_GROUPS: 3 \ No newline at end of file diff --git a/theodolite-benchmarks/docker-test/uc4-kstreams/test.sh b/theodolite-benchmarks/docker-test/uc4-kstreams/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..9b9dee7dc78e7a587b9f2e5b778066e5bc099755 --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc4-kstreams/test.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +until docker-compose exec -T kcat kcat -L -b kafka:9092 -t output -J | jq -r '.topics[0].partitions | length' | grep "\b3\b"; do sleep 5s; done + +docker-compose exec -T kcat kcat -C -b kafka:9092 -t output -s key=s -s value=avro -r http://schema-registry:8081 -f '%k:%s\n' -c 32 | + tee /dev/stderr | + awk -F ':' '!/^%/ {print $1}' | + sort | + uniq | + wc -l | + grep "\b16\b" diff --git a/theodolite-benchmarks/uc4-flink/src/main/java/theodolite/uc4/application/AggregationServiceFlinkJob.java b/theodolite-benchmarks/uc4-flink/src/main/java/theodolite/uc4/application/AggregationServiceFlinkJob.java index 63bef67f459b776fe4a26ba2c51f9832e89d94e5..48eadff101905b746ab239f6030decb728c12475 100644 --- a/theodolite-benchmarks/uc4-flink/src/main/java/theodolite/uc4/application/AggregationServiceFlinkJob.java +++ b/theodolite-benchmarks/uc4-flink/src/main/java/theodolite/uc4/application/AggregationServiceFlinkJob.java @@ -14,6 +14,7 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows; import org.apache.flink.streaming.api.windowing.time.Time; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer; +import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumerBase; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer; import org.apache.kafka.common.serialization.Serdes; import org.slf4j.Logger; @@ -100,9 +101,11 @@ public final class AggregationServiceFlinkJob { this.env.getConfig().registerTypeWithKryoSerializer(Set.of(1, 2, 3, 4).getClass(), // NOCS new ImmutableSetSerializer()); - this.env.getConfig().getRegisteredTypesWithKryoSerializers() - .forEach((c, s) -> LOGGER.info("Class " + c.getName() + " registered with serializer " - + s.getSerializer().getClass().getName())); + this.env + .getConfig() + .getRegisteredTypesWithKryoSerializers() + .forEach((c, s) -> LOGGER.info("Class '{}' registered with serializer '{}'.", c.getName(), + s.getSerializer().getClass().getName())); } private void buildPipeline() { @@ -131,12 +134,13 @@ public final class AggregationServiceFlinkJob { final FlinkKafkaConsumer<AggregatedActivePowerRecord> kafkaOutputSource = kafkaConnector.createConsumer(outputTopic, AggregatedActivePowerRecord.class); - final FlinkKafkaConsumer<Tuple2<Event, String>> kafkaConfigSource = + final FlinkKafkaConsumerBase<Tuple2<Event, String>> kafkaConfigSource = kafkaConnector.createConsumer( configurationTopic, EventSerde::serde, Serdes::String, - TupleType.of(TypeInformation.of(Event.class), Types.STRING)); + TupleType.of(TypeInformation.of(Event.class), Types.STRING)) + .setStartFromEarliest(); // Sink to output topic with SensorId, AggregatedActivePowerRecord final FlinkKafkaProducer<Tuple2<String, AggregatedActivePowerRecord>> kafkaAggregationSink =