diff --git a/theodolite-benchmarks/docker-test/uc2-hazelcastjet-docker-compose/docker-compose.yml b/theodolite-benchmarks/docker-test/uc2-hazelcastjet-docker-compose/docker-compose.yml index 051ff1f7c4ae05bae21c3f06ce85d1889033ba6b..7c35f64f21c424a5b6c852bb84e454cd9ce82557 100644 --- a/theodolite-benchmarks/docker-test/uc2-hazelcastjet-docker-compose/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc2-hazelcastjet-docker-compose/docker-compose.yml @@ -1,4 +1,4 @@ -version: '2' +version: '2.2' services: zookeeper: image: confluentinc/cp-zookeeper @@ -19,19 +19,24 @@ services: KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" - KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1" + KAFKA_CREATE_TOPICS: "input:3:1,output:3:1" + kcat: + image: edenhill/kcat:1.7.1 + entrypoint: "sh" + tty: true schema-registry: - image: confluentinc/cp-schema-registry:5.3.1 + image: confluentinc/cp-schema-registry:7.0.1 depends_on: - zookeeper - kafka + restart: "on-failure" expose: - "8081" #ports: # - 8081:8081 environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 benchmark: image: uc2-hazelcastjet depends_on: @@ -47,7 +52,7 @@ services: SCHEMA_REGISTRY_URL: http://schema-registry:8081 DOWNSAMPLE_INTERVAL: 5000 load-generator: - image: ghcr.io/cau-se/theodolite-uc2-workload-generator:latest + image: ghcr.io/cau-se/theodolite-uc2-workload-generator:${THEODOLITE_TAG:-latest} depends_on: - schema-registry - kafka diff --git a/theodolite-benchmarks/docker-test/uc2-hazelcastjet-docker-compose/test.sh b/theodolite-benchmarks/docker-test/uc2-hazelcastjet-docker-compose/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..be411755b74249d90756e445f7e67dc07bf5ebab --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc2-hazelcastjet-docker-compose/test.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +until docker-compose exec -T kcat kcat -L -b kafka:9092 -t output -J | jq -r '.topics[0].partitions | length' | grep "\b3\b"; do sleep 5s; done + +docker-compose exec -T kcat kcat -C -b kafka:9092 -t output -s key=s -s value=s -r http://schema-registry:8081 -f '%k:%s\n' -c 20 | + tee /dev/stderr | + awk -F ':' '!/^%/ {print $1}' | + sort | + uniq | + wc -l | + grep "\b10\b" \ No newline at end of file diff --git a/theodolite-benchmarks/docker-test/uc3-hazelcastjet-docker-compose/docker-compose.yml b/theodolite-benchmarks/docker-test/uc3-hazelcastjet-docker-compose/docker-compose.yml index 27a6713c238224a89899d11f1654d611498e5427..e4fd72601bfdd1958b6cc03aa7a73d3c7487d25e 100644 --- a/theodolite-benchmarks/docker-test/uc3-hazelcastjet-docker-compose/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc3-hazelcastjet-docker-compose/docker-compose.yml @@ -1,4 +1,4 @@ -version: '2' +version: '2.2' services: zookeeper: image: confluentinc/cp-zookeeper @@ -19,19 +19,24 @@ services: KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" - KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1" + KAFKA_CREATE_TOPICS: "input:3:1,output:3:1" + kcat: + image: edenhill/kcat:1.7.1 + entrypoint: "sh" + tty: true schema-registry: - image: confluentinc/cp-schema-registry:5.3.1 + image: confluentinc/cp-schema-registry:7.0.1 depends_on: - zookeeper - kafka + restart: "on-failure" expose: - "8081" #ports: # - 8081:8081 environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 benchmark: image: uc3-hazelcastjet depends_on: @@ -48,7 +53,7 @@ services: WINDOW_SIZE_IN_SECONDS: 50 HOPPING_SIZE_IN_SECONDS: 5 load-generator: - image: ghcr.io/cau-se/theodolite-uc3-workload-generator:latest + image: ghcr.io/cau-se/theodolite-uc3-workload-generator:${THEODOLITE_TAG:-latest} depends_on: - schema-registry - kafka diff --git a/theodolite-benchmarks/docker-test/uc3-hazelcastjet-docker-compose/test.sh b/theodolite-benchmarks/docker-test/uc3-hazelcastjet-docker-compose/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..217a730f73fa1fee3f875da34edd9047ed9221db --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc3-hazelcastjet-docker-compose/test.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +until docker-compose exec -T kcat kcat -L -b kafka:9092 -t output -J | jq -r '.topics[0].partitions | length' | grep "\b3\b"; do sleep 5s; done + +docker-compose exec -T kcat kcat -C -b kafka:9092 -t output -s key=s -s value=s -r http://schema-registry:8081 -f '%k:%s\n' -c 600 | + tee /dev/stderr | + awk -F ':' '!/^%/ {print $1}' | + sort | + uniq | + wc -l | + grep "\b10\b" \ No newline at end of file diff --git a/theodolite-benchmarks/docker-test/uc4-hazelcastjet-docker-compose/docker-compose.yml b/theodolite-benchmarks/docker-test/uc4-hazelcastjet-docker-compose/docker-compose.yml index 9045f061fddb095706b4a4a499e20c8a50420438..532f38e9ec5ef79a976bb5d05edb301aef8d665b 100644 --- a/theodolite-benchmarks/docker-test/uc4-hazelcastjet-docker-compose/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc4-hazelcastjet-docker-compose/docker-compose.yml @@ -1,4 +1,4 @@ -version: '2' +version: '2.2' services: zookeeper: image: confluentinc/cp-zookeeper @@ -20,18 +20,23 @@ services: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1" + kcat: + image: edenhill/kcat:1.7.1 + entrypoint: "sh" + tty: true schema-registry: - image: confluentinc/cp-schema-registry:5.3.1 + image: confluentinc/cp-schema-registry:7.0.1 depends_on: - zookeeper - kafka + restart: "on-failure" expose: - "8081" #ports: # - 8081:8081 environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 benchmark: image: uc4-hazelcastjet depends_on: @@ -47,7 +52,7 @@ services: SCHEMA_REGISTRY_URL: http://schema-registry:8081 WINDOW_SIZE_UC4: 5000 load-generator: - image: ghcr.io/cau-se/theodolite-uc4-workload-generator:latest + image: ghcr.io/cau-se/theodolite-uc4-workload-generator:${THEODOLITE_TAG:-latest} depends_on: - schema-registry - kafka diff --git a/theodolite-benchmarks/docker-test/uc4-hazelcastjet-docker-compose/test.sh b/theodolite-benchmarks/docker-test/uc4-hazelcastjet-docker-compose/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..731310b017a4c66f08ca2196e8429ccb7238e1c9 --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc4-hazelcastjet-docker-compose/test.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +until docker-compose exec -T kcat kcat -L -b kafka:9092 -t output -J | jq -r '.topics[0].partitions | length' | grep "\b3\b"; do sleep 5s; done + +docker-compose exec -T kcat kcat -C -b kafka:9092 -t output -s key=s -s value=avro -r http://schema-registry:8081 -f '%k:%s\n' -c 500 | + tee /dev/stderr | + awk -F ':' '!/^%/ {print $1}' | + sort | + uniq | + wc -l | + grep "\b21\b" \ No newline at end of file diff --git a/theodolite-benchmarks/uc4-hazelcastjet/Dockerfile b/theodolite-benchmarks/uc4-hazelcastjet/Dockerfile index 4e6e69935fc7ee8148a0ea72446cf5fad37195e4..a09c59d007a4de426a5046221662cdf1e912ee56 100644 --- a/theodolite-benchmarks/uc4-hazelcastjet/Dockerfile +++ b/theodolite-benchmarks/uc4-hazelcastjet/Dockerfile @@ -4,4 +4,4 @@ ADD build/distributions/uc4-hazelcastjet.tar / CMD JAVA_OPTS="$JAVA_OPTS -Dorg.slf4j.simpleLogger.defaultLogLevel=$LOG_LEVEL" \ - /uc3-hazelcastjet/bin/uc4-hazelcastjet \ No newline at end of file + /uc4-hazelcastjet/bin/uc4-hazelcastjet \ No newline at end of file