From e24c5ddab2273f5d82b688d0b85fd11667b228b7 Mon Sep 17 00:00:00 2001
From: lorenz <stu203404@mail.uni-kiel.de>
Date: Wed, 16 Feb 2022 15:45:16 +0100
Subject: [PATCH] Smoke test hazelcast uc 2-3, fix uc4 docker + first version
not working version of uc4 smoke test
---
.../docker-compose.yml | 15 ++++++++++-----
.../uc2-hazelcastjet-docker-compose/test.sh | 11 +++++++++++
.../docker-compose.yml | 15 ++++++++++-----
.../uc3-hazelcastjet-docker-compose/test.sh | 11 +++++++++++
.../docker-compose.yml | 13 +++++++++----
.../uc4-hazelcastjet-docker-compose/test.sh | 11 +++++++++++
theodolite-benchmarks/uc4-hazelcastjet/Dockerfile | 2 +-
7 files changed, 63 insertions(+), 15 deletions(-)
create mode 100755 theodolite-benchmarks/docker-test/uc2-hazelcastjet-docker-compose/test.sh
create mode 100755 theodolite-benchmarks/docker-test/uc3-hazelcastjet-docker-compose/test.sh
create mode 100755 theodolite-benchmarks/docker-test/uc4-hazelcastjet-docker-compose/test.sh
diff --git a/theodolite-benchmarks/docker-test/uc2-hazelcastjet-docker-compose/docker-compose.yml b/theodolite-benchmarks/docker-test/uc2-hazelcastjet-docker-compose/docker-compose.yml
index 051ff1f7c..7c35f64f2 100644
--- a/theodolite-benchmarks/docker-test/uc2-hazelcastjet-docker-compose/docker-compose.yml
+++ b/theodolite-benchmarks/docker-test/uc2-hazelcastjet-docker-compose/docker-compose.yml
@@ -1,4 +1,4 @@
-version: '2'
+version: '2.2'
services:
zookeeper:
image: confluentinc/cp-zookeeper
@@ -19,19 +19,24 @@ services:
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
- KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
+ KAFKA_CREATE_TOPICS: "input:3:1,output:3:1"
+ kcat:
+ image: edenhill/kcat:1.7.1
+ entrypoint: "sh"
+ tty: true
schema-registry:
- image: confluentinc/cp-schema-registry:5.3.1
+ image: confluentinc/cp-schema-registry:7.0.1
depends_on:
- zookeeper
- kafka
+ restart: "on-failure"
expose:
- "8081"
#ports:
# - 8081:8081
environment:
SCHEMA_REGISTRY_HOST_NAME: schema-registry
- SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
+ SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092
benchmark:
image: uc2-hazelcastjet
depends_on:
@@ -47,7 +52,7 @@ services:
SCHEMA_REGISTRY_URL: http://schema-registry:8081
DOWNSAMPLE_INTERVAL: 5000
load-generator:
- image: ghcr.io/cau-se/theodolite-uc2-workload-generator:latest
+ image: ghcr.io/cau-se/theodolite-uc2-workload-generator:${THEODOLITE_TAG:-latest}
depends_on:
- schema-registry
- kafka
diff --git a/theodolite-benchmarks/docker-test/uc2-hazelcastjet-docker-compose/test.sh b/theodolite-benchmarks/docker-test/uc2-hazelcastjet-docker-compose/test.sh
new file mode 100755
index 000000000..be411755b
--- /dev/null
+++ b/theodolite-benchmarks/docker-test/uc2-hazelcastjet-docker-compose/test.sh
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+until docker-compose exec -T kcat kcat -L -b kafka:9092 -t output -J | jq -r '.topics[0].partitions | length' | grep "\b3\b"; do sleep 5s; done
+
+docker-compose exec -T kcat kcat -C -b kafka:9092 -t output -s key=s -s value=s -r http://schema-registry:8081 -f '%k:%s\n' -c 20 |
+ tee /dev/stderr |
+ awk -F ':' '!/^%/ {print $1}' |
+ sort |
+ uniq |
+ wc -l |
+ grep "\b10\b"
\ No newline at end of file
diff --git a/theodolite-benchmarks/docker-test/uc3-hazelcastjet-docker-compose/docker-compose.yml b/theodolite-benchmarks/docker-test/uc3-hazelcastjet-docker-compose/docker-compose.yml
index 27a6713c2..e4fd72601 100644
--- a/theodolite-benchmarks/docker-test/uc3-hazelcastjet-docker-compose/docker-compose.yml
+++ b/theodolite-benchmarks/docker-test/uc3-hazelcastjet-docker-compose/docker-compose.yml
@@ -1,4 +1,4 @@
-version: '2'
+version: '2.2'
services:
zookeeper:
image: confluentinc/cp-zookeeper
@@ -19,19 +19,24 @@ services:
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
- KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
+ KAFKA_CREATE_TOPICS: "input:3:1,output:3:1"
+ kcat:
+ image: edenhill/kcat:1.7.1
+ entrypoint: "sh"
+ tty: true
schema-registry:
- image: confluentinc/cp-schema-registry:5.3.1
+ image: confluentinc/cp-schema-registry:7.0.1
depends_on:
- zookeeper
- kafka
+ restart: "on-failure"
expose:
- "8081"
#ports:
# - 8081:8081
environment:
SCHEMA_REGISTRY_HOST_NAME: schema-registry
- SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
+ SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092
benchmark:
image: uc3-hazelcastjet
depends_on:
@@ -48,7 +53,7 @@ services:
WINDOW_SIZE_IN_SECONDS: 50
HOPPING_SIZE_IN_SECONDS: 5
load-generator:
- image: ghcr.io/cau-se/theodolite-uc3-workload-generator:latest
+ image: ghcr.io/cau-se/theodolite-uc3-workload-generator:${THEODOLITE_TAG:-latest}
depends_on:
- schema-registry
- kafka
diff --git a/theodolite-benchmarks/docker-test/uc3-hazelcastjet-docker-compose/test.sh b/theodolite-benchmarks/docker-test/uc3-hazelcastjet-docker-compose/test.sh
new file mode 100755
index 000000000..217a730f7
--- /dev/null
+++ b/theodolite-benchmarks/docker-test/uc3-hazelcastjet-docker-compose/test.sh
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+until docker-compose exec -T kcat kcat -L -b kafka:9092 -t output -J | jq -r '.topics[0].partitions | length' | grep "\b3\b"; do sleep 5s; done
+
+docker-compose exec -T kcat kcat -C -b kafka:9092 -t output -s key=s -s value=s -r http://schema-registry:8081 -f '%k:%s\n' -c 600 |
+ tee /dev/stderr |
+ awk -F ':' '!/^%/ {print $1}' |
+ sort |
+ uniq |
+ wc -l |
+ grep "\b10\b"
\ No newline at end of file
diff --git a/theodolite-benchmarks/docker-test/uc4-hazelcastjet-docker-compose/docker-compose.yml b/theodolite-benchmarks/docker-test/uc4-hazelcastjet-docker-compose/docker-compose.yml
index 9045f061f..532f38e9e 100644
--- a/theodolite-benchmarks/docker-test/uc4-hazelcastjet-docker-compose/docker-compose.yml
+++ b/theodolite-benchmarks/docker-test/uc4-hazelcastjet-docker-compose/docker-compose.yml
@@ -1,4 +1,4 @@
-version: '2'
+version: '2.2'
services:
zookeeper:
image: confluentinc/cp-zookeeper
@@ -20,18 +20,23 @@ services:
KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
+ kcat:
+ image: edenhill/kcat:1.7.1
+ entrypoint: "sh"
+ tty: true
schema-registry:
- image: confluentinc/cp-schema-registry:5.3.1
+ image: confluentinc/cp-schema-registry:7.0.1
depends_on:
- zookeeper
- kafka
+ restart: "on-failure"
expose:
- "8081"
#ports:
# - 8081:8081
environment:
SCHEMA_REGISTRY_HOST_NAME: schema-registry
- SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
+ SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092
benchmark:
image: uc4-hazelcastjet
depends_on:
@@ -47,7 +52,7 @@ services:
SCHEMA_REGISTRY_URL: http://schema-registry:8081
WINDOW_SIZE_UC4: 5000
load-generator:
- image: ghcr.io/cau-se/theodolite-uc4-workload-generator:latest
+ image: ghcr.io/cau-se/theodolite-uc4-workload-generator:${THEODOLITE_TAG:-latest}
depends_on:
- schema-registry
- kafka
diff --git a/theodolite-benchmarks/docker-test/uc4-hazelcastjet-docker-compose/test.sh b/theodolite-benchmarks/docker-test/uc4-hazelcastjet-docker-compose/test.sh
new file mode 100755
index 000000000..731310b01
--- /dev/null
+++ b/theodolite-benchmarks/docker-test/uc4-hazelcastjet-docker-compose/test.sh
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+until docker-compose exec -T kcat kcat -L -b kafka:9092 -t output -J | jq -r '.topics[0].partitions | length' | grep "\b3\b"; do sleep 5s; done
+
+docker-compose exec -T kcat kcat -C -b kafka:9092 -t output -s key=s -s value=avro -r http://schema-registry:8081 -f '%k:%s\n' -c 500 |
+ tee /dev/stderr |
+ awk -F ':' '!/^%/ {print $1}' |
+ sort |
+ uniq |
+ wc -l |
+ grep "\b21\b"
\ No newline at end of file
diff --git a/theodolite-benchmarks/uc4-hazelcastjet/Dockerfile b/theodolite-benchmarks/uc4-hazelcastjet/Dockerfile
index 4e6e69935..a09c59d00 100644
--- a/theodolite-benchmarks/uc4-hazelcastjet/Dockerfile
+++ b/theodolite-benchmarks/uc4-hazelcastjet/Dockerfile
@@ -4,4 +4,4 @@ ADD build/distributions/uc4-hazelcastjet.tar /
CMD JAVA_OPTS="$JAVA_OPTS -Dorg.slf4j.simpleLogger.defaultLogLevel=$LOG_LEVEL" \
- /uc3-hazelcastjet/bin/uc4-hazelcastjet
\ No newline at end of file
+ /uc4-hazelcastjet/bin/uc4-hazelcastjet
\ No newline at end of file
--
GitLab