From 2932f1d4740cbd1e6e9b8587d84fced594c83d16 Mon Sep 17 00:00:00 2001 From: lorenz <stu203404@mail.uni-kiel.de> Date: Fri, 3 Dec 2021 15:15:58 +0100 Subject: [PATCH] Add beam-flink dockerfiles for uc2-4 --- .../uc2-beam-flink/docker-compose.yml | 52 +++++++++++++++++++ .../uc3-beam-flink/docker-compose.yml | 52 +++++++++++++++++++ .../uc4-beam-flink/docker-compose.yml | 52 +++++++++++++++++++ .../uc2-beam-flink/Dockerfile | 6 +-- .../uc3-beam-flink/Dockerfile | 6 +-- .../uc4-beam-flink/Dockerfile | 6 +-- 6 files changed, 165 insertions(+), 9 deletions(-) create mode 100644 theodolite-benchmarks/docker-test/uc2-beam-flink/docker-compose.yml create mode 100644 theodolite-benchmarks/docker-test/uc3-beam-flink/docker-compose.yml create mode 100644 theodolite-benchmarks/docker-test/uc4-beam-flink/docker-compose.yml diff --git a/theodolite-benchmarks/docker-test/uc2-beam-flink/docker-compose.yml b/theodolite-benchmarks/docker-test/uc2-beam-flink/docker-compose.yml new file mode 100644 index 000000000..cdcecde27 --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc2-beam-flink/docker-compose.yml @@ -0,0 +1,52 @@ +version: '2' +services: + zookeeper: + image: confluentinc/cp-zookeeper + expose: + - "9092" + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + kafka: + image: wurstmeister/kafka + expose: + - "9092" + #ports: + # - 19092:19092 + environment: + KAFKA_LISTENERS: PLAINTEXT://:9092,CONNECTIONS_FROM_HOST://:19092 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,CONNECTIONS_FROM_HOST://localhost:19092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 + KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1" + schema-registry: + image: confluentinc/cp-schema-registry:5.3.1 + depends_on: + - zookeeper + - kafka + expose: + - "8081" + #ports: + # - 8081:8081 + environment: + SCHEMA_REGISTRY_HOST_NAME: schema-registry + SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + benchmark: + image: uc2-beam-flink:latest + depends_on: + - schema-registry + - kafka + environment: + KAFKA_BOOTSTRAP_SERVERS: kafka:9092 + SCHEMA_REGISTRY_URL: http://schema-registry:8081 + load-generator: + image: ghcr.io/cau-se/theodolite-uc1-workload-generator:latest + depends_on: + - schema-registry + - kafka + environment: + BOOTSTRAP_SERVER: load-generator:5701 + PORT: 5701 + KAFKA_BOOTSTRAP_SERVERS: kafka:9092 + SCHEMA_REGISTRY_URL: http://schema-registry:8081 + NUM_SENSORS: 10 diff --git a/theodolite-benchmarks/docker-test/uc3-beam-flink/docker-compose.yml b/theodolite-benchmarks/docker-test/uc3-beam-flink/docker-compose.yml new file mode 100644 index 000000000..586789db5 --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc3-beam-flink/docker-compose.yml @@ -0,0 +1,52 @@ +version: '2' +services: + zookeeper: + image: confluentinc/cp-zookeeper + expose: + - "9092" + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + kafka: + image: wurstmeister/kafka + expose: + - "9092" + #ports: + # - 19092:19092 + environment: + KAFKA_LISTENERS: PLAINTEXT://:9092,CONNECTIONS_FROM_HOST://:19092 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,CONNECTIONS_FROM_HOST://localhost:19092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 + KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1" + schema-registry: + image: confluentinc/cp-schema-registry:5.3.1 + depends_on: + - zookeeper + - kafka + expose: + - "8081" + #ports: + # - 8081:8081 + environment: + SCHEMA_REGISTRY_HOST_NAME: schema-registry + SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + benchmark: + image: uc3-beam-flink:latest + depends_on: + - schema-registry + - kafka + environment: + KAFKA_BOOTSTRAP_SERVERS: kafka:9092 + SCHEMA_REGISTRY_URL: http://schema-registry:8081 + load-generator: + image: ghcr.io/cau-se/theodolite-uc1-workload-generator:latest + depends_on: + - schema-registry + - kafka + environment: + BOOTSTRAP_SERVER: load-generator:5701 + PORT: 5701 + KAFKA_BOOTSTRAP_SERVERS: kafka:9092 + SCHEMA_REGISTRY_URL: http://schema-registry:8081 + NUM_SENSORS: 10 diff --git a/theodolite-benchmarks/docker-test/uc4-beam-flink/docker-compose.yml b/theodolite-benchmarks/docker-test/uc4-beam-flink/docker-compose.yml new file mode 100644 index 000000000..7874ce917 --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc4-beam-flink/docker-compose.yml @@ -0,0 +1,52 @@ +version: '2' +services: + zookeeper: + image: confluentinc/cp-zookeeper + expose: + - "9092" + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + kafka: + image: wurstmeister/kafka + expose: + - "9092" + #ports: + # - 19092:19092 + environment: + KAFKA_LISTENERS: PLAINTEXT://:9092,CONNECTIONS_FROM_HOST://:19092 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,CONNECTIONS_FROM_HOST://localhost:19092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 + KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1" + schema-registry: + image: confluentinc/cp-schema-registry:5.3.1 + depends_on: + - zookeeper + - kafka + expose: + - "8081" + #ports: + # - 8081:8081 + environment: + SCHEMA_REGISTRY_HOST_NAME: schema-registry + SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + benchmark: + image: uc4-beam-flink:latest + depends_on: + - schema-registry + - kafka + environment: + KAFKA_BOOTSTRAP_SERVERS: kafka:9092 + SCHEMA_REGISTRY_URL: http://schema-registry:8081 + load-generator: + image: ghcr.io/cau-se/theodolite-uc1-workload-generator:latest + depends_on: + - schema-registry + - kafka + environment: + BOOTSTRAP_SERVER: load-generator:5701 + PORT: 5701 + KAFKA_BOOTSTRAP_SERVERS: kafka:9092 + SCHEMA_REGISTRY_URL: http://schema-registry:8081 + NUM_SENSORS: 10 diff --git a/theodolite-benchmarks/uc2-beam-flink/Dockerfile b/theodolite-benchmarks/uc2-beam-flink/Dockerfile index baee343ad..901da308b 100644 --- a/theodolite-benchmarks/uc2-beam-flink/Dockerfile +++ b/theodolite-benchmarks/uc2-beam-flink/Dockerfile @@ -1,5 +1,5 @@ -FROM openjdk:8-slim +FROM openjdk:11-slim -ADD build/distributions/uc3-application-flink.tar / +ADD build/distributions/uc3-beam-flink.tar / -CMD /uc3-application-flink/bin/uc3-application-flink --runner=FlinkRunner --flinkMaster=flink-jobmanager:8081 --streaming --parallelism=$PARALLELISM --disableMetrics=true --fasterCopy \ No newline at end of file +CMD /uc3-beam-flink/bin/uc3-beam-flink --runner=FlinkRunner --streaming --disableMetrics=true --fasterCopy diff --git a/theodolite-benchmarks/uc3-beam-flink/Dockerfile b/theodolite-benchmarks/uc3-beam-flink/Dockerfile index f16ec202e..4892ff32f 100644 --- a/theodolite-benchmarks/uc3-beam-flink/Dockerfile +++ b/theodolite-benchmarks/uc3-beam-flink/Dockerfile @@ -1,5 +1,5 @@ -FROM openjdk:8-slim +FROM openjdk:11-slim -ADD build/distributions/uc4-application-flink.tar / +ADD build/distributions/uc4-beam-flink.tar / -CMD /uc4-application-flink/bin/uc4-application-flink --runner=FlinkRunner --flinkMaster=flink-jobmanager:8081 --streaming --parallelism=$PARALLELISM --disableMetrics=true --fasterCopy --stateBackend=rocksdb --stateBackendStoragePath=file:///data/flink/checkpoints \ No newline at end of file +CMD /uc4-beam-flink/bin/uc4-beam-flink --runner=FlinkRunner --streaming --disableMetrics=true --fasterCopy --stateBackend=rocksdb --stateBackendStoragePath=file:///data/flink/checkpoints diff --git a/theodolite-benchmarks/uc4-beam-flink/Dockerfile b/theodolite-benchmarks/uc4-beam-flink/Dockerfile index 4817f0df1..a9824e67f 100644 --- a/theodolite-benchmarks/uc4-beam-flink/Dockerfile +++ b/theodolite-benchmarks/uc4-beam-flink/Dockerfile @@ -1,5 +1,5 @@ -FROM openjdk:8-slim +FROM openjdk:11-slim -ADD build/distributions/uc2-application-flink.tar / +ADD build/distributions/uc2-beam-flink.tar / -CMD /uc2-application-flink/bin/uc2-application-flink --runner=FlinkRunner --flinkMaster=flink-jobmanager:8081 --streaming --parallelism=$PARALLELISM --disableMetrics=true --fasterCopy --stateBackend=rocksdb --stateBackendStoragePath=file:///data/flink/checkpoints +CMD /uc2-beam-flink/bin/uc2-beam-flink --runner=FlinkRunner --streaming --disableMetrics=true --fasterCopy --stateBackend=rocksdb --stateBackendStoragePath=file:///data/flink/checkpoints -- GitLab