From 5e3059f61bc86e2871c4576d5d83704a9b3f4dec Mon Sep 17 00:00:00 2001
From: lorenz <stu203404@mail.uni-kiel.de>
Date: Fri, 17 Dec 2021 12:05:42 +0100
Subject: [PATCH] Beam Flink use task and job manager in docker-compose
---
.../uc1-beam-flink/docker-compose.yml | 37 ++++++++++++++----
.../uc2-beam-flink/docker-compose.yml | 39 ++++++++++++++-----
.../uc3-beam-flink/docker-compose.yml | 39 +++++++++++++++----
.../uc4-beam-flink/docker-compose.yml | 10 ++++-
4 files changed, 98 insertions(+), 27 deletions(-)
diff --git a/theodolite-benchmarks/docker-test/uc1-beam-flink/docker-compose.yml b/theodolite-benchmarks/docker-test/uc1-beam-flink/docker-compose.yml
index 8ee83ba0e..fcfc15d7f 100644
--- a/theodolite-benchmarks/docker-test/uc1-beam-flink/docker-compose.yml
+++ b/theodolite-benchmarks/docker-test/uc1-beam-flink/docker-compose.yml
@@ -31,14 +31,6 @@ services:
environment:
SCHEMA_REGISTRY_HOST_NAME: schema-registry
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
- benchmark:
- image: uc1-beam-flink:latest
- depends_on:
- - schema-registry
- - kafka
- environment:
- KAFKA_BOOTSTRAP_SERVERS: kafka:9092
- SCHEMA_REGISTRY_URL: http://schema-registry:8081
load-generator:
image: ghcr.io/cau-se/theodolite-uc1-workload-generator:latest
depends_on:
@@ -50,3 +42,32 @@ services:
KAFKA_BOOTSTRAP_SERVERS: kafka:9092
SCHEMA_REGISTRY_URL: http://schema-registry:8081
NUM_SENSORS: 10
+ benchmark-jobmanager:
+ image: uc1-beam-flink:latest
+ ports:
+ - "8080:8081"
+ command: >
+ standalone-job --job-classname application.Uc1BeamFlink
+ --streaming
+ --disableMetrics=true
+ --fasterCopy
+ environment:
+ - KAFKA_BOOTSTRAP_SERVERS=kafka:9092
+ - SCHEMA_REGISTRY_URL=http://schema-registry:8081
+ - |
+ FLINK_PROPERTIES=
+ jobmanager.rpc.address: benchmark-jobmanager
+ parallelism.default: 1
+ depends_on:
+ - schema-registry
+ - kafka
+ benchmark-taskmanager:
+ image: uc1-beam-flink:latest
+ command: taskmanager
+ environment:
+ - |
+ FLINK_PROPERTIES=
+ jobmanager.rpc.address: benchmark-jobmanager
+ depends_on:
+ - schema-registry
+ - kafka
diff --git a/theodolite-benchmarks/docker-test/uc2-beam-flink/docker-compose.yml b/theodolite-benchmarks/docker-test/uc2-beam-flink/docker-compose.yml
index 72913c015..0327a0e38 100644
--- a/theodolite-benchmarks/docker-test/uc2-beam-flink/docker-compose.yml
+++ b/theodolite-benchmarks/docker-test/uc2-beam-flink/docker-compose.yml
@@ -31,15 +31,6 @@ services:
environment:
SCHEMA_REGISTRY_HOST_NAME: schema-registry
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
- benchmark:
- image: uc2-beam-flink:latest
- depends_on:
- - schema-registry
- - kafka
- environment:
- KAFKA_BOOTSTRAP_SERVERS: kafka:9092
- SCHEMA_REGISTRY_URL: http://schema-registry:8081
- KAFKA_WINDOW_DURATION_MINUTES: 60
load-generator:
image: ghcr.io/cau-se/theodolite-uc2-workload-generator:latest
depends_on:
@@ -51,3 +42,33 @@ services:
KAFKA_BOOTSTRAP_SERVERS: kafka:9092
SCHEMA_REGISTRY_URL: http://schema-registry:8081
NUM_SENSORS: 10
+ benchmark-jobmanager:
+ image: uc2-beam-flink:latest
+ ports:
+ - "8080:8081"
+ command: >
+ standalone-job --job-classname application.Uc2BeamFlink
+ --streaming
+ --disableMetrics=true
+ --fasterCopy
+ environment:
+ - KAFKA_WINDOW_DURATION_MINUTES=60
+ - KAFKA_BOOTSTRAP_SERVERS=kafka:9092
+ - SCHEMA_REGISTRY_URL=http://schema-registry:8081
+ - |
+ FLINK_PROPERTIES=
+ jobmanager.rpc.address: benchmark-jobmanager
+ parallelism.default: 1
+ depends_on:
+ - schema-registry
+ - kafka
+ benchmark-taskmanager:
+ image: uc2-beam-flink:latest
+ command: taskmanager
+ environment:
+ - |
+ FLINK_PROPERTIES=
+ jobmanager.rpc.address: benchmark-jobmanager
+ depends_on:
+ - schema-registry
+ - kafka
diff --git a/theodolite-benchmarks/docker-test/uc3-beam-flink/docker-compose.yml b/theodolite-benchmarks/docker-test/uc3-beam-flink/docker-compose.yml
index e4f3a3c9f..3dd8115fc 100644
--- a/theodolite-benchmarks/docker-test/uc3-beam-flink/docker-compose.yml
+++ b/theodolite-benchmarks/docker-test/uc3-beam-flink/docker-compose.yml
@@ -31,14 +31,6 @@ services:
environment:
SCHEMA_REGISTRY_HOST_NAME: schema-registry
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
- benchmark:
- image: uc3-beam-flink:latest
- depends_on:
- - schema-registry
- - kafka
- environment:
- KAFKA_BOOTSTRAP_SERVERS: kafka:9092
- SCHEMA_REGISTRY_URL: http://schema-registry:8081
load-generator:
image: ghcr.io/cau-se/theodolite-uc3-workload-generator:latest
depends_on:
@@ -50,3 +42,34 @@ services:
KAFKA_BOOTSTRAP_SERVERS: kafka:9092
SCHEMA_REGISTRY_URL: http://schema-registry:8081
NUM_SENSORS: 10
+ benchmark-jobmanager:
+ image: uc3-beam-flink:latest
+ ports:
+ - "8080:8081"
+ command: >
+ standalone-job --job-classname application.Uc3BeamFlink
+ --streaming
+ --disableMetrics=true
+ --fasterCopy
+ --stateBackend=rocksdb
+ --stateBackendStoragePath=file:///data/flink/checkpoints
+ environment:
+ - KAFKA_BOOTSTRAP_SERVERS=kafka:9092
+ - SCHEMA_REGISTRY_URL=http://schema-registry:8081
+ - |
+ FLINK_PROPERTIES=
+ jobmanager.rpc.address: benchmark-jobmanager
+ parallelism.default: 1
+ depends_on:
+ - schema-registry
+ - kafka
+ benchmark-taskmanager:
+ image: uc3-beam-flink:latest
+ command: taskmanager
+ environment:
+ - |
+ FLINK_PROPERTIES=
+ jobmanager.rpc.address: benchmark-jobmanager
+ depends_on:
+ - schema-registry
+ - kafka
diff --git a/theodolite-benchmarks/docker-test/uc4-beam-flink/docker-compose.yml b/theodolite-benchmarks/docker-test/uc4-beam-flink/docker-compose.yml
index 030b28468..e32bf9f4d 100644
--- a/theodolite-benchmarks/docker-test/uc4-beam-flink/docker-compose.yml
+++ b/theodolite-benchmarks/docker-test/uc4-beam-flink/docker-compose.yml
@@ -47,9 +47,15 @@ services:
image: uc4-beam-flink:latest
ports:
- "8080:8081"
- command: standalone-job --job-classname application.Uc4BeamFlink
+ command: >
+ standalone-job
+ --job-classname application.Uc4BeamFlink
+ --streaming
+ --disableMetrics=true
+ --fasterCopy
+ --stateBackend=rocksdb
+ --stateBackendStoragePath=file:///data/flink/checkpoints
environment:
- - PARALLELISM=1
- KAFKA_BOOTSTRAP_SERVERS=kafka:9092
- SCHEMA_REGISTRY_URL=http://schema-registry:8081
- |
--
GitLab