Skip to content
Snippets Groups Projects
Commit 5e3059f6 authored by Lorenz Boguhn's avatar Lorenz Boguhn
Browse files

Beam Flink use task and job manager in docker-compose

parent f7a47710
No related branches found
No related tags found
1 merge request!187Migrate Beam benchmark implementation
Pipeline #5667 passed
...@@ -31,14 +31,6 @@ services: ...@@ -31,14 +31,6 @@ services:
environment: environment:
SCHEMA_REGISTRY_HOST_NAME: schema-registry SCHEMA_REGISTRY_HOST_NAME: schema-registry
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
benchmark:
image: uc1-beam-flink:latest
depends_on:
- schema-registry
- kafka
environment:
KAFKA_BOOTSTRAP_SERVERS: kafka:9092
SCHEMA_REGISTRY_URL: http://schema-registry:8081
load-generator: load-generator:
image: ghcr.io/cau-se/theodolite-uc1-workload-generator:latest image: ghcr.io/cau-se/theodolite-uc1-workload-generator:latest
depends_on: depends_on:
...@@ -50,3 +42,32 @@ services: ...@@ -50,3 +42,32 @@ services:
KAFKA_BOOTSTRAP_SERVERS: kafka:9092 KAFKA_BOOTSTRAP_SERVERS: kafka:9092
SCHEMA_REGISTRY_URL: http://schema-registry:8081 SCHEMA_REGISTRY_URL: http://schema-registry:8081
NUM_SENSORS: 10 NUM_SENSORS: 10
benchmark-jobmanager:
image: uc1-beam-flink:latest
ports:
- "8080:8081"
command: >
standalone-job --job-classname application.Uc1BeamFlink
--streaming
--disableMetrics=true
--fasterCopy
environment:
- KAFKA_BOOTSTRAP_SERVERS=kafka:9092
- SCHEMA_REGISTRY_URL=http://schema-registry:8081
- |
FLINK_PROPERTIES=
jobmanager.rpc.address: benchmark-jobmanager
parallelism.default: 1
depends_on:
- schema-registry
- kafka
benchmark-taskmanager:
image: uc1-beam-flink:latest
command: taskmanager
environment:
- |
FLINK_PROPERTIES=
jobmanager.rpc.address: benchmark-jobmanager
depends_on:
- schema-registry
- kafka
...@@ -31,15 +31,6 @@ services: ...@@ -31,15 +31,6 @@ services:
environment: environment:
SCHEMA_REGISTRY_HOST_NAME: schema-registry SCHEMA_REGISTRY_HOST_NAME: schema-registry
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
benchmark:
image: uc2-beam-flink:latest
depends_on:
- schema-registry
- kafka
environment:
KAFKA_BOOTSTRAP_SERVERS: kafka:9092
SCHEMA_REGISTRY_URL: http://schema-registry:8081
KAFKA_WINDOW_DURATION_MINUTES: 60
load-generator: load-generator:
image: ghcr.io/cau-se/theodolite-uc2-workload-generator:latest image: ghcr.io/cau-se/theodolite-uc2-workload-generator:latest
depends_on: depends_on:
...@@ -51,3 +42,33 @@ services: ...@@ -51,3 +42,33 @@ services:
KAFKA_BOOTSTRAP_SERVERS: kafka:9092 KAFKA_BOOTSTRAP_SERVERS: kafka:9092
SCHEMA_REGISTRY_URL: http://schema-registry:8081 SCHEMA_REGISTRY_URL: http://schema-registry:8081
NUM_SENSORS: 10 NUM_SENSORS: 10
benchmark-jobmanager:
image: uc2-beam-flink:latest
ports:
- "8080:8081"
command: >
standalone-job --job-classname application.Uc2BeamFlink
--streaming
--disableMetrics=true
--fasterCopy
environment:
- KAFKA_WINDOW_DURATION_MINUTES=60
- KAFKA_BOOTSTRAP_SERVERS=kafka:9092
- SCHEMA_REGISTRY_URL=http://schema-registry:8081
- |
FLINK_PROPERTIES=
jobmanager.rpc.address: benchmark-jobmanager
parallelism.default: 1
depends_on:
- schema-registry
- kafka
benchmark-taskmanager:
image: uc2-beam-flink:latest
command: taskmanager
environment:
- |
FLINK_PROPERTIES=
jobmanager.rpc.address: benchmark-jobmanager
depends_on:
- schema-registry
- kafka
...@@ -31,14 +31,6 @@ services: ...@@ -31,14 +31,6 @@ services:
environment: environment:
SCHEMA_REGISTRY_HOST_NAME: schema-registry SCHEMA_REGISTRY_HOST_NAME: schema-registry
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
benchmark:
image: uc3-beam-flink:latest
depends_on:
- schema-registry
- kafka
environment:
KAFKA_BOOTSTRAP_SERVERS: kafka:9092
SCHEMA_REGISTRY_URL: http://schema-registry:8081
load-generator: load-generator:
image: ghcr.io/cau-se/theodolite-uc3-workload-generator:latest image: ghcr.io/cau-se/theodolite-uc3-workload-generator:latest
depends_on: depends_on:
...@@ -50,3 +42,34 @@ services: ...@@ -50,3 +42,34 @@ services:
KAFKA_BOOTSTRAP_SERVERS: kafka:9092 KAFKA_BOOTSTRAP_SERVERS: kafka:9092
SCHEMA_REGISTRY_URL: http://schema-registry:8081 SCHEMA_REGISTRY_URL: http://schema-registry:8081
NUM_SENSORS: 10 NUM_SENSORS: 10
benchmark-jobmanager:
image: uc3-beam-flink:latest
ports:
- "8080:8081"
command: >
standalone-job --job-classname application.Uc3BeamFlink
--streaming
--disableMetrics=true
--fasterCopy
--stateBackend=rocksdb
--stateBackendStoragePath=file:///data/flink/checkpoints
environment:
- KAFKA_BOOTSTRAP_SERVERS=kafka:9092
- SCHEMA_REGISTRY_URL=http://schema-registry:8081
- |
FLINK_PROPERTIES=
jobmanager.rpc.address: benchmark-jobmanager
parallelism.default: 1
depends_on:
- schema-registry
- kafka
benchmark-taskmanager:
image: uc3-beam-flink:latest
command: taskmanager
environment:
- |
FLINK_PROPERTIES=
jobmanager.rpc.address: benchmark-jobmanager
depends_on:
- schema-registry
- kafka
...@@ -47,9 +47,15 @@ services: ...@@ -47,9 +47,15 @@ services:
image: uc4-beam-flink:latest image: uc4-beam-flink:latest
ports: ports:
- "8080:8081" - "8080:8081"
command: standalone-job --job-classname application.Uc4BeamFlink command: >
standalone-job
--job-classname application.Uc4BeamFlink
--streaming
--disableMetrics=true
--fasterCopy
--stateBackend=rocksdb
--stateBackendStoragePath=file:///data/flink/checkpoints
environment: environment:
- PARALLELISM=1
- KAFKA_BOOTSTRAP_SERVERS=kafka:9092 - KAFKA_BOOTSTRAP_SERVERS=kafka:9092
- SCHEMA_REGISTRY_URL=http://schema-registry:8081 - SCHEMA_REGISTRY_URL=http://schema-registry:8081
- | - |
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment