From 78cf80e82ddc46fc7e31fca87bd6279999ce8b44 Mon Sep 17 00:00:00 2001 From: lorenz <stu203404@mail.uni-kiel.de> Date: Fri, 3 Dec 2021 14:58:03 +0100 Subject: [PATCH] Add dockerfiles for uc1 --- .../uc1-beam-flink/docker-compose.yml | 52 +++++++++++++++++++ .../uc1-beam-samza/docker-compose.yml | 52 +++++++++++++++++++ .../uc1-beam-flink/Dockerfile | 7 +-- .../uc1-beam-samza/Dockerfile | 8 +-- .../samza-standalone.properties | 17 ++++++ 5 files changed, 129 insertions(+), 7 deletions(-) create mode 100644 theodolite-benchmarks/docker-test/uc1-beam-flink/docker-compose.yml create mode 100644 theodolite-benchmarks/docker-test/uc1-beam-samza/docker-compose.yml create mode 100644 theodolite-benchmarks/uc1-beam-samza/samza-standalone.properties diff --git a/theodolite-benchmarks/docker-test/uc1-beam-flink/docker-compose.yml b/theodolite-benchmarks/docker-test/uc1-beam-flink/docker-compose.yml new file mode 100644 index 000000000..2548472b4 --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc1-beam-flink/docker-compose.yml @@ -0,0 +1,52 @@ +version: '2' +services: + zookeeper: + image: confluentinc/cp-zookeeper + expose: + - "9092" + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + kafka: + image: wurstmeister/kafka + expose: + - "9092" + #ports: + # - 19092:19092 + environment: + KAFKA_LISTENERS: PLAINTEXT://:9092,CONNECTIONS_FROM_HOST://:19092 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,CONNECTIONS_FROM_HOST://localhost:19092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 + KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1" + schema-registry: + image: confluentinc/cp-schema-registry:5.3.1 + depends_on: + - zookeeper + - kafka + expose: + - "8081" + #ports: + # - 8081:8081 + environment: + SCHEMA_REGISTRY_HOST_NAME: schema-registry + SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + benchmark: + image: uc1-beam-flink:latest + depends_on: + - schema-registry + - kafka + environment: + KAFKA_BOOTSTRAP_SERVERS: kafka:9092 + SCHEMA_REGISTRY_URL: http://schema-registry:8081 + load-generator: + image: ghcr.io/cau-se/theodolite-uc1-workload-generator:latest + depends_on: + - schema-registry + - kafka + environment: + BOOTSTRAP_SERVER: load-generator:5701 + PORT: 5701 + KAFKA_BOOTSTRAP_SERVERS: kafka:9092 + SCHEMA_REGISTRY_URL: http://schema-registry:8081 + NUM_SENSORS: 10 diff --git a/theodolite-benchmarks/docker-test/uc1-beam-samza/docker-compose.yml b/theodolite-benchmarks/docker-test/uc1-beam-samza/docker-compose.yml new file mode 100644 index 000000000..88cc156ce --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc1-beam-samza/docker-compose.yml @@ -0,0 +1,52 @@ +version: '2' +services: + zookeeper: + image: confluentinc/cp-zookeeper + expose: + - "9092" + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + kafka: + image: wurstmeister/kafka + expose: + - "9092" + #ports: + # - 19092:19092 + environment: + KAFKA_LISTENERS: PLAINTEXT://:9092,CONNECTIONS_FROM_HOST://:19092 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,CONNECTIONS_FROM_HOST://localhost:19092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 + KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1" + schema-registry: + image: confluentinc/cp-schema-registry:5.3.1 + depends_on: + - zookeeper + - kafka + expose: + - "8081" + #ports: + # - 8081:8081 + environment: + SCHEMA_REGISTRY_HOST_NAME: schema-registry + SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + benchmark: + image: uc1-beam-samza:latest + depends_on: + - schema-registry + - kafka + environment: + KAFKA_BOOTSTRAP_SERVERS: kafka:9092 + SCHEMA_REGISTRY_URL: http://schema-registry:8081 + load-generator: + image: ghcr.io/cau-se/theodolite-uc1-workload-generator:latest + depends_on: + - schema-registry + - kafka + environment: + BOOTSTRAP_SERVER: load-generator:5701 + PORT: 5701 + KAFKA_BOOTSTRAP_SERVERS: kafka:9092 + SCHEMA_REGISTRY_URL: http://schema-registry:8081 + NUM_SENSORS: 10 diff --git a/theodolite-benchmarks/uc1-beam-flink/Dockerfile b/theodolite-benchmarks/uc1-beam-flink/Dockerfile index 67fbb025d..99afa663a 100644 --- a/theodolite-benchmarks/uc1-beam-flink/Dockerfile +++ b/theodolite-benchmarks/uc1-beam-flink/Dockerfile @@ -1,5 +1,6 @@ -FROM openjdk:8-slim +FROM openjdk:11-slim -ADD build/distributions/uc1-application-flink.tar / +ADD build/distributions/uc1-beam-flink.tar / + +CMD /uc1-beam-flink/bin/uc1-beam-flink --runner=FlinkRunner --streaming --disableMetrics=true --fasterCopy -CMD /uc1-application-flink/bin/uc1-application-flink --runner=FlinkRunner --flinkMaster=flink-jobmanager:8081 --streaming --parallelism=$PARALLELISM --disableMetrics=true --fasterCopy \ No newline at end of file diff --git a/theodolite-benchmarks/uc1-beam-samza/Dockerfile b/theodolite-benchmarks/uc1-beam-samza/Dockerfile index ef09939bb..75571f382 100644 --- a/theodolite-benchmarks/uc1-beam-samza/Dockerfile +++ b/theodolite-benchmarks/uc1-beam-samza/Dockerfile @@ -1,7 +1,7 @@ -FROM openjdk:8-slim +FROM openjdk:11-slim -ADD build/distributions/uc1-application-samza.tar / -ADD ../config/samza-standalone.properties / +ADD build/distributions/uc1-beam-samza.tar / +ADD samza-standalone.properties / +CMD /uc1-beam-samza/bin/uc1-beam-samza --configFactory=org.apache.samza.config.factories.PropertiesConfigFactory --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --enableMetrics=false -CMD /uc1-application-samza/bin/uc1-application-samza --configFactory=org.apache.samza.config.factories.PropertiesConfigFactory --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --maxSourceParallelism=$MAX_SOURCE_PARALLELISM --enableMetrics=false diff --git a/theodolite-benchmarks/uc1-beam-samza/samza-standalone.properties b/theodolite-benchmarks/uc1-beam-samza/samza-standalone.properties new file mode 100644 index 000000000..8044fe3d7 --- /dev/null +++ b/theodolite-benchmarks/uc1-beam-samza/samza-standalone.properties @@ -0,0 +1,17 @@ +# runner +app.runner.class=org.apache.samza.runtime.LocalApplicationRunner + +# zk +job.coordinator.factory=org.apache.samza.zk.ZkJobCoordinatorFactory +#job.coordinator.zk.connect=my-confluent-cp-zookeeper:2181 +job.coordinator.zk.connect=zookeeper:2181 +task.name.grouper.factory=org.apache.samza.container.grouper.task.GroupByContainerIdsFactory + +# default system +job.default.system=kafka +systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory +systems.kafka.consumer.zookeeper.connect=zookeeper:2181 +#systems.kafka.consumer.zookeeper.connect=my-confluent-cp-zookeeper:2181 +systems.kafka.producer.bootstrap.servers=kafka:9092 +#systems.kafka.producer.bootstrap.servers=my-confluent-cp-kafka:9092 +systems.kafka.default.stream.replication.factor=1 -- GitLab