Skip to content
Snippets Groups Projects
Commit 2be456a4 authored by Lorenz Boguhn's avatar Lorenz Boguhn
Browse files

Add beam-samza dockerfiles for uc2-4 

 + Moved samza-standalone-properties in every folder arcorrdingly
parent 2932f1d4
No related branches found
No related tags found
1 merge request!187Migrate Beam benchmark implementation
Hide whitespace changes
Inline Side-by-side
Showing
with 219 additions and 51 deletions
theodolite-benchmarks/config/samza-standalone.properties deleted 100644 → 0
+ 0
− 40
View file @ 2932f1d4
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# runner
app.runner.class=org.apache.samza.runtime.LocalApplicationRunner
# zk
job.coordinator.factory=org.apache.samza.zk.ZkJobCoordinatorFactory
job.coordinator.zk.connect=my-confluent-cp-zookeeper:2181
#job.coordinator.zk.connect=localhost:2181 #uncomment for local execution
task.name.grouper.factory=org.apache.samza.container.grouper.task.GroupByContainerIdsFactory
# default system
job.default.system=kafka
systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory
#systems.kafka.consumer.zookeeper.connect=localhost:2181 #uncomment for local execution
systems.kafka.consumer.zookeeper.connect=my-confluent-cp-zookeeper:2181
#systems.kafka.producer.bootstrap.servers=localhost:9092 #uncomment for local execution
systems.kafka.producer.bootstrap.servers=my-confluent-cp-kafka:9092
systems.kafka.default.stream.replication.factor=1
theodolite-benchmarks/docker-test/uc2-beam-samza/docker-compose.yml 0 → 100644
+ 52
− 0
View file @ 2be456a4
version: '2'
services:
zookeeper:
image: confluentinc/cp-zookeeper
expose:
- "9092"
environment:
ZOOKEEPER_CLIENT_PORT: 2181
kafka:
image: wurstmeister/kafka
expose:
- "9092"
#ports:
# - 19092:19092
environment:
KAFKA_LISTENERS: PLAINTEXT://:9092,CONNECTIONS_FROM_HOST://:19092
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,CONNECTIONS_FROM_HOST://localhost:19092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
schema-registry:
image: confluentinc/cp-schema-registry:5.3.1
depends_on:
- zookeeper
- kafka
expose:
- "8081"
#ports:
# - 8081:8081
environment:
SCHEMA_REGISTRY_HOST_NAME: schema-registry
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
benchmark:
image: uc2-beam-samza:latest
depends_on:
- schema-registry
- kafka
environment:
KAFKA_BOOTSTRAP_SERVERS: kafka:9092
SCHEMA_REGISTRY_URL: http://schema-registry:8081
load-generator:
image: ghcr.io/cau-se/theodolite-uc1-workload-generator:latest
depends_on:
- schema-registry
- kafka
environment:
BOOTSTRAP_SERVER: load-generator:5701
PORT: 5701
KAFKA_BOOTSTRAP_SERVERS: kafka:9092
SCHEMA_REGISTRY_URL: http://schema-registry:8081
NUM_SENSORS: 10
theodolite-benchmarks/docker-test/uc3-beam-samza/docker-compose.yml 0 → 100644
+ 52
− 0
View file @ 2be456a4
version: '2'
services:
zookeeper:
image: confluentinc/cp-zookeeper
expose:
- "9092"
environment:
ZOOKEEPER_CLIENT_PORT: 2181
kafka:
image: wurstmeister/kafka
expose:
- "9092"
#ports:
# - 19092:19092
environment:
KAFKA_LISTENERS: PLAINTEXT://:9092,CONNECTIONS_FROM_HOST://:19092
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,CONNECTIONS_FROM_HOST://localhost:19092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
schema-registry:
image: confluentinc/cp-schema-registry:5.3.1
depends_on:
- zookeeper
- kafka
expose:
- "8081"
#ports:
# - 8081:8081
environment:
SCHEMA_REGISTRY_HOST_NAME: schema-registry
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
benchmark:
image: uc3-beam-samza:latest
depends_on:
- schema-registry
- kafka
environment:
KAFKA_BOOTSTRAP_SERVERS: kafka:9092
SCHEMA_REGISTRY_URL: http://schema-registry:8081
load-generator:
image: ghcr.io/cau-se/theodolite-uc1-workload-generator:latest
depends_on:
- schema-registry
- kafka
environment:
BOOTSTRAP_SERVER: load-generator:5701
PORT: 5701
KAFKA_BOOTSTRAP_SERVERS: kafka:9092
SCHEMA_REGISTRY_URL: http://schema-registry:8081
NUM_SENSORS: 10
theodolite-benchmarks/docker-test/uc4-beam-samza/docker-compose.yml 0 → 100644
+ 52
− 0
View file @ 2be456a4
version: '2'
services:
zookeeper:
image: confluentinc/cp-zookeeper
expose:
- "9092"
environment:
ZOOKEEPER_CLIENT_PORT: 2181
kafka:
image: wurstmeister/kafka
expose:
- "9092"
#ports:
# - 19092:19092
environment:
KAFKA_LISTENERS: PLAINTEXT://:9092,CONNECTIONS_FROM_HOST://:19092
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,CONNECTIONS_FROM_HOST://localhost:19092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
schema-registry:
image: confluentinc/cp-schema-registry:5.3.1
depends_on:
- zookeeper
- kafka
expose:
- "8081"
#ports:
# - 8081:8081
environment:
SCHEMA_REGISTRY_HOST_NAME: schema-registry
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
benchmark:
image: uc4-beam-samza:latest
depends_on:
- schema-registry
- kafka
environment:
KAFKA_BOOTSTRAP_SERVERS: kafka:9092
SCHEMA_REGISTRY_URL: http://schema-registry:8081
load-generator:
image: ghcr.io/cau-se/theodolite-uc1-workload-generator:latest
depends_on:
- schema-registry
- kafka
environment:
BOOTSTRAP_SERVER: load-generator:5701
PORT: 5701
KAFKA_BOOTSTRAP_SERVERS: kafka:9092
SCHEMA_REGISTRY_URL: http://schema-registry:8081
NUM_SENSORS: 10
theodolite-benchmarks/uc2-beam-samza/Dockerfile
+ 4
− 3
View file @ 2be456a4
FROM openjdk:8-slim
FROM openjdk:11-slim
ADD build/distributions/uc3-application-flink.tar /
ADD build/distributions/uc2-beam-samza.tar /
ADD samza-standalone.properties /
CMD /uc3-application-flink/bin/uc3-application-flink --runner=FlinkRunner --flinkMaster=flink-jobmanager:8081 --streaming --parallelism=$PARALLELISM --disableMetrics=true --fasterCopy
\ No newline at end of file
CMD /uc2-beam-samza/bin/uc2-beam-samza --configFactory=org.apache.samza.config.factories.PropertiesConfigFactory --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --enableMetrics=false
theodolite-benchmarks/uc2-beam-samza/samza-standalone.properties 0 → 100644
+ 17
− 0
View file @ 2be456a4
# runner
app.runner.class=org.apache.samza.runtime.LocalApplicationRunner
# zk
job.coordinator.factory=org.apache.samza.zk.ZkJobCoordinatorFactory
#job.coordinator.zk.connect=my-confluent-cp-zookeeper:2181
job.coordinator.zk.connect=zookeeper:2181
task.name.grouper.factory=org.apache.samza.container.grouper.task.GroupByContainerIdsFactory
# default system
job.default.system=kafka
systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory
systems.kafka.consumer.zookeeper.connect=zookeeper:2181
#systems.kafka.consumer.zookeeper.connect=my-confluent-cp-zookeeper:2181
systems.kafka.producer.bootstrap.servers=kafka:9092
#systems.kafka.producer.bootstrap.servers=my-confluent-cp-kafka:9092
systems.kafka.default.stream.replication.factor=1
theodolite-benchmarks/uc3-beam-samza/Dockerfile
+ 4
− 3
View file @ 2be456a4
FROM openjdk:8-slim
FROM openjdk:11-slim
ADD build/distributions/uc4-application-flink.tar /
ADD build/distributions/uc3-beam-samza.tar /
ADD samza-standalone.properties /
CMD /uc4-application-flink/bin/uc4-application-flink --runner=FlinkRunner --flinkMaster=flink-jobmanager:8081 --streaming --parallelism=$PARALLELISM --disableMetrics=true --fasterCopy --stateBackend=rocksdb --stateBackendStoragePath=file:///data/flink/checkpoints
\ No newline at end of file
CMD /uc3-beam-samza/bin/uc3-beam-samza --configFactory=org.apache.samza.config.factories.PropertiesConfigFactory --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --enableMetrics=false
theodolite-benchmarks/uc3-beam-samza/samza-standalone.properties 0 → 100644
+ 17
− 0
View file @ 2be456a4
# runner
app.runner.class=org.apache.samza.runtime.LocalApplicationRunner
# zk
job.coordinator.factory=org.apache.samza.zk.ZkJobCoordinatorFactory
#job.coordinator.zk.connect=my-confluent-cp-zookeeper:2181
job.coordinator.zk.connect=zookeeper:2181
task.name.grouper.factory=org.apache.samza.container.grouper.task.GroupByContainerIdsFactory
# default system
job.default.system=kafka
systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory
systems.kafka.consumer.zookeeper.connect=zookeeper:2181
#systems.kafka.consumer.zookeeper.connect=my-confluent-cp-zookeeper:2181
systems.kafka.producer.bootstrap.servers=kafka:9092
#systems.kafka.producer.bootstrap.servers=my-confluent-cp-kafka:9092
systems.kafka.default.stream.replication.factor=1
theodolite-benchmarks/uc4-beam-samza/Dockerfile
+ 4
− 5
View file @ 2be456a4
FROM openjdk:8-slim
FROM openjdk:11-slim
ADD build/distributions/uc2-application-samza.tar /
ADD ../config/standalone.properties /
ADD build/distributions/uc4-beam-samza.tar /
ADD samza-standalone.properties /
CMD /uc2-application-samza/bin/uc2-application-samza --configFactory=org.apache.samza.config.factories.PropertiesConfigFactory --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --maxSourceParallelism=$MAX_SOURCE_PARALLELISM --enableMetrics=false
CMD /uc4-beam-samza/bin/uc4-beam-samza --configFactory=org.apache.samza.config.factories.PropertiesConfigFactory --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --enableMetrics=false
theodolite-benchmarks/uc4-beam-samza/samza-standalone.properties 0 → 100644
+ 17
− 0
View file @ 2be456a4
# runner
app.runner.class=org.apache.samza.runtime.LocalApplicationRunner
# zk
job.coordinator.factory=org.apache.samza.zk.ZkJobCoordinatorFactory
#job.coordinator.zk.connect=my-confluent-cp-zookeeper:2181
job.coordinator.zk.connect=zookeeper:2181
task.name.grouper.factory=org.apache.samza.container.grouper.task.GroupByContainerIdsFactory
# default system
job.default.system=kafka
systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory
systems.kafka.consumer.zookeeper.connect=zookeeper:2181
#systems.kafka.consumer.zookeeper.connect=my-confluent-cp-zookeeper:2181
systems.kafka.producer.bootstrap.servers=kafka:9092
#systems.kafka.producer.bootstrap.servers=my-confluent-cp-kafka:9092
systems.kafka.default.stream.replication.factor=1
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment