diff --git a/theodolite-benchmarks/config/samza-standalone.properties b/theodolite-benchmarks/config/samza-standalone.properties deleted file mode 100644 index 04b68b048434ada3cf7a467c275d7d397955eb2f..0000000000000000000000000000000000000000 --- a/theodolite-benchmarks/config/samza-standalone.properties +++ /dev/null @@ -1,40 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - - - -# runner -app.runner.class=org.apache.samza.runtime.LocalApplicationRunner - -# zk -job.coordinator.factory=org.apache.samza.zk.ZkJobCoordinatorFactory -job.coordinator.zk.connect=my-confluent-cp-zookeeper:2181 -#job.coordinator.zk.connect=localhost:2181 #uncomment for local execution -task.name.grouper.factory=org.apache.samza.container.grouper.task.GroupByContainerIdsFactory - -# default system -job.default.system=kafka -systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory -#systems.kafka.consumer.zookeeper.connect=localhost:2181 #uncomment for local execution -systems.kafka.consumer.zookeeper.connect=my-confluent-cp-zookeeper:2181 -#systems.kafka.producer.bootstrap.servers=localhost:9092 #uncomment for local execution -systems.kafka.producer.bootstrap.servers=my-confluent-cp-kafka:9092 -systems.kafka.default.stream.replication.factor=1 - - - diff --git a/theodolite-benchmarks/docker-test/uc2-beam-samza/docker-compose.yml b/theodolite-benchmarks/docker-test/uc2-beam-samza/docker-compose.yml new file mode 100644 index 0000000000000000000000000000000000000000..ece1b47d1923df0f8a68b8da6873a8553bbf5541 --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc2-beam-samza/docker-compose.yml @@ -0,0 +1,52 @@ +version: '2' +services: + zookeeper: + image: confluentinc/cp-zookeeper + expose: + - "9092" + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + kafka: + image: wurstmeister/kafka + expose: + - "9092" + #ports: + # - 19092:19092 + environment: + KAFKA_LISTENERS: PLAINTEXT://:9092,CONNECTIONS_FROM_HOST://:19092 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,CONNECTIONS_FROM_HOST://localhost:19092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 + KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1" + schema-registry: + image: confluentinc/cp-schema-registry:5.3.1 + depends_on: + - zookeeper + - kafka + expose: + - "8081" + #ports: + # - 8081:8081 + environment: + SCHEMA_REGISTRY_HOST_NAME: schema-registry + SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + benchmark: + image: uc2-beam-samza:latest + depends_on: + - schema-registry + - kafka + environment: + KAFKA_BOOTSTRAP_SERVERS: kafka:9092 + SCHEMA_REGISTRY_URL: http://schema-registry:8081 + load-generator: + image: ghcr.io/cau-se/theodolite-uc1-workload-generator:latest + depends_on: + - schema-registry + - kafka + environment: + BOOTSTRAP_SERVER: load-generator:5701 + PORT: 5701 + KAFKA_BOOTSTRAP_SERVERS: kafka:9092 + SCHEMA_REGISTRY_URL: http://schema-registry:8081 + NUM_SENSORS: 10 diff --git a/theodolite-benchmarks/docker-test/uc3-beam-samza/docker-compose.yml b/theodolite-benchmarks/docker-test/uc3-beam-samza/docker-compose.yml new file mode 100644 index 0000000000000000000000000000000000000000..91bad2b59a5c3efc451106e7039f884c40515829 --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc3-beam-samza/docker-compose.yml @@ -0,0 +1,52 @@ +version: '2' +services: + zookeeper: + image: confluentinc/cp-zookeeper + expose: + - "9092" + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + kafka: + image: wurstmeister/kafka + expose: + - "9092" + #ports: + # - 19092:19092 + environment: + KAFKA_LISTENERS: PLAINTEXT://:9092,CONNECTIONS_FROM_HOST://:19092 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,CONNECTIONS_FROM_HOST://localhost:19092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 + KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1" + schema-registry: + image: confluentinc/cp-schema-registry:5.3.1 + depends_on: + - zookeeper + - kafka + expose: + - "8081" + #ports: + # - 8081:8081 + environment: + SCHEMA_REGISTRY_HOST_NAME: schema-registry + SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + benchmark: + image: uc3-beam-samza:latest + depends_on: + - schema-registry + - kafka + environment: + KAFKA_BOOTSTRAP_SERVERS: kafka:9092 + SCHEMA_REGISTRY_URL: http://schema-registry:8081 + load-generator: + image: ghcr.io/cau-se/theodolite-uc1-workload-generator:latest + depends_on: + - schema-registry + - kafka + environment: + BOOTSTRAP_SERVER: load-generator:5701 + PORT: 5701 + KAFKA_BOOTSTRAP_SERVERS: kafka:9092 + SCHEMA_REGISTRY_URL: http://schema-registry:8081 + NUM_SENSORS: 10 diff --git a/theodolite-benchmarks/docker-test/uc4-beam-samza/docker-compose.yml b/theodolite-benchmarks/docker-test/uc4-beam-samza/docker-compose.yml new file mode 100644 index 0000000000000000000000000000000000000000..83e8e05abbf92f6e59575d9fa37558c10759a4dc --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc4-beam-samza/docker-compose.yml @@ -0,0 +1,52 @@ +version: '2' +services: + zookeeper: + image: confluentinc/cp-zookeeper + expose: + - "9092" + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + kafka: + image: wurstmeister/kafka + expose: + - "9092" + #ports: + # - 19092:19092 + environment: + KAFKA_LISTENERS: PLAINTEXT://:9092,CONNECTIONS_FROM_HOST://:19092 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,CONNECTIONS_FROM_HOST://localhost:19092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 + KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1" + schema-registry: + image: confluentinc/cp-schema-registry:5.3.1 + depends_on: + - zookeeper + - kafka + expose: + - "8081" + #ports: + # - 8081:8081 + environment: + SCHEMA_REGISTRY_HOST_NAME: schema-registry + SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + benchmark: + image: uc4-beam-samza:latest + depends_on: + - schema-registry + - kafka + environment: + KAFKA_BOOTSTRAP_SERVERS: kafka:9092 + SCHEMA_REGISTRY_URL: http://schema-registry:8081 + load-generator: + image: ghcr.io/cau-se/theodolite-uc1-workload-generator:latest + depends_on: + - schema-registry + - kafka + environment: + BOOTSTRAP_SERVER: load-generator:5701 + PORT: 5701 + KAFKA_BOOTSTRAP_SERVERS: kafka:9092 + SCHEMA_REGISTRY_URL: http://schema-registry:8081 + NUM_SENSORS: 10 diff --git a/theodolite-benchmarks/uc2-beam-samza/Dockerfile b/theodolite-benchmarks/uc2-beam-samza/Dockerfile index baee343ad95295b44ee1d459a867a052c45b24c9..bb517b9e2b1021ad75c1f6714ca9afecfcbbd154 100644 --- a/theodolite-benchmarks/uc2-beam-samza/Dockerfile +++ b/theodolite-benchmarks/uc2-beam-samza/Dockerfile @@ -1,5 +1,6 @@ -FROM openjdk:8-slim +FROM openjdk:11-slim -ADD build/distributions/uc3-application-flink.tar / +ADD build/distributions/uc2-beam-samza.tar / +ADD samza-standalone.properties / -CMD /uc3-application-flink/bin/uc3-application-flink --runner=FlinkRunner --flinkMaster=flink-jobmanager:8081 --streaming --parallelism=$PARALLELISM --disableMetrics=true --fasterCopy \ No newline at end of file +CMD /uc2-beam-samza/bin/uc2-beam-samza --configFactory=org.apache.samza.config.factories.PropertiesConfigFactory --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --enableMetrics=false diff --git a/theodolite-benchmarks/uc2-beam-samza/samza-standalone.properties b/theodolite-benchmarks/uc2-beam-samza/samza-standalone.properties new file mode 100644 index 0000000000000000000000000000000000000000..8044fe3d7516591b1a0eefb73585f6acc5382409 --- /dev/null +++ b/theodolite-benchmarks/uc2-beam-samza/samza-standalone.properties @@ -0,0 +1,17 @@ +# runner +app.runner.class=org.apache.samza.runtime.LocalApplicationRunner + +# zk +job.coordinator.factory=org.apache.samza.zk.ZkJobCoordinatorFactory +#job.coordinator.zk.connect=my-confluent-cp-zookeeper:2181 +job.coordinator.zk.connect=zookeeper:2181 +task.name.grouper.factory=org.apache.samza.container.grouper.task.GroupByContainerIdsFactory + +# default system +job.default.system=kafka +systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory +systems.kafka.consumer.zookeeper.connect=zookeeper:2181 +#systems.kafka.consumer.zookeeper.connect=my-confluent-cp-zookeeper:2181 +systems.kafka.producer.bootstrap.servers=kafka:9092 +#systems.kafka.producer.bootstrap.servers=my-confluent-cp-kafka:9092 +systems.kafka.default.stream.replication.factor=1 diff --git a/theodolite-benchmarks/uc3-beam-samza/Dockerfile b/theodolite-benchmarks/uc3-beam-samza/Dockerfile index f16ec202eedc33062f29a2a5856b70c0b7fd43ec..3e68ef01d7a927c93c782c6990f114b8d286f476 100644 --- a/theodolite-benchmarks/uc3-beam-samza/Dockerfile +++ b/theodolite-benchmarks/uc3-beam-samza/Dockerfile @@ -1,5 +1,6 @@ -FROM openjdk:8-slim +FROM openjdk:11-slim -ADD build/distributions/uc4-application-flink.tar / +ADD build/distributions/uc3-beam-samza.tar / +ADD samza-standalone.properties / -CMD /uc4-application-flink/bin/uc4-application-flink --runner=FlinkRunner --flinkMaster=flink-jobmanager:8081 --streaming --parallelism=$PARALLELISM --disableMetrics=true --fasterCopy --stateBackend=rocksdb --stateBackendStoragePath=file:///data/flink/checkpoints \ No newline at end of file +CMD /uc3-beam-samza/bin/uc3-beam-samza --configFactory=org.apache.samza.config.factories.PropertiesConfigFactory --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --enableMetrics=false diff --git a/theodolite-benchmarks/uc3-beam-samza/samza-standalone.properties b/theodolite-benchmarks/uc3-beam-samza/samza-standalone.properties new file mode 100644 index 0000000000000000000000000000000000000000..8044fe3d7516591b1a0eefb73585f6acc5382409 --- /dev/null +++ b/theodolite-benchmarks/uc3-beam-samza/samza-standalone.properties @@ -0,0 +1,17 @@ +# runner +app.runner.class=org.apache.samza.runtime.LocalApplicationRunner + +# zk +job.coordinator.factory=org.apache.samza.zk.ZkJobCoordinatorFactory +#job.coordinator.zk.connect=my-confluent-cp-zookeeper:2181 +job.coordinator.zk.connect=zookeeper:2181 +task.name.grouper.factory=org.apache.samza.container.grouper.task.GroupByContainerIdsFactory + +# default system +job.default.system=kafka +systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory +systems.kafka.consumer.zookeeper.connect=zookeeper:2181 +#systems.kafka.consumer.zookeeper.connect=my-confluent-cp-zookeeper:2181 +systems.kafka.producer.bootstrap.servers=kafka:9092 +#systems.kafka.producer.bootstrap.servers=my-confluent-cp-kafka:9092 +systems.kafka.default.stream.replication.factor=1 diff --git a/theodolite-benchmarks/uc4-beam-samza/Dockerfile b/theodolite-benchmarks/uc4-beam-samza/Dockerfile index 4505849ce764648f19c080a1f7534abc5486279d..e7f4e10b5ea5829bc7e1d4b545718d5be19760a4 100644 --- a/theodolite-benchmarks/uc4-beam-samza/Dockerfile +++ b/theodolite-benchmarks/uc4-beam-samza/Dockerfile @@ -1,7 +1,6 @@ -FROM openjdk:8-slim +FROM openjdk:11-slim -ADD build/distributions/uc2-application-samza.tar / -ADD ../config/standalone.properties / +ADD build/distributions/uc4-beam-samza.tar / +ADD samza-standalone.properties / - -CMD /uc2-application-samza/bin/uc2-application-samza --configFactory=org.apache.samza.config.factories.PropertiesConfigFactory --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --maxSourceParallelism=$MAX_SOURCE_PARALLELISM --enableMetrics=false +CMD /uc4-beam-samza/bin/uc4-beam-samza --configFactory=org.apache.samza.config.factories.PropertiesConfigFactory --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --enableMetrics=false diff --git a/theodolite-benchmarks/uc4-beam-samza/samza-standalone.properties b/theodolite-benchmarks/uc4-beam-samza/samza-standalone.properties new file mode 100644 index 0000000000000000000000000000000000000000..8044fe3d7516591b1a0eefb73585f6acc5382409 --- /dev/null +++ b/theodolite-benchmarks/uc4-beam-samza/samza-standalone.properties @@ -0,0 +1,17 @@ +# runner +app.runner.class=org.apache.samza.runtime.LocalApplicationRunner + +# zk +job.coordinator.factory=org.apache.samza.zk.ZkJobCoordinatorFactory +#job.coordinator.zk.connect=my-confluent-cp-zookeeper:2181 +job.coordinator.zk.connect=zookeeper:2181 +task.name.grouper.factory=org.apache.samza.container.grouper.task.GroupByContainerIdsFactory + +# default system +job.default.system=kafka +systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory +systems.kafka.consumer.zookeeper.connect=zookeeper:2181 +#systems.kafka.consumer.zookeeper.connect=my-confluent-cp-zookeeper:2181 +systems.kafka.producer.bootstrap.servers=kafka:9092 +#systems.kafka.producer.bootstrap.servers=my-confluent-cp-kafka:9092 +systems.kafka.default.stream.replication.factor=1