diff --git a/theodolite-benchmarks/docker-test/uc1-beam-flink/docker-compose.yml b/theodolite-benchmarks/docker-test/uc1-beam-flink/docker-compose.yml new file mode 100644 index 0000000000000000000000000000000000000000..2548472b41935bc83831e9da76e1a69fdb32e54a --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc1-beam-flink/docker-compose.yml @@ -0,0 +1,52 @@ +version: '2' +services: + zookeeper: + image: confluentinc/cp-zookeeper + expose: + - "9092" + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + kafka: + image: wurstmeister/kafka + expose: + - "9092" + #ports: + # - 19092:19092 + environment: + KAFKA_LISTENERS: PLAINTEXT://:9092,CONNECTIONS_FROM_HOST://:19092 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,CONNECTIONS_FROM_HOST://localhost:19092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 + KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1" + schema-registry: + image: confluentinc/cp-schema-registry:5.3.1 + depends_on: + - zookeeper + - kafka + expose: + - "8081" + #ports: + # - 8081:8081 + environment: + SCHEMA_REGISTRY_HOST_NAME: schema-registry + SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + benchmark: + image: uc1-beam-flink:latest + depends_on: + - schema-registry + - kafka + environment: + KAFKA_BOOTSTRAP_SERVERS: kafka:9092 + SCHEMA_REGISTRY_URL: http://schema-registry:8081 + load-generator: + image: ghcr.io/cau-se/theodolite-uc1-workload-generator:latest + depends_on: + - schema-registry + - kafka + environment: + BOOTSTRAP_SERVER: load-generator:5701 + PORT: 5701 + KAFKA_BOOTSTRAP_SERVERS: kafka:9092 + SCHEMA_REGISTRY_URL: http://schema-registry:8081 + NUM_SENSORS: 10 diff --git a/theodolite-benchmarks/docker-test/uc1-beam-samza/docker-compose.yml b/theodolite-benchmarks/docker-test/uc1-beam-samza/docker-compose.yml new file mode 100644 index 0000000000000000000000000000000000000000..88cc156cee3b09a4f9706c86638085109fc17681 --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc1-beam-samza/docker-compose.yml @@ -0,0 +1,52 @@ +version: '2' +services: + zookeeper: + image: confluentinc/cp-zookeeper + expose: + - "9092" + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + kafka: + image: wurstmeister/kafka + expose: + - "9092" + #ports: + # - 19092:19092 + environment: + KAFKA_LISTENERS: PLAINTEXT://:9092,CONNECTIONS_FROM_HOST://:19092 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,CONNECTIONS_FROM_HOST://localhost:19092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 + KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1" + schema-registry: + image: confluentinc/cp-schema-registry:5.3.1 + depends_on: + - zookeeper + - kafka + expose: + - "8081" + #ports: + # - 8081:8081 + environment: + SCHEMA_REGISTRY_HOST_NAME: schema-registry + SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + benchmark: + image: uc1-beam-samza:latest + depends_on: + - schema-registry + - kafka + environment: + KAFKA_BOOTSTRAP_SERVERS: kafka:9092 + SCHEMA_REGISTRY_URL: http://schema-registry:8081 + load-generator: + image: ghcr.io/cau-se/theodolite-uc1-workload-generator:latest + depends_on: + - schema-registry + - kafka + environment: + BOOTSTRAP_SERVER: load-generator:5701 + PORT: 5701 + KAFKA_BOOTSTRAP_SERVERS: kafka:9092 + SCHEMA_REGISTRY_URL: http://schema-registry:8081 + NUM_SENSORS: 10 diff --git a/theodolite-benchmarks/uc1-beam-flink/Dockerfile b/theodolite-benchmarks/uc1-beam-flink/Dockerfile index 67fbb025d7de1c0355409a24a6d5119782937d8b..99afa663aef13098ae9745742dd6dcc43a3dd0fa 100644 --- a/theodolite-benchmarks/uc1-beam-flink/Dockerfile +++ b/theodolite-benchmarks/uc1-beam-flink/Dockerfile @@ -1,5 +1,6 @@ -FROM openjdk:8-slim +FROM openjdk:11-slim -ADD build/distributions/uc1-application-flink.tar / +ADD build/distributions/uc1-beam-flink.tar / + +CMD /uc1-beam-flink/bin/uc1-beam-flink --runner=FlinkRunner --streaming --disableMetrics=true --fasterCopy -CMD /uc1-application-flink/bin/uc1-application-flink --runner=FlinkRunner --flinkMaster=flink-jobmanager:8081 --streaming --parallelism=$PARALLELISM --disableMetrics=true --fasterCopy \ No newline at end of file diff --git a/theodolite-benchmarks/uc1-beam-samza/Dockerfile b/theodolite-benchmarks/uc1-beam-samza/Dockerfile index ef09939bb35e8a81bdab2590b120cb6173e7a9ab..75571f382cea7da371a70b28fd693e599c3b526a 100644 --- a/theodolite-benchmarks/uc1-beam-samza/Dockerfile +++ b/theodolite-benchmarks/uc1-beam-samza/Dockerfile @@ -1,7 +1,7 @@ -FROM openjdk:8-slim +FROM openjdk:11-slim -ADD build/distributions/uc1-application-samza.tar / -ADD ../config/samza-standalone.properties / +ADD build/distributions/uc1-beam-samza.tar / +ADD samza-standalone.properties / +CMD /uc1-beam-samza/bin/uc1-beam-samza --configFactory=org.apache.samza.config.factories.PropertiesConfigFactory --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --enableMetrics=false -CMD /uc1-application-samza/bin/uc1-application-samza --configFactory=org.apache.samza.config.factories.PropertiesConfigFactory --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --maxSourceParallelism=$MAX_SOURCE_PARALLELISM --enableMetrics=false diff --git a/theodolite-benchmarks/uc1-beam-samza/samza-standalone.properties b/theodolite-benchmarks/uc1-beam-samza/samza-standalone.properties new file mode 100644 index 0000000000000000000000000000000000000000..8044fe3d7516591b1a0eefb73585f6acc5382409 --- /dev/null +++ b/theodolite-benchmarks/uc1-beam-samza/samza-standalone.properties @@ -0,0 +1,17 @@ +# runner +app.runner.class=org.apache.samza.runtime.LocalApplicationRunner + +# zk +job.coordinator.factory=org.apache.samza.zk.ZkJobCoordinatorFactory +#job.coordinator.zk.connect=my-confluent-cp-zookeeper:2181 +job.coordinator.zk.connect=zookeeper:2181 +task.name.grouper.factory=org.apache.samza.container.grouper.task.GroupByContainerIdsFactory + +# default system +job.default.system=kafka +systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory +systems.kafka.consumer.zookeeper.connect=zookeeper:2181 +#systems.kafka.consumer.zookeeper.connect=my-confluent-cp-zookeeper:2181 +systems.kafka.producer.bootstrap.servers=kafka:9092 +#systems.kafka.producer.bootstrap.servers=my-confluent-cp-kafka:9092 +systems.kafka.default.stream.replication.factor=1