Skip to content
Snippets Groups Projects
Commit 5da264ed authored by Lorenz Boguhn's avatar Lorenz Boguhn
Browse files

merge EnvVariables with Rest

parents 7f0610af 13f311b9
No related branches found
No related tags found
1 merge request!187Migrate Beam benchmark implementation
Pipeline #5828 passed
Stage: build
Stage: test
Stage: check
Hide whitespace changes
Inline Side-by-side
Showing
with 52 additions and 54 deletions
theodolite-benchmarks/beam-commons/src/main/java/theodolite/commons/beam/AbstractBeamService.java
+ 1
− 1
View file @ 5da264ed
......@@ -8,7 +8,7 @@ import org.slf4j.LoggerFactory;
import titan.ccp.common.configuration.ServiceConfigurations;
/**
* Abstraction of a beam microservice.
* Abstraction of a Beam microservice.
* Encapsulates the corresponding {@link PipelineOptions} and the beam Runner.
*/
public class AbstractBeamService {
......
theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.beam.samza.gradle
+ 3
− 1
View file @ 5da264ed
......@@ -3,5 +3,7 @@ plugins {
}
dependencies {
compile group: 'org.apache.beam', name: 'beam-runners-samza', version: '2.22.0'
implementation('org.apache.beam:beam-runners-samza:2.22.0') {
exclude group: 'org.apache.samza', module: 'samza-yarn_2.11'
}
}
\ No newline at end of file
theodolite-benchmarks/docker-test/uc1-beam-samza/docker-compose.yml
+ 2
− 0
View file @ 5da264ed
......@@ -4,6 +4,8 @@ services:
image: confluentinc/cp-zookeeper
expose:
- "2181"
#ports:
# - 2181:2181
environment:
ZOOKEEPER_CLIENT_PORT: 2181
kafka:
......
theodolite-benchmarks/docker-test/uc2-beam-samza/docker-compose.yml
+ 2
− 0
View file @ 5da264ed
......@@ -4,6 +4,8 @@ services:
image: confluentinc/cp-zookeeper
expose:
- "2181"
#ports:
# - 2181:2181
environment:
ZOOKEEPER_CLIENT_PORT: 2181
kafka:
......
theodolite-benchmarks/docker-test/uc3-beam-samza/docker-compose.yml
+ 2
− 0
View file @ 5da264ed
......@@ -4,6 +4,8 @@ services:
image: confluentinc/cp-zookeeper
expose:
- "2181"
#ports:
# - 2181:2181
environment:
ZOOKEEPER_CLIENT_PORT: 2181
kafka:
......
theodolite-benchmarks/docker-test/uc4-beam-samza/docker-compose.yml
+ 2
− 0
View file @ 5da264ed
......@@ -4,6 +4,8 @@ services:
image: confluentinc/cp-zookeeper
expose:
- "2181"
#ports:
# - 2181:2181
environment:
ZOOKEEPER_CLIENT_PORT: 2181
kafka:
......
theodolite-benchmarks/uc1-beam-samza/Dockerfile
+ 1
− 1
View file @ 5da264ed
......@@ -5,5 +5,5 @@ ENV MAX_SOURCE_PARALLELISM=1024
ADD build/distributions/uc1-beam-samza.tar /
ADD samza-standalone.properties /
CMD /uc1-beam-samza/bin/uc1-beam-samza --configFactory=org.apache.samza.config.factories.PropertiesConfigFactory --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --maxSourceParallelism=$MAX_SOURCE_PARALLELISM --enableMetrics=false --jobInstance=$HOSTNAME
CMD /uc1-beam-samza/bin/uc1-beam-samza --configFactory=org.apache.samza.config.factories.PropertiesConfigFactory --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --maxSourceParallelism=$MAX_SOURCE_PARALLELISM --enableMetrics=false
theodolite-benchmarks/uc1-beam-samza/samza-standalone.properties
+ 8
− 11
View file @ 5da264ed
# runner
app.runner.class=org.apache.samza.runtime.LocalApplicationRunner
# Set EnvironmentRewriter
job.config.rewriters=env-config
job.config.rewriter.env-config.class=org.apache.samza.config.EnvironmentConfigRewriter
# zk
# Configure ZooKeeper for coordination
job.coordinator.factory=org.apache.samza.zk.ZkJobCoordinatorFactory
#job.coordinator.zk.connect=my-confluent-cp-zookeeper:2181
job.coordinator.zk.connect=zookeeper:2181
job.coordinator.zk.connect=localhost:2181
# Use GroupByContainerIds
task.name.grouper.factory=org.apache.samza.container.grouper.task.GroupByContainerIdsFactory
# default system
# Configure Kafka as "system"
job.default.system=kafka
systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory
systems.kafka.consumer.zookeeper.connect=zookeeper:2181
#systems.kafka.consumer.zookeeper.connect=my-confluent-cp-zookeeper:2181
systems.kafka.producer.bootstrap.servers=kafka:9092
#systems.kafka.producer.bootstrap.servers=my-confluent-cp-kafka:9092
systems.kafka.consumer.bootstrap.servers=localhost:9092
systems.kafka.producer.bootstrap.servers=localhost:9092
systems.kafka.default.stream.replication.factor=1
theodolite-benchmarks/uc1-beam-samza/src/main/java/application/Uc1BeamSamza.java
+ 3
− 3
View file @ 5da264ed
......@@ -6,10 +6,10 @@ import theodolite.commons.beam.AbstractBeamService;
/**
* Implementation of the use case Database Storage using Apache Beam with the Samza Runner. To
* execute locally in standalone start Kafka, Zookeeper, the schema-registry and the workload
* generator using the delayed_startup.sh script. Add
* generator. Add
* --configFactory=org.apache.samza.config.factories.PropertiesConfigFactory
* --configFilePath=${workspace_loc:uc1-application-samza}/config/standalone_local.properties
* --samzaExecutionEnvironment=STANDALONE --maxSourceParallelism=1024 --as program arguments. To
* --configFilePath=samza-standalone.properties
* --samzaExecutionEnvironment=STANDALONE --maxSourceParallelism=1024 as program arguments. To
* persist logs add ${workspace_loc:/uc4-application-samza/eclipseConsoleLogs.log} as Output File
* under Standard Input Output in Common in the Run Configuration Start via Eclipse Run.
*/
......
theodolite-benchmarks/uc2-beam-samza/Dockerfile
+ 1
− 1
View file @ 5da264ed
......@@ -5,4 +5,4 @@ ENV MAX_SOURCE_PARALLELISM=1024
ADD build/distributions/uc2-beam-samza.tar /
ADD samza-standalone.properties /
CMD /uc2-beam-samza/bin/uc2-beam-samza --configFactory=org.apache.samza.config.factories.PropertiesConfigFactory --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --maxSourceParallelism=$MAX_SOURCE_PARALLELISM --enableMetrics=false --jobInstance=$HOSTNAME
CMD /uc2-beam-samza/bin/uc2-beam-samza --configFactory=org.apache.samza.config.factories.PropertiesConfigFactory --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --maxSourceParallelism=$MAX_SOURCE_PARALLELISM --enableMetrics=false
theodolite-benchmarks/uc2-beam-samza/samza-standalone.properties
+ 8
− 11
View file @ 5da264ed
# runner
app.runner.class=org.apache.samza.runtime.LocalApplicationRunner
# Set EnvironmentRewriter
job.config.rewriters=env-config
job.config.rewriter.env-config.class=org.apache.samza.config.EnvironmentConfigRewriter
# zk
# Configure ZooKeeper for coordination
job.coordinator.factory=org.apache.samza.zk.ZkJobCoordinatorFactory
#job.coordinator.zk.connect=my-confluent-cp-zookeeper:2181
job.coordinator.zk.connect=zookeeper:2181
job.coordinator.zk.connect=localhost:2181
# Use GroupByContainerIds
task.name.grouper.factory=org.apache.samza.container.grouper.task.GroupByContainerIdsFactory
# default system
# Configure Kafka as "system"
job.default.system=kafka
systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory
systems.kafka.consumer.zookeeper.connect=zookeeper:2181
#systems.kafka.consumer.zookeeper.connect=my-confluent-cp-zookeeper:2181
systems.kafka.producer.bootstrap.servers=kafka:9092
#systems.kafka.producer.bootstrap.servers=my-confluent-cp-kafka:9092
systems.kafka.consumer.bootstrap.servers=localhost:9092
systems.kafka.producer.bootstrap.servers=localhost:9092
systems.kafka.default.stream.replication.factor=1
theodolite-benchmarks/uc3-beam-samza/Dockerfile
+ 1
− 1
View file @ 5da264ed
......@@ -5,4 +5,4 @@ ENV MAX_SOURCE_PARALLELISM=1024
ADD build/distributions/uc3-beam-samza.tar /
ADD samza-standalone.properties /
CMD /uc3-beam-samza/bin/uc3-beam-samza --configFactory=org.apache.samza.config.factories.PropertiesConfigFactory --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --maxSourceParallelism=$MAX_SOURCE_PARALLELISM --enableMetrics=false --jobInstance=$HOSTNAME
CMD /uc3-beam-samza/bin/uc3-beam-samza --configFactory=org.apache.samza.config.factories.PropertiesConfigFactory --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --maxSourceParallelism=$MAX_SOURCE_PARALLELISM --enableMetrics=false
theodolite-benchmarks/uc3-beam-samza/samza-standalone.properties
+ 8
− 11
View file @ 5da264ed
# runner
app.runner.class=org.apache.samza.runtime.LocalApplicationRunner
# Set EnvironmentRewriter
job.config.rewriters=env-config
job.config.rewriter.env-config.class=org.apache.samza.config.EnvironmentConfigRewriter
# zk
# Configure ZooKeeper for coordination
job.coordinator.factory=org.apache.samza.zk.ZkJobCoordinatorFactory
#job.coordinator.zk.connect=my-confluent-cp-zookeeper:2181
job.coordinator.zk.connect=zookeeper:2181
job.coordinator.zk.connect=localhost:2181
# Use GroupByContainerIds
task.name.grouper.factory=org.apache.samza.container.grouper.task.GroupByContainerIdsFactory
# default system
# Configure Kafka as "system"
job.default.system=kafka
systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory
systems.kafka.consumer.zookeeper.connect=zookeeper:2181
#systems.kafka.consumer.zookeeper.connect=my-confluent-cp-zookeeper:2181
systems.kafka.producer.bootstrap.servers=kafka:9092
#systems.kafka.producer.bootstrap.servers=my-confluent-cp-kafka:9092
systems.kafka.consumer.bootstrap.servers=localhost:9092
systems.kafka.producer.bootstrap.servers=localhost:9092
systems.kafka.default.stream.replication.factor=1
theodolite-benchmarks/uc4-beam-samza/Dockerfile
+ 1
− 1
View file @ 5da264ed
......@@ -5,4 +5,4 @@ ENV MAX_SOURCE_PARALLELISM=1024
ADD build/distributions/uc4-beam-samza.tar /
ADD samza-standalone.properties /
CMD /uc4-beam-samza/bin/uc4-beam-samza --configFactory=org.apache.samza.config.factories.PropertiesConfigFactory --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --maxSourceParallelism=$MAX_SOURCE_PARALLELISM --enableMetrics=false --jobInstance=$HOSTNAME
CMD /uc4-beam-samza/bin/uc4-beam-samza --configFactory=org.apache.samza.config.factories.PropertiesConfigFactory --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --maxSourceParallelism=$MAX_SOURCE_PARALLELISM --enableMetrics=false
theodolite-benchmarks/uc4-beam-samza/samza-standalone.properties
+ 9
− 12
View file @ 5da264ed
# runner
app.runner.class=org.apache.samza.runtime.LocalApplicationRunner
# Set EnvironmentRewriter
job.config.rewriters=env-config
job.config.rewriter.env-config.class=org.apache.samza.config.EnvironmentConfigRewriter
# zk
# Configure ZooKeeper for coordination
job.coordinator.factory=org.apache.samza.zk.ZkJobCoordinatorFactory
#job.coordinator.zk.connect=my-confluent-cp-zookeeper:2181
job.coordinator.zk.connect=zookeeper:2181
job.coordinator.zk.connect=localhost:2181
# Use GroupByContainerIds
task.name.grouper.factory=org.apache.samza.container.grouper.task.GroupByContainerIdsFactory
# default system
# Configure Kafka as "system"
job.default.system=kafka
systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory
systems.kafka.consumer.zookeeper.connect=zookeeper:2181
#systems.kafka.consumer.zookeeper.connect=my-confluent-cp-zookeeper:2181
systems.kafka.producer.bootstrap.servers=kafka:9092
#systems.kafka.producer.bootstrap.servers=my-confluent-cp-kafka:9092
systems.kafka.consumer.bootstrap.servers=localhost:9092
systems.kafka.producer.bootstrap.servers=localhost:9092
systems.kafka.default.stream.replication.factor=1
# Configure serialization and stores
serializers.registry.string.class=org.apache.samza.serializers.StringSerdeFactory
stores.my-store.factory=org.apache.samza.storage.kv.RocksDbKeyValueStorageEngineFactory
stores.my-store.key.serde=string
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment