Skip to content
Snippets Groups Projects
Commit 0dfca34d authored by Sören Henning's avatar Sören Henning
Browse files

Merge branch 'master' of git.se.informatik.uni-kiel.de:she/theodolite

parents 1bf07839 851f4bd7
No related branches found
No related tags found
No related merge requests found
Pipeline #6044 canceled
Showing
with 561 additions and 5 deletions
version: '2.2'
services:
zookeeper:
image: confluentinc/cp-zookeeper
expose:
- "2181"
#ports:
# - 2181:2181
environment:
ZOOKEEPER_CLIENT_PORT: 2181
kafka:
image: wurstmeister/kafka
expose:
- "9092"
#ports:
# - 19092:19092
environment:
KAFKA_LISTENERS: PLAINTEXT://:9092,CONNECTIONS_FROM_HOST://:19092
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,CONNECTIONS_FROM_HOST://localhost:19092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1"
schema-registry:
image: confluentinc/cp-schema-registry:5.3.1
depends_on:
- zookeeper
- kafka
expose:
- "8081"
#ports:
# - 8081:8081
environment:
SCHEMA_REGISTRY_HOST_NAME: schema-registry
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
benchmark:
image: ghcr.io/cau-se/theodolite-uc4-beam-samza:latest
scale: 1
depends_on:
- schema-registry
- kafka
environment:
SAMZA_JOB_COORDINATOR_ZK_CONNECT: zookeeper:2181
SAMZA_SYSTEMS_KAFKA_PRODUCER_BOOTSTRAP_SERVERS: kafka:9092
SAMZA_SYSTEMS_KAFKA_CONSUMER_BOOTSTRAP_SERVERS: kafka:9092
KAFKA_BOOTSTRAP_SERVERS: kafka:9092
SCHEMA_REGISTRY_URL: http://schema-registry:8081
load-generator:
image: ghcr.io/cau-se/theodolite-uc4-workload-generator:latest
depends_on:
- schema-registry
- kafka
environment:
BOOTSTRAP_SERVER: load-generator:5701
PORT: 5701
KAFKA_BOOTSTRAP_SERVERS: kafka:9092
SCHEMA_REGISTRY_URL: http://schema-registry:8081
NUM_SENSORS: 4
NUM_NESTED_GROUPS: 4
......@@ -8,7 +8,7 @@ ext {
}
repositories {
jcenter()
mavenCentral()
maven {
url "https://oss.sonatype.org/content/repositories/snapshots/"
}
......
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-6.8.3-bin.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
......@@ -3,7 +3,7 @@ plugins {
}
repositories {
jcenter()
mavenCentral()
maven {
url "https://oss.sonatype.org/content/repositories/snapshots/"
}
......
......@@ -3,7 +3,7 @@ plugins {
}
repositories {
jcenter()
mavenCentral()
maven {
url "https://oss.sonatype.org/content/repositories/snapshots/"
}
......
......@@ -22,7 +22,7 @@ public final class LoadGenerator {
private static final int THREADS_DEFAULT = 4;
private static final String SCHEMA_REGISTRY_URL_DEFAULT = "http://localhost:8081";
private static final String KAFKA_TOPIC_DEFAULT = "input";
private static final String KAFKA_BOOTSTRAP_SERVERS_DEFAULT = "localhost:19092"; // NOPMD
private static final String KAFKA_BOOTSTRAP_SERVERS_DEFAULT = "localhost:9092"; // NOPMD
private ClusterConfig clusterConfig;
private WorkloadDefinition loadDefinition;
......
......@@ -3,19 +3,34 @@ rootProject.name = 'theodolite-benchmarks'
include 'load-generator-commons'
include 'kstreams-commons'
include 'flink-commons'
include 'beam-commons'
include 'uc1-beam'
include 'uc2-beam'
include 'uc3-beam'
include 'uc4-beam'
include 'uc1-load-generator'
include 'uc1-kstreams'
include 'uc1-flink'
include 'uc1-beam-flink'
include 'uc1-beam-samza'
include 'uc2-load-generator'
include 'uc2-kstreams'
include 'uc2-flink'
include 'uc2-beam-flink'
include 'uc2-beam-samza'
include 'uc3-load-generator'
include 'uc3-kstreams'
include 'uc3-flink'
include 'uc3-beam-flink'
include 'uc3-beam-samza'
include 'uc4-load-generator'
include 'uc4-kstreams'
include 'uc4-flink'
include 'uc4-beam-flink'
include 'uc4-beam-samza'
FROM flink:1.13-java11
ADD build/distributions/uc1-beam-flink.tar /opt/flink/usrlib/artifacts/uc1-beam-flink.tar
plugins {
id 'theodolite.beam.flink'
}
dependencies {
implementation project(':uc1-beam')
}
mainClassName = "application.Uc1BeamFlink"
package application;
import org.apache.beam.runners.flink.FlinkRunner;
import theodolite.commons.beam.AbstractBeamService;
/**
* Implementation of the use case Database Storage using Apache Beam with the Flink Runner. To
* execute locally in standalone start Kafka, Zookeeper, the schema-registry and the workload
* generator using the delayed_startup.sh script. Start a Flink cluster and pass its REST adress
* using--flinkMaster as run parameter. To persist logs add
* ${workspace_loc:/uc1-application-samza/eclipseConsoleLogs.log} as Output File under Standard
* Input Output in Common in the Run Configuration Start via Eclipse Run.
*/
public final class Uc1BeamFlink extends AbstractBeamService {
/**
* Private constructor setting specific options for this use case.
*/
private Uc1BeamFlink(final String[] args) { //NOPMD
super(args);
this.options.setRunner(FlinkRunner.class);
}
/**
* Main method.
*/
public static void main(final String[] args) {
// Create application via configurations
final Uc1BeamFlink uc1 = new Uc1BeamFlink(args);
// Create pipeline with configurations
final Uc1BeamPipeline pipeline = new Uc1BeamPipeline(uc1.options, uc1.getConfig());
// Submit job and start execution
pipeline.run().waitUntilFinish();
}
}
application.name=theodolite-uc1-application
application.version=0.0.1
kafka.bootstrap.servers=localhost:9092
kafka.input.topic=input
kafka.output.topic=output
schema.registry.url=http://localhost:8081
num.threads=1
commit.interval.ms=1000
cache.max.bytes.buffering=-1
specific.avro.reader=True
enable.auto.commit.config=True
auto.offset.reset.config=earliest
\ No newline at end of file
state
\ No newline at end of file
FROM openjdk:11-slim
ENV MAX_SOURCE_PARALLELISM=1024
ADD build/distributions/uc1-beam-samza.tar /
ADD samza-standalone.properties /
CMD /uc1-beam-samza/bin/uc1-beam-samza --configFactory=org.apache.samza.config.factories.PropertiesConfigFactory --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --maxSourceParallelism=$MAX_SOURCE_PARALLELISM --enableMetrics=false --configOverride="{\"job.coordinator.zk.connect\":\"$SAMZA_JOB_COORDINATOR_ZK_CONNECT\"}"
plugins {
id 'theodolite.beam.samza'
}
dependencies {
implementation project(':uc1-beam')
}
mainClassName = "application.Uc1BeamSamza"
# Set EnvironmentRewriter
job.config.rewriters=env-config
job.config.rewriter.env-config.class=org.apache.samza.config.EnvironmentConfigRewriter
# Configure ZooKeeper for coordination
job.coordinator.factory=org.apache.samza.zk.ZkJobCoordinatorFactory
job.coordinator.zk.connect=localhost:2181
# Use GroupByContainerIds
task.name.grouper.factory=org.apache.samza.container.grouper.task.GroupByContainerIdsFactory
# Configure Kafka as "system"
job.default.system=kafka
systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory
systems.kafka.consumer.bootstrap.servers=localhost:9092
systems.kafka.producer.bootstrap.servers=localhost:9092
systems.kafka.default.stream.replication.factor=1
package application;
import org.apache.beam.runners.samza.SamzaRunner;
import theodolite.commons.beam.AbstractBeamService;
/**
* Implementation of the use case Database Storage using Apache Beam with the Samza Runner. To
* execute locally in standalone start Kafka, Zookeeper, the schema-registry and the workload
* generator. Add
* --configFactory=org.apache.samza.config.factories.PropertiesConfigFactory
* --configFilePath=samza-standalone.properties
* --samzaExecutionEnvironment=STANDALONE --maxSourceParallelism=1024 as program arguments. To
* persist logs add ${workspace_loc:/uc4-application-samza/eclipseConsoleLogs.log} as Output File
* under Standard Input Output in Common in the Run Configuration Start via Eclipse Run.
*/
public final class Uc1BeamSamza extends AbstractBeamService {
/**
* Private constructor setting specific options for this use case.
*/
private Uc1BeamSamza(final String[] args) { //NOPMD
super(args);
this.options.setRunner(SamzaRunner.class);
}
/**
* Main method.
*/
public static void main(final String[] args) {
// Create application via configurations
final Uc1BeamSamza uc1 = new Uc1BeamSamza(args);
// Create pipeline with configurations
final Uc1BeamPipeline pipeline = new Uc1BeamPipeline(uc1.options, uc1.getConfig());
// Submit job and start execution
pipeline.run().waitUntilFinish();
}
}
application.name=theodolite-uc1-application
application.version=0.0.1
kafka.bootstrap.servers=localhost:9092
kafka.input.topic=input
kafka.output.topic=output
schema.registry.url=http://localhost:8081
num.threads=1
commit.interval.ms=1000
cache.max.bytes.buffering=-1
specific.avro.reader=True
enable.auto.commit.config=True
auto.offset.reset.config=earliest
\ No newline at end of file
cleanup.add_all=false
cleanup.add_default_serial_version_id=true
cleanup.add_generated_serial_version_id=false
cleanup.add_missing_annotations=true
cleanup.add_missing_deprecated_annotations=true
cleanup.add_missing_methods=false
cleanup.add_missing_nls_tags=false
cleanup.add_missing_override_annotations=true
cleanup.add_missing_override_annotations_interface_methods=true
cleanup.add_serial_version_id=false
cleanup.always_use_blocks=true
cleanup.always_use_parentheses_in_expressions=false
cleanup.always_use_this_for_non_static_field_access=true
cleanup.always_use_this_for_non_static_method_access=true
cleanup.array_with_curly=false
cleanup.arrays_fill=false
cleanup.bitwise_conditional_expression=false
cleanup.boolean_literal=false
cleanup.boolean_value_rather_than_comparison=true
cleanup.break_loop=false
cleanup.collection_cloning=false
cleanup.comparing_on_criteria=false
cleanup.comparison_statement=false
cleanup.controlflow_merge=false
cleanup.convert_functional_interfaces=false
cleanup.convert_to_enhanced_for_loop=true
cleanup.convert_to_enhanced_for_loop_if_loop_var_used=true
cleanup.convert_to_switch_expressions=false
cleanup.correct_indentation=true
cleanup.do_while_rather_than_while=true
cleanup.double_negation=false
cleanup.else_if=false
cleanup.embedded_if=false
cleanup.evaluate_nullable=false
cleanup.extract_increment=false
cleanup.format_source_code=true
cleanup.format_source_code_changes_only=false
cleanup.hash=false
cleanup.if_condition=false
cleanup.insert_inferred_type_arguments=false
cleanup.instanceof=false
cleanup.instanceof_keyword=false
cleanup.invert_equals=false
cleanup.join=false
cleanup.lazy_logical_operator=false
cleanup.make_local_variable_final=true
cleanup.make_parameters_final=true
cleanup.make_private_fields_final=true
cleanup.make_type_abstract_if_missing_method=false
cleanup.make_variable_declarations_final=true
cleanup.map_cloning=false
cleanup.merge_conditional_blocks=false
cleanup.multi_catch=false
cleanup.never_use_blocks=false
cleanup.never_use_parentheses_in_expressions=true
cleanup.no_string_creation=false
cleanup.no_super=false
cleanup.number_suffix=false
cleanup.objects_equals=false
cleanup.one_if_rather_than_duplicate_blocks_that_fall_through=true
cleanup.operand_factorization=false
cleanup.organize_imports=true
cleanup.overridden_assignment=false
cleanup.plain_replacement=false
cleanup.precompile_regex=false
cleanup.primitive_comparison=false
cleanup.primitive_parsing=false
cleanup.primitive_rather_than_wrapper=true
cleanup.primitive_serialization=false
cleanup.pull_out_if_from_if_else=false
cleanup.pull_up_assignment=false
cleanup.push_down_negation=false
cleanup.qualify_static_field_accesses_with_declaring_class=false
cleanup.qualify_static_member_accesses_through_instances_with_declaring_class=true
cleanup.qualify_static_member_accesses_through_subtypes_with_declaring_class=true
cleanup.qualify_static_member_accesses_with_declaring_class=true
cleanup.qualify_static_method_accesses_with_declaring_class=false
cleanup.reduce_indentation=false
cleanup.redundant_comparator=false
cleanup.redundant_falling_through_block_end=false
cleanup.remove_private_constructors=true
cleanup.remove_redundant_modifiers=false
cleanup.remove_redundant_semicolons=true
cleanup.remove_redundant_type_arguments=true
cleanup.remove_trailing_whitespaces=true
cleanup.remove_trailing_whitespaces_all=true
cleanup.remove_trailing_whitespaces_ignore_empty=false
cleanup.remove_unnecessary_array_creation=false
cleanup.remove_unnecessary_casts=true
cleanup.remove_unnecessary_nls_tags=true
cleanup.remove_unused_imports=true
cleanup.remove_unused_local_variables=false
cleanup.remove_unused_private_fields=true
cleanup.remove_unused_private_members=false
cleanup.remove_unused_private_methods=true
cleanup.remove_unused_private_types=true
cleanup.return_expression=false
cleanup.simplify_lambda_expression_and_method_ref=false
cleanup.single_used_field=false
cleanup.sort_members=false
cleanup.sort_members_all=false
cleanup.standard_comparison=false
cleanup.static_inner_class=false
cleanup.strictly_equal_or_different=false
cleanup.stringbuffer_to_stringbuilder=false
cleanup.stringbuilder=false
cleanup.stringbuilder_for_local_vars=true
cleanup.substring=false
cleanup.switch=false
cleanup.system_property=false
cleanup.system_property_boolean=false
cleanup.system_property_file_encoding=false
cleanup.system_property_file_separator=false
cleanup.system_property_line_separator=false
cleanup.system_property_path_separator=false
cleanup.ternary_operator=false
cleanup.try_with_resource=false
cleanup.unlooped_while=false
cleanup.unreachable_block=false
cleanup.use_anonymous_class_creation=false
cleanup.use_autoboxing=false
cleanup.use_blocks=true
cleanup.use_blocks_only_for_return_and_throw=false
cleanup.use_directly_map_method=false
cleanup.use_lambda=true
cleanup.use_parentheses_in_expressions=true
cleanup.use_string_is_blank=false
cleanup.use_this_for_non_static_field_access=true
cleanup.use_this_for_non_static_field_access_only_if_necessary=false
cleanup.use_this_for_non_static_method_access=true
cleanup.use_this_for_non_static_method_access_only_if_necessary=false
cleanup.use_unboxing=false
cleanup.use_var=false
cleanup.useless_continue=false
cleanup.useless_return=false
cleanup.valueof_rather_than_instantiation=false
cleanup_profile=_CAU-SE-Style
cleanup_settings_version=2
eclipse.preferences.version=1
editor_save_participant_org.eclipse.jdt.ui.postsavelistener.cleanup=true
formatter_profile=_CAU-SE-Style
formatter_settings_version=21
org.eclipse.jdt.ui.ignorelowercasenames=true
org.eclipse.jdt.ui.importorder=
org.eclipse.jdt.ui.ondemandthreshold=99
org.eclipse.jdt.ui.staticondemandthreshold=99
sp_cleanup.add_all=false
sp_cleanup.add_default_serial_version_id=true
sp_cleanup.add_generated_serial_version_id=false
sp_cleanup.add_missing_annotations=true
sp_cleanup.add_missing_deprecated_annotations=true
sp_cleanup.add_missing_methods=false
sp_cleanup.add_missing_nls_tags=false
sp_cleanup.add_missing_override_annotations=true
sp_cleanup.add_missing_override_annotations_interface_methods=true
sp_cleanup.add_serial_version_id=false
sp_cleanup.always_use_blocks=true
sp_cleanup.always_use_parentheses_in_expressions=false
sp_cleanup.always_use_this_for_non_static_field_access=true
sp_cleanup.always_use_this_for_non_static_method_access=true
sp_cleanup.array_with_curly=false
sp_cleanup.arrays_fill=false
sp_cleanup.bitwise_conditional_expression=false
sp_cleanup.boolean_literal=false
sp_cleanup.boolean_value_rather_than_comparison=false
sp_cleanup.break_loop=false
sp_cleanup.collection_cloning=false
sp_cleanup.comparing_on_criteria=false
sp_cleanup.comparison_statement=false
sp_cleanup.controlflow_merge=false
sp_cleanup.convert_functional_interfaces=false
sp_cleanup.convert_to_enhanced_for_loop=false
sp_cleanup.convert_to_enhanced_for_loop_if_loop_var_used=false
sp_cleanup.convert_to_switch_expressions=false
sp_cleanup.correct_indentation=false
sp_cleanup.do_while_rather_than_while=false
sp_cleanup.double_negation=false
sp_cleanup.else_if=false
sp_cleanup.embedded_if=false
sp_cleanup.evaluate_nullable=false
sp_cleanup.extract_increment=false
sp_cleanup.format_source_code=true
sp_cleanup.format_source_code_changes_only=false
sp_cleanup.hash=false
sp_cleanup.if_condition=false
sp_cleanup.insert_inferred_type_arguments=false
sp_cleanup.instanceof=false
sp_cleanup.instanceof_keyword=false
sp_cleanup.invert_equals=false
sp_cleanup.join=false
sp_cleanup.lazy_logical_operator=false
sp_cleanup.make_local_variable_final=true
sp_cleanup.make_parameters_final=false
sp_cleanup.make_private_fields_final=true
sp_cleanup.make_type_abstract_if_missing_method=false
sp_cleanup.make_variable_declarations_final=true
sp_cleanup.map_cloning=false
sp_cleanup.merge_conditional_blocks=false
sp_cleanup.multi_catch=false
sp_cleanup.never_use_blocks=false
sp_cleanup.never_use_parentheses_in_expressions=true
sp_cleanup.no_string_creation=false
sp_cleanup.no_super=false
sp_cleanup.number_suffix=false
sp_cleanup.objects_equals=false
sp_cleanup.on_save_use_additional_actions=true
sp_cleanup.one_if_rather_than_duplicate_blocks_that_fall_through=false
sp_cleanup.operand_factorization=false
sp_cleanup.organize_imports=true
sp_cleanup.overridden_assignment=false
sp_cleanup.plain_replacement=false
sp_cleanup.precompile_regex=false
sp_cleanup.primitive_comparison=false
sp_cleanup.primitive_parsing=false
sp_cleanup.primitive_rather_than_wrapper=false
sp_cleanup.primitive_serialization=false
sp_cleanup.pull_out_if_from_if_else=false
sp_cleanup.pull_up_assignment=false
sp_cleanup.push_down_negation=false
sp_cleanup.qualify_static_field_accesses_with_declaring_class=false
sp_cleanup.qualify_static_member_accesses_through_instances_with_declaring_class=true
sp_cleanup.qualify_static_member_accesses_through_subtypes_with_declaring_class=true
sp_cleanup.qualify_static_member_accesses_with_declaring_class=true
sp_cleanup.qualify_static_method_accesses_with_declaring_class=false
sp_cleanup.reduce_indentation=false
sp_cleanup.redundant_comparator=false
sp_cleanup.redundant_falling_through_block_end=false
sp_cleanup.remove_private_constructors=true
sp_cleanup.remove_redundant_modifiers=false
sp_cleanup.remove_redundant_semicolons=true
sp_cleanup.remove_redundant_type_arguments=true
sp_cleanup.remove_trailing_whitespaces=false
sp_cleanup.remove_trailing_whitespaces_all=true
sp_cleanup.remove_trailing_whitespaces_ignore_empty=false
sp_cleanup.remove_unnecessary_array_creation=false
sp_cleanup.remove_unnecessary_casts=true
sp_cleanup.remove_unnecessary_nls_tags=false
sp_cleanup.remove_unused_imports=false
sp_cleanup.remove_unused_local_variables=false
sp_cleanup.remove_unused_private_fields=true
sp_cleanup.remove_unused_private_members=false
sp_cleanup.remove_unused_private_methods=true
sp_cleanup.remove_unused_private_types=true
sp_cleanup.return_expression=false
sp_cleanup.simplify_lambda_expression_and_method_ref=false
sp_cleanup.single_used_field=false
sp_cleanup.sort_members=false
sp_cleanup.sort_members_all=false
sp_cleanup.standard_comparison=false
sp_cleanup.static_inner_class=false
sp_cleanup.strictly_equal_or_different=false
sp_cleanup.stringbuffer_to_stringbuilder=false
sp_cleanup.stringbuilder=false
sp_cleanup.stringbuilder_for_local_vars=true
sp_cleanup.substring=false
sp_cleanup.switch=false
sp_cleanup.system_property=false
sp_cleanup.system_property_boolean=false
sp_cleanup.system_property_file_encoding=false
sp_cleanup.system_property_file_separator=false
sp_cleanup.system_property_line_separator=false
sp_cleanup.system_property_path_separator=false
sp_cleanup.ternary_operator=false
sp_cleanup.try_with_resource=true
sp_cleanup.unlooped_while=false
sp_cleanup.unreachable_block=false
sp_cleanup.use_anonymous_class_creation=false
sp_cleanup.use_autoboxing=false
sp_cleanup.use_blocks=true
sp_cleanup.use_blocks_only_for_return_and_throw=false
sp_cleanup.use_directly_map_method=false
sp_cleanup.use_lambda=true
sp_cleanup.use_parentheses_in_expressions=true
sp_cleanup.use_string_is_blank=false
sp_cleanup.use_this_for_non_static_field_access=true
sp_cleanup.use_this_for_non_static_field_access_only_if_necessary=false
sp_cleanup.use_this_for_non_static_method_access=true
sp_cleanup.use_this_for_non_static_method_access_only_if_necessary=false
sp_cleanup.use_unboxing=false
sp_cleanup.use_var=false
sp_cleanup.useless_continue=false
sp_cleanup.useless_return=false
sp_cleanup.valueof_rather_than_instantiation=false
plugins {
id 'theodolite.beam'
}
package application;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.values.KV;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Logs all Key Value pairs.
*/
@SuppressWarnings({"unused"})
public class LogKeyValue extends DoFn<KV<String, String>, KV<String, String>> {
private static final long serialVersionUID = 4328743;
private static final Logger LOGGER = LoggerFactory.getLogger(LogKeyValue.class);
/**
* Logs all key value pairs it processes.
*/
@ProcessElement
public void processElement(@Element final KV<String, String> kv,
final OutputReceiver<KV<String, String>> out) {
if (LOGGER.isInfoEnabled()) {
LOGGER.info("Key: {}, Value: {}", kv.getKey(), kv.getValue());
}
out.output(kv);
}
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please to comment