Skip to content
Snippets Groups Projects
Commit 94a3999e authored by Simon Ehrenstein's avatar Simon Ehrenstein
Browse files

Implement uc1 wg with new wg

parent 3cd0b988
No related branches found
No related tags found
1 merge request!6Add Distributed Workload Generator
Showing
with 92 additions and 682 deletions
package common.dimensions.copy;
import java.util.concurrent.TimeUnit;
import common.generators.WorkloadGenerator;
/**
* Wrapper class for the definition of period to use for the {@link WorkloadGenerator}.
*/
public class Period extends Dimension {
private final int period;
private final TimeUnit timeUnit;
/**
* Define a new period.
*
* @param period the period
* @param timeUnit the time unit that applies to the specified {@code period}
*/
public Period(final int period, final TimeUnit timeUnit) {
super();
this.period = period;
this.timeUnit = timeUnit;
}
public int getDuration() {
return this.period;
}
public TimeUnit getTimeUnit() {
return this.timeUnit;
}
}
package common.dimensions.copy2;
/*
* Base class for workload dimensions.
*/
public abstract class Dimension {
}
package common.dimensions.copy2;
import java.util.concurrent.TimeUnit;
import common.generators.WorkloadGenerator;
/**
* Wrapper class for the definition of the duration for the {@link WorkloadGenerator}.
*/
public class Duration extends Dimension {
private final int duration;
private final TimeUnit timeUnit;
/**
* Define a new duration.
*
* @param duration the duration
* @param timeUnit the time unit that applies to the specified {@code duration}
*/
public Duration(final int duration, final TimeUnit timeUnit) {
super();
this.duration = duration;
this.timeUnit = timeUnit;
}
public int getDuration() {
return this.duration;
}
public TimeUnit getTimeUnit() {
return this.timeUnit;
}
}
package common.dimensions.copy2;
import common.generators.WorkloadGenerator;
/**
* Wrapper class for the definition of the Keys that should be used by the
* {@link WorkloadGenerator}.
*/
public class KeySpace extends Dimension {
private final String prefix;
private final int min;
private final int max;
/**
* Create a new key space. All keys will have the prefix {@code prefix}. The remaining part of
* each key will be determined by a number of the interval ({@code min}, {@code max}-1).
*
* @param prefix the prefix to use for all keys
* @param min the lower bound (inclusive) to start counting from
* @param max the upper bound (exclusive) to count to
*/
public KeySpace(final String prefix, final int min, final int max) {
if (prefix == null || prefix.contains(";")) {
throw new IllegalArgumentException(
"The prefix must not be null and must not contain the ';' character.");
}
this.prefix = prefix;
this.min = min;
this.max = max;
}
public KeySpace(final String prefix, final int numberOfKeys) {
this(prefix, 0, numberOfKeys - 1);
}
public KeySpace(final int numberOfKeys) {
this("sensor_", 0, numberOfKeys - 1);
}
public String getPrefix() {
return this.prefix;
}
public int getMin() {
return this.min;
}
public int getMax() {
return this.max;
}
}
package common.dimensions.copy2;
import java.util.concurrent.TimeUnit;
import common.generators.WorkloadGenerator;
/**
* Wrapper class for the definition of period to use for the {@link WorkloadGenerator}.
*/
public class Period extends Dimension {
private final int period;
private final TimeUnit timeUnit;
/**
* Define a new period.
*
* @param period the period
* @param timeUnit the time unit that applies to the specified {@code period}
*/
public Period(final int period, final TimeUnit timeUnit) {
super();
this.period = period;
this.timeUnit = timeUnit;
}
public int getDuration() {
return this.period;
}
public TimeUnit getTimeUnit() {
return this.timeUnit;
}
}
package common.functions.copy;
@FunctionalInterface
public interface BeforeAction {
public void run();
}
package common.functions.copy;
import common.messages.OutputMessage;
import kieker.common.record.IMonitoringRecord;
@FunctionalInterface
public interface MessageGenerator<T extends IMonitoringRecord> {
OutputMessage<T> generateMessage(final String key);
}
package common.functions.copy;
import common.messages.OutputMessage;
import kieker.common.record.IMonitoringRecord;
@FunctionalInterface
public interface Transport<T extends IMonitoringRecord> {
public void transport(final OutputMessage<T> message);
}
......@@ -5,6 +5,7 @@ import common.dimensions.KeySpace;
import common.dimensions.Period;
import common.functions.BeforeAction;
import common.functions.MessageGenerator;
import common.misc.ZooKeeper;
import communication.kafka.KafkaRecordSender;
import kieker.common.record.IMonitoringRecord;
......@@ -19,6 +20,7 @@ public class KafkaWorkloadGenerator<T extends IMonitoringRecord> extends Workloa
* Create a new workload generator.
*
* @param keySpace the key space to generate the workload for.
* @param threads tha amount of threads to use per instance.
* @param period the period how often a message is generated for each key specified in the
* {@code keySpace}
* @param duration the duration how long the workload generator will emit messages.
......@@ -29,22 +31,23 @@ public class KafkaWorkloadGenerator<T extends IMonitoringRecord> extends Workloa
* @param recordSender the record sender which is used to send the generated messages to kafka.
*/
public KafkaWorkloadGenerator(
final ZooKeeper zooKeeper,
final KeySpace keySpace,
final int threads,
final Period period,
final Duration duration,
final BeforeAction beforeAction,
final MessageGenerator<T> generatorFunction,
final KafkaRecordSender<T> recordSender) {
super(keySpace, period, duration, beforeAction, generatorFunction, o -> {
System.out.println(o.getKey());
});
super(zooKeeper, keySpace, threads, period, duration, beforeAction, generatorFunction,
recordSender);
this.recordSender = recordSender;
}
@Override
public void stop() {
// this.recordSender.terminate();
this.recordSender.terminate();
super.stop();
}
......
......@@ -6,13 +6,18 @@ import common.dimensions.KeySpace;
import common.dimensions.Period;
import common.functions.BeforeAction;
import common.functions.MessageGenerator;
import common.misc.ZooKeeper;
import communication.kafka.KafkaRecordSender;
import kieker.common.record.IMonitoringRecord;
public class KafkaWorkloadGeneratorBuilder<T extends IMonitoringRecord> {
private ZooKeeper zooKeeper;
private KeySpace keySpace;
private int threads;
private Period period;
private Duration duration;
......@@ -32,10 +37,15 @@ public class KafkaWorkloadGeneratorBuilder<T extends IMonitoringRecord> {
*
* @return the builder.
*/
public static KafkaWorkloadGeneratorBuilder<IMonitoringRecord> builder() {
public static <T extends IMonitoringRecord> KafkaWorkloadGeneratorBuilder<T> builder() {
return new KafkaWorkloadGeneratorBuilder<>();
}
public KafkaWorkloadGeneratorBuilder<T> setZooKeeper(final ZooKeeper zooKeeper) {
this.zooKeeper = zooKeeper;
return this;
}
/**
* Set the before action for the {@link KafkaWorkloadGenerator}.
*
......@@ -58,6 +68,17 @@ public class KafkaWorkloadGeneratorBuilder<T extends IMonitoringRecord> {
return this;
}
/**
* Set the key space for the {@link KafkaWorkloadGenerator}.
*
* @param keySpace the {@link KeySpace}.
* @return the builder.
*/
public KafkaWorkloadGeneratorBuilder<T> setThreads(final int threads) {
this.threads = threads;
return this;
}
/**
* Set the period for the {@link KafkaWorkloadGenerator}.
*
......@@ -118,15 +139,24 @@ public class KafkaWorkloadGeneratorBuilder<T extends IMonitoringRecord> {
* @return the built instance of the {@link KafkaWorkloadGenerator}.
*/
public KafkaWorkloadGenerator<T> build() {
Objects.requireNonNull(this.zooKeeper, "Please specify the ZooKeeper instance.");
this.threads = Objects.requireNonNullElse(this.threads, 1);
Objects.requireNonNull(this.keySpace, "Please specify the key space.");
Objects.requireNonNull(this.period, "Please specify the period.");
Objects.requireNonNull(this.duration, "Please specify the duration.");
final BeforeAction beforeAction = Objects.requireNonNullElse(this.beforeAction, () -> {
this.beforeAction = Objects.requireNonNullElse(this.beforeAction, () -> {
});
Objects.requireNonNull(this.generatorFunction, "Please specify the generator function.");
// Objects.requireNonNull(this.kafkaRecordSender, "Please specify the kafka record sender.");
return new KafkaWorkloadGenerator<>(this.keySpace, this.period, this.duration, beforeAction,
this.generatorFunction, this.kafkaRecordSender);
Objects.requireNonNull(this.kafkaRecordSender, "Please specify the kafka record sender.");
return new KafkaWorkloadGenerator<>(
this.zooKeeper,
this.keySpace,
this.threads,
this.period,
this.duration,
this.beforeAction,
this.generatorFunction,
this.kafkaRecordSender);
}
}
......@@ -7,6 +7,8 @@ import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.function.BiConsumer;
import java.util.function.BiFunction;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import common.dimensions.Duration;
import common.dimensions.KeySpace;
import common.dimensions.Period;
......@@ -17,13 +19,20 @@ import common.messages.OutputMessage;
import common.misc.Worker;
import common.misc.WorkloadDefinition;
import common.misc.WorkloadEntity;
import common.misc.ZooKeeper;
import communication.zookeeper.WorkloadDistributor;
import kieker.common.record.IMonitoringRecord;
public abstract class WorkloadGenerator<T extends IMonitoringRecord> implements IWorkloadGenerator {
private static final Logger LOGGER = LoggerFactory.getLogger(WorkloadGenerator.class);
private final ZooKeeper zooKeeper;
private final KeySpace keySpace;
private final int threads;
private final Period period;
private final Duration duration;
......@@ -42,7 +51,7 @@ public abstract class WorkloadGenerator<T extends IMonitoringRecord> implements
/**
* Start the workload generation. The generation terminates automatically after the specified
* {@code duration}.s
* {@code duration}.
*/
@Override
public void start() {
......@@ -55,13 +64,17 @@ public abstract class WorkloadGenerator<T extends IMonitoringRecord> implements
}
public WorkloadGenerator(
final ZooKeeper zooKeeper,
final KeySpace keySpace,
final int threads,
final Period period,
final Duration duration,
final BeforeAction beforeAction,
final MessageGenerator<T> generatorFunction,
final Transport<T> transport) {
this.zooKeeper = zooKeeper;
this.period = period;
this.threads = threads;
this.keySpace = keySpace;
this.duration = duration;
this.beforeAction = beforeAction;
......@@ -80,29 +93,27 @@ public abstract class WorkloadGenerator<T extends IMonitoringRecord> implements
};
this.transport = transport;
final int threads = 10; // env
this.executor = Executors.newScheduledThreadPool(threads);
final Random random = new Random();
final int periodMs = period.getDuration();
final int periodMs = period.getPeriod();
final BiConsumer<WorkloadDefinition, Worker> workerAction = (declaration, worker) -> {
final List<WorkloadEntity<T>> entities = this.workloadSelector.apply(declaration, worker);
System.out.println("Beginning of Experiment...");
System.out.println("Experiment is going to be executed for the specified duration...");
LOGGER.info("Beginning of Experiment...");
LOGGER.info("Experiment is going to be executed for the specified duration...");
entities.forEach(entity -> {
final OutputMessage<T> message = entity.generateMessage();
final long initialDelay = random.nextInt(periodMs);
this.executor.scheduleAtFixedRate(() -> this.transport.transport(message), initialDelay,
periodMs, period.getTimeUnit());
});
try {
this.executor.awaitTermination(duration.getDuration(), duration.getTimeUnit());
System.out.println("Terminating now...");
LOGGER.info("Terminating now...");
this.stop();
} catch (final InterruptedException e) {
// TODO Auto-generated catch block
......@@ -111,6 +122,6 @@ public abstract class WorkloadGenerator<T extends IMonitoringRecord> implements
};
this.workloadDistributor =
new WorkloadDistributor(this.keySpace, this.beforeAction, workerAction);
new WorkloadDistributor(this.zooKeeper, this.keySpace, this.beforeAction, workerAction);
}
}
package common.generators.copy;
/**
* Base methods for workload generators.
*/
public interface IWorkloadGenerator {
/**
* Start the workload generation.
*/
void start();
/**
* Stop the workload generation.
*/
void stop();
}
package common.generators.copy;
import common.dimensions.Duration;
import common.dimensions.KeySpace;
import common.dimensions.Period;
import common.functions.BeforeAction;
import common.functions.MessageGenerator;
import communication.kafka.KafkaRecordSender;
import kieker.common.record.IMonitoringRecord;
/**
* Workload generator for generating load for the kafka messaging system.
*/
public class KafkaWorkloadGenerator<T extends IMonitoringRecord> extends WorkloadGenerator<T> {
private final KafkaRecordSender<T> recordSender;
/**
* Create a new workload generator.
*
* @param keySpace the key space to generate the workload for.
* @param period the period how often a message is generated for each key specified in the
* {@code keySpace}
* @param duration the duration how long the workload generator will emit messages.
* @param beforeAction the action which will be performed before the workload generator starts
* generating messages. If {@code null}, no before action will be performed.
* @param generatorFunction the generator function. This function is executed, each time a message
* is generated.
* @param recordSender the record sender which is used to send the generated messages to kafka.
*/
public KafkaWorkloadGenerator(
final KeySpace keySpace,
final Period period,
final Duration duration,
final BeforeAction beforeAction,
final MessageGenerator<T> generatorFunction,
final KafkaRecordSender<T> recordSender) {
super(keySpace, period, duration, beforeAction, generatorFunction, o -> {
System.out.println(o.getKey());
});
this.recordSender = recordSender;
}
@Override
public void stop() {
// this.recordSender.terminate();
super.stop();
}
}
package common.generators.copy;
import java.util.Objects;
import common.dimensions.Duration;
import common.dimensions.KeySpace;
import common.dimensions.Period;
import common.functions.BeforeAction;
import common.functions.MessageGenerator;
import communication.kafka.KafkaRecordSender;
import kieker.common.record.IMonitoringRecord;
public class KafkaWorkloadGeneratorBuilder<T extends IMonitoringRecord> {
private KeySpace keySpace;
private Period period;
private Duration duration;
private BeforeAction beforeAction;
private MessageGenerator<T> generatorFunction;
private KafkaRecordSender<T> kafkaRecordSender;
private KafkaWorkloadGeneratorBuilder() {
}
/**
* Get a builder for the {@link KafkaWorkloadGenerator}.
*
* @return the builder.
*/
public static KafkaWorkloadGeneratorBuilder<IMonitoringRecord> builder() {
return new KafkaWorkloadGeneratorBuilder<>();
}
/**
* Set the before action for the {@link KafkaWorkloadGenerator}.
*
* @param beforeAction the {@link BeforeAction}.
* @return the builder.
*/
public KafkaWorkloadGeneratorBuilder<T> setBeforeAction(final BeforeAction beforeAction) {
this.beforeAction = beforeAction;
return this;
}
/**
* Set the key space for the {@link KafkaWorkloadGenerator}.
*
* @param keySpace the {@link KeySpace}.
* @return the builder.
*/
public KafkaWorkloadGeneratorBuilder<T> setKeySpace(final KeySpace keySpace) {
this.keySpace = keySpace;
return this;
}
/**
* Set the period for the {@link KafkaWorkloadGenerator}.
*
* @param period the {@link Period}
* @return the builder.
*/
public KafkaWorkloadGeneratorBuilder<T> setPeriod(final Period period) {
this.period = period;
return this;
}
/**
* Set the durtion for the {@link KafkaWorkloadGenerator}.
*
* @param duration the {@link Duration}.
* @return the builder.
*/
public KafkaWorkloadGeneratorBuilder<T> setDuration(final Duration duration) {
this.duration = duration;
return this;
}
/**
* Set the generator function for the {@link KafkaWorkloadGenerator}.
*
* @param generatorFunction the generator function.
* @return the builder.
*/
public KafkaWorkloadGeneratorBuilder<T> setGeneratorFunction(
final MessageGenerator<T> generatorFunction) {
this.generatorFunction = generatorFunction;
return this;
}
/**
* Set the {@link KafkaRecordSender} for the {@link KafkaWorkloadGenerator}.
*
* @param kafkaRecordSender the record sender to use.
* @return the builder.
*/
public KafkaWorkloadGeneratorBuilder<T> setKafkaRecordSender(
final KafkaRecordSender<T> kafkaRecordSender) {
this.kafkaRecordSender = kafkaRecordSender;
return this;
}
/**
* Build the actual {@link KafkaWorkloadGenerator}. The following parameters are must be
* specicified before this method is called:
* <ul>
* <li>key space</li>
* <li>period</li>
* <li>duration</li>
* <li>generator function</li>
* <li>kafka record sender</li>
* </ul>
*
* @return the built instance of the {@link KafkaWorkloadGenerator}.
*/
public KafkaWorkloadGenerator<T> build() {
Objects.requireNonNull(this.keySpace, "Please specify the key space.");
Objects.requireNonNull(this.period, "Please specify the period.");
Objects.requireNonNull(this.duration, "Please specify the duration.");
final BeforeAction beforeAction = Objects.requireNonNullElse(this.beforeAction, () -> {
});
Objects.requireNonNull(this.generatorFunction, "Please specify the generator function.");
// Objects.requireNonNull(this.kafkaRecordSender, "Please specify the kafka record sender.");
return new KafkaWorkloadGenerator<>(this.keySpace, this.period, this.duration, beforeAction,
this.generatorFunction, this.kafkaRecordSender);
}
}
package common.generators.copy;
import java.util.LinkedList;
import java.util.List;
import java.util.Random;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.function.BiConsumer;
import java.util.function.BiFunction;
import common.dimensions.Duration;
import common.dimensions.KeySpace;
import common.dimensions.Period;
import common.functions.BeforeAction;
import common.functions.MessageGenerator;
import common.functions.Transport;
import common.messages.OutputMessage;
import common.misc.Worker;
import common.misc.WorkloadDefinition;
import common.misc.WorkloadEntity;
import communication.zookeeper.WorkloadDistributor;
import kieker.common.record.IMonitoringRecord;
public abstract class WorkloadGenerator<T extends IMonitoringRecord> implements IWorkloadGenerator {
private final KeySpace keySpace;
private final Period period;
private final Duration duration;
private final BeforeAction beforeAction;
private final BiFunction<WorkloadDefinition, Worker, List<WorkloadEntity<T>>> workloadSelector;
private final MessageGenerator<T> generatorFunction;
private final Transport<T> transport;
private WorkloadDistributor workloadDistributor;
private final ScheduledExecutorService executor;
/**
* Start the workload generation. The generation terminates automatically after the specified
* {@code duration}.s
*/
@Override
public void start() {
this.workloadDistributor.start();
}
@Override
public void stop() {
this.workloadDistributor.stop();
}
public WorkloadGenerator(
final KeySpace keySpace,
final Period period,
final Duration duration,
final BeforeAction beforeAction,
final MessageGenerator<T> generatorFunction,
final Transport<T> transport) {
this.period = period;
this.keySpace = keySpace;
this.duration = duration;
this.beforeAction = beforeAction;
this.generatorFunction = generatorFunction;
this.workloadSelector = (workloadDeclaration, worker) -> {
final List<WorkloadEntity<T>> workloadEntities = new LinkedList<>();
for (int i =
workloadDeclaration.getKeySpace().getMin() + worker.getId(); i <= workloadDeclaration
.getKeySpace().getMax(); i += workloadDeclaration.getNumberOfWorkers()) {
final String id = workloadDeclaration.getKeySpace().getPrefix() + i;
workloadEntities.add(new WorkloadEntity<>(id, this.generatorFunction));
}
return workloadEntities;
};
this.transport = transport;
final int threads = 10; // env
this.executor = Executors.newScheduledThreadPool(threads);
final Random random = new Random();
final int periodMs = period.getDuration();
final BiConsumer<WorkloadDefinition, Worker> workerAction = (declaration, worker) -> {
final List<WorkloadEntity<T>> entities = this.workloadSelector.apply(declaration, worker);
System.out.println("Beginning of Experiment...");
System.out.println("Experiment is going to be executed for the specified duration...");
entities.forEach(entity -> {
final OutputMessage<T> message = entity.generateMessage();
final long initialDelay = random.nextInt(periodMs);
this.executor.scheduleAtFixedRate(() -> this.transport.transport(message), initialDelay,
periodMs, period.getTimeUnit());
});
try {
this.executor.awaitTermination(duration.getDuration(), duration.getTimeUnit());
System.out.println("Terminating now...");
this.stop();
} catch (final InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
};
this.workloadDistributor =
new WorkloadDistributor(this.keySpace, this.beforeAction, workerAction);
}
}
package common.messages.copy;
import kieker.common.record.IMonitoringRecord;
/*
* Wrapper class for messages within the messaging system.
*/
public class OutputMessage<T extends IMonitoringRecord> {
private final String key;
private final T value;
/***
* Create a new Message.
*
* @param key the key of the message.
* @param value the value of the message.
*/
public OutputMessage(final String key, final T value) {
super();
this.key = key;
this.value = value;
}
public String getKey() {
return this.key;
}
public T getValue() {
return this.value;
}
}
package common.messages.copy2;
import kieker.common.record.IMonitoringRecord;
/*
* Wrapper class for messages within the messaging system.
*/
public class OutputMessage<T extends IMonitoringRecord> {
private final String key;
private final T value;
/***
* Create a new Message.
*
* @param key the key of the message.
* @param value the value of the message.
*/
public OutputMessage(final String key, final T value) {
super();
this.key = key;
this.value = value;
}
public String getKey() {
return this.key;
}
public T getValue() {
return this.value;
}
}
package common.misc;
/*
* Wrapper for connection information for ZooKeeper.
*/
public class ZooKeeper {
private final String host;
private final int port;
/**
* Create a new representation of an ZooKeeper instance.
*
* @param host of zookeeper.
* @param port of zookeeper.
*/
public ZooKeeper(final String host, final int port) {
super();
this.host = host;
this.port = port;
}
public String getHost() {
return this.host;
}
public int getPort() {
return this.port;
}
}
package common.misc.copy;
/*
* Wrapper class for a worker.
*/
public class Worker {
private final int id;
/**
* Create a new worker with an {@code id}
*
* @param id the id of the worker.
*/
public Worker(final int id) {
super();
this.id = id;
}
public int getId() {
return this.id;
}
}
package common.misc.copy;
import common.dimensions.KeySpace;
/*
* The central class that contains all information that needs to be exchanged between the nodes for
* distributed workload generation.
*/
public class WorkloadDefinition {
private final KeySpace keySpace;
private final int numberOfWorkers;
/**
* Create a new workload definition.
*
* @param keySpace the key space to use.
* @param numberOfWorkers the number of workers participating in the workload generation.
*/
public WorkloadDefinition(final KeySpace keySpace, final int numberOfWorkers) {
this.keySpace = keySpace;
this.numberOfWorkers = numberOfWorkers;
}
public KeySpace getKeySpace() {
return this.keySpace;
}
public int getNumberOfWorkers() {
return this.numberOfWorkers;
}
/**
* Simple method for encoding all information of the workload definition into one string.
*
* @return a string that encodes all information of the workload generation in a compact format.
* The format is 'keySpace;keySpace.min;keySpace.max;numberOfWorkers'.
*/
@Override
public String toString() {
return this.getKeySpace().getPrefix() + ";" + this.getKeySpace().getMin() + ";"
+ this.getKeySpace().getMax() + ";" + this.getNumberOfWorkers();
}
/**
* Parse a workload generation from a previously encoded string with the format returned by
* {@link WorkloadDefinition#toString()}.
*
* @param workloadDefinitionString the workload definition string.
* @return the parsed workload definition.
*/
public static WorkloadDefinition fromString(final String workloadDefinitionString) {
final String[] deserialized = workloadDefinitionString.split(";");
if (deserialized.length != 4) {
throw new IllegalArgumentException(
"Wrong workload definition string when trying to parse the workload generation.");
}
return new WorkloadDefinition(new KeySpace(deserialized[0], Integer.valueOf(deserialized[1]),
Integer.valueOf(deserialized[2])), Integer.valueOf(deserialized[3]));
}
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment