Skip to content
Snippets Groups Projects
Commit 754d95dd authored by Marcel Samir Becker's avatar Marcel Samir Becker
Browse files

Comments and Clean-up

parent c993e3b0
No related branches found
No related tags found
1 merge request!215Redesign Strategy, Load, and Resources data types
This commit is part of merge request !215. Comments created here will be created in the context of that merge request.
Showing
with 132 additions and 106 deletions
......@@ -29,7 +29,7 @@ spec:
benchmark:
description: The name of the benchmark this execution is referring to.
type: string
load: # definition of the load dimension
loads: # definition of the load dimension
description: Specifies the load values that are benchmarked.
type: object
required: ["loadType", "loadValues"]
......
......@@ -4,7 +4,7 @@ metadata:
name: theodolite-example-execution
spec:
benchmark: "example-benchmark"
load:
loads:
loadType: "NumSensors"
loadValues: [25000, 50000, 75000, 100000, 125000, 150000]
resources:
......
......@@ -6,8 +6,8 @@ import theodolite.util.PatcherDefinition
/**
* A Benchmark contains:
* - The [Resource]s that can be scaled for the benchmark.
* - The [LoadDimension]s that can be scaled the benchmark.
* - The Resource that can be scaled for the benchmark.
* - The Load that can be scaled the benchmark.
* - additional [ConfigurationOverride]s.
*/
@RegisterForReflection
......
......@@ -12,8 +12,8 @@ import kotlin.properties.Delegates
* A BenchmarkExecution consists of:
* - A [name].
* - The [benchmark] that should be executed.
* - The [load] that should be checked in the benchmark.
* - The resources that should be checked in the benchmark.
* - The [loads]s that should be checked in the benchmark.
* - The [resources] that should be checked in the benchmark.
* - A list of [slos] that are used for the evaluation of the experiments.
* - An [execution] that encapsulates: the strategy, the duration, and the restrictions
* for the execution of the benchmark.
......@@ -28,7 +28,7 @@ class BenchmarkExecution : KubernetesResource {
var executionId: Int = 0
lateinit var name: String
lateinit var benchmark: String
lateinit var load: LoadDefinition
lateinit var loads: LoadDefinition
lateinit var resources: ResourceDefinition
lateinit var slos: List<Slo>
lateinit var execution: Execution
......@@ -41,7 +41,7 @@ class BenchmarkExecution : KubernetesResource {
@JsonDeserialize
@RegisterForReflection
class Execution : KubernetesResource {
var metric = "demand" //irgendwie mag er es nicht mit den default laden, wenn lateinit dann gibt es bei den tests fehler und muss bei var setzen
var metric = "demand"
lateinit var strategy: Strategy
var duration by Delegates.notNull<Long>()
var repetitions by Delegates.notNull<Int>()
......@@ -50,7 +50,9 @@ class BenchmarkExecution : KubernetesResource {
}
/**
* This Strategy encapsulates the [restrictions] which is used for restricting the resources.
* This Strategy encapsulates the [restrictions], [guessStrategy] and [searchStrategy],
* which are used for restricting the resources, the guess Strategy for the [InitialGuessSearchStrategy] and
* the actual [SearchStrategy] which is used.
*/
@JsonDeserialize
@RegisterForReflection
......@@ -80,8 +82,8 @@ class BenchmarkExecution : KubernetesResource {
}
/**
* Represents a Load that should be created and checked.
* It can be set to [loadValues].
* Represents the Loads that should be created and checked if the demand metric is in use or
* represents a Load that can be scaled to [loadValues] if the capacity metric is in use.
*/
@JsonDeserialize
@RegisterForReflection
......@@ -91,7 +93,8 @@ class BenchmarkExecution : KubernetesResource {
}
/**
* Represents a resource that can be scaled to [resourceValues].
* Represents a resource that can be scaled to [resourceValues] if the demand metric is in use or
* represents the Resources that should be created and checked if the capacity metric is in use.
*/
@JsonDeserialize
@RegisterForReflection
......
......@@ -8,7 +8,6 @@ import io.quarkus.runtime.annotations.RegisterForReflection
import mu.KotlinLogging
import theodolite.k8s.K8sManager
import theodolite.k8s.resourceLoader.K8sResourceLoader
import theodolite.patcher.Patcher
import theodolite.patcher.PatcherFactory
import theodolite.util.*
......@@ -76,10 +75,11 @@ class KubernetesBenchmark : KubernetesResource, Benchmark {
/**
* Builds a deployment.
* First loads all required resources and then patches them to the concrete load and resources for the experiment.
* Afterwards patches additional configurations(cluster depending) into the resources.
* @param load concrete load that will be benchmarked in this experiment.
* @param res concrete resource that will be scaled for this experiment.
* First loads all required resources and then patches them to the concrete load and resources for the experiment for the demand metric
* or loads all loads and then patches them to the concrete load and resources for the experiment.
* Afterwards patches additional configurations(cluster depending) into the resources (or loads).
* @param load concrete load that will be benchmarked in this experiment (demand metric), or scaled (capacity metric).
* @param resource concrete resource that will be scaled for this experiment (demand metric), or benchmarked (capacity metric).
* @param configurationOverrides
* @return a [BenchmarkDeployment]
*/
......
......@@ -40,10 +40,12 @@ abstract class BenchmarkExecutor(
* Run a experiment for the given parametrization, evaluate the
* experiment and save the result.
*
* @param load load to be tested.
* @param res resources to be tested.
* @param load to be tested.
* @param resource to be tested.
* @return True, if the number of resources are suitable for the
* given load, false otherwise.
* given load, false otherwise (demand metric), or
* True, if there is a load suitable for the
* given resource, false otherwise.
*/
abstract fun runExperiment(load: Int, resource: Int): Boolean
......
......@@ -56,7 +56,6 @@ class BenchmarkExecutorImpl(
break
}
}
/**
* Analyse the experiment, if [run] is true, otherwise the experiment was canceled by the user.
*/
......@@ -71,16 +70,10 @@ class BenchmarkExecutorImpl(
}
result = (false !in experimentResults)
// differentiate metric here on first/second ele pairs, also wenn demand so und wenn capacity dann mit (resource,load)
// so könnten wir die Methoden in Results so lassen und müssten keine Dopplung einbauen
// wird alles sehr undurchsichtig damit wenn man die vertauscht, evtl mit metric zu den Results klarer machen
this.results.setResult(Pair(load, resource), result)
}
if(!this.run.get()) {
} else {
throw ExecutionFailedException("The execution was interrupted")
}
return result
}
......
......@@ -51,7 +51,7 @@ class TheodoliteExecutor(
val loadDimensionPatcherDefinition =
PatcherDefinitionFactory().createPatcherDefinition(
config.load.loadType,
config.loads.loadType,
this.kubernetesBenchmark.loadTypes
)
......@@ -71,11 +71,11 @@ class TheodoliteExecutor(
resourcePatcherDefinitions = resourcePatcherDefinition
)
if (config.load.loadValues != config.load.loadValues.sorted()) {
config.load.loadValues = config.load.loadValues.sorted()
if (config.loads.loadValues != config.loads.loadValues.sorted()) {
config.loads.loadValues = config.loads.loadValues.sorted()
logger.info {
"Load values are not sorted correctly, Theodolite sorts them in ascending order." +
"New order is: ${config.load.loadValues}"
"New order is: ${config.loads.loadValues}"
}
}
......@@ -88,7 +88,7 @@ class TheodoliteExecutor(
}
return Config(
loads = config.load.loadValues,
loads = config.loads.loadValues,
loadPatcherDefinitions = loadDimensionPatcherDefinition,
resources = config.resources.resourceValues,
resourcePatcherDefinitions = resourcePatcherDefinition,
......
......@@ -16,27 +16,31 @@ class StrategyFactory {
* Create a [SearchStrategy].
*
* @param executor The [theodolite.execution.BenchmarkExecutor] that executes individual experiments.
* @param searchStrategyString Specifies the [SearchStrategy]. Must either be the string 'LinearSearch',
* or 'BinarySearch'.
* @param searchStrategyObject Specifies the [SearchStrategy]. Must either be an object with name 'FullSearch',
* 'LinearSearch', 'BinarySearch', 'RestrictionSearch' or 'InitialGuessSearch'.
* @param results The [Results] saves the state of the Theodolite benchmark run.
*
* @throws IllegalArgumentException if the [SearchStrategy] was not one of the allowed options.
*/
fun createSearchStrategy(executor: BenchmarkExecutor, searchStrategyObject: BenchmarkExecution.Strategy, results: Results): SearchStrategy {
fun createSearchStrategy(executor: BenchmarkExecutor, searchStrategyObject: BenchmarkExecution.Strategy,
results: Results): SearchStrategy {
var strategy : SearchStrategy = when (searchStrategyObject.name) {
"FullSearch" -> FullSearch(executor)
"LinearSearch" -> LinearSearch(executor)
"BinarySearch" -> BinarySearch(executor)
"RestrictionSearch" -> when (searchStrategyObject.searchStrategy){
//TODO: Do we only need LinearSearch here as valid searchstrat? Or actually just allow all?
// If we dont have restriction Strat specified but still RestrictionSearch just do normal Search
"FullSearch" -> composeSearchRestrictionStrategy(executor, FullSearch(executor), results, searchStrategyObject.restrictions)
"LinearSearch" -> composeSearchRestrictionStrategy(executor, LinearSearch(executor), results, searchStrategyObject.restrictions)
"BinarySearch" -> composeSearchRestrictionStrategy(executor, BinarySearch(executor), results, searchStrategyObject.restrictions)
else -> throw IllegalArgumentException("Search Strategy ${searchStrategyObject.searchStrategy} for RestrictionSearch not found")
"FullSearch" -> composeSearchRestrictionStrategy(executor, FullSearch(executor), results,
searchStrategyObject.restrictions)
"LinearSearch" -> composeSearchRestrictionStrategy(executor, LinearSearch(executor), results,
searchStrategyObject.restrictions)
"BinarySearch" -> composeSearchRestrictionStrategy(executor, BinarySearch(executor), results,
searchStrategyObject.restrictions)
else -> throw IllegalArgumentException(
"Search Strategy ${searchStrategyObject.searchStrategy} for RestrictionSearch not found")
}
"InitialGuessSearch" -> when (searchStrategyObject.guessStrategy){
"PrevResourceMinGuess" -> InitialGuessSearchStrategy(executor,PrevResourceMinGuess(), results)
"PrevResourceMinGuess" -> InitialGuessSearchStrategy(executor,PrevInstanceOptGuess(), results)
else -> throw IllegalArgumentException("Guess Strategy ${searchStrategyObject.guessStrategy} not found")
}
else -> throw IllegalArgumentException("Search Strategy $searchStrategyObject not found")
......@@ -50,8 +54,8 @@ class StrategyFactory {
*
* @param results The [Results] saves the state of the Theodolite benchmark run.
* @param restrictionStrings Specifies the list of [RestrictionStrategy] that are used to restrict the amount
* of Resource for a fixed LoadDimension. Must equal the string
* 'LowerBound'.
* of Resource for a fixed load or resource (depending on the metric).
* Must equal the string 'LowerBound'.
*
* @throws IllegalArgumentException if param searchStrategyString was not one of the allowed options.
*/
......@@ -70,9 +74,9 @@ class StrategyFactory {
* searchStrategy.
*
* @param executor The [theodolite.execution.BenchmarkExecutor] that executes individual experiments.
* @param searchStrategy The [SearchStrategy] to use
* @param searchStrategy The [SearchStrategy] to use.
* @param results The [Results] saves the state of the Theodolite benchmark run.
* @param restrictions The [RestrictionStrategy]'s to use
* @param restrictions The [RestrictionStrategy]'s to use.
*/
private fun composeSearchRestrictionStrategy(executor: BenchmarkExecutor, searchStrategy: SearchStrategy,
results: Results, restrictions: List<String>): SearchStrategy {
......
......@@ -3,8 +3,10 @@ package theodolite.strategies.restriction
import theodolite.util.Results
/**
* The [LowerBoundRestriction] sets the lower bound of the resources to be examined to the value
* needed to successfully execute the next smaller load.
* The [LowerBoundRestriction] sets the lower bound of the resources to be examined in the experiment to the value
* needed to successfully execute the previous smaller load (demand metric), or sets the lower bound of the loads
* to be examined in the experiment to the largest value, which still successfully executed the previous smaller
* resource (capacity metric).
*
* @param results [Result] object used as a basis to restrict the resources.
*/
......
......@@ -4,7 +4,7 @@ import io.quarkus.runtime.annotations.RegisterForReflection
import theodolite.util.Results
/**
* A 'Restriction Strategy' restricts a list of resources based on the current
* A 'Restriction Strategy' restricts a list of resources or loads depending on the metric based on the current
* results of all previously performed benchmarks.
*
* @param results the [Results] object
......@@ -12,10 +12,11 @@ import theodolite.util.Results
@RegisterForReflection
abstract class RestrictionStrategy(val results: Results) {
/**
* Apply the restriction of the given resource list for the given load based on the results object.
* Apply the restriction of the given resource list for the given load based on the results object (demand metric),
* or apply the restriction of the given load list for the given resource based on the results object (capacity metric).
*
* @param load LoadDimension for which a subset of resources is required.
* @param resources List of Resource s to be restricted.
* @param xValue The value to be examined in the experiment, can be load (demand metric) or resource (capacity metric).
* @param yValues List of values to be restricted, can be resources (demand metric) or loads (capacity metric).
* @return Returns a list containing only elements that have not been filtered out by the
* restriction (possibly empty).
*/
......
......@@ -28,12 +28,12 @@ class BinarySearch(benchmarkExecutor: BenchmarkExecutor) : SearchStrategy(benchm
}
/**
* Apply binary search for metric demand.
* Apply binary search for the demand metric.
*
* @param load the load dimension to perform experiments for
* @param resources the list in which binary search is performed
* @param lower lower bound for binary search (inclusive)
* @param upper upper bound for binary search (inclusive)
* @param load the load to perform experiments for.
* @param resources the list of resources in which binary search is performed.
* @param lower lower bound for binary search (inclusive).
* @param upper upper bound for binary search (inclusive).
*/
private fun binarySearchDemand(load: Int, resources: List<Int>, lower: Int, upper: Int): Int {
if (lower > upper) {
......@@ -68,12 +68,12 @@ class BinarySearch(benchmarkExecutor: BenchmarkExecutor) : SearchStrategy(benchm
/**
* Apply binary search for metric capacity.
* Apply binary search for the capacity metric.
*
* @param resource the load dimension to perform experiments for
* @param loads the list in which binary search is performed
* @param lower lower bound for binary search (inclusive)
* @param upper upper bound for binary search (inclusive)
* @param resource the resource to perform experiments for.
* @param loads the list of loads in which binary search is performed.
* @param lower lower bound for binary search (inclusive).
* @param upper upper bound for binary search (inclusive).
*/
private fun binarySearchCapacity(resource: Int, loads: List<Int>, lower: Int, upper: Int): Int {
if (lower > upper) {
......
......@@ -6,10 +6,11 @@ import theodolite.execution.BenchmarkExecutor
private val logger = KotlinLogging.logger {}
/**
* [SearchStrategy] that executes experiment for provides resources in a linear-search-like fashion, but **without
* stopping** once a suitable resource amount is found.
* [SearchStrategy] that executes an experiment for a load and a resource list (demand metric) or for a resource and a
* load list (capacity metric) in a linear-search-like fashion, but **without stopping** once the desired
* resource (demand) or load (capacity) is found.
*
* @see LinearSearch for a SearchStrategy that stops once a suitable resource amount is found.
* @see LinearSearch for a SearchStrategy that stops once the desired resource (demand) or load (capacity) is found.
*
* @param benchmarkExecutor Benchmark executor which runs the individual benchmarks.
*/
......
......@@ -4,18 +4,19 @@ import io.quarkus.runtime.annotations.RegisterForReflection
/**
* Base class for the implementation of Guess strategies. Guess strategies are strategies to determine the resource
* demand we start with in our initial guess search strategy.
* demand (demand metric) or load (capacity metric) we start with in our initial guess search strategy.
*/
@RegisterForReflection
abstract class GuessStrategy {
/**
* Computing the resource demand for the initial guess search strategy to start with.
* Computing the resource demand (demand metric) or load (capacity metric) for the initial guess search strategy
* to start with.
*
* @param resources List of all possible resources.
* @param lastLowestResource Previous resource demand needed for the given load.
* @param valuesToCheck List of all possible resources/loads.
* @param lastOptValue Previous minimal/maximal resource/load value for the given load/resource.
*
* @return Returns the resource demand to start the initial guess search strategy with or null
* @return the resource/load to start the initial guess search strategy with or null
*/
abstract fun firstGuess(resources: List<Int>, lastLowestResource: Int?): Int?
abstract fun firstGuess(valuesToCheck: List<Int>, lastOptValue: Int?): Int?
}
\ No newline at end of file
......@@ -6,7 +6,6 @@ import theodolite.util.Results
private val logger = KotlinLogging.logger {}
// TODO: Is actually just a heuristic approach. Not ensured to have opt solution. Maybe Talk to Sören about it.
/**
* Search strategy implementation for determining the smallest suitable resource demand.
* Starting with a resource amount provided by a guess strategy.
......
......@@ -6,7 +6,8 @@ import theodolite.execution.BenchmarkExecutor
private val logger = KotlinLogging.logger {}
/**
* Linear-search-like implementation for determining the smallest suitable number of instances.
* Linear-search-like implementation for determining the smallest/biggest suitable number of resources/loads,
* depending on the metric.
*
* @param benchmarkExecutor Benchmark executor which runs the individual benchmarks.
*/
......
package theodolite.strategies.searchstrategy
/**
* [PrevInstanceOptGuess] is a [GuessStrategy] that implements the function [firstGuess],
* where it returns the optimal result of the previous run.
*/
class PrevInstanceOptGuess() : GuessStrategy(){
/**
* If the metric is
*
* - "demand", [valuesToCheck] is a List of resources and [lastOptValue] a resource value.
* - "capacity", [valuesToCheck] is a List of loads and [lastOptValue] a load value.
*
* @param valuesToCheck List of all possible resources/loads.
* @param lastOptValue Previous minimal/maximal resource/load value for the given load/resource.
*
* @return the value of [lastOptValue] if given otherwise the first element of the [valuesToCheck] list or null
*/
override fun firstGuess(valuesToCheck: List<Int>, lastOptValue: Int?): Int? {
if (lastOptValue != null) return lastOptValue
else if(valuesToCheck.isNotEmpty()) return valuesToCheck[0]
else return null
}
}
\ No newline at end of file
package theodolite.strategies.searchstrategy
/**
* This Guess strategy takes the minimal resource demand of the previous load, which is given as an argument for the
* firstGuess function.
*/
class PrevResourceMinGuess() : GuessStrategy(){
/**
* @param resources List of all possible Resources.
* @param lastLowestResource Previous resource demand needed for the given load.
*
* @return the value of lastLowestResource if given otherwise the first element of the resource list or null
*/
// TODO verallgemeinern für loads
override fun firstGuess(resources: List<Int>, lastLowestResource: Int?): Int? {
if (lastLowestResource != null) return lastLowestResource
else if(resources.isNotEmpty()) return resources[0]
else return null
}
}
\ No newline at end of file
......@@ -7,17 +7,21 @@ import theodolite.strategies.restriction.RestrictionStrategy
/**
* Strategy that combines a SearchStrategy and a set of RestrictionStrategy.
*
* @param searchStrategy the [SearchStrategy] that is executed as part of this [RestrictionSearch].
* @param restrictionStrategies the set of [RestrictionStrategy] that are connected conjunctive to restrict the Resource
* @param benchmarkExecutor Benchmark executor which runs the individual benchmarks.
* @param searchStrategy the [SearchStrategy] that is executed as part of this [RestrictionSearch].
* @param restrictionStrategies the set of [RestrictionStrategy] that are connected conjunctive to restrict the Resource.
*
*/
@RegisterForReflection
class RestrictionSearch(
benchmarkExecutor: BenchmarkExecutor,
private val searchStrategy: SearchStrategy,
val restrictionStrategies: Set<RestrictionStrategy>
private val restrictionStrategies: Set<RestrictionStrategy>
) : SearchStrategy(benchmarkExecutor) {
/**
* Restricting the possible resources and calling findSuitableResource of the given [SearchStrategy].
*/
override fun findSuitableResource(load: Int, resources: List<Int>): Int? {
var restrictedResources = resources
for (strategy in this.restrictionStrategies) {
......@@ -26,6 +30,9 @@ class RestrictionSearch(
return this.searchStrategy.findSuitableResource(load, restrictedResources)
}
/**
* Restricting the possible loads and calling findSuitableLoad of the given [SearchStrategy].
*/
override fun findSuitableLoad(resource: Int, loads: List<Int>): Int? {
var restrictedLoads = loads
for (strategy in this.restrictionStrategies) {
......
......@@ -6,7 +6,8 @@ import theodolite.strategies.Metric
import theodolite.util.Results
/**
* Base class for the implementation for SearchStrategies. SearchStrategies determine the smallest suitable number of instances.
* Base class for the implementation for SearchStrategies. SearchStrategies determine the smallest suitable number
* of resources/loads for a load/resource (depending on the metric).
*
* @param benchmarkExecutor Benchmark executor which runs the individual benchmarks.
* @param guessStrategy Guess strategy for the initial resource amount in case the InitialGuessStrategy is selected.
......@@ -17,6 +18,13 @@ abstract class SearchStrategy(val benchmarkExecutor: BenchmarkExecutor, val gues
val results: Results? = null) {
/**
* Calling findSuitableResource or findSuitableLoad for each load/resource depending on the chosen metric.
*
* @param loads List of possible loads for the experiments.
* @param resources List of possible resources for the experiments.
* @param metric The [Metric] for the experiments, either "demand" or "capacity".
*/
fun applySearchStrategyByMetric(loads: List<Int>, resources: List<Int>, metric: Metric) {
if (metric.value == "demand") {
......@@ -37,7 +45,7 @@ abstract class SearchStrategy(val benchmarkExecutor: BenchmarkExecutor, val gues
}
/**
* Find smallest suitable resource from the specified resource list for the given load.
* Find the smallest suitable resource from the specified resource list for the given load.
*
* @param load the load to be tested.
* @param resources List of all possible resources.
......@@ -47,7 +55,7 @@ abstract class SearchStrategy(val benchmarkExecutor: BenchmarkExecutor, val gues
abstract fun findSuitableResource(load: Int, resources: List<Int>): Int?
/**
* Find biggest suitable load from the specified load list for the given resource amount.
* Find the biggest suitable load from the specified load list for the given resource amount.
*
* @param resource the resource to be tested.
* @param loads List of all possible loads.
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment