Skip to content
Snippets Groups Projects
Commit 6ed62994 authored by Sören Henning's avatar Sören Henning
Browse files

Merge branch 'master' into issue-274

parents d8a47087 9f6cc90a
No related branches found
No related tags found
1 merge request!251Move definition of SLOs to Benchmark
Pipeline #8467 passed
Showing
with 468 additions and 239 deletions
......@@ -1838,7 +1838,7 @@ Contains the Kafka configuration.
</td>
<td>true</td>
</tr><tr>
<td><b><a href="#executionspecload">load</a></b></td>
<td><b><a href="#executionspecloads">loads</a></b></td>
<td>object</td>
<td>
Specifies the load values that are benchmarked.<br/>
......@@ -1979,35 +1979,83 @@ Defines the overall parameter for the execution.
<td><b>repetitions</b></td>
<td>integer</td>
<td>
Numper of repititions for each experiments.<br/>
Number of repititions for each experiment.<br/>
</td>
<td>true</td>
</tr><tr>
<td><b>restrictions</b></td>
<td>[]string</td>
<td><b><a href="#executionspecexecutionstrategy">strategy</a></b></td>
<td>object</td>
<td>
List of restriction strategys used to delimit the search space.<br/>
Defines the used strategy for the execution, either 'LinearSearch', 'BinarySearch' or 'InitialGuessSearch'.<br/>
</td>
<td>true</td>
</tr><tr>
<td><b>strategy</b></td>
<td><b>loadGenerationDelay</b></td>
<td>integer</td>
<td>
Seconds to wait between the start of the SUT and the load generator.<br/>
</td>
<td>false</td>
</tr><tr>
<td><b>metric</b></td>
<td>string</td>
<td>
<br/>
</td>
<td>false</td>
</tr></tbody>
</table>
### execution.spec.execution.strategy
<sup><sup>[↩ Parent](#executionspecexecution)</sup></sup>
Defines the used strategy for the execution, either 'LinearSearch', 'BinarySearch' or 'InitialGuessSearch'.
<table>
<thead>
<tr>
<th>Name</th>
<th>Type</th>
<th>Description</th>
<th>Required</th>
</tr>
</thead>
<tbody><tr>
<td><b>name</b></td>
<td>string</td>
<td>
Defines the used strategy for the execution, either 'LinearSearch' or 'BinarySearch'<br/>
<br/>
</td>
<td>true</td>
</tr><tr>
<td><b>loadGenerationDelay</b></td>
<td>integer</td>
<td><b>guessStrategy</b></td>
<td>string</td>
<td>
Seconds to wait between the start of the SUT and the load generator.<br/>
<br/>
</td>
<td>false</td>
</tr><tr>
<td><b>restrictions</b></td>
<td>[]string</td>
<td>
List of restriction strategies used to delimit the search space.<br/>
</td>
<td>false</td>
</tr><tr>
<td><b>searchStrategy</b></td>
<td>string</td>
<td>
<br/>
</td>
<td>false</td>
</tr></tbody>
</table>
### execution.spec.load
### execution.spec.loads
<sup><sup>[↩ Parent](#executionspec)</sup></sup>
......
......@@ -20,7 +20,7 @@ spec:
properties:
spec:
type: object
required: ["benchmark", "load", "resources", "execution", "configOverrides"]
required: ["benchmark", "loads", "resources", "execution", "configOverrides"]
properties:
name:
description: This field exists only for technical reasons and should not be set by the user. The value of the field will be overwritten.
......@@ -29,7 +29,7 @@ spec:
benchmark:
description: The name of the benchmark this execution is referring to.
type: string
load: # definition of the load dimension
loads: # definition of the load dimension
description: Specifies the load values that are benchmarked.
type: object
required: ["loadType", "loadValues"]
......@@ -74,25 +74,35 @@ spec:
execution: # def execution config
description: Defines the overall parameter for the execution.
type: object
required: ["strategy", "duration", "repetitions", "restrictions"]
required: ["strategy", "duration", "repetitions"]
properties:
strategy:
description: Defines the used strategy for the execution, either 'LinearSearch' or 'BinarySearch'
metric:
type: string
strategy:
description: Defines the used strategy for the execution, either 'LinearSearch', 'BinarySearch' or 'InitialGuessSearch'.
type: object
required: ["name"]
properties:
name:
type: string
restrictions:
description: List of restriction strategies used to delimit the search space.
type: array
items:
type: string
guessStrategy:
type: string
searchStrategy:
type: string
duration:
description: Defines the duration of each experiment in seconds.
type: integer
repetitions:
description: Numper of repititions for each experiments.
description: Number of repititions for each experiment.
type: integer
loadGenerationDelay:
description: Seconds to wait between the start of the SUT and the load generator.
type: integer
restrictions:
description: List of restriction strategys used to delimit the search space.
type: array
items:
type: string
configOverrides:
description: List of patchers that are used to override existing configurations.
type: array
......
......@@ -4,7 +4,7 @@ metadata:
name: theodolite-example-execution
spec:
benchmark: "example-benchmark"
load:
loads:
loadType: "NumSensors"
loadValues: [25000, 50000, 75000, 100000, 125000, 150000]
resources:
......@@ -15,12 +15,14 @@ spec:
properties:
threshold: 2000
execution:
strategy: "LinearSearch"
strategy:
name: "RestrictionSearch"
restrictions:
- "LowerBound"
searchStrategy: "LinearSearch"
duration: 300 # in seconds
repetitions: 1
loadGenerationDelay: 30 # in seconds
restrictions:
- "LowerBound"
configOverrides: []
# - patcher:
# type: "NodeSelectorPatcher"
......
......@@ -2,13 +2,12 @@ package theodolite.benchmark
import io.quarkus.runtime.annotations.RegisterForReflection
import theodolite.util.ConfigurationOverride
import theodolite.util.LoadDimension
import theodolite.util.Resource
import theodolite.util.PatcherDefinition
/**
* A Benchmark contains:
* - The [Resource]s that can be scaled for the benchmark.
* - The [LoadDimension]s that can be scaled the benchmark.
* - The Resource that can be scaled for the benchmark.
* - The Load that can be scaled the benchmark.
* - additional [ConfigurationOverride]s.
*/
@RegisterForReflection
......@@ -22,10 +21,12 @@ interface Benchmark {
* @return a BenchmarkDeployment.
*/
fun buildDeployment(
load: LoadDimension,
res: Resource,
configurationOverrides: List<ConfigurationOverride?>,
loadGenerationDelay: Long,
afterTeardownDelay: Long
load: Int,
loadPatcherDefinitions: List<PatcherDefinition>,
resource: Int,
resourcePatcherDefinitions: List<PatcherDefinition>,
configurationOverrides: List<ConfigurationOverride?>,
loadGenerationDelay: Long,
afterTeardownDelay: Long
): BenchmarkDeployment
}
......@@ -12,7 +12,7 @@ import kotlin.properties.Delegates
* A BenchmarkExecution consists of:
* - A [name].
* - The [benchmark] that should be executed.
* - The [load] that should be checked in the benchmark.
* - The [loads]s that should be checked in the benchmark.
* - The [resources] that should be checked in the benchmark.
* - The [slos] further restrict the Benchmark SLOs for the evaluation of the experiments.
* - An [execution] that encapsulates: the strategy, the duration, and the restrictions
......@@ -28,27 +28,42 @@ class BenchmarkExecution : KubernetesResource {
var executionId: Int = 0
lateinit var name: String
lateinit var benchmark: String
lateinit var load: LoadDefinition
lateinit var loads: LoadDefinition
lateinit var resources: ResourceDefinition
lateinit var slos: List<SloConfiguration>
lateinit var execution: Execution
lateinit var configOverrides: MutableList<ConfigurationOverride?>
/**
* This execution encapsulates the [strategy], the [duration], the [repetitions], and the [restrictions]
* This execution encapsulates the [strategy], the [duration], and the [repetitions],
* which are used for the concrete benchmark experiments.
*/
@JsonDeserialize
@RegisterForReflection
class Execution : KubernetesResource {
lateinit var strategy: String
var metric = "demand"
lateinit var strategy: Strategy
var duration by Delegates.notNull<Long>()
var repetitions by Delegates.notNull<Int>()
lateinit var restrictions: List<String>
var loadGenerationDelay = 0L
var afterTeardownDelay = 5L
}
/**
* This Strategy encapsulates the [restrictions], [guessStrategy] and [searchStrategy],
* which are used for restricting the resources, the guess Strategy for the
* [theodolite.strategies.searchstrategy.InitialGuessSearchStrategy] and the name of the actual
* [theodolite.strategies.searchstrategy.SearchStrategy] which is used.
*/
@JsonDeserialize
@RegisterForReflection
class Strategy : KubernetesResource {
lateinit var name: String
var restrictions = emptyList<String>()
var guessStrategy = ""
var searchStrategy = ""
}
/**
* Further SLO configurations for the SLOs specified in the Benchmark.
*/
......@@ -60,8 +75,8 @@ class BenchmarkExecution : KubernetesResource {
}
/**
* Represents a Load that should be created and checked.
* It can be set to [loadValues].
* Represents the Loads that should be created and checked if the demand metric is in use or
* represents a Load that can be scaled to [loadValues] if the capacity metric is in use.
*/
@JsonDeserialize
@RegisterForReflection
......@@ -71,7 +86,8 @@ class BenchmarkExecution : KubernetesResource {
}
/**
* Represents a resource that can be scaled to [resourceValues].
* Represents a resource that can be scaled to [resourceValues] if the demand metric is in use or
* represents the Resources that should be created and checked if the capacity metric is in use.
*/
@JsonDeserialize
@RegisterForReflection
......
......@@ -83,19 +83,22 @@ class KubernetesBenchmark : KubernetesResource, Benchmark {
/**
* Builds a deployment.
* First loads all required resources and then patches them to the concrete load and resources for the experiment.
* Afterwards patches additional configurations(cluster depending) into the resources.
* @param load concrete load that will be benchmarked in this experiment.
* @param res concrete resource that will be scaled for this experiment.
* First loads all required resources and then patches them to the concrete load and resources for the experiment for the demand metric
* or loads all loads and then patches them to the concrete load and resources for the experiment.
* Afterwards patches additional configurations(cluster depending) into the resources (or loads).
* @param load concrete load that will be benchmarked in this experiment (demand metric), or scaled (capacity metric).
* @param resource concrete resource that will be scaled for this experiment (demand metric), or benchmarked (capacity metric).
* @param configurationOverrides
* @return a [BenchmarkDeployment]
*/
override fun buildDeployment(
load: LoadDimension,
res: Resource,
load: Int,
loadPatcherDefinitions: List<PatcherDefinition>,
resource: Int,
resourcePatcherDefinitions: List<PatcherDefinition>,
configurationOverrides: List<ConfigurationOverride?>,
loadGenerationDelay: Long,
afterTeardownDelay: Long,
afterTeardownDelay: Long
): BenchmarkDeployment {
logger.info { "Using $namespace as namespace." }
......@@ -103,14 +106,14 @@ class KubernetesBenchmark : KubernetesResource, Benchmark {
val appResources = loadResources(this.sut.resources).toResourceMap()
val loadGenResources = loadResources(this.loadGenerator.resources).toResourceMap()
load.getType().forEach { patcherDefinition ->
// patch the load dimension the resources
loadPatcherDefinitions.forEach { patcherDefinition ->
loadGenResources[patcherDefinition.resource] =
PatchHandler.patchResource(loadGenResources, patcherDefinition, load.get().toString())
PatchHandler.patchResource(loadGenResources, patcherDefinition, load.toString())
}
res.getType().forEach { patcherDefinition ->
resourcePatcherDefinitions.forEach { patcherDefinition ->
appResources[patcherDefinition.resource] =
PatchHandler.patchResource(appResources, patcherDefinition, res.get().toString())
PatchHandler.patchResource(appResources, patcherDefinition, resource.toString())
}
configurationOverrides.forEach { override ->
......
package theodolite.evaluation
import theodolite.benchmark.Slo
import theodolite.strategies.Metric
import theodolite.util.EvaluationFailedException
import theodolite.util.IOHandler
import theodolite.util.LoadDimension
import theodolite.util.Resource
import java.text.Normalizer
import java.time.Duration
import java.time.Instant
......@@ -29,17 +28,17 @@ class AnalysisExecutor(
* Analyses an experiment via prometheus data.
* First fetches data from prometheus, then documents them and afterwards evaluate it via a [slo].
* @param load of the experiment.
* @param res of the experiment.
* @param resource of the experiment.
* @param executionIntervals list of start and end points of experiments
* @return true if the experiment succeeded.
*/
fun analyze(load: LoadDimension, res: Resource, executionIntervals: List<Pair<Instant, Instant>>): Boolean {
fun analyze(load: Int, resource: Int, executionIntervals: List<Pair<Instant, Instant>>, metric: Metric): Boolean {
var repetitionCounter = 1
try {
val ioHandler = IOHandler()
val resultsFolder = ioHandler.getResultFolderURL()
val fileURL = "${resultsFolder}exp${executionId}_${load.get()}_${res.get()}_${slo.sloType.toSlug()}"
val fileURL = "${resultsFolder}exp${executionId}_${load}_${resource}_${slo.sloType.toSlug()}"
val prometheusData = executionIntervals
.map { interval ->
......@@ -61,13 +60,15 @@ class AnalysisExecutor(
val sloChecker = SloCheckerFactory().create(
sloType = slo.sloType,
properties = slo.properties,
load = load
load = load,
resource = resource,
metric = metric
)
return sloChecker.evaluate(prometheusData)
} catch (e: Exception) {
throw EvaluationFailedException("Evaluation failed for resource '${res.get()}' and load '${load.get()}", e)
throw EvaluationFailedException("Evaluation failed for resource '$resource' and load '$load ", e)
}
}
......
package theodolite.evaluation
import theodolite.util.LoadDimension
import theodolite.strategies.Metric
/**
* Factory used to potentially create different [SloChecker]s.
......@@ -41,7 +42,9 @@ class SloCheckerFactory {
fun create(
sloType: String,
properties: MutableMap<String, String>,
load: LoadDimension
load: Int,
resource: Int,
metric: Metric
): SloChecker {
return when (SloTypes.from(sloType)) {
SloTypes.GENERIC -> ExternalSloChecker(
......@@ -76,7 +79,7 @@ class SloCheckerFactory {
throw IllegalArgumentException("Threshold ratio needs to be an Double greater or equal 0.0")
}
// cast to int, as rounding is not really necessary
val threshold = (load.get() * thresholdRatio).toInt()
val threshold = (load * thresholdRatio).toInt()
ExternalSloChecker(
externalSlopeURL = properties["externalSloUrl"]
......
......@@ -4,8 +4,7 @@ import mu.KotlinLogging
import theodolite.benchmark.Benchmark
import theodolite.benchmark.Slo
import theodolite.util.ConfigurationOverride
import theodolite.util.LoadDimension
import theodolite.util.Resource
import theodolite.util.PatcherDefinition
import theodolite.util.Results
import java.time.Duration
import java.util.concurrent.atomic.AtomicBoolean
......@@ -30,7 +29,9 @@ abstract class BenchmarkExecutor(
val executionId: Int,
val loadGenerationDelay: Long,
val afterTeardownDelay: Long,
val executionName: String
val executionName: String,
val loadPatcherDefinitions: List<PatcherDefinition>,
val resourcePatcherDefinitions: List<PatcherDefinition>
) {
var run: AtomicBoolean = AtomicBoolean(true)
......@@ -39,12 +40,14 @@ abstract class BenchmarkExecutor(
* Run a experiment for the given parametrization, evaluate the
* experiment and save the result.
*
* @param load load to be tested.
* @param res resources to be tested.
* @param load to be tested.
* @param resource to be tested.
* @return True, if the number of resources are suitable for the
* given load, false otherwise.
* given load, false otherwise (demand metric), or
* True, if there is a load suitable for the
* given resource, false otherwise.
*/
abstract fun runExperiment(load: LoadDimension, res: Resource): Boolean
abstract fun runExperiment(load: Int, resource: Int): Boolean
/**
* Wait while the benchmark is running and log the number of minutes executed every 1 minute.
......
......@@ -14,16 +14,18 @@ private val logger = KotlinLogging.logger {}
@RegisterForReflection
class BenchmarkExecutorImpl(
benchmark: Benchmark,
results: Results,
executionDuration: Duration,
configurationOverrides: List<ConfigurationOverride?>,
slos: List<Slo>,
repetitions: Int,
executionId: Int,
loadGenerationDelay: Long,
afterTeardownDelay: Long,
executionName: String
benchmark: Benchmark,
results: Results,
executionDuration: Duration,
configurationOverrides: List<ConfigurationOverride?>,
slos: List<Slo>,
repetitions: Int,
executionId: Int,
loadGenerationDelay: Long,
afterTeardownDelay: Long,
executionName: String,
loadPatcherDefinitions: List<PatcherDefinition>,
resourcePatcherDefinitions: List<PatcherDefinition>
) : BenchmarkExecutor(
benchmark,
results,
......@@ -34,24 +36,26 @@ class BenchmarkExecutorImpl(
executionId,
loadGenerationDelay,
afterTeardownDelay,
executionName
executionName,
loadPatcherDefinitions,
resourcePatcherDefinitions
) {
private val eventCreator = EventCreator()
private val mode = Configuration.EXECUTION_MODE
override fun runExperiment(load: LoadDimension, res: Resource): Boolean {
override fun runExperiment(load: Int, resource: Int): Boolean {
var result = false
val executionIntervals: MutableList<Pair<Instant, Instant>> = ArrayList()
for (i in 1.rangeTo(repetitions)) {
if (this.run.get()) {
logger.info { "Run repetition $i/$repetitions" }
executionIntervals.add(runSingleExperiment(load, res))
executionIntervals.add(runSingleExperiment(
load, resource))
} else {
break
}
}
/**
* Analyse the experiment, if [run] is true, otherwise the experiment was canceled by the user.
*/
......@@ -60,26 +64,26 @@ class BenchmarkExecutorImpl(
AnalysisExecutor(slo = it, executionId = executionId)
.analyze(
load = load,
res = res,
executionIntervals = executionIntervals
resource = resource,
executionIntervals = executionIntervals,
metric = this.results.metric
)
}
result = (false !in experimentResults)
this.results.setResult(Pair(load, res), result)
}
if(!this.run.get()) {
this.results.setResult(Pair(load, resource), result)
} else {
throw ExecutionFailedException("The execution was interrupted")
}
return result
}
private fun runSingleExperiment(load: LoadDimension, res: Resource): Pair<Instant, Instant> {
private fun runSingleExperiment(load: Int, resource: Int): Pair<Instant, Instant> {
val benchmarkDeployment = benchmark.buildDeployment(
load,
res,
this.loadPatcherDefinitions,
resource,
this.resourcePatcherDefinitions,
this.configurationOverrides,
this.loadGenerationDelay,
this.afterTeardownDelay
......@@ -94,7 +98,7 @@ class BenchmarkExecutorImpl(
executionName = executionName,
type = "NORMAL",
reason = "Start experiment",
message = "load: ${load.get()}, resources: ${res.get()}")
message = "load: $load, resources: $resource")
}
} catch (e: Exception) {
this.run.set(false)
......@@ -104,7 +108,7 @@ class BenchmarkExecutorImpl(
executionName = executionName,
type = "WARNING",
reason = "Start experiment failed",
message = "load: ${load.get()}, resources: ${res.get()}")
message = "load: $load, resources: $resource")
}
throw ExecutionFailedException("Error during setup the experiment", e)
}
......
......@@ -3,8 +3,6 @@ package theodolite.execution
import mu.KotlinLogging
import theodolite.benchmark.BenchmarkExecution
import theodolite.benchmark.KubernetesBenchmark
import theodolite.util.LoadDimension
import theodolite.util.Resource
private val logger = KotlinLogging.logger {}
......@@ -26,8 +24,10 @@ class Shutdown(private val benchmarkExecution: BenchmarkExecution, private val b
logger.info { "Received shutdown signal -> Shutting down" }
val deployment =
benchmark.buildDeployment(
load = LoadDimension(0, emptyList()),
res = Resource(0, emptyList()),
load = 0,
loadPatcherDefinitions = emptyList(),
resource = 0,
resourcePatcherDefinitions = emptyList(),
configurationOverrides = benchmarkExecution.configOverrides,
loadGenerationDelay = 0L,
afterTeardownDelay = 5L
......
......@@ -4,8 +4,8 @@ import mu.KotlinLogging
import theodolite.benchmark.BenchmarkExecution
import theodolite.benchmark.KubernetesBenchmark
import theodolite.patcher.PatcherDefinitionFactory
import theodolite.strategies.Metric
import theodolite.strategies.StrategyFactory
import theodolite.strategies.searchstrategy.CompositeStrategy
import theodolite.util.*
import java.io.File
import java.time.Duration
......@@ -21,8 +21,8 @@ private val logger = KotlinLogging.logger {}
* @constructor Create empty Theodolite executor
*/
class TheodoliteExecutor(
private val benchmarkExecution: BenchmarkExecution,
private val kubernetesBenchmark: KubernetesBenchmark
private val benchmarkExecution: BenchmarkExecution,
private val kubernetesBenchmark: KubernetesBenchmark
) {
/**
* An executor object, configured with the specified benchmark, evaluation method, experiment duration
......@@ -33,12 +33,12 @@ class TheodoliteExecutor(
/**
* Creates all required components to start Theodolite.
*
* @return a [Config], that contains a list of [LoadDimension]s,
* a list of [Resource]s , and the [CompositeStrategy].
* The [CompositeStrategy] is configured and able to find the minimum required resource for the given load.
* @return a [Config], that contains a list of LoadDimension s,
* a list of Resource s , and the [restrictionSearch].
* The [searchStrategy] is configured and able to find the minimum required resource for the given load.
*/
private fun buildConfig(): Config {
val results = Results()
val results = Results(Metric.from(benchmarkExecution.execution.metric))
val strategyFactory = StrategyFactory()
val executionDuration = Duration.ofSeconds(benchmarkExecution.execution.duration)
......@@ -51,7 +51,7 @@ class TheodoliteExecutor(
val loadDimensionPatcherDefinition =
PatcherDefinitionFactory().createPatcherDefinition(
benchmarkExecution.load.loadType,
benchmarkExecution.loads.loadType,
this.kubernetesBenchmark.loadTypes
)
......@@ -68,41 +68,34 @@ class TheodoliteExecutor(
executionId = benchmarkExecution.executionId,
loadGenerationDelay = benchmarkExecution.execution.loadGenerationDelay,
afterTeardownDelay = benchmarkExecution.execution.afterTeardownDelay,
executionName = benchmarkExecution.name
executionName = benchmarkExecution.name,
loadPatcherDefinitions = loadDimensionPatcherDefinition,
resourcePatcherDefinitions = resourcePatcherDefinition
)
if (benchmarkExecution.load.loadValues != benchmarkExecution.load.loadValues.sorted()) {
benchmarkExecution.load.loadValues = benchmarkExecution.load.loadValues.sorted()
if (benchmarkExecution.loads.loadValues != benchmarkExecution.loads.loadValues.sorted()) {
benchmarkExecution.loads.loadValues = benchmarkExecution.loads.loadValues.sorted()
logger.info {
"Load values are not sorted correctly, Theodolite sorts them in ascending order." +
"New order is: ${benchmarkExecution.load.loadValues}"
"Load values are not sorted correctly. Theodolite sorts them in ascending order." +
"New order is: ${benchmarkExecution.loads.loadValues}."
}
}
if (benchmarkExecution.resources.resourceValues != benchmarkExecution.resources.resourceValues.sorted()) {
benchmarkExecution.resources.resourceValues = benchmarkExecution.resources.resourceValues.sorted()
logger.info {
"Load values are not sorted correctly, Theodolite sorts them in ascending order." +
"New order is: ${benchmarkExecution.resources.resourceValues}"
"Load values are not sorted correctly. Theodolite sorts them in ascending order." +
"New order is: ${benchmarkExecution.resources.resourceValues}."
}
}
return Config(
loads = benchmarkExecution.load.loadValues.map { load -> LoadDimension(load, loadDimensionPatcherDefinition) },
resources = benchmarkExecution.resources.resourceValues.map { resource ->
Resource(
resource,
resourcePatcherDefinition
)
},
compositeStrategy = CompositeStrategy(
benchmarkExecutor = executor,
searchStrategy = strategyFactory.createSearchStrategy(executor, benchmarkExecution.execution.strategy),
restrictionStrategies = strategyFactory.createRestrictionStrategy(
results,
benchmarkExecution.execution.restrictions
)
)
loads = benchmarkExecution.loads.loadValues,
loadPatcherDefinitions = loadDimensionPatcherDefinition,
resources = benchmarkExecution.resources.resourceValues,
resourcePatcherDefinitions = resourcePatcherDefinition,
searchStrategy = strategyFactory.createSearchStrategy(executor, benchmarkExecution.execution.strategy, results),
metric = Metric.from(benchmarkExecution.execution.metric)
)
}
......@@ -127,24 +120,31 @@ class TheodoliteExecutor(
)
val config = buildConfig()
// execute benchmarks for each load
//execute benchmarks for each load for the demand metric, or for each resource amount for capacity metric
try {
for (load in config.loads) {
if (executor.run.get()) {
config.compositeStrategy.findSuitableResource(load, config.resources)
}
}
config.searchStrategy.applySearchStrategyByMetric(config.loads, config.resources, config.metric)
} finally {
ioHandler.writeToJSONFile(
config.compositeStrategy.benchmarkExecutor.results,
config.searchStrategy.benchmarkExecutor.results,
"${resultsFolder}exp${this.benchmarkExecution.executionId}-result"
)
// Create expXYZ_demand.csv file
ioHandler.writeToCSVFile(
"${resultsFolder}exp${this.benchmarkExecution.executionId}_demand",
calculateDemandMetric(config.loads, config.compositeStrategy.benchmarkExecutor.results),
listOf("load","resources")
)
// Create expXYZ_demand.csv file or expXYZ_capacity.csv depending on metric
when(config.metric) {
Metric.DEMAND ->
ioHandler.writeToCSVFile(
"${resultsFolder}exp${this.benchmarkExecution.executionId}_demand",
calculateMetric(config.loads, config.searchStrategy.benchmarkExecutor.results),
listOf("load","resources")
)
Metric.CAPACITY ->
ioHandler.writeToCSVFile(
"${resultsFolder}exp${this.benchmarkExecution.executionId}_capacity",
calculateMetric(config.resources, config.searchStrategy.benchmarkExecutor.results),
listOf("resource", "loads")
)
}
}
kubernetesBenchmark.teardownInfrastructure()
}
......@@ -159,8 +159,8 @@ class TheodoliteExecutor(
return executionID
}
private fun calculateDemandMetric(loadDimensions: List<LoadDimension>, results: Results): List<List<String>> {
return loadDimensions.map { listOf(it.get().toString(), results.getMinRequiredInstances(it).get().toString()) }
private fun calculateMetric(xValues: List<Int>, results: Results): List<List<String>> {
return xValues.map { listOf(it.toString(), results.getOptYDimensionValue(it).toString()) }
}
}
package theodolite.strategies
enum class Metric(val value: String) {
DEMAND("demand"),
CAPACITY("capacity");
companion object {
fun from(metric: String): Metric =
values().find { it.value == metric } ?: throw IllegalArgumentException("Requested Metric does not exist")
}
}
\ No newline at end of file
package theodolite.strategies
import theodolite.benchmark.BenchmarkExecution
import theodolite.execution.BenchmarkExecutor
import theodolite.strategies.restriction.LowerBoundRestriction
import theodolite.strategies.restriction.RestrictionStrategy
import theodolite.strategies.searchstrategy.BinarySearch
import theodolite.strategies.searchstrategy.FullSearch
import theodolite.strategies.searchstrategy.LinearSearch
import theodolite.strategies.searchstrategy.SearchStrategy
import theodolite.strategies.searchstrategy.*
import theodolite.util.Results
/**
......@@ -18,18 +16,37 @@ class StrategyFactory {
* Create a [SearchStrategy].
*
* @param executor The [theodolite.execution.BenchmarkExecutor] that executes individual experiments.
* @param searchStrategyString Specifies the [SearchStrategy]. Must either be the string 'LinearSearch',
* or 'BinarySearch'.
* @param searchStrategyObject Specifies the [SearchStrategy]. Must either be an object with name 'FullSearch',
* 'LinearSearch', 'BinarySearch', 'RestrictionSearch' or 'InitialGuessSearch'.
* @param results The [Results] saves the state of the Theodolite benchmark run.
*
* @throws IllegalArgumentException if the [SearchStrategy] was not one of the allowed options.
*/
fun createSearchStrategy(executor: BenchmarkExecutor, searchStrategyString: String): SearchStrategy {
return when (searchStrategyString) {
fun createSearchStrategy(executor: BenchmarkExecutor, searchStrategyObject: BenchmarkExecution.Strategy,
results: Results): SearchStrategy {
var strategy : SearchStrategy = when (searchStrategyObject.name) {
"FullSearch" -> FullSearch(executor)
"LinearSearch" -> LinearSearch(executor)
"BinarySearch" -> BinarySearch(executor)
else -> throw IllegalArgumentException("Search Strategy $searchStrategyString not found")
"RestrictionSearch" -> when (searchStrategyObject.searchStrategy){
"FullSearch" -> composeSearchRestrictionStrategy(executor, FullSearch(executor), results,
searchStrategyObject.restrictions)
"LinearSearch" -> composeSearchRestrictionStrategy(executor, LinearSearch(executor), results,
searchStrategyObject.restrictions)
"BinarySearch" -> composeSearchRestrictionStrategy(executor, BinarySearch(executor), results,
searchStrategyObject.restrictions)
else -> throw IllegalArgumentException(
"Search Strategy ${searchStrategyObject.searchStrategy} for RestrictionSearch not found")
}
"InitialGuessSearch" -> when (searchStrategyObject.guessStrategy){
"PrevResourceMinGuess" -> InitialGuessSearchStrategy(executor,PrevInstanceOptGuess(), results)
else -> throw IllegalArgumentException("Guess Strategy ${searchStrategyObject.guessStrategy} not found")
}
else -> throw IllegalArgumentException("Search Strategy $searchStrategyObject not found")
}
return strategy
}
/**
......@@ -37,12 +54,12 @@ class StrategyFactory {
*
* @param results The [Results] saves the state of the Theodolite benchmark run.
* @param restrictionStrings Specifies the list of [RestrictionStrategy] that are used to restrict the amount
* of [theodolite.util.Resource] for a fixed LoadDimension. Must equal the string
* 'LowerBound'.
* of Resource for a fixed load or resource (depending on the metric).
* Must equal the string 'LowerBound'.
*
* @throws IllegalArgumentException if param searchStrategyString was not one of the allowed options.
*/
fun createRestrictionStrategy(results: Results, restrictionStrings: List<String>): Set<RestrictionStrategy> {
private fun createRestrictionStrategy(results: Results, restrictionStrings: List<String>): Set<RestrictionStrategy> {
return restrictionStrings
.map { restriction ->
when (restriction) {
......@@ -51,4 +68,21 @@ class StrategyFactory {
}
}.toSet()
}
/**
* Create a RestrictionSearch, if the provided restriction list is not empty. Otherwise just return the given
* searchStrategy.
*
* @param executor The [theodolite.execution.BenchmarkExecutor] that executes individual experiments.
* @param searchStrategy The [SearchStrategy] to use.
* @param results The [Results] saves the state of the Theodolite benchmark run.
* @param restrictions The [RestrictionStrategy]'s to use.
*/
private fun composeSearchRestrictionStrategy(executor: BenchmarkExecutor, searchStrategy: SearchStrategy,
results: Results, restrictions: List<String>): SearchStrategy {
if(restrictions.isNotEmpty()){
return RestrictionSearch(executor,searchStrategy,createRestrictionStrategy(results, restrictions))
}
return searchStrategy
}
}
package theodolite.strategies.restriction
import theodolite.util.LoadDimension
import theodolite.util.Resource
import theodolite.util.Results
/**
* The [LowerBoundRestriction] sets the lower bound of the resources to be examined to the value
* needed to successfully execute the next smaller load.
* The [LowerBoundRestriction] sets the lower bound of the resources to be examined in the experiment to the value
* needed to successfully execute the previous smaller load (demand metric), or sets the lower bound of the loads
* to be examined in the experiment to the largest value, which still successfully executed the previous smaller
* resource (capacity metric).
*
* @param results [Result] object used as a basis to restrict the resources.
*/
class LowerBoundRestriction(results: Results) : RestrictionStrategy(results) {
override fun apply(load: LoadDimension, resources: List<Resource>): List<Resource> {
val maxLoad: LoadDimension? = this.results.getMaxBenchmarkedLoad(load)
var lowerBound: Resource? = this.results.getMinRequiredInstances(maxLoad)
override fun apply(xValue: Int, yValues: List<Int>): List<Int> {
val maxXValue: Int? = this.results.getMaxBenchmarkedXDimensionValue(xValue)
var lowerBound: Int? = this.results.getOptYDimensionValue(maxXValue)
if (lowerBound == null) {
lowerBound = resources[0]
lowerBound = yValues[0]
}
return resources.filter { x -> x.get() >= lowerBound.get() }
return yValues.filter { x -> x >= lowerBound }
}
}
package theodolite.strategies.restriction
import io.quarkus.runtime.annotations.RegisterForReflection
import theodolite.util.LoadDimension
import theodolite.util.Resource
import theodolite.util.Results
/**
* A 'Restriction Strategy' restricts a list of resources based on the current
* A 'Restriction Strategy' restricts a list of resources or loads depending on the metric based on the current
* results of all previously performed benchmarks.
*
* @param results the [Results] object
......@@ -14,12 +12,13 @@ import theodolite.util.Results
@RegisterForReflection
abstract class RestrictionStrategy(val results: Results) {
/**
* Apply the restriction of the given resource list for the given load based on the results object.
* Apply the restriction of the given resource list for the given load based on the results object (demand metric),
* or apply the restriction of the given load list for the given resource based on the results object (capacity metric).
*
* @param load [LoadDimension] for which a subset of resources are required.
* @param resources List of [Resource]s to be restricted.
* @param xValue The value to be examined in the experiment, can be load (demand metric) or resource (capacity metric).
* @param yValues List of values to be restricted, can be resources (demand metric) or loads (capacity metric).
* @return Returns a list containing only elements that have not been filtered out by the
* restriction (possibly empty).
*/
abstract fun apply(load: LoadDimension, resources: List<Resource>): List<Resource>
abstract fun apply(xValue: Int, yValues: List<Int>): List<Int>
}
......@@ -2,8 +2,6 @@ package theodolite.strategies.searchstrategy
import mu.KotlinLogging
import theodolite.execution.BenchmarkExecutor
import theodolite.util.LoadDimension
import theodolite.util.Resource
private val logger = KotlinLogging.logger {}
......@@ -13,48 +11,97 @@ private val logger = KotlinLogging.logger {}
* @param benchmarkExecutor Benchmark executor which runs the individual benchmarks.
*/
class BinarySearch(benchmarkExecutor: BenchmarkExecutor) : SearchStrategy(benchmarkExecutor) {
override fun findSuitableResource(load: LoadDimension, resources: List<Resource>): Resource? {
val result = binarySearch(load, resources, 0, resources.size - 1)
override fun findSuitableResource(load: Int, resources: List<Int>): Int? {
val result = binarySearchDemand(load, resources, 0, resources.size - 1)
if (result == -1) {
return null
}
return resources[result]
}
override fun findSuitableLoad(resource: Int, loads: List<Int>): Int? {
val result = binarySearchCapacity(resource, loads, 0, loads.size - 1)
if (result == -1) {
return null
}
return loads[result]
}
/**
* Apply binary search.
* Apply binary search for the demand metric.
*
* @param load the load dimension to perform experiments for
* @param resources the list in which binary search is performed
* @param lower lower bound for binary search (inclusive)
* @param upper upper bound for binary search (inclusive)
* @param load the load to perform experiments for.
* @param resources the list of resources in which binary search is performed.
* @param lower lower bound for binary search (inclusive).
* @param upper upper bound for binary search (inclusive).
*/
private fun binarySearch(load: LoadDimension, resources: List<Resource>, lower: Int, upper: Int): Int {
private fun binarySearchDemand(load: Int, resources: List<Int>, lower: Int, upper: Int): Int {
if (lower > upper) {
throw IllegalArgumentException()
}
// special case: length == 1 or 2
// special case: length == 1, so lower and upper bounds are the same
if (lower == upper) {
val res = resources[lower]
logger.info { "Running experiment with load '${load.get()}' and resources '${res.get()}'" }
if (this.benchmarkExecutor.runExperiment(load, resources[lower])) return lower
logger.info { "Running experiment with load '$load' and resource '$res'" }
if (this.benchmarkExecutor.runExperiment(load, res)) return lower
else {
if (lower + 1 == resources.size) return -1
return lower + 1
}
} else {
// apply binary search for a list with
// length > 2 and adjust upper and lower depending on the result for `resources[mid]`
// length >= 2 and adjust upper and lower depending on the result for `resources[mid]`
val mid = (upper + lower) / 2
val res = resources[mid]
logger.info { "Running experiment with load '${load.get()}' and resources '${res.get()}'" }
if (this.benchmarkExecutor.runExperiment(load, resources[mid])) {
logger.info { "Running experiment with load '$load' and resource '$res'" }
if (this.benchmarkExecutor.runExperiment(load, res)) {
// case length = 2
if (mid == lower) {
return lower
}
return binarySearch(load, resources, lower, mid - 1)
return binarySearchDemand(load, resources, lower, mid - 1)
} else {
return binarySearchDemand(load, resources, mid + 1, upper)
}
}
}
/**
* Apply binary search for the capacity metric.
*
* @param resource the resource to perform experiments for.
* @param loads the list of loads in which binary search is performed.
* @param lower lower bound for binary search (inclusive).
* @param upper upper bound for binary search (inclusive).
*/
private fun binarySearchCapacity(resource: Int, loads: List<Int>, lower: Int, upper: Int): Int {
if (lower > upper) {
throw IllegalArgumentException()
}
// length = 1, so lower and upper bounds are the same
if (lower == upper) {
val load = loads[lower]
logger.info { "Running experiment with load '$load' and resource '$resource'" }
if (this.benchmarkExecutor.runExperiment(load, resource)) return lower
else {
if (lower + 1 == loads.size) return -1
return lower - 1
}
} else {
// apply binary search for a list with
// length > 2 and adjust upper and lower depending on the result for `resources[mid]`
val mid = (upper + lower + 1) / 2 //round to next int
val load = loads[mid]
logger.info { "Running experiment with load '$load' and resource '$resource'" }
if (this.benchmarkExecutor.runExperiment(load, resource)) {
// length = 2, so since we round down mid is equal to lower
if (mid == upper) {
return upper
}
return binarySearchCapacity(resource, loads, mid + 1, upper)
} else {
return binarySearch(load, resources, mid + 1, upper)
return binarySearchCapacity(resource, loads, lower, mid - 1)
}
}
}
......
......@@ -2,25 +2,24 @@ package theodolite.strategies.searchstrategy
import mu.KotlinLogging
import theodolite.execution.BenchmarkExecutor
import theodolite.util.LoadDimension
import theodolite.util.Resource
private val logger = KotlinLogging.logger {}
/**
* [SearchStrategy] that executes experiment for provides resources in a linear-search-like fashion, but **without
* stopping** once a suitable resource amount is found.
* [SearchStrategy] that executes an experiment for a load and a resource list (demand metric) or for a resource and a
* load list (capacity metric) in a linear-search-like fashion, but **without stopping** once the desired
* resource (demand) or load (capacity) is found.
*
* @see LinearSearch for a SearchStrategy that stops once a suitable resource amount is found.
* @see LinearSearch for a SearchStrategy that stops once the desired resource (demand) or load (capacity) is found.
*
* @param benchmarkExecutor Benchmark executor which runs the individual benchmarks.
*/
class FullSearch(benchmarkExecutor: BenchmarkExecutor) : SearchStrategy(benchmarkExecutor) {
override fun findSuitableResource(load: LoadDimension, resources: List<Resource>): Resource? {
var minimalSuitableResources: Resource? = null
override fun findSuitableResource(load: Int, resources: List<Int>): Int? {
var minimalSuitableResources: Int? = null
for (res in resources) {
logger.info { "Running experiment with load '${load.get()}' and resources '${res.get()}'" }
logger.info { "Running experiment with load '$load' and resources '$res'" }
val result = this.benchmarkExecutor.runExperiment(load, res)
if (result && minimalSuitableResources == null) {
minimalSuitableResources = res
......@@ -28,4 +27,13 @@ class FullSearch(benchmarkExecutor: BenchmarkExecutor) : SearchStrategy(benchmar
}
return minimalSuitableResources
}
override fun findSuitableLoad(resource: Int, loads: List<Int>): Int? {
var maxSuitableLoad: Int? = null
for (load in loads) {
logger.info { "Running experiment with resources '$resource' and load '$load'" }
if (this.benchmarkExecutor.runExperiment(load, resource)) maxSuitableLoad = load
}
return maxSuitableLoad
}
}
package theodolite.strategies.searchstrategy
import io.quarkus.runtime.annotations.RegisterForReflection
import theodolite.util.Resource
/**
* Base class for the implementation of Guess strategies. Guess strategies are strategies to determine the resource
* demand we start with in our initial guess search strategy.
* demand (demand metric) or load (capacity metric) we start with in our initial guess search strategy.
*/
@RegisterForReflection
abstract class GuessStrategy {
/**
* Computing the resource demand for the initial guess search strategy to start with.
* Computing the resource demand (demand metric) or load (capacity metric) for the initial guess search strategy
* to start with.
*
* @param resources List of all possible [Resource]s.
* @param lastLowestResource Previous resource demand needed for the given load.
* @param valuesToCheck List of all possible resources/loads.
* @param lastOptValue Previous minimal/maximal resource/load value for the given load/resource.
*
* @return Returns the resource demand to start the initial guess search strategy with or null
* @return the resource/load to start the initial guess search strategy with or null
*/
abstract fun firstGuess(resources: List<Resource>, lastLowestResource: Resource?): Resource?
abstract fun firstGuess(valuesToCheck: List<Int>, lastOptValue: Int?): Int?
}
\ No newline at end of file
......@@ -2,8 +2,6 @@ package theodolite.strategies.searchstrategy
import mu.KotlinLogging
import theodolite.execution.BenchmarkExecutor
import theodolite.util.LoadDimension
import theodolite.util.Resource
import theodolite.util.Results
private val logger = KotlinLogging.logger {}
......@@ -16,42 +14,28 @@ private val logger = KotlinLogging.logger {}
* @param guessStrategy Strategy that provides us with a guess for the first resource amount.
* @param results current results of all previously performed benchmarks.
*/
class InitialGuessSearchStrategy(benchmarkExecutor: BenchmarkExecutor, guessStrategy: GuessStrategy, results: Results) :
SearchStrategy(benchmarkExecutor, guessStrategy, results) {
class InitialGuessSearchStrategy(
benchmarkExecutor: BenchmarkExecutor,
private val guessStrategy: GuessStrategy,
private var results: Results
) : SearchStrategy(benchmarkExecutor) {
override fun findSuitableResource(load: LoadDimension, resources: List<Resource>): Resource? {
if(resources.isEmpty()) {
logger.info { "You need to specify resources to be checked for the InitialGuessSearchStrategy to work." }
return null
}
if(guessStrategy == null){
logger.info { "Your InitialGuessSearchStrategy doesn't have a GuessStrategy. This is not supported." }
return null
}
if(results == null){
logger.info { "The results need to be initialized." }
return null
}
override fun findSuitableResource(load: Int, resources: List<Int>): Int? {
var lastLowestResource : Resource? = null
var lastLowestResource : Int? = null
// Getting the lastLowestResource from results and calling firstGuess() with it
if (!results.isEmpty()) {
val maxLoad: LoadDimension? = this.results.getMaxBenchmarkedLoad(load)
lastLowestResource = this.results.getMinRequiredInstances(maxLoad)
if (lastLowestResource.get() == Int.MAX_VALUE) lastLowestResource = null
val maxLoad: Int? = this.results.getMaxBenchmarkedXDimensionValue(load)
lastLowestResource = this.results.getOptYDimensionValue(maxLoad)
}
lastLowestResource = this.guessStrategy.firstGuess(resources, lastLowestResource)
if (lastLowestResource != null) {
val resourcesToCheck: List<Resource>
val resourcesToCheck: List<Int>
val startIndex: Int = resources.indexOf(lastLowestResource)
logger.info { "Running experiment with load '${load.get()}' and resources '${lastLowestResource.get()}'" }
logger.info { "Running experiment with load '$load' and resources '$lastLowestResource'" }
// If the first experiment passes, starting downward linear search
// otherwise starting upward linear search
......@@ -60,10 +44,10 @@ class InitialGuessSearchStrategy(benchmarkExecutor: BenchmarkExecutor, guessStra
resourcesToCheck = resources.subList(0, startIndex).reversed()
if (resourcesToCheck.isEmpty()) return lastLowestResource
var currentMin: Resource = lastLowestResource
var currentMin: Int = lastLowestResource
for (res in resourcesToCheck) {
logger.info { "Running experiment with load '${load.get()}' and resources '${res.get()}'" }
logger.info { "Running experiment with load '$load' and resources '$res'" }
if (this.benchmarkExecutor.runExperiment(load, res)) {
currentMin = res
}
......@@ -79,14 +63,69 @@ class InitialGuessSearchStrategy(benchmarkExecutor: BenchmarkExecutor, guessStra
for (res in resourcesToCheck) {
logger.info { "Running experiment with load '${load.get()}' and resources '${res.get()}'" }
logger.info { "Running experiment with load '$load' and resources '$res'" }
if (this.benchmarkExecutor.runExperiment(load, res)) return res
}
}
}
else {
logger.info { "InitialGuessSearchStrategy called without lastLowestResource value, which is needed as a " +
"starting point!" }
logger.info { "lastLowestResource was null." }
}
return null
}
override fun findSuitableLoad(resource: Int, loads: List<Int>): Int?{
var lastMaxLoad : Int? = null
// Getting the lastLowestLoad from results and calling firstGuess() with it
if (!results.isEmpty()) {
val maxResource: Int? = this.results.getMaxBenchmarkedXDimensionValue(resource)
lastMaxLoad = this.results.getOptYDimensionValue(maxResource)
}
lastMaxLoad = this.guessStrategy.firstGuess(loads, lastMaxLoad)
if (lastMaxLoad != null) {
val loadsToCheck: List<Int>
val startIndex: Int = loads.indexOf(lastMaxLoad)
logger.info { "Running experiment with resource '$resource' and load '$lastMaxLoad'" }
// If the first experiment passes, starting upwards linear search
// otherwise starting downward linear search
if (!this.benchmarkExecutor.runExperiment(lastMaxLoad, resource)) {
// downward search
loadsToCheck = loads.subList(0, startIndex).reversed()
if (loadsToCheck.isNotEmpty()) {
for (load in loadsToCheck) {
logger.info { "Running experiment with resource '$resource' and load '$load'" }
if (this.benchmarkExecutor.runExperiment(load, resource)) {
return load
}
}
}
}
else {
// upward search
if (loads.size <= startIndex + 1) {
return lastMaxLoad
}
loadsToCheck = loads.subList(startIndex + 1, loads.size)
var currentMax: Int = lastMaxLoad
for (load in loadsToCheck) {
logger.info { "Running experiment with resource '$resource' and load '$load'" }
if (this.benchmarkExecutor.runExperiment(load, resource)) {
currentMax = load
}
}
return currentMax
}
}
else {
logger.info { "lastMaxLoad was null." }
}
return null
}
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment