Skip to content
Snippets Groups Projects
Commit 60064b67 authored by Benedikt Wetzel's avatar Benedikt Wetzel
Browse files

Introduce a PatcherFactory and a PatcherDefinitionFactory to make the API more understandable.

parent 71bd421f
No related branches found
No related tags found
4 merge requests!159Re-implementation of Theodolite with Kotlin/Quarkus,!157Update Graal Image in CI pipeline,!97Restructuring to create the PatcherDefinition inside the TheodoliteExecutor,!83WIP: Re-implementation of Theodolite with Kotlin/Quarkus
Showing
with 109 additions and 54 deletions
...@@ -4,7 +4,7 @@ import io.fabric8.kubernetes.api.model.KubernetesResource ...@@ -4,7 +4,7 @@ import io.fabric8.kubernetes.api.model.KubernetesResource
import io.fabric8.kubernetes.client.DefaultKubernetesClient import io.fabric8.kubernetes.client.DefaultKubernetesClient
import mu.KotlinLogging import mu.KotlinLogging
import theodolite.k8s.K8sResourceLoader import theodolite.k8s.K8sResourceLoader
import theodolite.patcher.PatcherManager import theodolite.patcher.PatcherFactory
import theodolite.util.* import theodolite.util.*
private val logger = KotlinLogging.logger {} private val logger = KotlinLogging.logger {}
...@@ -43,20 +43,20 @@ class KubernetesBenchmark : Benchmark { ...@@ -43,20 +43,20 @@ class KubernetesBenchmark : Benchmark {
configurationOverrides: List<ConfigurationOverride> configurationOverrides: List<ConfigurationOverride>
): BenchmarkDeployment { ): BenchmarkDeployment {
val resources = loadKubernetesResources(this.appResource + this.loadGenResource) val resources = loadKubernetesResources(this.appResource + this.loadGenResource)
val patcherManager = PatcherManager() val patcherFactory = PatcherFactory()
// patch res and load // patch the load dimension
patcherManager.createAndApplyPatcher(res.getType(), this.resourceTypes, resources, res.get()) load.getType().map { patcherDefinition -> patcherFactory.createPatcher(patcherDefinition, resources) }
patcherManager.createAndApplyPatcher(load.getType(), this.loadTypes, resources, load.get().toString()) .forEach { patcher -> patcher.patch(load.get().toString()) }
// patch overrides // patch the resources
res.getType().map { patcherDefinition -> patcherFactory.createPatcher(patcherDefinition, resources) }
.forEach { patcher -> patcher.patch(res.get().toString()) }
// Patch the given overrides
configurationOverrides.forEach { override -> configurationOverrides.forEach { override ->
patcherManager.applyPatcher( patcherFactory.createPatcher(override.patcher, resources).patch(override.value) }
listOf(override.patcher),
resources,
override.value
)
}
return KubernetesBenchmarkDeployment( return KubernetesBenchmarkDeployment(
namespace = namespace, namespace = namespace,
......
...@@ -2,6 +2,7 @@ package theodolite.execution ...@@ -2,6 +2,7 @@ package theodolite.execution
import theodolite.benchmark.BenchmarkExecution import theodolite.benchmark.BenchmarkExecution
import theodolite.benchmark.KubernetesBenchmark import theodolite.benchmark.KubernetesBenchmark
import theodolite.patcher.PatcherDefinitionFactory
import theodolite.strategies.StrategyFactory import theodolite.strategies.StrategyFactory
import theodolite.strategies.searchstrategy.CompositeStrategy import theodolite.strategies.searchstrategy.CompositeStrategy
import theodolite.util.Config import theodolite.util.Config
...@@ -21,11 +22,12 @@ class TheodoliteExecutor( ...@@ -21,11 +22,12 @@ class TheodoliteExecutor(
val executionDuration = Duration.ofSeconds(config.execution.duration) val executionDuration = Duration.ofSeconds(config.execution.duration)
val executor = BenchmarkExecutorImpl(kubernetesBenchmark, results, executionDuration, config.configOverrides) val executor = BenchmarkExecutorImpl(kubernetesBenchmark, results, executionDuration, config.configOverrides)
val resourcePatcherDefinition = PatcherDefinitionFactory().createPatcherDefinition(config.resources.resourceType, this.kubernetesBenchmark.resourceTypes)
val loadDimensionPatcherDefinition = PatcherDefinitionFactory().createPatcherDefinition(config.load.loadType, this.kubernetesBenchmark.loadTypes)
return Config( return Config(
loads = config.load.loadValues.map { load -> LoadDimension(load, config.load.loadType) }, loads = config.load.loadValues.map { load -> LoadDimension(load, loadDimensionPatcherDefinition) },
resources = config.resources.resourceValues.map resources = config.resources.resourceValues.map { resource -> Resource(resource, resourcePatcherDefinition) },
{ resource -> Resource(resource, config.resources.resourceType) },
compositeStrategy = CompositeStrategy( compositeStrategy = CompositeStrategy(
benchmarkExecutor = executor, benchmarkExecutor = executor,
searchStrategy = strategyFactory.createSearchStrategy(executor, config.execution.strategy), searchStrategy = strategyFactory.createSearchStrategy(executor, config.execution.strategy),
......
...@@ -5,6 +5,8 @@ import mu.KotlinLogging ...@@ -5,6 +5,8 @@ import mu.KotlinLogging
import theodolite.benchmark.BenchmarkExecution import theodolite.benchmark.BenchmarkExecution
import theodolite.benchmark.KubernetesBenchmark import theodolite.benchmark.KubernetesBenchmark
import theodolite.util.YamlParser import theodolite.util.YamlParser
import kotlin.system.exitProcess
private val logger = KotlinLogging.logger {} private val logger = KotlinLogging.logger {}
@QuarkusMain(name = "TheodoliteYamlExecutor") @QuarkusMain(name = "TheodoliteYamlExecutor")
...@@ -23,5 +25,6 @@ object TheodoliteYamlExecutor { ...@@ -23,5 +25,6 @@ object TheodoliteYamlExecutor {
val executor = TheodoliteExecutor(benchmarkExecution, benchmark) val executor = TheodoliteExecutor(benchmarkExecution, benchmark)
executor.run() executor.run()
logger.info { "Theodolite finished" } logger.info { "Theodolite finished" }
exitProcess(0)
} }
} }
package theodolite.patcher
import theodolite.util.PatcherDefinition
import theodolite.util.TypeName
class PatcherDefinitionFactory {
fun createPatcherDefinition(requiredType: String, patcherTypes: List<TypeName>) : List<PatcherDefinition> {
return patcherTypes
.filter { type -> type.typeName == requiredType }
.flatMap { type -> type.patchers }
}
fun getEmptyPatcherDefinition(): PatcherDefinition {
val emptyDef = PatcherDefinition()
emptyDef.type = ""
emptyDef.resource = ""
emptyDef.container = ""
emptyDef.variableName = ""
return emptyDef
}
}
\ No newline at end of file
...@@ -2,13 +2,10 @@ package theodolite.patcher ...@@ -2,13 +2,10 @@ package theodolite.patcher
import io.fabric8.kubernetes.api.model.KubernetesResource import io.fabric8.kubernetes.api.model.KubernetesResource
import theodolite.util.PatcherDefinition import theodolite.util.PatcherDefinition
import theodolite.util.TypeName
class PatcherManager { class PatcherFactory {
private fun createK8sPatcher( fun createPatcher(patcherDefinition: PatcherDefinition,
patcherDefinition: PatcherDefinition, k8sResources: List<Pair<String, KubernetesResource>>) : Patcher {
k8sResources: List<Pair<String, KubernetesResource>>
): Patcher {
val resource = val resource =
k8sResources.filter { it.first == patcherDefinition.resource }.map { resource -> resource.second }[0] k8sResources.filter { it.first == patcherDefinition.resource }.map { resource -> resource.second }[0]
return when (patcherDefinition.type) { return when (patcherDefinition.type) {
...@@ -28,46 +25,4 @@ class PatcherManager { ...@@ -28,46 +25,4 @@ class PatcherManager {
else -> throw IllegalArgumentException("Patcher type ${patcherDefinition.type} not found") else -> throw IllegalArgumentException("Patcher type ${patcherDefinition.type} not found")
} }
} }
}
private fun getPatcherDef(requiredType: String, patcherTypes: List<TypeName>): List<PatcherDefinition> { \ No newline at end of file
return patcherTypes
.filter { type -> type.typeName == requiredType }
.flatMap { type -> type.patchers }
}
/**
* This function first creates a patcher definition and
* then patches the list of resources based on this patcher definition
*
* @param type Patcher type, for example "EnvVarPatcher"
* @param patcherTypes List of patcher types definitions, for example for resources and threads
* @param resources List of K8s resources, a patcher takes the resources that are needed
* @param value The value to patch
*/
fun createAndApplyPatcher(
type: String,
patcherTypes: List<TypeName>,
resources: List<Pair<String, KubernetesResource>>,
value: Any
) {
this.getPatcherDef(type, patcherTypes)
.forEach { patcherDef ->
createK8sPatcher(patcherDef, resources).patch(value)
}
}
/**
* Patch a resource based on the given patcher definition, a list of resources and a value to patch
*
* @param patcherDefinition The patcher definition
* @param resources List of patcher types definitions, for example for resources and threads
* @param value The value to patch
*/
fun applyPatcher(
patcherDefinition: List<PatcherDefinition>,
resources: List<Pair<String, KubernetesResource>>,
value: Any
) {
patcherDefinition.forEach { def -> this.createK8sPatcher(def, resources).patch(value) }
}
}
package theodolite.strategies.restriction package theodolite.strategies.restriction
import theodolite.patcher.PatcherDefinitionFactory
import theodolite.util.LoadDimension import theodolite.util.LoadDimension
import theodolite.util.Resource import theodolite.util.Resource
import theodolite.util.Results import theodolite.util.Results
...@@ -13,7 +14,7 @@ import theodolite.util.Results ...@@ -13,7 +14,7 @@ import theodolite.util.Results
class LowerBoundRestriction(results: Results) : RestrictionStrategy(results) { class LowerBoundRestriction(results: Results) : RestrictionStrategy(results) {
override fun next(load: LoadDimension, resources: List<Resource>): List<Resource> { override fun next(load: LoadDimension, resources: List<Resource>): List<Resource> {
val maxLoad: LoadDimension? = this.results.getMaxBenchmarkedLoad(load) val maxLoad: LoadDimension? = this.results.getMaxBenchmarkedLoad(load)
var lowerBound: Resource? = this.results.getMinRequiredInstances(maxLoad, resources[0].getType()) var lowerBound: Resource? = this.results.getMinRequiredInstances(maxLoad, listOf(PatcherDefinitionFactory().getEmptyPatcherDefinition()))
if (lowerBound == null) { if (lowerBound == null) {
lowerBound = resources[0] lowerBound = resources[0]
} }
......
package theodolite.util package theodolite.util
data class LoadDimension(private val number: Int, private val type: String) { data class LoadDimension(private val number: Int, private val type: List<PatcherDefinition>) {
fun get(): Int { fun get(): Int {
return this.number return this.number
} }
fun getType(): String { fun getType(): List<PatcherDefinition> {
return this.type return this.type
} }
} }
package theodolite.util package theodolite.util
data class Resource(private val number: Int, private val type: String) { data class Resource(private val number: Int, private val type: List<PatcherDefinition>) {
fun get(): Int { fun get(): Int {
return this.number return this.number
} }
fun getType(): String { fun getType(): List<PatcherDefinition> {
return this.type return this.type
} }
} }
...@@ -11,7 +11,7 @@ class Results { ...@@ -11,7 +11,7 @@ class Results {
return this.results[experiment] return this.results[experiment]
} }
fun getMinRequiredInstances(load: LoadDimension?, resourceTyp: String): Resource? { fun getMinRequiredInstances(load: LoadDimension?, resourceTyp: List<PatcherDefinition>): Resource? {
if (this.results.isEmpty()) return Resource(Int.MIN_VALUE, resourceTyp) if (this.results.isEmpty()) return Resource(Int.MIN_VALUE, resourceTyp)
var requiredInstances: Resource? = Resource(Int.MAX_VALUE, resourceTyp) var requiredInstances: Resource? = Resource(Int.MAX_VALUE, resourceTyp)
......
...@@ -3,6 +3,7 @@ package theodolite ...@@ -3,6 +3,7 @@ package theodolite
import io.quarkus.test.junit.QuarkusTest import io.quarkus.test.junit.QuarkusTest
import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Test import org.junit.jupiter.api.Test
import theodolite.patcher.PatcherDefinitionFactory
import theodolite.strategies.restriction.LowerBoundRestriction import theodolite.strategies.restriction.LowerBoundRestriction
import theodolite.strategies.searchstrategy.BinarySearch import theodolite.strategies.searchstrategy.BinarySearch
import theodolite.strategies.searchstrategy.CompositeStrategy import theodolite.strategies.searchstrategy.CompositeStrategy
...@@ -25,8 +26,9 @@ class CompositeStrategyTest { ...@@ -25,8 +26,9 @@ class CompositeStrategyTest {
arrayOf(false, false, false, false, false, false, true), arrayOf(false, false, false, false, false, false, true),
arrayOf(false, false, false, false, false, false, false) arrayOf(false, false, false, false, false, false, false)
) )
val mockLoads: List<LoadDimension> = (0..6).map { number -> LoadDimension(number, "NumSensors") } val emptyPatcherDefinition = PatcherDefinitionFactory().getEmptyPatcherDefinition()
val mockResources: List<Resource> = (0..6).map { number -> Resource(number, "Instances") } val mockLoads: List<LoadDimension> = (0..6).map { number -> LoadDimension(number, listOf(emptyPatcherDefinition)) }
val mockResources: List<Resource> = (0..6).map { number -> Resource(number, listOf(emptyPatcherDefinition)) }
val results = Results() val results = Results()
val benchmark = TestBenchmark() val benchmark = TestBenchmark()
val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results) val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results)
...@@ -36,7 +38,7 @@ class CompositeStrategyTest { ...@@ -36,7 +38,7 @@ class CompositeStrategyTest {
CompositeStrategy(benchmarkExecutor, linearSearch, setOf(lowerBoundRestriction)) CompositeStrategy(benchmarkExecutor, linearSearch, setOf(lowerBoundRestriction))
val actual: ArrayList<Resource?> = ArrayList() val actual: ArrayList<Resource?> = ArrayList()
val expected: ArrayList<Resource?> = ArrayList(listOf(0, 2, 2, 3, 4, 6).map { x -> Resource(x, "Instances") }) val expected: ArrayList<Resource?> = ArrayList(listOf(0, 2, 2, 3, 4, 6).map { x -> Resource(x, listOf(emptyPatcherDefinition)) })
expected.add(null) expected.add(null)
for (load in mockLoads) { for (load in mockLoads) {
...@@ -57,8 +59,9 @@ class CompositeStrategyTest { ...@@ -57,8 +59,9 @@ class CompositeStrategyTest {
arrayOf(false, false, false, false, false, false, true), arrayOf(false, false, false, false, false, false, true),
arrayOf(false, false, false, false, false, false, false) arrayOf(false, false, false, false, false, false, false)
) )
val mockLoads: List<LoadDimension> = (0..6).map { number -> LoadDimension(number, "NumSensors") } val emptyPatcherDefinition = PatcherDefinitionFactory().getEmptyPatcherDefinition()
val mockResources: List<Resource> = (0..6).map { number -> Resource(number, "Instances") } val mockLoads: List<LoadDimension> = (0..6).map { number -> LoadDimension(number, listOf(emptyPatcherDefinition)) }
val mockResources: List<Resource> = (0..6).map { number -> Resource(number, listOf(emptyPatcherDefinition)) }
val results = Results() val results = Results()
val benchmark = TestBenchmark() val benchmark = TestBenchmark()
val benchmarkExecutorImpl = val benchmarkExecutorImpl =
...@@ -69,7 +72,7 @@ class CompositeStrategyTest { ...@@ -69,7 +72,7 @@ class CompositeStrategyTest {
CompositeStrategy(benchmarkExecutorImpl, binarySearch, setOf(lowerBoundRestriction)) CompositeStrategy(benchmarkExecutorImpl, binarySearch, setOf(lowerBoundRestriction))
val actual: ArrayList<Resource?> = ArrayList() val actual: ArrayList<Resource?> = ArrayList()
val expected: ArrayList<Resource?> = ArrayList(listOf(0, 2, 2, 3, 4, 6).map { x -> Resource(x, "Instances") }) val expected: ArrayList<Resource?> = ArrayList(listOf(0, 2, 2, 3, 4, 6).map { x -> Resource(x, listOf(emptyPatcherDefinition)) })
expected.add(null) expected.add(null)
for (load in mockLoads) { for (load in mockLoads) {
...@@ -90,8 +93,9 @@ class CompositeStrategyTest { ...@@ -90,8 +93,9 @@ class CompositeStrategyTest {
arrayOf(false, false, false, false, false, false, true, true), arrayOf(false, false, false, false, false, false, true, true),
arrayOf(false, false, false, false, false, false, false, true) arrayOf(false, false, false, false, false, false, false, true)
) )
val mockLoads: List<LoadDimension> = (0..6).map { number -> LoadDimension(number, "NumSensors") } val emptyPatcherDefinition = PatcherDefinitionFactory().getEmptyPatcherDefinition()
val mockResources: List<Resource> = (0..7).map { number -> Resource(number, "Instances") } val mockLoads: List<LoadDimension> = (0..6).map { number -> LoadDimension(number, listOf(emptyPatcherDefinition)) }
val mockResources: List<Resource> = (0..7).map { number -> Resource(number, listOf(emptyPatcherDefinition)) }
val results = Results() val results = Results()
val benchmark = TestBenchmark() val benchmark = TestBenchmark()
val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results) val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results)
...@@ -102,7 +106,7 @@ class CompositeStrategyTest { ...@@ -102,7 +106,7 @@ class CompositeStrategyTest {
val actual: ArrayList<Resource?> = ArrayList() val actual: ArrayList<Resource?> = ArrayList()
val expected: ArrayList<Resource?> = val expected: ArrayList<Resource?> =
ArrayList(listOf(0, 2, 2, 3, 4, 6, 7).map { x -> Resource(x, "Instances") }) ArrayList(listOf(0, 2, 2, 3, 4, 6, 7).map { x -> Resource(x, listOf(emptyPatcherDefinition)) })
for (load in mockLoads) { for (load in mockLoads) {
actual.add(strategy.findSuitableResource(load, mockResources)) actual.add(strategy.findSuitableResource(load, mockResources))
......
...@@ -6,7 +6,7 @@ import io.quarkus.test.junit.QuarkusTest ...@@ -6,7 +6,7 @@ import io.quarkus.test.junit.QuarkusTest
import io.smallrye.common.constraint.Assert.assertTrue import io.smallrye.common.constraint.Assert.assertTrue
import org.junit.jupiter.api.Test import org.junit.jupiter.api.Test
import theodolite.k8s.K8sResourceLoader import theodolite.k8s.K8sResourceLoader
import theodolite.patcher.PatcherManager import theodolite.patcher.PatcherFactory
import theodolite.util.PatcherDefinition import theodolite.util.PatcherDefinition
/** /**
...@@ -23,7 +23,7 @@ import theodolite.util.PatcherDefinition ...@@ -23,7 +23,7 @@ import theodolite.util.PatcherDefinition
class ResourceLimitPatcherTest { class ResourceLimitPatcherTest {
val testPath = "./src/main/resources/testYaml/" val testPath = "./src/main/resources/testYaml/"
val loader = K8sResourceLoader(DefaultKubernetesClient().inNamespace("")) val loader = K8sResourceLoader(DefaultKubernetesClient().inNamespace(""))
val manager = PatcherManager() val patcherFactory = PatcherFactory()
fun applyTest(fileName: String) { fun applyTest(fileName: String) {
val cpuValue = "50m" val cpuValue = "50m"
...@@ -42,16 +42,14 @@ class ResourceLimitPatcherTest { ...@@ -42,16 +42,14 @@ class ResourceLimitPatcherTest {
defMEM.container = "uc-application" defMEM.container = "uc-application"
defMEM.type = "ResourceLimitPatcher" defMEM.type = "ResourceLimitPatcher"
manager.applyPatcher( patcherFactory.createPatcher(
patcherDefinition = listOf(defCPU), patcherDefinition = defCPU,
resources = listOf(Pair("cpu-memory-deployment.yaml", k8sResource)), k8sResources = listOf(Pair("cpu-memory-deployment.yaml", k8sResource))
value = cpuValue ).patch(value = cpuValue)
) patcherFactory.createPatcher(
manager.applyPatcher( patcherDefinition = defMEM,
patcherDefinition = listOf(defMEM), k8sResources = listOf(Pair("cpu-memory-deployment.yaml", k8sResource))
resources = listOf(Pair("cpu-memory-deployment.yaml", k8sResource)), ).patch(value = memValue)
value = memValue
)
k8sResource.spec.template.spec.containers.filter { it.name == defCPU.container } k8sResource.spec.template.spec.containers.filter { it.name == defCPU.container }
.forEach { .forEach {
......
...@@ -6,7 +6,7 @@ import io.quarkus.test.junit.QuarkusTest ...@@ -6,7 +6,7 @@ import io.quarkus.test.junit.QuarkusTest
import io.smallrye.common.constraint.Assert.assertTrue import io.smallrye.common.constraint.Assert.assertTrue
import org.junit.jupiter.api.Test import org.junit.jupiter.api.Test
import theodolite.k8s.K8sResourceLoader import theodolite.k8s.K8sResourceLoader
import theodolite.patcher.PatcherManager import theodolite.patcher.PatcherFactory
import theodolite.util.PatcherDefinition import theodolite.util.PatcherDefinition
/** /**
...@@ -23,7 +23,7 @@ import theodolite.util.PatcherDefinition ...@@ -23,7 +23,7 @@ import theodolite.util.PatcherDefinition
class ResourceRequestPatcherTest { class ResourceRequestPatcherTest {
val testPath = "./src/main/resources/testYaml/" val testPath = "./src/main/resources/testYaml/"
val loader = K8sResourceLoader(DefaultKubernetesClient().inNamespace("")) val loader = K8sResourceLoader(DefaultKubernetesClient().inNamespace(""))
val manager = PatcherManager() val patcherFactory = PatcherFactory()
fun applyTest(fileName: String) { fun applyTest(fileName: String) {
val cpuValue = "50m" val cpuValue = "50m"
...@@ -42,16 +42,14 @@ class ResourceRequestPatcherTest { ...@@ -42,16 +42,14 @@ class ResourceRequestPatcherTest {
defMEM.container = "uc-application" defMEM.container = "uc-application"
defMEM.type = "ResourceRequestPatcher" defMEM.type = "ResourceRequestPatcher"
manager.applyPatcher( patcherFactory.createPatcher(
patcherDefinition = listOf(defCPU), patcherDefinition = defCPU,
resources = listOf(Pair("cpu-memory-deployment.yaml", k8sResource)), k8sResources = listOf(Pair("cpu-memory-deployment.yaml", k8sResource))
value = cpuValue ).patch(value = cpuValue)
) patcherFactory.createPatcher(
manager.applyPatcher( patcherDefinition = defMEM,
patcherDefinition = listOf(defMEM), k8sResources = listOf(Pair("cpu-memory-deployment.yaml", k8sResource))
resources = listOf(Pair("cpu-memory-deployment.yaml", k8sResource)), ).patch(value = memValue)
value = memValue
)
k8sResource.spec.template.spec.containers.filter { it.name == defCPU.container } k8sResource.spec.template.spec.containers.filter { it.name == defCPU.container }
.forEach { .forEach {
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment