Skip to content
Snippets Groups Projects
Commit 3889bc89 authored by Lorenz Boguhn's avatar Lorenz Boguhn
Browse files

Merge with master and localhost as internal ip

parents beba3c9c f1e49211
No related branches found
No related tags found
4 merge requests!159Re-implementation of Theodolite with Kotlin/Quarkus,!157Update Graal Image in CI pipeline,!83WIP: Re-implementation of Theodolite with Kotlin/Quarkus,!78Resolve "Implement Quarkus/Kotlin protype"
Showing
with 81 additions and 48 deletions
package theodolite.execution package theodolite.execution
import mu.KotlinLogging
import theodolite.execution.BenchmarkExecutor import theodolite.execution.BenchmarkExecutor
import theodolite.util.Benchmark import theodolite.util.Benchmark
import theodolite.util.LoadDimension import theodolite.util.LoadDimension
...@@ -13,8 +14,6 @@ class TestBenchmarkExecutor(private val mockResults: Array<Array<Boolean>>, benc ...@@ -13,8 +14,6 @@ class TestBenchmarkExecutor(private val mockResults: Array<Array<Boolean>>, benc
override fun runExperiment(load: LoadDimension, res: Resource): Boolean { override fun runExperiment(load: LoadDimension, res: Resource): Boolean {
val result = this.mockResults[load.get()][res.get()] val result = this.mockResults[load.get()][res.get()]
System.out.println("load :" + load.get().toString() + ", res: " + res.get().toString() + ", res: " + result)
this.results.setResult(Pair(load, res), result) this.results.setResult(Pair(load, res), result)
return result; return result;
} }
......
package theodolite.execution package theodolite.execution
import mu.KotlinLogging
import theodolite.k8s.UC1Benchmark import theodolite.k8s.UC1Benchmark
import theodolite.strategies.restriction.LowerBoundRestriction import theodolite.strategies.restriction.LowerBoundRestriction
import theodolite.strategies.searchstrategy.CompositeStrategy import theodolite.strategies.searchstrategy.CompositeStrategy
...@@ -10,6 +11,8 @@ import theodolite.util.Resource ...@@ -10,6 +11,8 @@ import theodolite.util.Resource
import theodolite.util.Results import theodolite.util.Results
import java.time.Duration import java.time.Duration
private val logger = KotlinLogging.logger {}
class TheodoliteExecutor() { class TheodoliteExecutor() {
val path = "/home/lorenz/git/spesb/theodolite-quarkus/src/main/resources/yaml" val path = "/home/lorenz/git/spesb/theodolite-quarkus/src/main/resources/yaml"
private fun loadConfig(): Config { private fun loadConfig(): Config {
...@@ -31,7 +34,9 @@ class TheodoliteExecutor() { ...@@ -31,7 +34,9 @@ class TheodoliteExecutor() {
) )
) )
val results: Results = Results() val results: Results = Results()
val executionDuration = Duration.ofSeconds(60 * 5) val executionDuration = Duration.ofSeconds(60 * 5)
val executor: BenchmarkExecutor = KafkaBenchmarkExecutor(benchmark, results, executionDuration) val executor: BenchmarkExecutor = KafkaBenchmarkExecutor(benchmark, results, executionDuration)
val restrictionStrategy = LowerBoundRestriction(results) val restrictionStrategy = LowerBoundRestriction(results)
...@@ -56,8 +61,7 @@ class TheodoliteExecutor() { ...@@ -56,8 +61,7 @@ class TheodoliteExecutor() {
// execute benchmarks for each load // execute benchmarks for each load
for (load in config.loads) { for (load in config.loads) {
config.compositeStrategy.findSuitableResources(load, config.resources) config.compositeStrategy.findSuitableResource(load, config.resources)
} }
} }
} }
\ No newline at end of file
package theodolite.k8s package theodolite.k8s
import mu.KotlinLogging
import org.apache.kafka.clients.admin.AdminClient import org.apache.kafka.clients.admin.AdminClient
import org.apache.kafka.clients.admin.AdminClientConfig import org.apache.kafka.clients.admin.AdminClientConfig
import org.apache.kafka.clients.admin.ListTopicsResult import org.apache.kafka.clients.admin.ListTopicsResult
import org.apache.kafka.clients.admin.NewTopic import org.apache.kafka.clients.admin.NewTopic
private val logger = KotlinLogging.logger {}
class TopicManager(boostrapIp: String) { class TopicManager(boostrapIp: String) {
val props = hashMapOf<String, Any>(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG to boostrapIp) val props = hashMapOf<String, Any>(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG to boostrapIp)
lateinit var kafkaAdmin: AdminClient lateinit var kafkaAdmin: AdminClient
...@@ -13,7 +16,7 @@ class TopicManager(boostrapIp: String) { ...@@ -13,7 +16,7 @@ class TopicManager(boostrapIp: String) {
try { try {
kafkaAdmin = AdminClient.create(props) kafkaAdmin = AdminClient.create(props)
} catch (e: Exception) { } catch (e: Exception) {
System.out.println(e.toString()) logger.error {e.toString()}
} }
} }
...@@ -25,7 +28,7 @@ class TopicManager(boostrapIp: String) { ...@@ -25,7 +28,7 @@ class TopicManager(boostrapIp: String) {
newTopics.add(tops) newTopics.add(tops)
} }
kafkaAdmin.createTopics(newTopics) kafkaAdmin.createTopics(newTopics)
System.out.println("Topics created") logger.info {"Topics created"}
} }
fun createTopics(topics: List<String>, numPartitions: Int, replicationfactor: Short) { fun createTopics(topics: List<String>, numPartitions: Int, replicationfactor: Short) {
...@@ -36,7 +39,7 @@ class TopicManager(boostrapIp: String) { ...@@ -36,7 +39,7 @@ class TopicManager(boostrapIp: String) {
newTopics.add(tops) newTopics.add(tops)
} }
kafkaAdmin.createTopics(newTopics) kafkaAdmin.createTopics(newTopics)
System.out.println("Creation of $topics started") logger.info {"Creation of $topics started"}
} }
fun deleteTopics(topics: List<String>) { fun deleteTopics(topics: List<String>) {
...@@ -46,9 +49,9 @@ class TopicManager(boostrapIp: String) { ...@@ -46,9 +49,9 @@ class TopicManager(boostrapIp: String) {
try { try {
result.all().get() result.all().get()
} catch (ex: Exception) { } catch (ex: Exception) {
System.out.println(ex.toString()) logger.error {ex.toString()}
} }
System.out.println("Topics deleted") logger.info {"Topics deleted"}
} }
fun getTopics(): ListTopicsResult? { fun getTopics(): ListTopicsResult? {
......
...@@ -67,8 +67,8 @@ class UC1Benchmark(config: UC1BenchmarkConfig) : Benchmark(config) { ...@@ -67,8 +67,8 @@ class UC1Benchmark(config: UC1BenchmarkConfig) : Benchmark(config) {
// set environment variables // set environment variables
val environmentVariables: MutableMap<String, String> = mutableMapOf() val environmentVariables: MutableMap<String, String> = mutableMapOf()
environmentVariables.put("KAFKA_BOOTSTRAP_SERVERS", this.config.kafkaIPConnectionString) //environmentVariables.put("KAFKA_BOOTSTRAP_SERVERS", this.config.kafkaIPConnectionString)
environmentVariables.put("SCHEMA_REGISTRY_URL", this.config.schemaRegistryConnectionString) //environmentVariables.put("SCHEMA_REGISTRY_URL", this.config.schemaRegistryConnectionString)
// setup deployment // setup deployment
...@@ -89,9 +89,9 @@ class UC1Benchmark(config: UC1BenchmarkConfig) : Benchmark(config) { ...@@ -89,9 +89,9 @@ class UC1Benchmark(config: UC1BenchmarkConfig) : Benchmark(config) {
// TODO ("calculate number of required instances") // TODO ("calculate number of required instances")
val requiredInstances: Int = 1 val requiredInstances: Int = 1
val environmentVariables: MutableMap<String, String> = mutableMapOf() val environmentVariables: MutableMap<String, String> = mutableMapOf()
environmentVariables.put("KAFKA_BOOTSTRAP_SERVERS", this.config.kafkaIPConnectionString) //environmentVariables.put("KAFKA_BOOTSTRAP_SERVERS", this.config.kafkaIPConnectionString)
environmentVariables.put("ZK_HOST", this.config.zookeeperConnectionString.split(":")[0]) //environmentVariables.put("ZK_HOST", this.config.zookeeperConnectionString.split(":")[0])
environmentVariables.put("ZK_PORT", this.config.zookeeperConnectionString.split(":")[1]) //environmentVariables.put("ZK_PORT", this.config.zookeeperConnectionString.split(":")[1])
environmentVariables.put("NUM_SENSORS", load.get().toString()) environmentVariables.put("NUM_SENSORS", load.get().toString())
environmentVariables.put("INSTANCES", requiredInstances.toString()) environmentVariables.put("INSTANCES", requiredInstances.toString())
......
package theodolite.k8s package theodolite.k8s
import mu.KotlinLogging
import org.apache.zookeeper.KeeperException import org.apache.zookeeper.KeeperException
import org.apache.zookeeper.WatchedEvent import org.apache.zookeeper.WatchedEvent
import org.apache.zookeeper.Watcher import org.apache.zookeeper.Watcher
import org.apache.zookeeper.ZooKeeper import org.apache.zookeeper.ZooKeeper
private val logger = KotlinLogging.logger {}
class WorkloadGeneratorStateCleaner(ip: String) { class WorkloadGeneratorStateCleaner(ip: String) {
val path = "/workload-generation" val path = "/workload-generation"
val sessionTimeout = 60 val sessionTimeout = 60
...@@ -16,7 +20,7 @@ class WorkloadGeneratorStateCleaner(ip: String) { ...@@ -16,7 +20,7 @@ class WorkloadGeneratorStateCleaner(ip: String) {
val watcher: Watcher = ZookeperWatcher() // defined below val watcher: Watcher = ZookeperWatcher() // defined below
zookeeperClient = ZooKeeper(ip, sessionTimeout, watcher) zookeeperClient = ZooKeeper(ip, sessionTimeout, watcher)
} catch (e: Exception) { } catch (e: Exception) {
System.out.println(e.toString()) logger.error {e.toString()}
} }
} }
...@@ -28,7 +32,7 @@ class WorkloadGeneratorStateCleaner(ip: String) { ...@@ -28,7 +32,7 @@ class WorkloadGeneratorStateCleaner(ip: String) {
try { try {
zookeeperClient.delete(path, -1) zookeeperClient.delete(path, -1)
} catch (ex: Exception) { } catch (ex: Exception) {
System.out.println(ex.toString()) logger.error {ex.toString()}
} }
try { try {
...@@ -42,15 +46,15 @@ class WorkloadGeneratorStateCleaner(ip: String) { ...@@ -42,15 +46,15 @@ class WorkloadGeneratorStateCleaner(ip: String) {
deleted = true deleted = true
} }
is InterruptedException -> { is InterruptedException -> {
System.out.println(ex.toString()) logger.error {ex.toString()}
} }
} }
} }
Thread.sleep(retryTime) Thread.sleep(retryTime)
System.out.println("ZooKeeper reset was not successful. Retrying in 5s") logger.info {"ZooKeeper reset was not successful. Retrying in 5s"}
} }
System.out.println("ZooKeeper reset was successful") logger.info {"ZooKeeper reset was successful"}
} }
private class ZookeperWatcher : Watcher { private class ZookeperWatcher : Watcher {
......
...@@ -5,5 +5,5 @@ import theodolite.util.LoadDimension ...@@ -5,5 +5,5 @@ import theodolite.util.LoadDimension
import theodolite.util.Resource import theodolite.util.Resource
abstract class RestrictionStrategy(val results: Results) { abstract class RestrictionStrategy(val results: Results) {
public abstract fun next(load: LoadDimension, resources: List<Resource>): List<Resource>; public abstract fun next(load: LoadDimension, resources: List<Resource>): List<Resource>
} }
\ No newline at end of file
...@@ -7,7 +7,7 @@ import theodolite.util.Results ...@@ -7,7 +7,7 @@ import theodolite.util.Results
import java.lang.IllegalArgumentException import java.lang.IllegalArgumentException
class BinarySearch(benchmarkExecutor: BenchmarkExecutor, results: Results) : SearchStrategy(benchmarkExecutor, results) { class BinarySearch(benchmarkExecutor: BenchmarkExecutor, results: Results) : SearchStrategy(benchmarkExecutor, results) {
override fun findSuitableResources(load: LoadDimension, resources: List<Resource>): Resource? { override fun findSuitableResource(load: LoadDimension, resources: List<Resource>): Resource? {
val result = search(load, resources, 0, resources.size - 1) val result = search(load, resources, 0, resources.size - 1)
if( result == -1 ) { if( result == -1 ) {
return null; return null;
...@@ -19,22 +19,23 @@ class BinarySearch(benchmarkExecutor: BenchmarkExecutor, results: Results) : Sea ...@@ -19,22 +19,23 @@ class BinarySearch(benchmarkExecutor: BenchmarkExecutor, results: Results) : Sea
if (lower > upper) { if (lower > upper) {
throw IllegalArgumentException() throw IllegalArgumentException()
} }
if (lower == upper ) { // special case: length == 1 or 2
if (this.benchmarkExecutor.runExperiment(load, resources[lower])) return lower; if (lower == upper) {
if (this.benchmarkExecutor.runExperiment(load, resources[lower])) return lower
else { else {
if (lower + 1 == resources.size) return - 1 if (lower + 1 == resources.size) return - 1
return lower + 1; return lower + 1
} }
} else { } else {
// (true, true), (false, true), (false, false) // (false, false, false, true, false, true, false, true) // apply binary search for a list with length > 2 and adjust upper and lower depending on the result for `resources[mid]`
val mid = (upper + lower) / 2 val mid = (upper + lower) / 2
if (this.benchmarkExecutor.runExperiment(load, resources[mid])) { if (this.benchmarkExecutor.runExperiment(load, resources[mid])) {
if (mid == lower) { if (mid == lower) {
return lower return lower
} }
return search(load, resources, lower, mid - 1 ); return search(load, resources, lower, mid - 1 )
} else { } else {
return search(load, resources, mid + 1 , upper); return search(load, resources, mid + 1 , upper)
} }
} }
} }
......
...@@ -8,11 +8,11 @@ import theodolite.util.Results ...@@ -8,11 +8,11 @@ import theodolite.util.Results
class CompositeStrategy(benchmarkExecutor: BenchmarkExecutor, val searchStrategy: SearchStrategy, val restrictionStrategies: Set<RestrictionStrategy>, results: Results) : SearchStrategy(benchmarkExecutor, results) { class CompositeStrategy(benchmarkExecutor: BenchmarkExecutor, val searchStrategy: SearchStrategy, val restrictionStrategies: Set<RestrictionStrategy>, results: Results) : SearchStrategy(benchmarkExecutor, results) {
override fun findSuitableResources(load: LoadDimension, resources: List<Resource>): Resource? { override fun findSuitableResource(load: LoadDimension, resources: List<Resource>): Resource? {
var restrictedResources = resources.toList() var restrictedResources = resources.toList()
for (strategy in this.restrictionStrategies) { for (strategy in this.restrictionStrategies) {
restrictedResources = restrictedResources.intersect(strategy.next(load, resources)).toList() restrictedResources = restrictedResources.intersect(strategy.next(load, resources)).toList()
} }
return this.searchStrategy.findSuitableResources(load, restrictedResources) return this.searchStrategy.findSuitableResource(load, restrictedResources)
} }
} }
\ No newline at end of file
...@@ -7,7 +7,7 @@ import theodolite.util.Results ...@@ -7,7 +7,7 @@ import theodolite.util.Results
class LinearSearch(benchmarkExecutor: BenchmarkExecutor, results: Results) : SearchStrategy(benchmarkExecutor, results) { class LinearSearch(benchmarkExecutor: BenchmarkExecutor, results: Results) : SearchStrategy(benchmarkExecutor, results) {
override fun findSuitableResources(load: LoadDimension, resources: List<Resource>): Resource? { override fun findSuitableResource(load: LoadDimension, resources: List<Resource>): Resource? {
for (res in resources) { for (res in resources) {
if (this.benchmarkExecutor.runExperiment(load, res)) return res if (this.benchmarkExecutor.runExperiment(load, res)) return res
} }
......
...@@ -6,5 +6,5 @@ import theodolite.util.Resource ...@@ -6,5 +6,5 @@ import theodolite.util.Resource
import theodolite.util.Results import theodolite.util.Results
abstract class SearchStrategy(val benchmarkExecutor: BenchmarkExecutor, val results: Results) { abstract class SearchStrategy(val benchmarkExecutor: BenchmarkExecutor, val results: Results) {
abstract fun findSuitableResources(load: LoadDimension, resources: List<Resource>): Resource?; abstract fun findSuitableResource(load: LoadDimension, resources: List<Resource>): Resource?;
} }
\ No newline at end of file
package theodolite.util package theodolite.util
//class TestBenchmark: Benchmark(config = emptyMap()) { import theodolite.k8s.UC1Benchmark
// override fun start() {
// TODO("Not yet implemented") class TestBenchmark : Benchmark(
// } UC1Benchmark.UC1BenchmarkConfig(
// zookeeperConnectionString = "",
// override fun clearClusterEnvironment() { kafkaIPConnectionString = "",
// TODO("Not yet implemented") schemaRegistryConnectionString = "",
// } kafkaTopics = emptyList(),
// kafkaReplication = 0,
// override fun startWorkloadGenerator(wg: String, dimValue: Int, ucId: String) { kafkaPartition = 0,
// TODO("Not yet implemented") ucServicePath = "",
// } ucDeploymentPath = "",
//} wgDeploymentPath = "",
\ No newline at end of file configMapPath = "",
ucImageURL = "",
wgImageURL = ""
)
) {
override fun initializeClusterEnvironment() {
TODO("Not yet implemented")
}
override fun clearClusterEnvironment() {
TODO("Not yet implemented")
}
override fun startSUT(resources: Resource) {
TODO("Not yet implemented")
}
override fun startWorkloadGenerator(load: LoadDimension) {
TODO("Not yet implemented")
}
}
...@@ -40,7 +40,7 @@ class CompositeStrategyTest { ...@@ -40,7 +40,7 @@ class CompositeStrategyTest {
expected.add(null) expected.add(null)
for(load in mockLoads) { for(load in mockLoads) {
actual.add(strategy.findSuitableResources(load, mockResources)) actual.add(strategy.findSuitableResource(load, mockResources))
} }
assertEquals(actual, expected) assertEquals(actual, expected)
...@@ -71,7 +71,7 @@ class CompositeStrategyTest { ...@@ -71,7 +71,7 @@ class CompositeStrategyTest {
expected.add(null) expected.add(null)
for(load in mockLoads) { for(load in mockLoads) {
actual.add(strategy.findSuitableResources(load, mockResources)) actual.add(strategy.findSuitableResource(load, mockResources))
} }
assertEquals(actual, expected) assertEquals(actual, expected)
...@@ -101,7 +101,7 @@ class CompositeStrategyTest { ...@@ -101,7 +101,7 @@ class CompositeStrategyTest {
val expected: ArrayList<Resource?> = ArrayList(listOf(0,2,2,3,4,6,7).map{ x -> Resource(x)}) val expected: ArrayList<Resource?> = ArrayList(listOf(0,2,2,3,4,6,7).map{ x -> Resource(x)})
for(load in mockLoads) { for(load in mockLoads) {
actual.add(strategy.findSuitableResources(load, mockResources)) actual.add(strategy.findSuitableResource(load, mockResources))
} }
assertEquals(actual, expected) assertEquals(actual, expected)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment