Merge commit '6654f2ce' into quarkus-upgrade

CITATION.cff

@@ -8,7 +8,7 @@ authors:
     given-names: Wilhelm
     orcid: "https://orcid.org/0000-0001-6625-4335"
 title: Theodolite
-version: "0.5.1"
+version: "0.6.0"
 repository-code: "https://github.com/cau-se/theodolite"
 license: "Apache-2.0"
 doi: "10.1016/j.bdr.2021.100209"

codemeta.json

@@ -5,16 +5,20 @@
     "codeRepository": "https://github.com/cau-se/theodolite",
     "dateCreated": "2020-03-13",
     "datePublished": "2020-07-27",
-    "dateModified": "2021-11-12",
+    "dateModified": "2022-01-12",
     "downloadUrl": "https://github.com/cau-se/theodolite/releases",
     "name": "Theodolite",
-    "version": "0.5.1",
-    "description": "Theodolite is a framework for benchmarking the horizontal and vertical scalability of stream processing engines.",
+    "version": "0.6.0",
+    "description": "Theodolite is a framework for benchmarking the horizontal and vertical scalability of cloud-native applications.",
     "developmentStatus": "active",
+    "relatedLink": [
+        "https://www.theodolite.rocks"
+    ],
     "referencePublication": "https://doi.org/10.1016/j.bdr.2021.100209",
     "programmingLanguage": [
-        "Python",
-        "Java"
+        "Kotlin",
+        "Java",
+        "Python"
     ],
     "runtimePlatform": [
         "Kubernetes"

docs/_config.yml

 title: "Theodolite"
 description: >-
   Theodolite is a framework for benchmarking the horizontal and vertical
-  scalability of stream processing engines.
+  scalability of cloud-native applications.
 
 remote_theme: pmarsceill/just-the-docs
 #color_scheme: "dark"

docs/development/release-process.md

@@ -23,13 +23,15 @@ again be merged into master.
         1. the default `helm/values.yaml` file,
         2. the example `execution/theodolite.yaml` job,
         3. the Kubernetes benchmark resources in `theodolite-benchmarks/definitions/**/resources` and
-        2. the Docker Compose files in `theodolite-benchmarks/docker-test`.
+        4. the Docker Compose files in `theodolite-benchmarks/docker-test`.
 
     2. Update both, the `version` and the `appVersion` fields, in the Helm `Charts.yaml` file to `0.3.1`.
 
-    3. Update `codemeta.json` to match the new version. In particular, make sure that `version` points to the version you are releasing and `dateModified` points to the date you are relasing this version. [CodeMeta generator](https://codemeta.github.io/codemeta-generator/) may help you in updating the file.
+    3. Update the `version` field of the `theodolite/build.gradle` file to `0.3.1`. Make sure to also adjust all references to the build artifact in the `theodolite/README.md`.
 
-    4. Update `CITATION.cff` to match the new version. At least update the `version` field.
+    4. Update `codemeta.json` to match the new version. In particular, make sure that `version` points to the version you are releasing and `dateModified` points to the date you are relasing this version. [CodeMeta generator](https://codemeta.github.io/codemeta-generator/) may help you in updating the file.
+
+    5. Update `CITATION.cff` to match the new version. At least update the `version` field.
 
 4. Create a Helm package by running `./build-package.sh` from the chart directory.
 
@@ -49,8 +51,10 @@ again be merged into master.
 
     1. Update the Helm `Charts.yaml` file to `0.4.0-SNAPSHOT` (see Step 3).
 
-    2. Update the `codemeta.json` file according to Step 3.
+    2. Update the Theodolite `build.gradle` and `README.md` files `0.4.0-SNAPSHOT` (see Step 3).
+
+    3. Update the `codemeta.json` file according to Step 3.
 
-    3. Update the `CITATION.cff` file according to Step 3.
+    4. Update the `CITATION.cff` file according to Step 3.
 
 12. Commit these changes to the `master` branch.

docs/index.yaml

 apiVersion: v1
 entries:
   theodolite:
+  - apiVersion: v2
+    appVersion: 0.6.0
+    created: "2022-01-12T13:53:08.413006558+01:00"
+    dependencies:
+    - condition: grafana.enabled
+      name: grafana
+      repository: https://grafana.github.io/helm-charts
+      version: 6.17.5
+    - condition: kube-prometheus-stack.enabled
+      name: kube-prometheus-stack
+      repository: https://prometheus-community.github.io/helm-charts
+      version: 20.0.1
+    - condition: cp-helm-charts.enabled
+      name: cp-helm-charts
+      repository: https://soerenhenning.github.io/cp-helm-charts
+      version: 0.6.0
+    - condition: kafka-lag-exporter.enabled
+      name: kafka-lag-exporter
+      repository: https://lightbend.github.io/kafka-lag-exporter/repo/
+      version: 0.6.7
+    description: Theodolite is a framework for benchmarking the horizontal and vertical
+      scalability of cloud-native applications.
+    digest: 53435304229582680d55360ad79a25050f6cc97641cbb88d691b35d91a54d354
+    home: https://www.theodolite.rocks
+    maintainers:
+    - email: soeren.henning@email.uni-kiel.de
+      name: Sören Henning
+      url: https://www.se.informatik.uni-kiel.de/en/team/soeren-henning-m-sc
+    name: theodolite
+    sources:
+    - https://github.com/cau-se/theodolite
+    type: application
+    urls:
+    - https://github.com/cau-se/theodolite/releases/download/v0.6.0/theodolite-0.6.0.tgz
+    version: 0.6.0
   - apiVersion: v2
     appVersion: 0.5.1
     created: "2021-11-12T16:15:01.629937292+01:00"
@@ -106,4 +141,4 @@ entries:
     urls:
     - https://github.com/cau-se/theodolite/releases/download/v0.4.0/theodolite-0.4.0.tgz
     version: 0.4.0
-generated: "2021-11-12T16:15:01.591258889+01:00"
+generated: "2022-01-12T13:53:08.367695997+01:00"

helm/Chart.yaml

 apiVersion: v2
 name: theodolite
-description: Theodolite is a framework for benchmarking the scalability of stream processing engines.
-home: https://cau-se.github.io/theodolite
+description: Theodolite is a framework for benchmarking the horizontal and vertical scalability of cloud-native applications.
+home: https://www.theodolite.rocks
 sources:
   - https://github.com/cau-se/theodolite
 maintainers:
@@ -29,6 +29,6 @@ dependencies:
     repository: https://lightbend.github.io/kafka-lag-exporter/repo/
     condition: kafka-lag-exporter.enabled
 
-version: 0.6.0-SNAPSHOT
+version: 0.7.0-SNAPSHOT
 
-appVersion: 0.6.0-SNAPSHOT
+appVersion: 0.7.0-SNAPSHOT

helm/templates/NOTES.txt

-Welcome to Theodolite!
+Welcome to Theodolite, a framework for benchmarking the scalability of cloud-native applications!
 
-Visit https://cau-se.github.io/theodolite for getting started and more information.
+Visit https://www.theodolite.rocks for getting started and more information.

theodolite-benchmarks/load-generator-commons/src/main/java/theodolite/commons/workloadgeneration/LoadGenerator.java

@@ -22,7 +22,7 @@ public final class LoadGenerator {
   private static final int THREADS_DEFAULT = 4;
   private static final String SCHEMA_REGISTRY_URL_DEFAULT = "http://localhost:8081";
   private static final String KAFKA_TOPIC_DEFAULT = "input";
-  private static final String KAFKA_BOOTSTRAP_SERVERS_DEFAULT = "localhost:19092"; // NOPMD
+  private static final String KAFKA_BOOTSTRAP_SERVERS_DEFAULT = "localhost:9092"; // NOPMD
 
   private ClusterConfig clusterConfig;
   private WorkloadDefinition loadDefinition;

theodolite/README.md

@@ -23,7 +23,7 @@ The application can be packaged using:
 ./gradlew build
 ```
 
-It produces the `theodolite-0.6.0-SNAPSHOT-runner.jar` file in the `/build` directory. Be aware that it’s not
+It produces the `theodolite-0.7.0-SNAPSHOT-runner.jar` file in the `/build` directory. Be aware that it’s not
 an _über-jar_ as the dependencies are copied into the `build/lib` directory.
 
 If you want to build an _über-jar_, execute the following command:
@@ -32,7 +32,7 @@ If you want to build an _über-jar_, execute the following command:
 ./gradlew build -Dquarkus.package.type=uber-jar
 ```
 
-The application is now runnable using `java -jar build/theodolite-0.6.0-SNAPSHOT-runner.jar`.
+The application is now runnable using `java -jar build/theodolite-0.7.0-SNAPSHOT-runner.jar`.
 
 ## Creating a native executable
 
@@ -51,7 +51,7 @@ Or, if you don't have GraalVM installed, you can run the native executable build
 ```
 
 You can then execute your native executable with:
-```./build/theodolite-0.6.0-SNAPSHOT-runner```
+```./build/theodolite-0.7.0-SNAPSHOT-runner```
 
 If you want to learn more about building native executables, please consult https://quarkus.io/guides/gradle-tooling.
 

theodolite/build.gradle

@@ -43,7 +43,7 @@ dependencies {
 }
 
 group 'theodolite'
-version '0.6.0-SNAPSHOT'
+version '0.7.0-SNAPSHOT'
 
 java {
     sourceCompatibility = JavaVersion.VERSION_11

theodolite/src/main/kotlin/theodolite/strategies/searchstrategy/GuessStrategy.kt

+package theodolite.strategies.searchstrategy
+
+import io.quarkus.runtime.annotations.RegisterForReflection
+import theodolite.util.Resource
+
+/**
+ * Base class for the implementation of Guess strategies. Guess strategies are strategies to determine the resource
+ * demand we start with in our initial guess search strategy.
+ */
+
+@RegisterForReflection
+abstract class GuessStrategy {
+    /**
+     * Computing the resource demand for the initial guess search strategy to start with.
+     *
+     * @param resources List of all possible [Resource]s.
+     * @param lastLowestResource Previous resource demand needed for the given load.
+     *
+     * @return Returns the resource demand to start the initial guess search strategy with or null
+     */
+    abstract fun firstGuess(resources: List<Resource>, lastLowestResource: Resource?): Resource?
+}
\ No newline at end of file

theodolite/src/main/kotlin/theodolite/strategies/searchstrategy/InitialGuessSearchStrategy.kt

+package theodolite.strategies.searchstrategy
+
+import mu.KotlinLogging
+import theodolite.execution.BenchmarkExecutor
+import theodolite.util.LoadDimension
+import theodolite.util.Resource
+import theodolite.util.Results
+
+private val logger = KotlinLogging.logger {}
+
+/**
+ *  Search strategy implementation for determining the smallest suitable resource demand.
+ *  Starting with a resource amount provided by a guess strategy.
+ *
+ * @param benchmarkExecutor Benchmark executor which runs the individual benchmarks.
+ * @param guessStrategy Strategy that provides us with a guess for the first resource amount.
+ * @param results current results of all previously performed benchmarks.
+ */
+class InitialGuessSearchStrategy(benchmarkExecutor: BenchmarkExecutor, guessStrategy: GuessStrategy, results: Results) :
+        SearchStrategy(benchmarkExecutor, guessStrategy, results) {
+
+    override fun findSuitableResource(load: LoadDimension, resources: List<Resource>): Resource? {
+
+        if(resources.isEmpty()) {
+            logger.info { "You need to specify resources to be checked for the InitialGuessSearchStrategy to work." }
+            return null
+        }
+
+        if(guessStrategy == null){
+            logger.info { "Your InitialGuessSearchStrategy doesn't have a GuessStrategy. This is not supported." }
+            return null
+        }
+
+        if(results == null){
+            logger.info { "The results need to be initialized." }
+            return null
+        }
+
+
+        var lastLowestResource : Resource? = null
+
+        // Getting the lastLowestResource from results and calling firstGuess() with it
+        if (!results.isEmpty()) {
+            val maxLoad: LoadDimension? = this.results.getMaxBenchmarkedLoad(load)
+            lastLowestResource = this.results.getMinRequiredInstances(maxLoad)
+            if (lastLowestResource.get() == Int.MAX_VALUE) lastLowestResource = null
+        }
+        lastLowestResource = this.guessStrategy.firstGuess(resources, lastLowestResource)
+
+        if (lastLowestResource != null) {
+            val resourcesToCheck: List<Resource>
+            val startIndex: Int = resources.indexOf(lastLowestResource)
+
+            logger.info { "Running experiment with load '${load.get()}' and resources '${lastLowestResource.get()}'" }
+
+            // If the first experiment passes, starting downward linear search
+            // otherwise starting upward linear search
+            if (this.benchmarkExecutor.runExperiment(load, lastLowestResource)) {
+
+                resourcesToCheck = resources.subList(0, startIndex).reversed()
+                if (resourcesToCheck.isEmpty()) return lastLowestResource
+
+                var currentMin: Resource = lastLowestResource
+                for (res in resourcesToCheck) {
+
+                    logger.info { "Running experiment with load '${load.get()}' and resources '${res.get()}'" }
+                    if (this.benchmarkExecutor.runExperiment(load, res)) {
+                        currentMin = res
+                    }
+                }
+                return currentMin
+            }
+            else {
+                if (resources.size <= startIndex + 1) {
+                    logger.info{ "No more resources left to check." }
+                    return null
+                }
+                resourcesToCheck = resources.subList(startIndex + 1, resources.size)
+
+                for (res in resourcesToCheck) {
+
+                    logger.info { "Running experiment with load '${load.get()}' and resources '${res.get()}'" }
+                    if (this.benchmarkExecutor.runExperiment(load, res)) return res
+                }
+            }
+        }
+        else {
+            logger.info { "InitialGuessSearchStrategy called without lastLowestResource value, which is needed as a " +
+                    "starting point!" }
+        }
+        return null
+    }
+}
\ No newline at end of file

theodolite/src/main/kotlin/theodolite/strategies/searchstrategy/PrevResourceMinGuess.kt

+package theodolite.strategies.searchstrategy
+
+import theodolite.util.Resource
+
+/**
+ * This Guess strategy takes the minimal resource demand of the previous load, which is given as an argument for the
+ * firstGuess function.
+ */
+
+class PrevResourceMinGuess() : GuessStrategy(){
+
+    /**
+     * @param resources List of all possible [Resource]s.
+     * @param lastLowestResource Previous resource demand needed for the given load.
+     *
+     * @return the value of lastLowestResource if given otherwise the first element of the resource list or null
+     */
+    override fun firstGuess(resources: List<Resource>, lastLowestResource: Resource?): Resource? {
+
+        if (lastLowestResource != null) return lastLowestResource
+        else if(resources.isNotEmpty()) return resources[0]
+        else return null
+    }
+}
\ No newline at end of file

theodolite/src/main/kotlin/theodolite/strategies/searchstrategy/SearchStrategy.kt

@@ -4,14 +4,18 @@ import io.quarkus.runtime.annotations.RegisterForReflection
 import theodolite.execution.BenchmarkExecutor
 import theodolite.util.LoadDimension
 import theodolite.util.Resource
+import theodolite.util.Results
 
 /**
  *  Base class for the implementation for SearchStrategies. SearchStrategies determine the smallest suitable number of instances.
  *
  * @param benchmarkExecutor Benchmark executor which runs the individual benchmarks.
+ * @param guessStrategy Guess strategy for the initial resource amount in case the InitialGuessStrategy is selected.
+ * @param results the [Results] object.
  */
 @RegisterForReflection
-abstract class SearchStrategy(val benchmarkExecutor: BenchmarkExecutor) {
+abstract class SearchStrategy(val benchmarkExecutor: BenchmarkExecutor, val guessStrategy: GuessStrategy? = null,
+                              val results: Results? = null) {
     /**
      * Find smallest suitable resource from the specified resource list for the given load.
      *

theodolite/src/main/kotlin/theodolite/util/Results.kt

@@ -3,7 +3,7 @@ package theodolite.util
 import io.quarkus.runtime.annotations.RegisterForReflection
 
 /**
- * Central class that saves the state of a execution of Theodolite. For an execution, it is used to save the result of
+ * Central class that saves the state of an execution of Theodolite. For an execution, it is used to save the result of
  * individual experiments. Further, it is used by the RestrictionStrategy to
  * perform the [theodolite.strategies.restriction.RestrictionStrategy].
  */
@@ -44,16 +44,16 @@ class Results {
      * If no experiments have been marked as either successful or unsuccessful
      * yet, a Resource with the constant value Int.MIN_VALUE is returned.
      */
-    fun getMinRequiredInstances(load: LoadDimension?): Resource? {
+    fun getMinRequiredInstances(load: LoadDimension?): Resource {
         if (this.results.isEmpty()) {
             return Resource(Int.MIN_VALUE, emptyList())
         }
 
-        var minRequiredInstances: Resource? = Resource(Int.MAX_VALUE, emptyList())
+        var minRequiredInstances = Resource(Int.MAX_VALUE, emptyList())
         for (experiment in results) {
             // Get all successful experiments for requested load
             if (experiment.key.first == load && experiment.value) {
-                if (minRequiredInstances == null || experiment.key.second.get() < minRequiredInstances.get()) {
+                if (experiment.key.second.get() < minRequiredInstances.get()) {
                     // Found new smallest resources
                     minRequiredInstances = experiment.key.second
                 }
@@ -83,4 +83,13 @@ class Results {
         }
         return maxBenchmarkedLoad
     }
+
+    /**
+     * Checks whether the results are empty.
+     *
+     * @return true if [results] is empty.
+     */
+    fun isEmpty(): Boolean{
+        return results.isEmpty()
+    }
 }

theodolite/src/test/kotlin/theodolite/InitialGuessSearchStrategyTest.kt

+package theodolite
+
+import io.quarkus.test.junit.QuarkusTest
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Test
+import theodolite.benchmark.BenchmarkExecution
+import theodolite.strategies.searchstrategy.InitialGuessSearchStrategy
+import theodolite.util.LoadDimension
+import theodolite.util.Resource
+import theodolite.util.Results
+import mu.KotlinLogging
+import theodolite.strategies.searchstrategy.PrevResourceMinGuess
+
+private val logger = KotlinLogging.logger {}
+
+@QuarkusTest
+class InitialGuessSearchStrategyTest {
+
+    @Test
+    fun testInitialGuessSearch() {
+        val mockResults = arrayOf(
+            arrayOf(true, true, true, true, true, true, true),
+            arrayOf(false, false, true, true, true, true, true),
+            arrayOf(false, false, true, true, true, true, true),
+            arrayOf(false, false, false, true, true, true, true),
+            arrayOf(false, false, false, false, true, true, true),
+            arrayOf(false, false, false, false, false, false, true),
+            arrayOf(false, false, false, false, false, false, false)
+        )
+        val mockLoads: List<LoadDimension> = (0..6).map { number -> LoadDimension(number, emptyList()) }
+        val mockResources: List<Resource> = (0..6).map { number -> Resource(number, emptyList()) }
+        val results = Results()
+        val benchmark = TestBenchmark()
+        val guessStrategy = PrevResourceMinGuess()
+        val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo()
+        val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, listOf(sloChecker), 0, 0, 5)
+        val strategy = InitialGuessSearchStrategy(benchmarkExecutor,guessStrategy, results)
+
+        val actual: ArrayList<Resource?> = ArrayList()
+        val expected: ArrayList<Resource?> = ArrayList(listOf(0, 2, 2, 3, 4, 6).map { x -> Resource(x, emptyList()) })
+        expected.add(null)
+
+        for (load in mockLoads) {
+            val returnVal : Resource? = strategy.findSuitableResource(load, mockResources)
+            if(returnVal != null) {
+                logger.info { "returnVal '${returnVal.get()}'" }
+            }
+            else {
+                logger.info { "returnVal is null." }
+            }
+            actual.add(returnVal)
+        }
+
+        assertEquals(actual, expected)
+    }
+
+    @Test
+    fun testInitialGuessSearchLowerResourceDemandHigherLoad() {
+        val mockResults = arrayOf(
+            arrayOf(true, true, true, true, true, true, true),
+            arrayOf(false, false, true, true, true, true, true),
+            arrayOf(false, false, true, true, true, true, true),
+            arrayOf(false, true, true, true, true, true, true),
+            arrayOf(false, false, false, false, true, true, true),
+            arrayOf(false, false, false, false, false, false, true),
+            arrayOf(false, false, false, false, false, false, false)
+        )
+        val mockLoads: List<LoadDimension> = (0..6).map { number -> LoadDimension(number, emptyList()) }
+        val mockResources: List<Resource> = (0..6).map { number -> Resource(number, emptyList()) }
+        val results = Results()
+        val benchmark = TestBenchmark()
+        val guessStrategy = PrevResourceMinGuess()
+        val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo()
+        val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, listOf(sloChecker), 0, 0, 5)
+        val strategy = InitialGuessSearchStrategy(benchmarkExecutor,guessStrategy, results)
+
+        val actual: ArrayList<Resource?> = ArrayList()
+        val expected: ArrayList<Resource?> = ArrayList(listOf(0, 2, 2, 1, 4, 6).map { x -> Resource(x, emptyList()) })
+        expected.add(null)
+
+        for (load in mockLoads) {
+            val returnVal : Resource? = strategy.findSuitableResource(load, mockResources)
+            if(returnVal != null) {
+                logger.info { "returnVal '${returnVal.get()}'" }
+            }
+            else {
+                logger.info { "returnVal is null." }
+            }
+            actual.add(returnVal)
+        }
+
+        assertEquals(actual, expected)
+    }
+
+    @Test
+    fun testInitialGuessSearchFirstNotDoable() {
+        val mockResults = arrayOf(
+                arrayOf(false, false, false, false, false, false, false),
+                arrayOf(false, false, true, true, true, true, true),
+                arrayOf(false, false, false, true, true, true, true),
+                arrayOf(true, true, true, true, true, true, true),
+                arrayOf(false, false, false, false, true, true, true),
+                arrayOf(false, false, false, false, false, false, true),
+                arrayOf(false, false, false, false, false, false, false)
+        )
+        val mockLoads: List<LoadDimension> = (0..6).map { number -> LoadDimension(number, emptyList()) }
+        val mockResources: List<Resource> = (0..6).map { number -> Resource(number, emptyList()) }
+        val results = Results()
+        val benchmark = TestBenchmark()
+        val guessStrategy = PrevResourceMinGuess()
+        val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo()
+        val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, listOf(sloChecker), 0, 0, 5)
+        val strategy = InitialGuessSearchStrategy(benchmarkExecutor, guessStrategy, results)
+
+        val actual: ArrayList<Resource?> = ArrayList()
+        var expected: ArrayList<Resource?> = ArrayList(listOf(2, 3, 0, 4, 6).map { x -> Resource(x, emptyList()) })
+        expected.add(null)
+        expected = ArrayList(listOf(null) + expected)
+
+        for (load in mockLoads) {
+            val returnVal : Resource? = strategy.findSuitableResource(load, mockResources)
+            if(returnVal != null) {
+                logger.info { "returnVal '${returnVal.get()}'" }
+            }
+            else {
+                logger.info { "returnVal is null." }
+            }
+            actual.add(returnVal)
+        }
+
+        assertEquals(actual, expected)
+    }
+}
\ No newline at end of file