merge upstream master

CITATION.cff

 cff-version: "1.1.0"
 message: "If you use Theodolite, please cite it using these metadata."
 authors: 
-  -
-    family-names: Henning
+  - family-names: Henning
     given-names: "Sören"
     orcid: "https://orcid.org/0000-0001-6912-2549"
-  -
-    family-names: Hasselbring
+  - family-names: Hasselbring
     given-names: Wilhelm
     orcid: "https://orcid.org/0000-0001-6625-4335"
 title: Theodolite
@@ -14,3 +12,18 @@ version: "0.4.0"
 repository-code: "https://github.com/cau-se/theodolite"
 license: "Apache-2.0"
 doi: "10.1016/j.bdr.2021.100209"
+preferred-citation:
+  type: article
+  authors: 
+    - family-names: Henning
+      given-names: "Sören"
+      orcid: "https://orcid.org/0000-0001-6912-2549"
+    - family-names: Hasselbring
+      given-names: Wilhelm
+      orcid: "https://orcid.org/0000-0001-6625-4335"
+  doi: "10.1016/j.bdr.2021.100209"
+  journal: "Big Data Research"
+  month: 7
+  title: "Theodolite: Scalability Benchmarking of Distributed Stream Processing Engines in Microservice Architectures"
+  volume: 25
+  year: 2021

helm/templates/theodolite/crd-execution.yaml

@@ -52,20 +52,23 @@ spec:
                 type: array
                 items:
                   type: object
-                  required: ["sloType", "threshold", "prometheusUrl", "externalSloUrl", "offset", "warmup"]
+                  required: ["sloType", "prometheusUrl", "offset"]
                   properties:
                     sloType:
+                      description: The type of the SLO. It must match 'lag trend'.
                       type: string
-                    threshold:
-                      type: integer
                     prometheusUrl:
-                      type: string
-                    externalSloUrl:
+                      description: Connection string for Promehteus.
                       type: string
                     offset:
+                      description: Hours by which the start and end timestamp will be shifted (for different timezones).
                       type: integer
-                    warmup:
-                      type: integer
+                    properties:
+                        description: (Optional) SLO specific additional arguments.
+                        type: object
+                        additionalProperties: true
+                        x-kubernetes-map-type: "granular"
+                        default: {}
               execution: # def execution config
                 type: object
                 required: ["strategy", "duration", "repetitions", "restrictions"]

theodolite/crd/crd-execution.yaml

@@ -60,26 +60,23 @@ spec:
                 type: array
                 items:
                   type: object
-                  required: ["sloType", "threshold", "prometheusUrl", "externalSloUrl", "offset", "warmup"]
+                  required: ["sloType", "prometheusUrl", "offset"]
                   properties:
                     sloType:
                       description: The type of the SLO. It must match 'lag trend'.
                       type: string
-                    threshold:
-                      description: The threshold the SUT should meet for a sucessful experiment.
-                      type: integer
                     prometheusUrl:
                       description: Connection string for Promehteus.
                       type: string
-                    externalSloUrl:
-                      description: Connection string for a external slo analysis.
-                      type: string
                     offset:
                       description: Hours by which the start and end timestamp will be shifted (for different timezones).
                       type: integer
-                    warmup:
-                      description:  Seconds of time that are ignored in the analysis.
-                      type: integer
+                    properties:
+                        description: (Optional) SLO specific additional arguments.
+                        type: object
+                        additionalProperties: true
+                        x-kubernetes-map-type: "granular"
+                        default: {}
               execution: # def execution config
                 description: Defines the overall parameter for the execution.
                 type: object
@@ -152,4 +149,4 @@ spec:
       jsonPath: .metadata.creationTimestamp
     subresources:
       status: {}
-  scope: Namespaced
+  scope: Namespaced
\ No newline at end of file

theodolite/examples/operator/example-execution.yaml

@@ -12,11 +12,12 @@ spec:
     resourceValues: [1, 2, 3, 4, 5]
   slos:
     - sloType: "lag trend"
-      threshold: 2000
-      prometheusUrl:  "http://localhost:9090" # "http://prometheus-operated:9090"
-      externalSloUrl: "http://localhost:80/evaluate-slope"
+      prometheusUrl: "http://prometheus-operated:9090"
       offset: 0
-      warmup: 60 # in seconds
+      properties:
+        threshold: 2000
+        externalSloUrl: "http://localhost:80/evaluate-slope"
+        warmup: 60 # in seconds
   execution:
     strategy: "LinearSearch"
     duration: 300 # in seconds

theodolite/examples/standalone/example-execution.yaml

@@ -8,11 +8,12 @@ resources:
   resourceValues: [1, 2, 3, 4, 5]
 slos:
   - sloType: "lag trend"
-    threshold: 2000
     prometheusUrl: "http://prometheus-operated:9090"
-    externalSloUrl: "http://localhost:80/evaluate-slope"
     offset: 0
-    warmup: 60 # in seconds
+    properties:
+      threshold: 2000
+      externalSloUrl: "http://localhost:80/evaluate-slope"
+      warmup: 60 # in seconds
 execution:
   strategy: "LinearSearch"
   duration: 300 # in seconds
@@ -20,4 +21,4 @@ execution:
   loadGenerationDelay: 30 # in seconds, optional field, default is 0 seconds
   restrictions:
     - "LowerBound"
-configOverrides: []
+configOverrides: []
\ No newline at end of file

theodolite/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt

@@ -62,11 +62,9 @@ class BenchmarkExecution : KubernetesResource {
     @RegisterForReflection
     class Slo : KubernetesResource {
         lateinit var sloType: String
-        var threshold by Delegates.notNull<Int>()
         lateinit var prometheusUrl: String
-        lateinit var externalSloUrl: String
         var offset by Delegates.notNull<Int>()
-        var warmup by Delegates.notNull<Int>()
+        lateinit var properties: MutableMap<String, String>
     }
 
     /**

theodolite/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt

@@ -64,9 +64,8 @@ class AnalysisExecutor(
 
             val sloChecker = SloCheckerFactory().create(
                 sloType = slo.sloType,
-                externalSlopeURL = slo.externalSloUrl,
-                threshold = slo.threshold,
-                warmup = slo.warmup
+                properties = slo.properties,
+                load = load
             )
 
             result = sloChecker.evaluate(prometheusData)

theodolite/src/main/kotlin/theodolite/evaluation/SloCheckerFactory.kt

 package theodolite.evaluation
 
+import theodolite.util.LoadDimension
+
 /**
  * Factory used to potentially create different [SloChecker]s.
  * Supports: lag type.
@@ -8,28 +10,66 @@ class SloCheckerFactory {
 
     /**
      * Creates different [SloChecker]s.
-     * Supports: lag type.
+     *
+     * Supports: `lag trend` and `lag trend percent` as arguments for `sloType`
+     *
+     * ### `lag trend`
+     * Creates an [ExternalSloChecker] with defined parameters.
+     *
+     * The properties map needs the following fields:
+     * - `externalSlopeURL`: Url to the concrete SLO checker service.
+     * - `threshold`: fixed value used for the slope.
+     * - `warmup`: time from the beginning to skip in the analysis.
+     *
+     *
+     * ### `lag trend percent`
+     * Creates an [ExternalSloChecker] with defined parameters.
+     * The required threshold is computed using a percentage and the load of the experiment.
+     *
+     * The properties map needs the following fields:
+     * - `externalSlopeURL`: Url to the concrete SLO checker service.
+     * - `percent`: of the executed load that is accepted for the slope.
+     * - `warmup`: time from the beginning to skip in the analysis.
      *
      * @param sloType Type of the [SloChecker].
-     * @param externalSlopeURL Url to the concrete [SloChecker].
-     * @param threshold for the [SloChecker].
-     * @param warmup for the [SloChecker].
+     * @param properties map of properties to use for the SLO checker creation.
+     * @param load that is executed in the experiment.
      *
      * @return A [SloChecker]
      * @throws IllegalArgumentException If [sloType] not supported.
      */
     fun create(
         sloType: String,
-        externalSlopeURL: String,
-        threshold: Int,
-        warmup: Int
+        properties: MutableMap<String, String>,
+        load: LoadDimension
     ): SloChecker {
         return when (sloType) {
             "lag trend" -> ExternalSloChecker(
-                externalSlopeURL = externalSlopeURL,
-                threshold = threshold,
-                warmup = warmup
+                externalSlopeURL = properties["externalSloUrl"]
+                    ?: throw IllegalArgumentException("externalSloUrl expected"),
+                threshold = properties["threshold"]?.toInt() ?: throw IllegalArgumentException("threshold expected"),
+                warmup = properties["warmup"]?.toInt() ?: throw IllegalArgumentException("warmup expected")
             )
+            "lag trend percent" -> {
+                if (!properties["loadType"].equals("NumSensors")) {
+                    throw IllegalArgumentException("Percent Threshold is only allowed with load type NumSensors")
+                }
+                var thresholdPercent =
+                    properties["percent"]?.toDouble()
+                        ?: throw IllegalArgumentException("percent for threshold expected")
+                if (thresholdPercent < 0.0 || thresholdPercent > 1.0) {
+                    throw IllegalArgumentException("Threshold percent need to be an Double in the range between 0.0 and 1.0 (inclusive)")
+                }
+                // cast to int, as rounding is not really necessary
+                var threshold = (load.get() * thresholdPercent).toInt()
+
+                ExternalSloChecker(
+                    externalSlopeURL = properties["externalSloUrl"]
+                        ?: throw IllegalArgumentException("externalSloUrl expected"),
+                    threshold = threshold,
+                    warmup = properties["warmup"]?.toInt() ?: throw IllegalArgumentException("warmup expected")
+                )
+            }
             else -> throw IllegalArgumentException("Slotype $sloType not found.")
         }
     }

theodolite/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt

@@ -55,6 +55,9 @@ class TheodoliteExecutor(
                 this.kubernetesBenchmark.loadTypes
             )
 
+        // Add load type to check if the percentage lag trend is applicable
+        config.slos.forEach { it.properties["loadType"] = config.load.loadType }
+
         executor =
             BenchmarkExecutorImpl(
                 benchmark = kubernetesBenchmark,

theodolite/src/test/resources/k8s-resource-files/test-execution-update.yaml

@@ -13,11 +13,12 @@ spec:
     resourceValues: [1, 2, 3, 4, 5]
   slos:
     - sloType: "lag trend"
-      threshold: 2000
       prometheusUrl: "http://prometheus-operated:9090"
-      externalSloUrl: "http://localhost:80/evaluate-slope"
       offset: 0
-      warmup: 60 # in seconds
+      properties:
+        threshold: 2000
+        externalSloUrl: "http://localhost:80/evaluate-slope"
+        warmup: 60 # in seconds
   execution:
     strategy: "LinearSearch"
     duration: 300 # in seconds

theodolite/src/test/resources/k8s-resource-files/test-execution.yaml

@@ -13,11 +13,12 @@ spec:
     resourceValues: [1, 2, 3, 4, 5]
   slos:
     - sloType: "lag trend"
-      threshold: 2000
       prometheusUrl: "http://prometheus-operated:9090"
-      externalSloUrl: "http://localhost:80/evaluate-slope"
       offset: 0
-      warmup: 60 # in seconds
+      properties:
+        threshold: 2000
+        externalSloUrl: "http://localhost:80/evaluate-slope"
+        warmup: 60 # in seconds
   execution:
     strategy: "LinearSearch"
     duration: 300 # in seconds