Merge branch 'master' into feature/147-add-beam-implementations

helm/README.md

@@ -44,7 +44,7 @@ helm install theodolite . -f preconfigs/one-broker-values.yaml
 
 ## Uninstall this Chart
 
-To uninstall/delete the `theodolite` deployment:
+To uninstall/delete the `theodolite` deployment (by default Helm will be install all CRDs (`execution` and `benchmark`) automatically. If Helm should not install these CRDs, use the flag `--skip-crds`)
 
 ```sh
 helm uninstall theodolite

helm/crds/benchmark.yaml

+./../../theodolite/crd/crd-benchmark.yaml
\ No newline at end of file

helm/crds/execution.yaml

+./../../theodolite/crd/crd-execution.yaml
\ No newline at end of file

helm/templates/theodolite/crd-benchmark.yaml

-{{- if .Values.operator.benchmarkCRD.create -}}
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  name: benchmarks.theodolite.com
-spec:
-  group: theodolite.com
-  names:
-    kind: benchmark
-    plural: benchmarks
-    shortNames:
-      - bench
-  versions:
-  - name: v1
-    served: true
-    storage: true
-    schema:
-      openAPIV3Schema:
-        type: object
-        required: ["spec"]
-        properties:
-          spec:
-            type: object
-            required: []
-            properties:
-              name:
-                type: string
-              appResource:
-                type: array
-                minItems: 1
-                items:
-                  type: string
-              loadGenResource:
-                type: array
-                minItems: 1
-                items:
-                  type: string
-              resourceTypes:
-                type: array
-                minItems: 1
-                items:
-                  type: object
-                  properties:
-                    typeName:
-                      type: string
-                    patchers:
-                      type: array
-                      minItems: 1
-                      items:
-                        type: object
-                        properties:
-                          type:
-                            type: string
-                            default: ""
-                          resource:
-                            type: string
-                            default: ""
-                          properties:
-                            type: object
-                            additionalProperties: true
-                            x-kubernetes-map-type: "granular"
-                            default: {}
-              loadTypes:
-                type: array
-                minItems: 1
-                items:
-                  type: object
-                  properties:
-                    typeName:
-                      type: string
-                    patchers:
-                      type: array
-                      minItems: 1
-                      items:
-                        type: object
-                        properties:
-                          type:
-                            type: string
-                            default: ""
-                          resource:
-                            type: string
-                            default: ""
-                          properties:
-                            type: object
-                            additionalProperties: true
-                            x-kubernetes-map-type: "granular"
-                            default: {}
-              kafkaConfig:
-                type: object
-                properties:
-                  bootstrapServer:
-                    type: string
-                  topics:
-                    type: array
-                    minItems: 1
-                    items:
-                      type: object
-                      required: []
-                      properties:
-                        name:
-                          type: string
-                          default: ""
-                        numPartitions:
-                          type: integer
-                          default: 0
-                        replicationFactor:
-                          type: integer
-                          default: 0
-                        removeOnly:
-                          type: boolean
-                          default: false
-    additionalPrinterColumns:
-    - name: Age
-      type: date
-      jsonPath: .metadata.creationTimestamp
-    subresources:
-      status: {}
-  scope: Namespaced
-{{- end }}

helm/templates/theodolite/crd-execution.yaml

-{{- if .Values.operator.executionCRD.create -}}
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  name: executions.theodolite.com
-spec:
-  group: theodolite.com
-  names:
-    kind: execution
-    plural: executions
-    shortNames:
-      - exec
-  versions:
-  - name: v1
-    served: true
-    storage: true
-    schema:
-      openAPIV3Schema:
-        type: object
-        required: ["spec"]
-        properties:
-          spec:
-            type: object
-            required: ["benchmark", "load", "resources", "slos", "execution", "configOverrides"]
-            properties:
-              name:
-                type: string
-                default: ""
-              benchmark:
-                type: string
-              load: # definition of the load dimension
-                type: object
-                required: ["loadType", "loadValues"]
-                properties:
-                  loadType:
-                   type: string
-                  loadValues:
-                    type: array
-                    items:
-                      type: integer
-              resources: # definition of the resource dimension
-                type: object
-                required: ["resourceType", "resourceValues"]
-                properties:
-                  resourceType:
-                    type: string
-                  resourceValues:
-                    type: array
-                    items:
-                      type: integer
-              slos: # def of service level objectives
-                type: array
-                items:
-                  type: object
-                  required: ["sloType", "prometheusUrl", "offset"]
-                  properties:
-                    sloType:
-                      description: The type of the SLO. It must match 'lag trend'.
-                      type: string
-                    prometheusUrl:
-                      description: Connection string for Promehteus.
-                      type: string
-                    offset:
-                      description: Hours by which the start and end timestamp will be shifted (for different timezones).
-                      type: integer
-                    properties:
-                        description: (Optional) SLO specific additional arguments.
-                        type: object
-                        additionalProperties: true
-                        x-kubernetes-map-type: "granular"
-                        default: {}
-              execution: # def execution config
-                type: object
-                required: ["strategy", "duration", "repetitions", "restrictions"]
-                properties:
-                  strategy:
-                    type: string
-                  duration:
-                    type: integer
-                  repetitions:
-                    type: integer
-                  loadGenerationDelay:
-                    type: integer
-                  restrictions:
-                    type: array
-                    items:
-                      type: string
-              configOverrides:
-                type: array
-                items:
-                  type: object
-                  properties:
-                    patcher:
-                      type: object
-                      properties:
-                        type:
-                          type: string
-                          default: ""
-                        resource:
-                          type: string
-                          default: ""
-                        properties:
-                            type: object
-                            additionalProperties: true
-                            x-kubernetes-map-type: "granular"
-                            default: {}
-                    value:
-                      type: string
-          status:
-            type: object
-            properties:
-              executionState:
-                description: ""
-                type: string
-              executionDuration:
-                description: "Duration of the execution in seconds"
-                type: string
-    additionalPrinterColumns:
-    - name: STATUS
-      type: string
-      description: State of the execution
-      jsonPath: .status.executionState
-    - name: Duration
-      type: string
-      description: Duration of the execution
-      jsonPath: .status.executionDuration
-    - name: Age
-      type: date
-      jsonPath: .metadata.creationTimestamp
-    subresources:
-      status: {}
-  scope: Namespaced
-{{- end }}

helm/values.yaml

@@ -250,11 +250,6 @@ operator:
   imageTag: latest
   imagePullPolicy: Always
 
-  executionCRD:
-    create: true
-  benchmarkCRD:
-    create: true
-
   sloChecker:
     lagTrend:
       enabled: true

theodolite-benchmarks/kstreams-commons/build.gradle

@@ -7,6 +7,9 @@ repositories {
   maven {
     url "https://oss.sonatype.org/content/repositories/snapshots/"
   }
+  maven {
+    url 'https://packages.confluent.io/maven/'
+  }
 }
 
 dependencies {

theodolite/crd/crd-benchmark.yaml

@@ -118,7 +118,7 @@ spec:
                     minItems: 1
                     items:
                       type: object
-                      required: ["name", "numPartitions", "replicationFactor"]
+                      required: ["name"]
                       properties:
                         name:
                           description: The name of the topic.

theodolite/src/main/kotlin/theodolite/evaluation/SloCheckerFactory.kt

@@ -22,13 +22,13 @@ class SloCheckerFactory {
      * - `warmup`: time from the beginning to skip in the analysis.
      *
      *
-     * ### `lag trend percent`
+     * ### `lag trend ratio`
      * Creates an [ExternalSloChecker] with defined parameters.
-     * The required threshold is computed using a percentage and the load of the experiment.
+     * The required threshold is computed using a ratio and the load of the experiment.
      *
      * The properties map needs the following fields:
      * - `externalSlopeURL`: Url to the concrete SLO checker service.
-     * - `percent`: of the executed load that is accepted for the slope.
+     * - `ratio`: of the executed load that is accepted for the slope.
      * - `warmup`: time from the beginning to skip in the analysis.
      *
      * @param sloType Type of the [SloChecker].
@@ -50,18 +50,15 @@ class SloCheckerFactory {
                 threshold = properties["threshold"]?.toInt() ?: throw IllegalArgumentException("threshold expected"),
                 warmup = properties["warmup"]?.toInt() ?: throw IllegalArgumentException("warmup expected")
             )
-            "lag trend percent" -> {
-                if (!properties["loadType"].equals("NumSensors")) {
-                    throw IllegalArgumentException("Percent Threshold is only allowed with load type NumSensors")
-                }
-                var thresholdPercent =
-                    properties["percent"]?.toDouble()
-                        ?: throw IllegalArgumentException("percent for threshold expected")
-                if (thresholdPercent < 0.0 || thresholdPercent > 1.0) {
-                    throw IllegalArgumentException("Threshold percent need to be an Double in the range between 0.0 and 1.0 (inclusive)")
+            "lag trend ratio" -> {
+                var thresholdRatio =
+                    properties["ratio"]?.toDouble()
+                        ?: throw IllegalArgumentException("ratio for threshold expected")
+                if (thresholdRatio < 0.0) {
+                    throw IllegalArgumentException("Threshold ratio needs to be an Double greater or equal 0.0")
                 }
                 // cast to int, as rounding is not really necessary
-                var threshold = (load.get() * thresholdPercent).toInt()
+                var threshold = (load.get() * thresholdRatio).toInt()
 
                 ExternalSloChecker(
                     externalSlopeURL = properties["externalSloUrl"]

theodolite/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt

@@ -55,9 +55,6 @@ class TheodoliteExecutor(
                 this.kubernetesBenchmark.loadTypes
             )
 
-        // Add load type to check if the percentage lag trend is applicable
-        config.slos.forEach { it.properties["loadType"] = config.load.loadType }
-
         executor =
             BenchmarkExecutorImpl(
                 benchmark = kubernetesBenchmark,