merge upstream master

docs/patchers.md

+## Patchers
+
+* **ReplicaPatcher**: Allows to modify the number of Replicas for a kubernetes deployment.
+  * **type**: "ReplicaPatcher"
+  * **resource**: "uc1-kstreams-deployment.yaml"
+
+* **NumSensorsLoadGeneratorReplicaPatcher**: Allows to scale the nummer of load generators. Scales arcording to the following formular: (value + 15_000 - 1) / 15_000
+  * **type**: "NumSensorsLoadGeneratorReplicaPatcher"
+  * **resource**: "uc1-load-generator-deployment.yaml"
+
+* **NumNestedGroupsLoadGeneratorReplicaPatcher**: Allows to scale the nummer of load generators. Scales arcording to the following formular: (4^(value) + 15_000 -1) /15_000
+  * **type**: "NumNestedGroupsLoadGeneratorReplicaPatcher"
+  * **resource**: "uc1-load-generator-deployment.yaml"
+
+* **ReplicaPatcher**: Allows to modify the number of Replicas for a kubernetes deployment.
+  * **type**: "ReplicaPatcher"
+  * **resource**: "uc1-kstreams-deployment.yaml"
+
+* **EnvVarPatcher**: Allows to modify the value of an environment variable for a container in a kubernetes deployment. 
+  * **type**: "EnvVarPatcher"
+  * **resource**: "uc1-load-generator-deployment.yaml"
+  * **properties**:
+    * container: "workload-generator"
+    * variableName: "NUM_SENSORS"
+
+* **NodeSelectorPatcher**: Changes the node selection field in kubernetes resources.
+  * **type**: "NodeSelectorPatcher"
+  * **resource**: "uc1-load-generator-deployment.yaml"
+  * **properties**:
+    * variableName: "env"
+  * **value**: "prod"
+
+* **ResourceLimitPatcher**: Changes the resource limit for a kubernetes resource.
+  * **resource**: "uc1-kstreams-deployment.yaml"
+  * **properties**:
+    * container: "uc-application"
+    * variableName: "cpu" or "memory"
+  * **value**:"1000m" or "2Gi"
+  
+* **SchedulerNamePatcher**: Changes the sheduler for kubernetes resources.
+  * **type**: "SchedulerNamePatcher"
+  * **resource**: "uc1-kstreams-deployment.yaml"
+  * **value**: "random-scheduler"
+
+* **ImagePatcher**: Changes the image of a kubernetes resource. Currently not fully implemented.
+  * **type**: "ImagePatcher"
+  * **resource**: "uc1-kstreams-deployment.yaml"
+  * **properties**:
+    * container: "uc-application"
+  * **value**: "dockerhubrepo/imagename"

execution/.dockerignore

-*
-!requirements.txt
-!uc-workload-generator
-!uc-application
-!strategies
-!lib
-!theodolite.py
-!run_uc.py
-!lag_analysis.py

execution/theodolite.yaml

@@ -25,11 +25,11 @@ spec:
             # - name: MODE
             #   value: yaml-executor # Default is `yaml-executor`
             - name: THEODOLITE_EXECUTION
-              value: /etc/execution/example-execution-yaml-resource.yaml # The name of this file must correspond to the filename of the execution, from which the config map is created.
+              value: "execution/execution.yaml" # The name of this file must correspond to the filename of the execution, from which the config map is created.
             - name: THEODOLITE_BENCHMARK
-              value: /etc/benchmark/example-benchmark-yaml-resource.yaml # The name of this file must correspond to the filename of the benchmark, from which the config map is created.
+              value: "benchmark/benchmark.yaml" # The name of this file must correspond to the filename of the benchmark, from which the config map is created.
             - name: THEODOLITE_APP_RESOURCES
-              value: /etc/app-resources
+              value: "benchmark-resources"
             - name: RESULTS_FOLDER # Folder for saving results
               value: results # Default is the pwd (/deployments)
             # - name: CREATE_RESULTS_FOLDER # Specify whether the specified result folder should be created if it does not exist.
@@ -37,11 +37,11 @@ spec:
           volumeMounts:
             - mountPath: "/deployments/results" # the mounted path must corresponds to the value of `RESULT_FOLDER`.
               name: theodolite-pv-storage
-            - mountPath: "/etc/app-resources" # must be corresponds to the value of `THEODOLITE_APP_RESOURCES`.
-              name: app-resources
-            - mountPath: "/etc/benchmark"  # must be corresponds to the value of `THEODOLITE_BENCHMARK`.
+            - mountPath: "/deployments/benchmark-resources" # must correspond to the value of `THEODOLITE_APP_RESOURCES`.
+              name: benchmark-resources
+            - mountPath: "/deployments/benchmark"  # must correspond to the value of `THEODOLITE_BENCHMARK`.
               name: benchmark
-            - mountPath: "/etc/execution" # must be corresponds to the value of `THEODOLITE_EXECUTION`.
+            - mountPath: "/deployments/execution" # must correspond to the value of `THEODOLITE_EXECUTION`.
               name: execution
       restartPolicy: Never
       # Uncomment if RBAC is enabled and configured
@@ -52,9 +52,9 @@ spec:
         - name: theodolite-pv-storage
           persistentVolumeClaim:
             claimName: theodolite-pv-claim
-        - name: app-resources
+        - name: benchmark-resources
           configMap:
-            name: app-resources-configmap
+            name: benchmark-resources-configmap
         - name: benchmark
           configMap:
             name: benchmark-configmap

theodolite-benchmarks/definitions/uc1-kstreams/uc1-benchmark-standalone.yaml

@@ -22,7 +22,7 @@ loadTypes:
       - type: NumSensorsLoadGeneratorReplicaPatcher
         resource: "uc1-load-generator-deployment.yaml"
         properties:
-          loadGenMaxRecords: "15000"
+          loadGenMaxRecords: "150000"
 kafkaConfig:
   bootstrapServer: "theodolite-cp-kafka:9092"
   topics:

theodolite-benchmarks/definitions/uc2-kstreams/uc2-benchmark-standalone.yaml

@@ -23,7 +23,7 @@ loadTypes:
       - type: NumSensorsLoadGeneratorReplicaPatcher
         resource: "uc2-load-generator-deployment.yaml"
         properties:
-          loadGenMaxRecords: "15000"
+          loadGenMaxRecords: "150000"
 kafkaConfig:
   bootstrapServer: "theodolite-cp-kafka:9092"
   topics:

theodolite-benchmarks/definitions/uc3-kstreams/uc3-benchmark-standalone.yaml

@@ -23,7 +23,7 @@ loadTypes:
       - type: NumSensorsLoadGeneratorReplicaPatcher
         resource: "uc3-load-generator-deployment.yaml"
         properties:
-          loadGenMaxRecords: "15000"
+          loadGenMaxRecords: "150000"
 kafkaConfig:
   bootstrapServer: "theodolite-cp-kafka:9092"
   topics:

theodolite-benchmarks/definitions/uc4-kstreams/uc4-benchmark-standalone.yaml

@@ -23,7 +23,7 @@ loadTypes:
       - type: "NumNestedGroupsLoadGeneratorReplicaPatcher"
         resource: "uc4-load-generator-deployment.yaml"
         properties:
-          loadGenMaxRecords: "15000"
+          loadGenMaxRecords: "150000"
           numSensors: "4.0"
 kafkaConfig:
   bootstrapServer: "theodolite-cp-kafka:9092"

theodolite/.dockerignore

@@ -3,4 +3,3 @@
 !build/*-runner.jar
 !build/lib/*
 !build/quarkus-app/*
-!config/*
\ No newline at end of file

theodolite/config/README.md

-## The Benchmark Object
-
-The *benchmark* object defines all static components of an execution of a benchmark with Theodolite.
-An exapmle for a benchmark object is given in [example-benchmark-yaml-resource](example-benchmark-yaml-resource.yaml).
-
-
-A **Benchmark** is a [*standard tool for the competitive evaluation and comparison of competing systems or components according to specific characteristics, such as performance, dependability, or security*](https://doi.org/10.1145/2668930.2688819). In Theodolite, we have [specification-based benchmarks](https://doi.org/10.1145/2668930.2688819), or at least something very close to that. That is, our benchmarks are architectural descriptions---in our case---[of typical use cases of stream processing in microservices](https://doi.org/10.1016/j.bdr.2021.100209) (e.g. our UC1). Hence, we don't really have a piece of software, which represents a benchmark. We only have implementations of benchmarks, e.g. an implementation of UC1 with Kafka Streams. For simplification, we call these *benchmark implementations* simply *benchmarks*.
-
-```yaml
-name: String
-appResource:
-  - String
-  ...
-loadGenResource:
-  - String
-  ...
-resourceTypes:
-  - typeName: String
-    patchers:
-      - type: String
-        resources: String
-        properties:
-          <Patcher Arguments> ...
-      ...
-loadTypes:
-  - typeName: String
-  patchers:
-    - type: String
-      resources: String
-      properties:
-        <Patcher Arguments> ...
-    ...
-kafkaConfig:
-  bootstrapServer: String
-  topics:
-    - name: String
-      numPartitions: UnsignedInt
-      replicationFactor: UnsignedInt
-    - name: String
-      removeOnly: bool
-    ...
-```
-
-The properties have the following definitions:
-
-* **name**: The name of the *benchmark*
-* **appResource**: A list of file names that reference Kubernetes resources that are deployed on the cluster for the system under test (SUT).
-* **loadGenResources**: A list of file names that reference Kubernetes resources that are deployed on the cluster for the load generator.
-* **resourceTypes**: A list of resource types that can be scaled for this *benchmark*. For each resource type the concrete values are defined in the *execution* object. Each resource type has the following structure:
-    * **typeName**: Name of the resource type.
-    * **patchers**: List of [patchers](#Patchers) used to scale this resource type. Each patcher has the following structure:
-        * **type**: Type of the [patcher](#Patchers). The concrete types can be looked up in the list of [patchers](#Patchers). 
-        * **resources**: Specifies the Kubernetes resource to be patched.
-        *  **properties**: *Patcher Arguments*: (Optional) Patcher specific additional arguments.
-* **loadTypes**: A list of load types that can be scaled for this *benchmark*. For each load type the concrete values are defined in the *execution* object. Each load type has the following structure:
-    * **typeName**: Name of the load type.
-    * **patchers**: List of patchers used to scale * **resourceTypes**: A list of resource types that can be scaled for this *benchmark*. For each resource type the concrete values are defined in the *execution* resource object.Each resource type has the following structure:
-    * **typeName**: Name of the resource type.
-    * **patchers**: List of patchers used to scale this resource type. Each patcher has the following structure:
-        * **type**: Type of the Patcher. The concrete types can be looked up in the list of patchers. 
-        * **resources**: Specifies the Kubernetes resource to be patched.
-        * **properties**: *Patcher Arguments*: (Optional) Patcher specific additional arguments as Map<String, String>.
-* **kafkaConfig**: Contains the Kafka configuration.
-    * **bootstrapServers**: The bootstrap servers connection string.
-    * **topics**: List of topics to be created for each [experiment](#Experiment). Alternative theodolite offers the possibility to remove certain topics after each experiment.
-        * **name**: The name of the topic.
-        * **numPartitions**: The number of partitions of the topic.
-        * **replicationFactor**: The replication factor of the topic.
-        * **removeOnly**: determines if this topic should only be deleted after each experiement. For removeOnly topics the name can be a RegEx describing the topic.
-    
-
-## The Execution Object
-
-A benchmark can be executed for different SUTs, by different users and multiple times. We call such an execution of a benchmark simply an *execution*. The *execution* object defines all conrete values of an Execution.
-An exapmle for an execution object is given in [example-execution-yaml-resource](example-benchmark-yaml-resource.yaml).
-
-
-```yaml
-name: String
-benchmark: String
-load:
-  loadType: String
-  loadValues:
-    - UnsignedInt
-    ...
-resources:
-  resourceType: String
-  resourceValues:
-    - UnsignedInt
-    ...
-slos:
-  - sloType: String
-    threshold: UnsignedInt
-    prometheusUrl: String
-    externalSloUrl: String
-    offset: SignedInt
-    warmup: UnsignedInt
-  ...
-executions:
-  strategy: "LinearSearch" or "BinarySearch"
-  duration: UnsignedInt
-  repetition: UnsignedInt
-  restrictions:
-    - "LowerBound"
-    ...
-configurationOverrides:
-  - patcher:
-      type: String
-      resource: String
-      properties:
-        <Patcher Arguments> ...
-  ...
-```
-
-The properties have the following definitions:
-
-* **name**: The name of the *execution*
-* **benchmark**: The name of the *benchmark* this *execution* is referring to.
-* **load**: Specifies the load values that are benchmarked.
-  * **loadType**: The type of the load. It must match one of the load types specified in the referenced *benchmark*.
-  * **loadValues**: List of load values for the specified load type.
-* **resources**: Specifies the scaling resource that is benchmarked.
-  * **resourceType**: The type of the resource. It must match one of the resource types specified in the referenced *benchmark*.
-  * **resourceValues**: List of resource values for the specified resource type.
-* **slos**: List of the Service Level Objective (SLO) for this *execution*. Each SLO has the following fields:
-  * **sloType**: The type of the SLO. It must match 'lag trend'.
-  * **threshold**: The threshold the SUT should meet for a sucessful experiment.
-  * **prometheusUrl**: Connection string for promehteus.
-  * **externalSloUrl**: Connection string for a external slo analysis.
-  * **offset**: Hours by which the start and end timestamp will be shifted (for different timezones).
-  * **warmup**: Seconds of time that are ignored in the analysis.
-* **executions**: Defines the overall parameter for the execution.
-  * **strategy**: Defines the used strategy for the execution: either 'LinearSearch' or 'BinarySearch'
-  * **duration**: Defines the duration of each [experiment](#Experiment) in seconds.
-  * **repetition**: Unused.
-  * **restrictions**: List of restriction strategys used to delimit the search space.
-    **- LowerBound**: Currently only supported *restriction strategy*.
-* **configurationOverrides**: List of patchers that are used to override existing configurations.
-  * **patcher**: Patcher used to patch a resource. Each patcher has the following structure:
-        * **type**: Type of the Patcher. The concrete types can be looked up in the list of patchers. 
-        * **resources**: Specifies the Kubernetes resource to be patched.
-        * **properties**: *Patcher Arguments*: (Optional) Patcher specific additional arguments.
-
-## Patchers
-
-* **ReplicaPatcher**: Allows to modify the number of Replicas for a kubernetes deployment.
-  * **type**: "ReplicaPatcher"
-  * **resource**: "uc1-kstreams-deployment.yaml"
-
-* **NumSensorsLoadGeneratorReplicaPatcher**: Allows to scale the nummer of load generators. Scales arcording to the following formular: (value + 15_000 - 1) / 15_000
-  * **type**: "NumSensorsLoadGeneratorReplicaPatcher"
-  * **resource**: "uc1-load-generator-deployment.yaml"
-
-* **NumNestedGroupsLoadGeneratorReplicaPatcher**: Allows to scale the nummer of load generators. Scales arcording to the following formular: (4^(value) + 15_000 -1) /15_000
-  * **type**: "NumNestedGroupsLoadGeneratorReplicaPatcher"
-  * **resource**: "uc1-load-generator-deployment.yaml"
-
-* **ReplicaPatcher**: Allows to modify the number of Replicas for a kubernetes deployment.
-  * **type**: "ReplicaPatcher"
-  * **resource**: "uc1-kstreams-deployment.yaml"
-
-* **EnvVarPatcher**: Allows to modify the value of an environment variable for a container in a kubernetes deployment. 
-  * **type**: "EnvVarPatcher"
-  * **resource**: "uc1-load-generator-deployment.yaml"
-  * **properties**:
-    * container: "workload-generator"
-    * variableName: "NUM_SENSORS"
-
-* **NodeSelectorPatcher**: Changes the node selection field in kubernetes resources.
-  * **type**: "NodeSelectorPatcher"
-  * **resource**: "uc1-load-generator-deployment.yaml"
-  * **properties**:
-    * variableName: "env"
-  * **value**: "prod"
-
-* **ResourceLimitPatcher**: Changes the resource limit for a kubernetes resource.
-  * **resource**: "uc1-kstreams-deployment.yaml"
-  * **properties**:
-    * container: "uc-application"
-    * variableName: "cpu" or "memory"
-  * **value**:"1000m" or "2Gi"
-  
-* **SchedulerNamePatcher**: Changes the sheduler for kubernetes resources.
-  * **type**: "SchedulerNamePatcher"
-  * **resource**: "uc1-kstreams-deployment.yaml"
-  * **value**: "random-scheduler"
-
-* **ImagePatcher**: Changes the image of a kubernetes resource. Currently not fully implemented.
-  * **type**: "ImagePatcher"
-  * **resource**: "uc1-kstreams-deployment.yaml"
-  * **properties**:
-    * container: "uc-application"
-  * **value**: "dockerhubrepo/imagename"
-
-
-
-## Experiment
-According to [our benchmarking method](https://doi.org/10.1016/j.bdr.2021.100209), the execution of a benchmark requires performing multiple **Experiments**. I think what is actually done within/during an experiment is another level of detail. (But just for the sake of completeness: In an experiment, the benchmark implementation is deployed, load is generated according to the benchmark specification, some SLOs are monitored continuously, etc.)
-
-
-

theodolite/config/aggregation-service.yaml

-apiVersion: v1
-kind: Service
-metadata:  
-  name: titan-ccp-aggregation
-  labels:
-    app: titan-ccp-aggregation
-spec:
-  #type: NodePort
-  selector:    
-    app: titan-ccp-aggregation
-  ports:  
-  - name: http
-    port: 80
-    targetPort: 80
-    protocol: TCP
-  - name: metrics
-    port: 5556

theodolite/config/jmx-configmap.yaml

-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: aggregation-jmx-configmap
-data:
-  jmx-kafka-prometheus.yml: |+
-    jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi
-    lowercaseOutputName: true
-    lowercaseOutputLabelNames: true
-    ssl: false

theodolite/config/service-monitor.yaml

-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
-  labels:
-    app: titan-ccp-aggregation
-    appScope: titan-ccp
-  name: titan-ccp-aggregation
-spec:
-  selector:
-    matchLabels:
-        app: titan-ccp-aggregation
-  endpoints:
-    - port: metrics
-      interval: 10s

theodolite/config/uc1-kstreams-deployment.yaml

-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: titan-ccp-aggregation
-spec:
-  selector:
-    matchLabels:
-      app: titan-ccp-aggregation
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: titan-ccp-aggregation
-    spec:
-      terminationGracePeriodSeconds: 0
-      containers:
-        - name: uc-application
-          image: ghcr.io/cau-se/theodolite-uc1-kstreams-app:latest
-          ports:
-            - containerPort: 5555
-              name: jmx
-          env:
-            - name: KAFKA_BOOTSTRAP_SERVERS
-              value: "theodolite-cp-kafka:9092"
-            - name: SCHEMA_REGISTRY_URL
-              value: "http://theodolite-cp-schema-registry:8081"
-            - name: JAVA_OPTS
-              value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555"
-            - name: COMMIT_INTERVAL_MS # Set as default for the applications
-              value: "100"
-          resources:
-            limits:
-              memory: 4Gi
-              cpu: 1000m
-        - name: prometheus-jmx-exporter
-          image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143"
-          command:
-            - java
-            - -XX:+UnlockExperimentalVMOptions
-            - -XX:+UseCGroupMemoryLimitForHeap
-            - -XX:MaxRAMFraction=1
-            - -XshowSettings:vm
-            - -jar
-            - jmx_prometheus_httpserver.jar
-            - "5556"
-            - /etc/jmx-aggregation/jmx-kafka-prometheus.yml
-          ports:
-            - containerPort: 5556
-          volumeMounts:
-            - name: jmx-config
-              mountPath: /etc/jmx-aggregation
-      volumes:
-        - name: jmx-config
-          configMap:
-            name: aggregation-jmx-configmap
\ No newline at end of file

theodolite/config/uc1-load-generator-deployment.yaml

-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: titan-ccp-load-generator
-spec:
-  selector:
-    matchLabels:
-      app: titan-ccp-load-generator
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: titan-ccp-load-generator
-    spec:
-      terminationGracePeriodSeconds: 0
-      containers:
-        - name: workload-generator
-          image: ghcr.io/cau-se/theodolite-uc1-workload-generator:latest
-          ports:
-            - containerPort: 5701
-              name: coordination
-          env:
-            - name: NUM_SENSORS
-              value: "25000"
-            - name: NUM_NESTED_GROUPS
-              value: "5"
-            - name: KUBERNETES_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-            - name: KUBERNETES_DNS_NAME
-              value: "titan-ccp-load-generator.$(KUBERNETES_NAMESPACE).svc.cluster.local"
-            - name: KAFKA_BOOTSTRAP_SERVERS
-              value: "theodolite-cp-kafka:9092"
-            - name: SCHEMA_REGISTRY_URL
-              value: "http://theodolite-cp-schema-registry:8081"

theodolite/config/uc1-load-generator-service.yaml

-apiVersion: v1
-kind: Service
-metadata:
-  name: titan-ccp-load-generator
-  labels:
-    app: titan-ccp-load-generator
-spec:
-  type: ClusterIP
-  clusterIP: None
-  selector:
-    app: titan-ccp-load-generator
-  ports:
-    - name: coordination
-      port: 5701
-      targetPort: 5701
-      protocol: TCP

theodolite/config/uc1-service-monitor.yaml

-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
-  labels:
-    app: titan-ccp-aggregation
-    appScope: titan-ccp
-  name: titan-ccp-aggregation
-spec:
-  selector:
-    matchLabels:
-        app: titan-ccp-aggregation
-  endpoints:
-    - port: metrics
-      interval: 10s

theodolite/src/main/docker/Dockerfile.jvm

@@ -44,7 +44,6 @@ RUN microdnf install curl ca-certificates ${JAVA_PACKAGE} \
 ENV JAVA_OPTIONS="-Dquarkus.http.host=0.0.0.0 -Djava.util.logging.manager=org.jboss.logmanager.LogManager"
 COPY build/lib/* /deployments/lib/
 COPY build/*-runner.jar /deployments/app.jar
-COPY config/ /deployments/config/
 
 EXPOSE 8080
 USER 1001

theodolite/src/main/docker/Dockerfile.native

@@ -20,7 +20,6 @@ RUN chown 1001 /deployments \
     && chmod "g+rwX" /deployments \
     && chown 1001:root /deployments
 COPY --chown=1001:root build/*-runner /deployments/application
-COPY config/ /deployments/config/
 
 EXPOSE 8080
 USER 1001

theodolite/src/main/kotlin/theodolite/benchmark/Benchmark.kt

 package theodolite.benchmark
 
-import io.fabric8.kubernetes.api.model.KubernetesResource
-import io.fabric8.kubernetes.api.model.Namespaced
-import io.fabric8.kubernetes.client.CustomResource
 import io.quarkus.runtime.annotations.RegisterForReflection
 import theodolite.util.ConfigurationOverride
 import theodolite.util.LoadDimension

theodolite/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt

@@ -18,7 +18,7 @@ import kotlin.properties.Delegates
  *  - An [execution] that encapsulates: the strategy, the duration, and the restrictions
  *  for the execution of the benchmark.
  *  - [configOverrides] additional configurations.
- *  This class is used for parsing(in [theodolite.execution.TheodoliteYamlExecutor]) and
+ *  This class is used for parsing(in [theodolite.execution.TheodoliteStandalone]) and
  *  for the deserializing in the [theodolite.execution.operator.TheodoliteOperator].
  *  @constructor construct an empty BenchmarkExecution.
  */