diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 01630912c52ccc728870d72c33a8f02cdf7c2c7f..f2f1bd6263e917ff4c9c78f7d851b61cacc6c57f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -307,11 +307,23 @@ deploy-theodolite:
# Theodolite SLO Checker: Lag Trend
+test-slo-checker-lag-trend:
+ stage: test
+ image: python:3.7-slim
+ tags:
+ - exec-docker
+ script:
+ - cd slope-evaluator
+ - pip install -r requirements.txt
+ - cd app
+ - python -m unittest
+
deploy-slo-checker-lag-trend:
stage: deploy
extends:
- .dind
- needs: []
+ needs:
+ - test-slo-checker-lag-trend
script:
- DOCKER_TAG_NAME=$(echo $CI_COMMIT_REF_SLUG- | sed 's/^master-$//')
- docker build --pull -t theodolite-slo-checker-lag-trend slope-evaluator
diff --git a/execution/helm/.gitignore b/execution/helm/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..80bf7fc709ac6d08e703fe9f24d7d5776e26830e
--- /dev/null
+++ b/execution/helm/.gitignore
@@ -0,0 +1 @@
+charts
\ No newline at end of file
diff --git a/execution/helm/templates/grafana/dashboard-config-map.yaml b/execution/helm/templates/grafana/dashboard-config-map.yaml
index 1125d7833cc62e78c049436f38b854d926e2a216..41365e5efefaddc92a9f2f25f867a9d895e4ca3d 100644
--- a/execution/helm/templates/grafana/dashboard-config-map.yaml
+++ b/execution/helm/templates/grafana/dashboard-config-map.yaml
@@ -253,7 +253,7 @@ data:
"steppedLine": false,
"targets": [
{
- "expr": "sum by(group, topic) (kafka_consumergroup_group_lag > 0)",
+ "expr": "sum by(group, topic) (kafka_consumergroup_group_lag >= 0)",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{topic}}",
@@ -436,7 +436,7 @@ data:
"steppedLine": false,
"targets": [
{
- "expr": "sum by(group,topic) (kafka_consumergroup_group_offset > 0)",
+ "expr": "sum by(group,topic) (kafka_consumergroup_group_offset >= 0)",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{topic}}",
@@ -527,7 +527,7 @@ data:
"steppedLine": false,
"targets": [
{
- "expr": "count by(group,topic) (kafka_consumergroup_group_offset > 0)",
+ "expr": "count by(group,topic) (kafka_consumergroup_group_offset >= 0)",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{topic}}",
@@ -892,7 +892,7 @@ data:
"steppedLine": false,
"targets": [
{
- "expr": "sum by(group) (kafka_consumergroup_group_lag > 0)",
+ "expr": "sum by(group) (kafka_consumergroup_group_lag >= 0)",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "total lag",
diff --git a/execution/helm/templates/theodolite/random-scheduler/cluster-role-binding.yaml b/execution/helm/templates/theodolite/random-scheduler/cluster-role-binding.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..658f75c8c5018fe5b9f47cf9619bb4ee5b26b8e5
--- /dev/null
+++ b/execution/helm/templates/theodolite/random-scheduler/cluster-role-binding.yaml
@@ -0,0 +1,14 @@
+{{- if .Values.randomScheduler.rbac.create -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: {{ include "theodolite.fullname" . }}-random-scheduler
+subjects:
+- kind: ServiceAccount
+ name: {{ include "theodolite.fullname" . }}-random-scheduler
+ namespace: kube-system
+roleRef:
+ kind: ClusterRole
+ apiGroup: rbac.authorization.k8s.io
+ name: system:kube-scheduler
+{{- end }}
diff --git a/execution/helm/templates/theodolite/random-scheduler/deployment.yaml b/execution/helm/templates/theodolite/random-scheduler/deployment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..35a6ad027b93446a2bb97e2ebd67f2e27652e99a
--- /dev/null
+++ b/execution/helm/templates/theodolite/random-scheduler/deployment.yaml
@@ -0,0 +1,30 @@
+{{- if .Values.randomScheduler.enabled -}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: {{ include "theodolite.fullname" . }}-random-scheduler
+ labels:
+ app: {{ include "theodolite.fullname" . }}
+ component: random-scheduler
+ namespace: kube-system
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: {{ include "theodolite.fullname" . }}
+ component: random-scheduler
+ template:
+ metadata:
+ labels:
+ app: {{ include "theodolite.fullname" . }}
+ component: random-scheduler
+ spec:
+ serviceAccount: {{ include "theodolite.fullname" . }}-random-scheduler
+ containers:
+ - name: random-scheduler
+ image: ghcr.io/cau-se/theodolite-random-scheduler:theodolite-kotlin-latest
+ #imagePullPolicy: Always
+ env:
+ - name: TARGET_NAMESPACE
+ value: {{ .Release.Namespace }}
+{{- end }}
diff --git a/execution/helm/templates/theodolite/random-scheduler/service-account.yaml b/execution/helm/templates/theodolite/random-scheduler/service-account.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..babfff17b46d62e7e820fcb9dc8a35d73b4e6538
--- /dev/null
+++ b/execution/helm/templates/theodolite/random-scheduler/service-account.yaml
@@ -0,0 +1,10 @@
+{{- if .Values.randomScheduler.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ namespace: kube-system
+ name: {{ include "theodolite.fullname" . }}-random-scheduler
+ labels:
+ app: {{ include "theodolite.fullname" . }}
+ component: random-scheduler
+{{- end }}
diff --git a/execution/helm/values.yaml b/execution/helm/values.yaml
index 67dab74c3931ce13a1ab0f7504a946a208b4dfb8..d245e37e6f50d3874a1a5c06bd0156009777a999 100644
--- a/execution/helm/values.yaml
+++ b/execution/helm/values.yaml
@@ -25,8 +25,16 @@ grafana:
adminUser: admin
adminPassword: admin
grafana.ini:
+ #org_name: Theodolite
+ auth.anonymous:
+ # enable anonymous access
+ enabled: true
+ org_role: Admin # Role for unauthenticated users, other valid values are `Viewer`, `Editor` and `Admin`
users:
default_theme: light
+ #dashboards: # the following doesn't work but is planed
+ # Path to the default home dashboard. If this value is empty, then Grafana uses StaticRootPath + "dashboards/home.json"
+ #default_home_dashboard_path: "/tmp/dashboards/k8s-dashboard.json"
## Sidecars that collect the configmaps with specified label and stores the included files them into the respective folders
## Requires at least Grafana 5 to work and can't be used together with parameters dashboardProviders, datasources and dashboards
sidecar:
@@ -92,8 +100,9 @@ cp-helm-charts:
"replica.fetch.max.bytes": "134217728" # 128 MB
#default.replication.factor: 1
# "min.insync.replicas": 2
- # "auto.create.topics.enable": false
- "log.retention.ms": "10000" # 10s
+ "auto.create.topics.enable": false
+ #"log.retention.ms": "10000" # 10s
+ "log.retention.ms": "7200000" # 2h
"metrics.sample.window.ms": "5000" #5s
"advertised.listeners": |-
EXTERNAL://${HOST_IP}:$((31090 + ${KAFKA_BROKER_ID}))
@@ -248,3 +257,10 @@ serviceAccount:
rbac:
create: true
+
+randomScheduler:
+ enabled: true
+ rbac:
+ create: true
+ serviceAccount:
+ create: true
diff --git a/slope-evaluator/README.md b/slope-evaluator/README.md
index 12e8a61783532efce9e8722f500cf5ff880bace7..cd9e6820ed46452ce44d57d0c7e5cd5ae05e5a3b 100644
--- a/slope-evaluator/README.md
+++ b/slope-evaluator/README.md
@@ -5,7 +5,7 @@
For development:
```sh
-uvicorn main:app --reload
+uvicorn main:app --reload # run this command inside the app/ folder
```
## Build the docker image:
@@ -32,7 +32,7 @@ The running webserver provides a REST API with the following route:
* /evaluate-slope
* Method: POST
* Body:
- * total_lag
+ * total_lags
* threshold
* warmup
@@ -40,14 +40,16 @@ The body of the request must be a JSON string that satisfies the following condi
* **total_lag**: This property is based on the [Range Vector type](https://www.prometheus.io/docs/prometheus/latest/querying/api/#range-vectors) from Prometheus and must have the following JSON structure:
```
- {
- "metric": {
- "group": "<label_value>"
- },
- "values": [
- [
- <unix_timestamp>,
- "<sample_value>"
+ {
+ [
+ "metric": {
+ "group": "<label_value>"
+ },
+ "values": [
+ [
+ <unix_timestamp>,
+ "<sample_value>"
+ ]
]
]
}
diff --git a/slope-evaluator/app/main.py b/slope-evaluator/app/main.py
index 5995a1175f781c4eb51ab3d3083a665fbc02d6fd..6f6788f0ca84b7710be5b509ca4f0641047e963d 100644
--- a/slope-evaluator/app/main.py
+++ b/slope-evaluator/app/main.py
@@ -5,6 +5,7 @@ import os
import pandas as pd
import json
import sys
+from statistics import median
app = FastAPI()
@@ -20,7 +21,7 @@ elif os.getenv('LOG_LEVEL') == 'WARNING':
elif os.getenv('LOG_LEVEL') == 'DEBUG':
logger.setLevel(logging.DEBUG)
-def execute(results, threshold, warmup):
+def calculate_slope_trend(results, warmup):
d = []
for result in results:
group = result['metric']['group']
@@ -39,13 +40,16 @@ def execute(results, threshold, warmup):
logger.error('Mark this subexperiment as not successful and continue benchmark.')
return False
- result = trend_slope < threshold
- logger.info("Computed lag trend slope is '%s'. Result is: %s", trend_slope, result)
- return result
+ logger.info("Computed lag trend slope is '%s'", trend_slope)
+ return trend_slope
+
+def check_service_level_objective(results, threshold):
+ return median(results) < threshold
@app.post("/evaluate-slope",response_model=bool)
async def evaluate_slope(request: Request):
data = json.loads(await request.body())
- return execute(data['total_lag'], data['threshold'], data['warmup'])
+ results = [calculate_slope_trend(total_lag, data['warmup']) for total_lag in data['total_lags']]
+ return check_service_level_objective(results=results, threshold=data["threshold"])
-logger.info("Slope evaluator is online")
\ No newline at end of file
+logger.info("SLO evaluator is online")
\ No newline at end of file
diff --git a/slope-evaluator/app/test.py b/slope-evaluator/app/test.py
new file mode 100644
index 0000000000000000000000000000000000000000..9b165ea479bb9a552edaba7692df4fd4ef3f4ab4
--- /dev/null
+++ b/slope-evaluator/app/test.py
@@ -0,0 +1,30 @@
+import unittest
+from main import app, check_service_level_objective
+import json
+from fastapi.testclient import TestClient
+
+class TestSloEvaluation(unittest.TestCase):
+ client = TestClient(app)
+
+ def test_1_rep(self):
+ with open('../resources/test-1-rep-success.json') as json_file:
+ data = json.load(json_file)
+ response = self.client.post("/evaluate-slope", json=data)
+ self.assertEquals(response.json(), True)
+
+ def test_3_rep(self):
+ with open('../resources/test-3-rep-success.json') as json_file:
+ data = json.load(json_file)
+ response = self.client.post("/evaluate-slope", json=data)
+ self.assertEquals(response.json(), True)
+
+ def test_check_service_level_objective(self):
+ list = [1,2,3,4]
+ self.assertEquals(check_service_level_objective(list, 2), False)
+ self.assertEquals(check_service_level_objective(list, 3), True)
+ list = [1,2,3,4,5]
+ self.assertEquals(check_service_level_objective(list, 2), False)
+ self.assertEquals(check_service_level_objective(list, 4), True)
+
+if __name__ == '__main__':
+ unittest.main()
\ No newline at end of file
diff --git a/slope-evaluator/requirements.txt b/slope-evaluator/requirements.txt
index ca77b6c891136b1388aaf56c5ae269d6ee4b5729..670815f35b18361951a2fa7b2142eee6bc86b01d 100644
--- a/slope-evaluator/requirements.txt
+++ b/slope-evaluator/requirements.txt
@@ -1,3 +1,5 @@
fastapi==0.55.1
scikit-learn==0.20.3
pandas==1.0.3
+uvicorn
+requests
diff --git a/slope-evaluator/resources/test-1-rep-success.json b/slope-evaluator/resources/test-1-rep-success.json
new file mode 100644
index 0000000000000000000000000000000000000000..9e315c707be7b2a874c58fcb1093aa86f7676560
--- /dev/null
+++ b/slope-evaluator/resources/test-1-rep-success.json
@@ -0,0 +1,139 @@
+{
+ "total_lags": [
+ [
+ {
+ "metric": {
+ "group": "theodolite-uc1-application-0.0.1"
+ },
+ "values": [
+ [
+ 1.621008960827E9,
+ "234"
+ ],
+ [
+ 1.621008965827E9,
+ "234"
+ ],
+ [
+ 1.621008970827E9,
+ "234"
+ ],
+ [
+ 1.621008975827E9,
+ "719"
+ ],
+ [
+ 1.621008980827E9,
+ "719"
+ ],
+ [
+ 1.621008985827E9,
+ "719"
+ ],
+ [
+ 1.621008990827E9,
+ "1026"
+ ],
+ [
+ 1.621008995827E9,
+ "1026"
+ ],
+ [
+ 1.621009000827E9,
+ "1026"
+ ],
+ [
+ 1.621009005827E9,
+ "534"
+ ],
+ [
+ 1.621009010827E9,
+ "534"
+ ],
+ [
+ 1.621009015827E9,
+ "534"
+ ],
+ [
+ 1.621009020827E9,
+ "943"
+ ],
+ [
+ 1.621009025827E9,
+ "943"
+ ],
+ [
+ 1.621009030827E9,
+ "943"
+ ],
+ [
+ 1.621009035827E9,
+ "66"
+ ],
+ [
+ 1.621009040827E9,
+ "66"
+ ],
+ [
+ 1.621009045827E9,
+ "66"
+ ],
+ [
+ 1.621009050827E9,
+ "841"
+ ],
+ [
+ 1.621009055827E9,
+ "841"
+ ],
+ [
+ 1.621009060827E9,
+ "841"
+ ],
+ [
+ 1.621009065827E9,
+ "405"
+ ],
+ [
+ 1.621009070827E9,
+ "405"
+ ],
+ [
+ 1.621009075827E9,
+ "405"
+ ],
+ [
+ 1.621009080827E9,
+ "201"
+ ],
+ [
+ 1.621009085827E9,
+ "201"
+ ],
+ [
+ 1.621009090827E9,
+ "201"
+ ],
+ [
+ 1.621009095827E9,
+ "227"
+ ],
+ [
+ 1.621009100827E9,
+ "227"
+ ],
+ [
+ 1.621009105827E9,
+ "227"
+ ],
+ [
+ 1.621009110827E9,
+ "943"
+ ]
+ ]
+ }
+ ]
+ ],
+ "threshold": 2000,
+ "warmup": 0
+}
\ No newline at end of file
diff --git a/slope-evaluator/resources/test-3-rep-success.json b/slope-evaluator/resources/test-3-rep-success.json
new file mode 100644
index 0000000000000000000000000000000000000000..485966cba40f01e4a646e626914510ba49b707bc
--- /dev/null
+++ b/slope-evaluator/resources/test-3-rep-success.json
@@ -0,0 +1,289 @@
+{
+ "total_lags": [
+ [
+ {
+ "metric": {
+ "group": "theodolite-uc1-application-0.0.1"
+ },
+ "values": [
+ [
+ 1.621012384232E9,
+ "6073"
+ ],
+ [
+ 1.621012389232E9,
+ "6073"
+ ],
+ [
+ 1.621012394232E9,
+ "6073"
+ ],
+ [
+ 1.621012399232E9,
+ "227"
+ ],
+ [
+ 1.621012404232E9,
+ "227"
+ ],
+ [
+ 1.621012409232E9,
+ "227"
+ ],
+ [
+ 1.621012414232E9,
+ "987"
+ ],
+ [
+ 1.621012419232E9,
+ "987"
+ ],
+ [
+ 1.621012424232E9,
+ "987"
+ ],
+ [
+ 1.621012429232E9,
+ "100"
+ ],
+ [
+ 1.621012434232E9,
+ "100"
+ ],
+ [
+ 1.621012439232E9,
+ "100"
+ ],
+ [
+ 1.621012444232E9,
+ "959"
+ ],
+ [
+ 1.621012449232E9,
+ "959"
+ ],
+ [
+ 1.621012454232E9,
+ "959"
+ ],
+ [
+ 1.621012459232E9,
+ "625"
+ ],
+ [
+ 1.621012464232E9,
+ "625"
+ ],
+ [
+ 1.621012469232E9,
+ "625"
+ ],
+ [
+ 1.621012474232E9,
+ "683"
+ ],
+ [
+ 1.621012479232E9,
+ "683"
+ ],
+ [
+ 1.621012484232E9,
+ "683"
+ ],
+ [
+ 1.621012489232E9,
+ "156"
+ ]
+ ]
+ }
+ ],
+ [
+ {
+ "metric": {
+ "group": "theodolite-uc1-application-0.0.1"
+ },
+ "values": [
+ [
+ 1.621012545211E9,
+ "446"
+ ],
+ [
+ 1.621012550211E9,
+ "446"
+ ],
+ [
+ 1.621012555211E9,
+ "446"
+ ],
+ [
+ 1.621012560211E9,
+ "801"
+ ],
+ [
+ 1.621012565211E9,
+ "801"
+ ],
+ [
+ 1.621012570211E9,
+ "801"
+ ],
+ [
+ 1.621012575211E9,
+ "773"
+ ],
+ [
+ 1.621012580211E9,
+ "773"
+ ],
+ [
+ 1.621012585211E9,
+ "773"
+ ],
+ [
+ 1.621012590211E9,
+ "509"
+ ],
+ [
+ 1.621012595211E9,
+ "509"
+ ],
+ [
+ 1.621012600211E9,
+ "509"
+ ],
+ [
+ 1.621012605211E9,
+ "736"
+ ],
+ [
+ 1.621012610211E9,
+ "736"
+ ],
+ [
+ 1.621012615211E9,
+ "736"
+ ],
+ [
+ 1.621012620211E9,
+ "903"
+ ],
+ [
+ 1.621012625211E9,
+ "903"
+ ],
+ [
+ 1.621012630211E9,
+ "903"
+ ],
+ [
+ 1.621012635211E9,
+ "512"
+ ],
+ [
+ 1.621012640211E9,
+ "512"
+ ],
+ [
+ 1.621012645211E9,
+ "512"
+ ]
+ ]
+ }
+ ],
+ [
+ {
+ "metric": {
+ "group": "theodolite-uc1-application-0.0.1"
+ },
+ "values": [
+ [
+ 1.621012700748E9,
+ "6484"
+ ],
+ [
+ 1.621012705748E9,
+ "6484"
+ ],
+ [
+ 1.621012710748E9,
+ "6484"
+ ],
+ [
+ 1.621012715748E9,
+ "505"
+ ],
+ [
+ 1.621012720748E9,
+ "505"
+ ],
+ [
+ 1.621012725748E9,
+ "505"
+ ],
+ [
+ 1.621012730748E9,
+ "103"
+ ],
+ [
+ 1.621012735748E9,
+ "103"
+ ],
+ [
+ 1.621012740748E9,
+ "103"
+ ],
+ [
+ 1.621012745748E9,
+ "201"
+ ],
+ [
+ 1.621012750748E9,
+ "201"
+ ],
+ [
+ 1.621012755748E9,
+ "201"
+ ],
+ [
+ 1.621012760748E9,
+ "965"
+ ],
+ [
+ 1.621012765748E9,
+ "965"
+ ],
+ [
+ 1.621012770748E9,
+ "965"
+ ],
+ [
+ 1.621012775748E9,
+ "876"
+ ],
+ [
+ 1.621012780748E9,
+ "876"
+ ],
+ [
+ 1.621012785748E9,
+ "876"
+ ],
+ [
+ 1.621012790748E9,
+ "380"
+ ],
+ [
+ 1.621012795748E9,
+ "380"
+ ],
+ [
+ 1.621012800748E9,
+ "380"
+ ]
+ ]
+ }
+ ]
+ ],
+ "threshold": 2000,
+ "warmup": 0
+}
\ No newline at end of file
diff --git a/theodolite-benchmarks/uc1-flink/src/main/java/theodolite/uc1/application/ConfigurationKeys.java b/theodolite-benchmarks/uc1-flink/src/main/java/theodolite/uc1/application/ConfigurationKeys.java
index ed961bab733a409dc07b1be7fa35562103c3e2f4..382525cfe75f82dbbe8fbcc85308b0e7788a43bc 100644
--- a/theodolite-benchmarks/uc1-flink/src/main/java/theodolite/uc1/application/ConfigurationKeys.java
+++ b/theodolite-benchmarks/uc1-flink/src/main/java/theodolite/uc1/application/ConfigurationKeys.java
@@ -19,6 +19,8 @@ public final class ConfigurationKeys {
public static final String CHECKPOINTING = "checkpointing";
+ public static final String PARALLELISM = "parallelism";
+
private ConfigurationKeys() {}
}
diff --git a/theodolite-benchmarks/uc1-flink/src/main/java/theodolite/uc1/application/HistoryServiceFlinkJob.java b/theodolite-benchmarks/uc1-flink/src/main/java/theodolite/uc1/application/HistoryServiceFlinkJob.java
index 6655b52ec3020f46bb8a37c7124ee870fa663573..8d9832e40253fe9e3178bfc25047ed2b376abe76 100644
--- a/theodolite-benchmarks/uc1-flink/src/main/java/theodolite/uc1/application/HistoryServiceFlinkJob.java
+++ b/theodolite-benchmarks/uc1-flink/src/main/java/theodolite/uc1/application/HistoryServiceFlinkJob.java
@@ -1,6 +1,7 @@
package theodolite.uc1.application;
import org.apache.commons.configuration2.Configuration;
+import org.apache.flink.api.common.typeinfo.Types;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;
@@ -42,6 +43,14 @@ public final class HistoryServiceFlinkJob {
if (checkpointing) {
this.env.enableCheckpointing(commitIntervalMs);
}
+
+ // Parallelism
+ final Integer parallelism = this.config.getInteger(ConfigurationKeys.PARALLELISM, null);
+ if (parallelism != null) {
+ LOGGER.error("Set parallelism: {}.", parallelism);
+ this.env.setParallelism(parallelism);
+ }
+
}
private void buildPipeline() {
@@ -61,7 +70,8 @@ public final class HistoryServiceFlinkJob {
stream
.rebalance()
.map(new GsonMapper())
- .flatMap((record, c) -> LOGGER.info("Record: {}", record));
+ .flatMap((record, c) -> LOGGER.info("Record: {}", record))
+ .returns(Types.GENERIC(Object.class)); // Will never be used
}
/**
diff --git a/theodolite-benchmarks/uc2-flink/src/main/java/theodolite/uc2/application/ConfigurationKeys.java b/theodolite-benchmarks/uc2-flink/src/main/java/theodolite/uc2/application/ConfigurationKeys.java
index 9ba56c828a0ae5c6147aadd90d449c7cf2324992..e8261062689ce4c586a4e6fbde02878a28f48e97 100644
--- a/theodolite-benchmarks/uc2-flink/src/main/java/theodolite/uc2/application/ConfigurationKeys.java
+++ b/theodolite-benchmarks/uc2-flink/src/main/java/theodolite/uc2/application/ConfigurationKeys.java
@@ -30,6 +30,8 @@ public final class ConfigurationKeys {
public static final String CHECKPOINTING = "checkpointing";
+ public static final String PARALLELISM = "parallelism";
+
private ConfigurationKeys() {}
}
diff --git a/theodolite-benchmarks/uc2-flink/src/main/java/theodolite/uc2/application/HistoryServiceFlinkJob.java b/theodolite-benchmarks/uc2-flink/src/main/java/theodolite/uc2/application/HistoryServiceFlinkJob.java
index b8452847df800226ad481f9309323a2a9a532939..1068267086892c4538001b6afc670b3b0cd043ef 100644
--- a/theodolite-benchmarks/uc2-flink/src/main/java/theodolite/uc2/application/HistoryServiceFlinkJob.java
+++ b/theodolite-benchmarks/uc2-flink/src/main/java/theodolite/uc2/application/HistoryServiceFlinkJob.java
@@ -56,6 +56,13 @@ public final class HistoryServiceFlinkJob {
this.env.enableCheckpointing(commitIntervalMs);
}
+ // Parallelism
+ final Integer parallelism = this.config.getInteger(ConfigurationKeys.PARALLELISM, null);
+ if (parallelism != null) {
+ LOGGER.error("Set parallelism: {}.", parallelism);
+ this.env.setParallelism(parallelism);
+ }
+
// State Backend
final StateBackend stateBackend = StateBackends.fromConfiguration(this.config);
this.env.setStateBackend(stateBackend);
diff --git a/theodolite-benchmarks/uc3-flink/src/main/java/theodolite/uc3/application/ConfigurationKeys.java b/theodolite-benchmarks/uc3-flink/src/main/java/theodolite/uc3/application/ConfigurationKeys.java
index a895c74d89c5d788c47b3b78dc70500b4b5a6f5b..bc4e0b9d2d230026e9d2b6df0a11e4fb68380aed 100644
--- a/theodolite-benchmarks/uc3-flink/src/main/java/theodolite/uc3/application/ConfigurationKeys.java
+++ b/theodolite-benchmarks/uc3-flink/src/main/java/theodolite/uc3/application/ConfigurationKeys.java
@@ -34,6 +34,8 @@ public final class ConfigurationKeys {
public static final String CHECKPOINTING = "checkpointing";
+ public static final String PARALLELISM = "parallelism";
+
private ConfigurationKeys() {}
}
diff --git a/theodolite-benchmarks/uc3-flink/src/main/java/theodolite/uc3/application/HistoryServiceFlinkJob.java b/theodolite-benchmarks/uc3-flink/src/main/java/theodolite/uc3/application/HistoryServiceFlinkJob.java
index 0f26d37652924a16be1840fd759b3cd5b023f338..d69ee47d8c831f2e5e74abdd8c33393c8ee6e07e 100644
--- a/theodolite-benchmarks/uc3-flink/src/main/java/theodolite/uc3/application/HistoryServiceFlinkJob.java
+++ b/theodolite-benchmarks/uc3-flink/src/main/java/theodolite/uc3/application/HistoryServiceFlinkJob.java
@@ -63,6 +63,13 @@ public final class HistoryServiceFlinkJob {
this.env.enableCheckpointing(commitIntervalMs);
}
+ // Parallelism
+ final Integer parallelism = this.config.getInteger(ConfigurationKeys.PARALLELISM, null);
+ if (parallelism != null) {
+ LOGGER.error("Set parallelism: {}.", parallelism);
+ this.env.setParallelism(parallelism);
+ }
+
// State Backend
final StateBackend stateBackend = StateBackends.fromConfiguration(this.config);
this.env.setStateBackend(stateBackend);
diff --git a/theodolite-benchmarks/uc4-flink/src/main/java/theodolite/uc4/application/AggregationServiceFlinkJob.java b/theodolite-benchmarks/uc4-flink/src/main/java/theodolite/uc4/application/AggregationServiceFlinkJob.java
index 0db5a3d524f74fbf22304e8f9b44fa55eead321a..45c7ff1ad1faeec6357e4ac3871dec7a51306698 100644
--- a/theodolite-benchmarks/uc4-flink/src/main/java/theodolite/uc4/application/AggregationServiceFlinkJob.java
+++ b/theodolite-benchmarks/uc4-flink/src/main/java/theodolite/uc4/application/AggregationServiceFlinkJob.java
@@ -76,6 +76,13 @@ public final class AggregationServiceFlinkJob {
this.env.enableCheckpointing(commitIntervalMs);
}
+ // Parallelism
+ final Integer parallelism = this.config.getInteger(ConfigurationKeys.PARALLELISM, null);
+ if (parallelism != null) {
+ LOGGER.error("Set parallelism: {}.", parallelism);
+ this.env.setParallelism(parallelism);
+ }
+
// State Backend
final StateBackend stateBackend = StateBackends.fromConfiguration(this.config);
this.env.setStateBackend(stateBackend);
diff --git a/theodolite-benchmarks/uc4-flink/src/main/java/theodolite/uc4/application/ConfigurationKeys.java b/theodolite-benchmarks/uc4-flink/src/main/java/theodolite/uc4/application/ConfigurationKeys.java
index 6497f6b055ef115c4a681499c5fa38657bb5d29e..448e8b095ef15c434655ca3c76a9e2de21244054 100644
--- a/theodolite-benchmarks/uc4-flink/src/main/java/theodolite/uc4/application/ConfigurationKeys.java
+++ b/theodolite-benchmarks/uc4-flink/src/main/java/theodolite/uc4/application/ConfigurationKeys.java
@@ -15,7 +15,7 @@ public final class ConfigurationKeys {
public static final String KAFKA_OUTPUT_TOPIC = "kafka.output.topic";
public static final String KAFKA_INPUT_TOPIC = "kafka.input.topic";
-
+
public static final String SCHEMA_REGISTRY_URL = "schema.registry.url";
public static final String WINDOW_SIZE_MS = "window.size.ms";
@@ -28,13 +28,15 @@ public final class ConfigurationKeys {
public static final String FLINK_STATE_BACKEND_PATH = "flink.state.backend.path";
- public static final String FLINK_STATE_BACKEND_MEMORY_SIZE = //NOPMD
+ public static final String FLINK_STATE_BACKEND_MEMORY_SIZE = // NOPMD
"flink.state.backend.memory.size";
public static final String DEBUG = "debug";
public static final String CHECKPOINTING = "checkpointing";
+ public static final String PARALLELISM = "parallelism";
+
private ConfigurationKeys() {}
}
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/Benchmark.kt b/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/Benchmark.kt
index 95af36d1b14eb4d664ffd0d3d463df17fd8fb534..05d021b1bcfb77fa8ffeb0522510d49e39ef501c 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/Benchmark.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/Benchmark.kt
@@ -22,6 +22,7 @@ interface Benchmark {
load: LoadDimension,
res: Resource,
configurationOverrides: List<ConfigurationOverride?>,
- delay: Long
+ loadGenerationDelay: Long,
+ afterTeardownDelay: Long
): BenchmarkDeployment
}
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt b/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt
index 40ab663e179735be61356947df7c37e9edd0a2ea..38d0f0389ce92a8720df05e892d11cf4f1ac480a 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt
@@ -48,6 +48,7 @@ class BenchmarkExecution : CustomResource(), Namespaced {
var repetitions by Delegates.notNull<Int>()
lateinit var restrictions: List<String>
var loadGenerationDelay = 0L
+ var afterTeardownDelay = 5L
}
/**
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt b/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt
index b5316a3df78e7291569ec1d36e561a5445b6f86d..c89e9e85323d6e51c104b1d34ff1ef9d8d4d60cd 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt
@@ -71,7 +71,8 @@ class KubernetesBenchmark : Benchmark, CustomResource(), Namespaced {
load: LoadDimension,
res: Resource,
configurationOverrides: List<ConfigurationOverride?>,
- loadGenerationDelay: Long
+ loadGenerationDelay: Long,
+ afterTeardownDelay: Long
): BenchmarkDeployment {
logger.info { "Using $namespace as namespace." }
logger.info { "Using $path as resource path." }
@@ -100,6 +101,7 @@ class KubernetesBenchmark : Benchmark, CustomResource(), Namespaced {
appResources = appResources.map { it.second },
loadGenResources = loadGenResources.map { it.second },
loadGenerationDelay = loadGenerationDelay,
+ afterTeardownDelay = afterTeardownDelay,
kafkaConfig = hashMapOf("bootstrap.servers" to kafkaConfig.bootstrapServer),
topics = kafkaConfig.topics,
client = DefaultKubernetesClient().inNamespace(namespace)
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmarkDeployment.kt b/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmarkDeployment.kt
index 7d1c3f4ba727b8c149e3c5678ad406fa293c8f5b..6cf239676ddb24752f4754a85fc62657f9eb6603 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmarkDeployment.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmarkDeployment.kt
@@ -26,6 +26,7 @@ class KubernetesBenchmarkDeployment(
val appResources: List<KubernetesResource>,
val loadGenResources: List<KubernetesResource>,
private val loadGenerationDelay: Long,
+ private val afterTeardownDelay: Long,
private val kafkaConfig: HashMap<String, Any>,
private val topics: List<KafkaConfig.TopicWrapper>,
private val client: NamespacedKubernetesClient
@@ -33,7 +34,6 @@ class KubernetesBenchmarkDeployment(
private val kafkaController = TopicManager(this.kafkaConfig)
private val kubernetesManager = K8sManager(client)
private val LAG_EXPORTER_POD_LABEL = "app.kubernetes.io/name=kafka-lag-exporter"
- private val SLEEP_AFTER_TEARDOWN = 5000L
/**
* Setup a [KubernetesBenchmark] using the [TopicManager] and the [K8sManager]:
@@ -61,7 +61,7 @@ class KubernetesBenchmarkDeployment(
appResources.forEach { kubernetesManager.remove(it) }
kafkaController.removeTopics(this.topics.map { topic -> topic.name })
KafkaLagExporterRemover(client).remove(LAG_EXPORTER_POD_LABEL)
- logger.info { "Teardown complete. Wait $SLEEP_AFTER_TEARDOWN ms to let everything come down." }
- Thread.sleep(SLEEP_AFTER_TEARDOWN)
+ logger.info { "Teardown complete. Wait $afterTeardownDelay ms to let everything come down." }
+ Thread.sleep(Duration.ofSeconds(afterTeardownDelay).toMillis())
}
}
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt
index 96f32f634f1d0164b1e25b36fe44de2162c86bfe..ef4d371173c7099eb091f90cddbe26d31e6522be 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt
@@ -35,23 +35,27 @@ class AnalysisExecutor(
* @param executionDuration of the experiment.
* @return true if the experiment succeeded.
*/
- fun analyze(load: LoadDimension, res: Resource, executionDuration: Duration): Boolean {
+ fun analyze(load: LoadDimension, res: Resource, executionIntervals: List<Pair<Instant, Instant>>): Boolean {
var result = false
+ var repetitionCounter = 1
try {
- val prometheusData = fetcher.fetchMetric(
- start = Instant.now().minus(executionDuration),
- end = Instant.now(),
- query = "sum by(group)(kafka_consumergroup_group_lag >= 0)"
- )
-
val ioHandler = IOHandler()
val resultsFolder: String = ioHandler.getResultFolderURL()
val fileURL = "${resultsFolder}exp${executionId}_${load.get()}_${res.get()}_${slo.sloType.toSlug()}"
- ioHandler.writeToCSVFile(
- fileURL = fileURL,
- data = prometheusData.getResultAsList(),
- columns = listOf("group", "timestamp", "value"))
+
+ val prometheusData = executionIntervals
+ .map { interval -> fetcher.fetchMetric(
+ start = interval.first,
+ end = interval.second,
+ query = "sum by(group)(kafka_consumergroup_group_lag >= 0)") }
+
+ prometheusData.forEach{ data ->
+ ioHandler.writeToCSVFile(
+ fileURL = "${fileURL}_${repetitionCounter++}",
+ data = data.getResultAsList(),
+ columns = listOf("group", "timestamp", "value"))
+ }
val sloChecker = SloCheckerFactory().create(
sloType = slo.sloType,
@@ -60,10 +64,7 @@ class AnalysisExecutor(
warmup = slo.warmup
)
- result = sloChecker.evaluate(
- start = Instant.now().minus(executionDuration),
- end = Instant.now(), fetchedData = prometheusData
- )
+ result = sloChecker.evaluate(prometheusData)
} catch (e: Exception) {
logger.error { "Evaluation failed for resource '${res.get()}' and load '${load.get()}'. Error: $e" }
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt
index 8eac05a815e0bc45e1abc38bcdf0352e61bd7730..f7ebee8faf740583dbe6a37381a599e9bde19280 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt
@@ -35,10 +35,10 @@ class ExternalSloChecker(
* @return true if the experiment was successful(the threshold was not exceeded.
* @throws ConnectException if the external service could not be reached.
*/
- override fun evaluate(start: Instant, end: Instant, fetchedData: PrometheusResponse): Boolean {
+ override fun evaluate(fetchedData: List<PrometheusResponse>): Boolean {
var counter = 0
val data = Gson().toJson(mapOf(
- "total_lag" to fetchedData.data?.result,
+ "total_lags" to fetchedData.map { it.data?.result},
"threshold" to threshold,
"warmup" to warmup))
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SloChecker.kt b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SloChecker.kt
index 758dcdefcaee79654d8ff65f0f798832aafc1294..9ee5fe7ef34ce5b6214882ce2c1d19677f1d7130 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SloChecker.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SloChecker.kt
@@ -1,14 +1,12 @@
package theodolite.evaluation
import theodolite.util.PrometheusResponse
-import java.time.Instant
/**
* A SloChecker can be used to evaluate data from Prometheus.
* @constructor Creates an empty SloChecker
*/
interface SloChecker {
-
/**
* Evaluates [fetchedData] and returns if the experiment was successful.
* Returns if the evaluated experiment was successful.
@@ -18,5 +16,5 @@ interface SloChecker {
* @param fetchedData from Prometheus that will be evaluated.
* @return true if experiment was successful. Otherwise false.
*/
- fun evaluate(start: Instant, end: Instant, fetchedData: PrometheusResponse): Boolean
+ fun evaluate(fetchedData: List<PrometheusResponse>): Boolean
}
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt
index 909ae77a95dfef7dfc1dafd7bd0a326a72ef1424..e7b511d8c83b5abccece1204aad2a4a9ecfdfd26 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt
@@ -26,8 +26,10 @@ abstract class BenchmarkExecutor(
val executionDuration: Duration,
val configurationOverrides: List<ConfigurationOverride?>,
val slo: BenchmarkExecution.Slo,
+ val repetitions: Int,
val executionId: Int,
- val loadGenerationDelay: Long
+ val loadGenerationDelay: Long,
+ val afterTeardownDelay: Long
) {
var run: AtomicBoolean = AtomicBoolean(true)
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt
index 22db5076bee51e03318aa9ac95e66c2d61d27f28..3afc85f0a8cb67011763498a662b447ce2c07f0f 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt
@@ -5,11 +5,9 @@ import mu.KotlinLogging
import theodolite.benchmark.Benchmark
import theodolite.benchmark.BenchmarkExecution
import theodolite.evaluation.AnalysisExecutor
-import theodolite.util.ConfigurationOverride
-import theodolite.util.LoadDimension
-import theodolite.util.Resource
-import theodolite.util.Results
+import theodolite.util.*
import java.time.Duration
+import java.time.Instant
private val logger = KotlinLogging.logger {}
@@ -20,41 +18,56 @@ class BenchmarkExecutorImpl(
executionDuration: Duration,
configurationOverrides: List<ConfigurationOverride?>,
slo: BenchmarkExecution.Slo,
+ repetitions: Int,
executionId: Int,
- loadGenerationDelay: Long
-) : BenchmarkExecutor(benchmark, results, executionDuration, configurationOverrides, slo, executionId, loadGenerationDelay) {
+ loadGenerationDelay: Long,
+ afterTeardownDelay: Long
+) : BenchmarkExecutor(benchmark, results, executionDuration, configurationOverrides, slo, repetitions, executionId, loadGenerationDelay, afterTeardownDelay) {
override fun runExperiment(load: LoadDimension, res: Resource): Boolean {
var result = false
- val benchmarkDeployment = benchmark.buildDeployment(load, res, configurationOverrides, loadGenerationDelay)
+ val executionIntervals: MutableList<Pair<Instant, Instant>> = ArrayList()
- try {
- benchmarkDeployment.setup()
- this.waitAndLog()
- } catch (e: Exception) {
- logger.error { "Error while setting up experiment with id ${this.executionId}." }
- logger.error { "Error is: $e" }
- this.run.set(false)
+ for (i in 1.rangeTo(repetitions)) {
+ logger.info { "Run repetition $i/$repetitions" }
+ if (this.run.get()) {
+ executionIntervals.add(runSingleExperiment(load,res))
+ } else {
+ break
+ }
}
/**
* Analyse the experiment, if [run] is true, otherwise the experiment was canceled by the user.
*/
if (this.run.get()) {
- result = AnalysisExecutor(slo = slo, executionId = executionId).analyze(
- load = load,
- res = res,
- executionDuration = executionDuration
- )
+ result =AnalysisExecutor(slo = slo, executionId = executionId)
+ .analyze(
+ load = load,
+ res = res,
+ executionIntervals = executionIntervals)
this.results.setResult(Pair(load, res), result)
}
+ return result
+ }
+ private fun runSingleExperiment(load: LoadDimension, res: Resource): Pair<Instant, Instant> {
+ val benchmarkDeployment = benchmark.buildDeployment(load, res, this.configurationOverrides, this.loadGenerationDelay, this.afterTeardownDelay)
+ val from = Instant.now()
+ try {
+ benchmarkDeployment.setup()
+ this.waitAndLog()
+ } catch (e: Exception) {
+ logger.error { "Error while setup experiment." }
+ logger.error { "Error is: $e" }
+ this.run.set(false)
+ }
+ val to = Instant.now()
try {
benchmarkDeployment.teardown()
} catch (e: Exception) {
logger.warn { "Error while tearing down the benchmark deployment." }
logger.debug { "Teardown failed, caused by: $e" }
}
-
- return result
+ return Pair(from,to)
}
}
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/Shutdown.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/Shutdown.kt
index 80e1ac2341d9f8608d70b39a0d99f7b88dd7f108..0ff8379a0af4b11154214dde021d7c60609631d1 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/Shutdown.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/Shutdown.kt
@@ -31,7 +31,8 @@ class Shutdown(private val benchmarkExecution: BenchmarkExecution, private val b
load = LoadDimension(0, emptyList()),
res = Resource(0, emptyList()),
configurationOverrides = benchmarkExecution.configOverrides,
- loadGenerationDelay = 0L
+ loadGenerationDelay = 0L,
+ afterTeardownDelay = 5L
)
deployment.teardown()
} catch (e: Exception) {
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt
index c437dd3b1b1adefc3657b7c43c671512f5713858..54201ca330db7882c8637ad6e8bb33816dc3372a 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt
@@ -1,6 +1,5 @@
package theodolite.execution
-import com.google.gson.GsonBuilder
import mu.KotlinLogging
import theodolite.benchmark.BenchmarkExecution
import theodolite.benchmark.KubernetesBenchmark
@@ -9,11 +8,6 @@ import theodolite.strategies.StrategyFactory
import theodolite.strategies.searchstrategy.CompositeStrategy
import theodolite.util.*
import java.io.File
-import java.io.PrintWriter
-import java.lang.IllegalArgumentException
-import java.lang.Thread.sleep
-import java.nio.file.Files
-import java.nio.file.Path
import java.time.Duration
@@ -68,8 +62,10 @@ class TheodoliteExecutor(
executionDuration = executionDuration,
configurationOverrides = config.configOverrides,
slo = config.slos[0],
+ repetitions = config.execution.repetitions,
executionId = config.executionId,
- loadGenerationDelay = config.execution.loadGenerationDelay
+ loadGenerationDelay = config.execution.loadGenerationDelay,
+ afterTeardownDelay = config.execution.afterTeardownDelay
)
return Config(
diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteYamlExecutor.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteYamlExecutor.kt
index 8b2909f7658f4dffcfd961cad8cd00eb013a160c..b9977029703c8012ada7fb3d7766bfa321a836c3 100644
--- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteYamlExecutor.kt
+++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteYamlExecutor.kt
@@ -31,8 +31,8 @@ class TheodoliteYamlExecutor {
fun start() {
logger.info { "Theodolite started" }
- val executionPath = System.getenv("THEODOLITE_EXECUTION") ?: "./config/BenchmarkExecution.yaml"
- val benchmarkPath = System.getenv("THEODOLITE_BENCHMARK") ?: "./config/BenchmarkType.yaml"
+ val executionPath = System.getenv("THEODOLITE_EXECUTION") ?: "./config/example-execution-yaml-resource.yaml"
+ val benchmarkPath = System.getenv("THEODOLITE_BENCHMARK") ?: "./config/example-benchmark-yaml-resource.yaml"
logger.info { "Using $executionPath for BenchmarkExecution" }
logger.info { "Using $benchmarkPath for BenchmarkType" }
diff --git a/theodolite-quarkus/src/test/kotlin/theodolite/CompositeStrategyTest.kt b/theodolite-quarkus/src/test/kotlin/theodolite/CompositeStrategyTest.kt
index 726d4490b053b5c56f0a0387f54ad557d4c1865c..c2b30ab7eb23d60db39778218ad9d6a4c12799a6 100644
--- a/theodolite-quarkus/src/test/kotlin/theodolite/CompositeStrategyTest.kt
+++ b/theodolite-quarkus/src/test/kotlin/theodolite/CompositeStrategyTest.kt
@@ -31,7 +31,7 @@ class CompositeStrategyTest {
val results = Results()
val benchmark = TestBenchmark()
val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo()
- val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, sloChecker, 0, 0)
+ val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, sloChecker, 0, 0, 5)
val linearSearch = LinearSearch(benchmarkExecutor)
val lowerBoundRestriction = LowerBoundRestriction(results)
val strategy =
@@ -65,7 +65,7 @@ class CompositeStrategyTest {
val benchmark = TestBenchmark()
val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo()
val benchmarkExecutorImpl =
- TestBenchmarkExecutorImpl(mockResults, benchmark, results, sloChecker, 0, 0)
+ TestBenchmarkExecutorImpl(mockResults, benchmark, results, sloChecker, 0, 0, 0)
val binarySearch = BinarySearch(benchmarkExecutorImpl)
val lowerBoundRestriction = LowerBoundRestriction(results)
val strategy =
@@ -98,7 +98,7 @@ class CompositeStrategyTest {
val results = Results()
val benchmark = TestBenchmark()
val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo()
- val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, sloChecker, 0, 0)
+ val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, sloChecker, 0, 0,0)
val binarySearch = BinarySearch(benchmarkExecutor)
val lowerBoundRestriction = LowerBoundRestriction(results)
val strategy =
diff --git a/theodolite-quarkus/src/test/kotlin/theodolite/TestBenchmark.kt b/theodolite-quarkus/src/test/kotlin/theodolite/TestBenchmark.kt
index 71e3cbcc78208cc4227e369d59d841e39adb87ed..6ddca296acced658609fcccb307a83047238e118 100644
--- a/theodolite-quarkus/src/test/kotlin/theodolite/TestBenchmark.kt
+++ b/theodolite-quarkus/src/test/kotlin/theodolite/TestBenchmark.kt
@@ -12,7 +12,8 @@ class TestBenchmark : Benchmark {
load: LoadDimension,
res: Resource,
configurationOverrides: List<ConfigurationOverride?>,
- loadGenerationDelay: Long
+ loadGenerationDelay: Long,
+ afterTeardownDelay: Long
): BenchmarkDeployment {
return TestBenchmarkDeployment()
}
diff --git a/theodolite-quarkus/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt b/theodolite-quarkus/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt
index 294727d6a252a29e312ff532937e7fdf9f079ac7..cbd2d5926d61b0bfd4de6fab0c14422ddf88f190 100644
--- a/theodolite-quarkus/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt
+++ b/theodolite-quarkus/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt
@@ -14,7 +14,8 @@ class TestBenchmarkExecutorImpl(
results: Results,
slo: BenchmarkExecution.Slo,
executionId: Int,
- loadGenerationDelay: Long
+ loadGenerationDelay: Long,
+ afterTeardownDelay: Long
) :
BenchmarkExecutor(
benchmark,
@@ -22,8 +23,10 @@ class TestBenchmarkExecutorImpl(
executionDuration = Duration.ofSeconds(1),
configurationOverrides = emptyList(),
slo = slo,
+ repetitions = 1,
executionId = executionId,
- loadGenerationDelay = loadGenerationDelay
+ loadGenerationDelay = loadGenerationDelay,
+ afterTeardownDelay = afterTeardownDelay
) {
override fun runExperiment(load: LoadDimension, res: Resource): Boolean {