diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 433facfa99765228af9b753cf88378fc9d939a88..c4b8495224ad81ae2eebe81b0087319422da1969 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -319,7 +319,18 @@ test-slo-checker-lag-trend:
tags:
- exec-docker
script:
- - cd slope-evaluator
+ - cd slo-checker/record-lag
+ - pip install -r requirements.txt
+ - cd app
+ - python -m unittest
+
+test-slo-checker-dropped-records-kstreams:
+ stage: test
+ image: python:3.7-slim
+ tags:
+ - exec-docker
+ script:
+ - cd slo-checker/dropped-records
- pip install -r requirements.txt
- cd app
- python -m unittest
@@ -332,7 +343,7 @@ deploy-slo-checker-lag-trend:
- test-slo-checker-lag-trend
script:
- DOCKER_TAG_NAME=$(echo $CI_COMMIT_REF_SLUG- | sed 's/^master-$//')
- - docker build --pull -t theodolite-slo-checker-lag-trend slope-evaluator
+ - docker build --pull -t theodolite-slo-checker-lag-trend slo-checker/record-lag
- "[ ! $CI_COMMIT_TAG ] && docker tag theodolite-slo-checker-lag-trend $CR_HOST/$CR_ORG/theodolite-slo-checker-lag-trend:${DOCKER_TAG_NAME}latest"
- "[ $CI_COMMIT_TAG ] && docker tag theodolite-slo-checker-lag-trend $CR_HOST/$CR_ORG/theodolite-slo-checker-lag-trend:$CI_COMMIT_TAG"
- echo $CR_PW | docker login $CR_HOST -u $CR_USER --password-stdin
@@ -342,7 +353,32 @@ deploy-slo-checker-lag-trend:
- if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW && $CI_COMMIT_TAG"
when: always
- changes:
- - slope-evaluator/**/*
+ - slo-checker/record-lag/**/*
+ if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW"
+ when: always
+ - if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW"
+ when: manual
+ allow_failure: true
+
+deploy-slo-checker-dropped-records-kstreams:
+ stage: deploy
+ extends:
+ - .dind
+ needs:
+ - test-slo-checker-dropped-records-kstreams
+ script:
+ - DOCKER_TAG_NAME=$(echo $CI_COMMIT_REF_SLUG- | sed 's/^master-$//')
+ - docker build --pull -t theodolite-slo-checker-dropped-records-kstreams slo-checker/dropped-records
+ - "[ ! $CI_COMMIT_TAG ] && docker tag theodolite-slo-checker-dropped-records-kstreams $CR_HOST/$CR_ORG/theodolite-slo-checker-dropped-records-kstreams:${DOCKER_TAG_NAME}latest"
+ - "[ $CI_COMMIT_TAG ] && docker tag theodolite-slo-checker-dropped-records-kstreams $CR_HOST/$CR_ORG/theodolite-slo-checker-dropped-records-kstreams:$CI_COMMIT_TAG"
+ - echo $CR_PW | docker login $CR_HOST -u $CR_USER --password-stdin
+ - docker push $CR_HOST/$CR_ORG/theodolite-slo-checker-dropped-records-kstreams
+ - docker logout
+ rules:
+ - if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW && $CI_COMMIT_TAG"
+ when: always
+ - changes:
+ - slo-checker/dropped-records/**/*
if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW"
when: always
- if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW"
diff --git a/CITATION.cff b/CITATION.cff
index 4ec1e6c14d466c84478208ecff81e0f24629af0f..ca94e1c5039d3aeac3a4535767d5217de4960a6f 100644
--- a/CITATION.cff
+++ b/CITATION.cff
@@ -8,7 +8,7 @@ authors:
given-names: Wilhelm
orcid: "https://orcid.org/0000-0001-6625-4335"
title: Theodolite
-version: "0.5.0"
+version: "0.5.1"
repository-code: "https://github.com/cau-se/theodolite"
license: "Apache-2.0"
doi: "10.1016/j.bdr.2021.100209"
diff --git a/codemeta.json b/codemeta.json
index fd2fbe137fc4f8e854452998c62ee988829693fb..a158e30eb7f1ab433779678aba3a1cc3b7e33c80 100644
--- a/codemeta.json
+++ b/codemeta.json
@@ -5,10 +5,10 @@
"codeRepository": "https://github.com/cau-se/theodolite",
"dateCreated": "2020-03-13",
"datePublished": "2020-07-27",
- "dateModified": "2021-03-18",
+ "dateModified": "2021-11-12",
"downloadUrl": "https://github.com/cau-se/theodolite/releases",
"name": "Theodolite",
- "version": "0.5.0",
+ "version": "0.5.1",
"description": "Theodolite is a framework for benchmarking the horizontal and vertical scalability of stream processing engines.",
"developmentStatus": "active",
"referencePublication": "https://doi.org/10.1016/j.bdr.2021.100209",
diff --git a/docs/index.yaml b/docs/index.yaml
index 635cf8e5fe879cd89927b1b7d87ae621187b3e94..54580ea45f1c678443dae96c7139f53fdac37f60 100644
--- a/docs/index.yaml
+++ b/docs/index.yaml
@@ -1,6 +1,41 @@
apiVersion: v1
entries:
theodolite:
+ - apiVersion: v2
+ appVersion: 0.5.1
+ created: "2021-11-12T16:15:01.629937292+01:00"
+ dependencies:
+ - condition: grafana.enabled
+ name: grafana
+ repository: https://grafana.github.io/helm-charts
+ version: 6.17.5
+ - condition: kube-prometheus-stack.enabled
+ name: kube-prometheus-stack
+ repository: https://prometheus-community.github.io/helm-charts
+ version: 12.0.0
+ - condition: cp-helm-charts.enabled
+ name: cp-helm-charts
+ repository: https://soerenhenning.github.io/cp-helm-charts
+ version: 0.6.0
+ - condition: kafka-lag-exporter.enabled
+ name: kafka-lag-exporter
+ repository: https://lightbend.github.io/kafka-lag-exporter/repo/
+ version: 0.6.6
+ description: Theodolite is a framework for benchmarking the scalability stream
+ processing engines.
+ digest: a67374c4cb2b0e8b2d711468364c6b4a486a910bd1c667dbf3c5614e36e0680c
+ home: https://cau-se.github.io/theodolite
+ maintainers:
+ - email: soeren.henning@email.uni-kiel.de
+ name: Sören Henning
+ url: https://www.se.informatik.uni-kiel.de/en/team/soeren-henning-m-sc
+ name: theodolite
+ sources:
+ - https://github.com/cau-se/theodolite
+ type: application
+ urls:
+ - https://github.com/cau-se/theodolite/releases/download/v0.5.1/theodolite-0.5.1.tgz
+ version: 0.5.1
- apiVersion: v2
appVersion: 0.5.0
created: "2021-11-04T17:45:14.153231798+01:00"
@@ -71,4 +106,4 @@ entries:
urls:
- https://github.com/cau-se/theodolite/releases/download/v0.4.0/theodolite-0.4.0.tgz
version: 0.4.0
-generated: "2021-11-04T17:45:14.120866992+01:00"
+generated: "2021-11-12T16:15:01.591258889+01:00"
diff --git a/helm/README.md b/helm/README.md
index fe5c15e6fd52122d52c016bff62b8a33e2a22018..1a3428b5e601de0c6c33f9dab236321e95592c6c 100644
--- a/helm/README.md
+++ b/helm/README.md
@@ -9,8 +9,6 @@ helm dependencies update .
helm install theodolite .
```
-Additionally, a PersistentVolumeClaim with the name `theodolite-pv-claim` needs to be created or the results volume can be disabled as done with the minimal configuration, described below.
-
## Customize Installation
As usual, the installation with Helm can be configured by passing a values YAML file:
diff --git a/helm/preconfigs/oci.yaml b/helm/preconfigs/oci.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..35fe5dcf423eed77cb2d3d4298088738125fa9fe
--- /dev/null
+++ b/helm/preconfigs/oci.yaml
@@ -0,0 +1,6 @@
+operator:
+ resultsVolume:
+ persistent:
+ enabled: true
+ storageClassName: "oci-bv"
+ size: 50Gi # minimal size in OCI
diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl
index f59f74d369b64ec89a44cbf2048fda9e844df92b..569cf454a950f3f621c23472f0346c8bbd52229d 100644
--- a/helm/templates/_helpers.tpl
+++ b/helm/templates/_helpers.tpl
@@ -60,3 +60,10 @@ Create the name of the service account to use
{{- default "default" .Values.serviceAccount.name }}
{{- end }}
{{- end }}
+
+{{/*
+Create the name of the results volume to use
+*/}}
+{{- define "theodolite.resultsClaimName" -}}
+{{- default (printf "%s-results" (include "theodolite.fullname" .)) .Values.operator.resultsVolume.persistent.existingClaim }}
+{{- end }}
diff --git a/helm/templates/theodolite/results-volume/pvc.yaml b/helm/templates/theodolite/results-volume/pvc.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..26ac56e42889ccaebbc669791ad4d318b8318fec
--- /dev/null
+++ b/helm/templates/theodolite/results-volume/pvc.yaml
@@ -0,0 +1,18 @@
+{{- if and .Values.operator.resultsVolume.persistent.enabled (not .Values.operator.resultsVolume.persistent.existingClaim) -}}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ name: {{ include "theodolite.resultsClaimName" . }}
+spec:
+ {{- if .Values.operator.resultsVolume.persistent.storageClassName }}
+ storageClassName: {{ .Values.operator.resultsVolume.persistent.storageClassName }}
+ {{- end }}
+ accessModes:
+ - ReadWriteOnce
+ {{- range .Values.operator.resultsVolume.persistent.accessModes }}
+ - {{ . | quote }}
+ {{- end }}
+ resources:
+ requests:
+ storage: {{ .Values.operator.resultsVolume.persistent.size | quote }}
+{{- end }}
\ No newline at end of file
diff --git a/helm/templates/theodolite/theodolite-operator.yaml b/helm/templates/theodolite/theodolite-operator.yaml
index 3487b9a4fabb16897b302d8be65f065a647ffb16..3d1d186306a5d1103a921b4c39e1425621548db8 100644
--- a/helm/templates/theodolite/theodolite-operator.yaml
+++ b/helm/templates/theodolite/theodolite-operator.yaml
@@ -31,10 +31,8 @@ spec:
- name: RESULTS_FOLDER
value: "./results"
volumeMounts:
- {{- if .Values.operator.resultsVolume.enabled }}
- - name: theodolite-pv-storage
+ - name: theodolite-results-volume
mountPath: "/deployments/results"
- {{- end }}
- name: benchmark-resources-uc1-kstreams
mountPath: /deployments/benchmark-resources/uc1-kstreams
- name: benchmark-resources-uc2-kstreams
@@ -64,7 +62,20 @@ spec:
- name: LOG_LEVEL
value: INFO
{{- end }}
- {{- if and .Values.operator.resultsVolume.enabled .Values.operator.resultsVolume.accessSidecar.enabled }}
+ {{- if .Values.operator.sloChecker.droppedRecordsKStreams.enabled }}
+ - name: slo-checker-dropped-records-kstreams
+ image: "{{ .Values.operator.sloChecker.droppedRecordsKStreams.image }}:{{ .Values.operator.sloChecker.droppedRecordsKStreams.imageTag }}"
+ imagePullPolicy: "{{ .Values.operator.sloChecker.droppedRecordsKStreams.imagePullPolicy }}"
+ ports:
+ - containerPort: 8081
+ name: analysis
+ env:
+ - name: PORT
+ value: "8081"
+ - name: LOG_LEVEL
+ value: INFO
+ {{- end }}
+ {{- if .Values.operator.resultsVolume.accessSidecar.enabled }}
- name: results-access
image: busybox:stable
image: "{{ .Values.operator.resultsVolume.accessSidecar.image }}:{{ .Values.operator.resultsVolume.accessSidecar.imageTag }}"
@@ -75,14 +86,16 @@ spec:
- exec tail -f /dev/null
volumeMounts:
- mountPath: /results
- name: theodolite-pv-storage
+ name: theodolite-results-volume
{{- end }}
volumes:
- {{- if .Values.operator.resultsVolume.enabled }}
- - name: theodolite-pv-storage
+ - name: theodolite-results-volume
+ {{- if .Values.operator.resultsVolume.persistent.enabled }}
persistentVolumeClaim:
- claimName: {{ .Values.operator.resultsVolume.persistentVolumeClaim.name | quote }}
- {{- end }}
+ claimName: {{ include "theodolite.resultsClaimName" . | quote }}
+ {{- else }}
+ emptyDir: {}
+ {{- end }}
- name: benchmark-resources-uc1-kstreams
configMap:
name: benchmark-resources-uc1-kstreams
diff --git a/helm/values.yaml b/helm/values.yaml
index c51c82d0063cbe0ccdc4e058658c88a6711eaf1b..c45e62aed8126a8b9f20a6dd0ca6083cd162bc2b 100644
--- a/helm/values.yaml
+++ b/helm/values.yaml
@@ -256,11 +256,20 @@ operator:
image: ghcr.io/cau-se/theodolite-slo-checker-lag-trend
imageTag: latest
imagePullPolicy: Always
+ droppedRecordsKStreams:
+ enabled: true
+ image: ghcr.io/cau-se/theodolite-slo-checker-dropped-records-kstreams
+ imageTag: latest
+ imagePullPolicy: Always
resultsVolume:
- enabled: true
- persistentVolumeClaim:
- name: theodolite-pv-claim
+ persistent:
+ enabled: false
+ # existingClaim:
+ # storageClassName:
+ accessModes:
+ - ReadWriteOnce
+ size: 1Gi
accessSidecar:
enabled: true
image: busybox
diff --git a/slope-evaluator/Dockerfile b/slo-checker/dropped-records/Dockerfile
similarity index 100%
rename from slope-evaluator/Dockerfile
rename to slo-checker/dropped-records/Dockerfile
diff --git a/slo-checker/dropped-records/README.md b/slo-checker/dropped-records/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..3a1ea982a399201143ad50f173c934ff58abbf4a
--- /dev/null
+++ b/slo-checker/dropped-records/README.md
@@ -0,0 +1,80 @@
+# Kafka Streams Dropped Record SLO Evaluator
+
+## Execution
+
+For development:
+
+```sh
+uvicorn main:app --reload --port 81 # run this command inside the app/ folder
+```
+
+## Build the docker image:
+
+```sh
+docker build . -t theodolite-evaluator
+```
+
+Run the Docker image:
+
+```sh
+docker run -p 80:81 theodolite-evaluator
+```
+
+## Configuration
+
+You can set the `HOST` and the `PORT` (and a lot of more parameters) via environment variables. Default is `0.0.0.0:80`.
+For more information see the [Gunicorn/FastAPI Docker docs](https://github.com/tiangolo/uvicorn-gunicorn-fastapi-docker#advanced-usage).
+
+## API Documentation
+
+The running webserver provides a REST API with the following route:
+
+* /dropped-records
+ * Method: POST
+ * Body:
+ * results
+ * metric-metadata
+ * values
+ * metadata
+ * threshold
+ * warmup
+
+The body of the request must be a JSON string that satisfies the following conditions:
+
+* **dropped records**: This property is based on the [Range Vector type](https://www.prometheus.io/docs/prometheus/latest/querying/api/#range-vectors) from Prometheus and must have the following JSON *structure*:
+
+ ```json
+ {
+ "results": [
+ [
+ {
+ "metric": {
+ "<label-name>": "<label-value>"
+ },
+ "values": [
+ [
+ <unix_timestamp>, // 1.634624989695E9
+ "<sample_value>" // integer
+ ]
+ ]
+ }
+ ]
+ ],
+ "metadata": {
+ "threshold": 2000000,
+ "warmup": 60
+ }
+ }
+ ```
+
+### description
+
+* results:
+ * metric-metadata:
+ * Labels of this metric. The `dropped-records` slo checker does not use labels in the calculation of the service level objective.
+ * results
+ * The `<unix_timestamp>` provided as the first element of each element in the "values" array must be the timestamp of the measurement value in seconds (with optional decimal precision)
+ * The `<sample_value>` must be the measurement value as string.
+* metadata: For the calculation of the service level objective require metadata.
+ * **threshold**: Must be an unsigned integer that specifies the threshold for the SLO evaluation. The SLO is considered fulfilled, if the result value is below the threshold. If the result value is equal or above the threshold, the SLO is considered not fulfilled.
+ * **warmup**: Specifieds the warmup time in seconds that are ignored for evaluating the SLO.
diff --git a/slo-checker/dropped-records/app/main.py b/slo-checker/dropped-records/app/main.py
new file mode 100644
index 0000000000000000000000000000000000000000..c1577f9c11ed5a2798ee0b4505ae0739301ab2a8
--- /dev/null
+++ b/slo-checker/dropped-records/app/main.py
@@ -0,0 +1,32 @@
+from fastapi import FastAPI,Request
+import logging
+import os
+import json
+import sys
+
+app = FastAPI()
+
+logging.basicConfig(stream=sys.stdout,
+ format="%(asctime)s %(levelname)s %(name)s: %(message)s")
+logger = logging.getLogger("API")
+
+
+if os.getenv('LOG_LEVEL') == 'INFO':
+ logger.setLevel(logging.INFO)
+elif os.getenv('LOG_LEVEL') == 'WARNING':
+ logger.setLevel(logging.WARNING)
+elif os.getenv('LOG_LEVEL') == 'DEBUG':
+ logger.setLevel(logging.DEBUG)
+
+
+def check_service_level_objective(results, threshold):
+ return max(results) < threshold
+
+@app.post("/dropped-records",response_model=bool)
+async def evaluate_slope(request: Request):
+ data = json.loads(await request.body())
+ warmup = int(data['results'][0][0]['values'][0][0]) + int(data['metadata']['warmup'])
+ results = [int(val[1]) if(int(val[0]>=warmup)) else 0 for result in data['results'] for r in result for val in r['values'] ]
+ return check_service_level_objective(results=results, threshold=data['metadata']["threshold"])
+
+logger.info("SLO evaluator is online")
\ No newline at end of file
diff --git a/slo-checker/dropped-records/app/test.py b/slo-checker/dropped-records/app/test.py
new file mode 100644
index 0000000000000000000000000000000000000000..3c657c914002066357d58d88d7f8e4afe920db45
--- /dev/null
+++ b/slo-checker/dropped-records/app/test.py
@@ -0,0 +1,23 @@
+import unittest
+from main import app, check_service_level_objective
+import numpy as np
+import json
+from fastapi.testclient import TestClient
+
+class TestSloEvaluation(unittest.TestCase):
+ client = TestClient(app)
+
+ def test_1_rep(self):
+ with open('../resources/test-1-rep-success.json') as json_file:
+ data = json.load(json_file)
+ response = self.client.post("/dropped-records", json=data)
+ self.assertEquals(response.json(), True)
+
+ def test_check_service_level_objective(self):
+ list = [ x for x in range(-100, 100) ]
+
+ self.assertEquals(check_service_level_objective(list, 90), False)
+ self.assertEquals(check_service_level_objective(list, 110), True)
+
+if __name__ == '__main__':
+ unittest.main()
\ No newline at end of file
diff --git a/slope-evaluator/requirements.txt b/slo-checker/dropped-records/requirements.txt
similarity index 100%
rename from slope-evaluator/requirements.txt
rename to slo-checker/dropped-records/requirements.txt
diff --git a/slo-checker/dropped-records/resources/test-1-rep-success.json b/slo-checker/dropped-records/resources/test-1-rep-success.json
new file mode 100644
index 0000000000000000000000000000000000000000..0964c30fed60e34c1ac4cf6b6b89f81d95a2f0eb
--- /dev/null
+++ b/slo-checker/dropped-records/resources/test-1-rep-success.json
@@ -0,0 +1,273 @@
+{
+ "results": [
+ [
+ {
+ "metric": {
+ "job": "titan-ccp-aggregation"
+ },
+ "values": [
+ [
+ 1.634624674695E9,
+ "0"
+ ],
+ [
+ 1.634624679695E9,
+ "0"
+ ],
+ [
+ 1.634624684695E9,
+ "0"
+ ],
+ [
+ 1.634624689695E9,
+ "0"
+ ],
+ [
+ 1.634624694695E9,
+ "0"
+ ],
+ [
+ 1.634624699695E9,
+ "0"
+ ],
+ [
+ 1.634624704695E9,
+ "0"
+ ],
+ [
+ 1.634624709695E9,
+ "0"
+ ],
+ [
+ 1.634624714695E9,
+ "0"
+ ],
+ [
+ 1.634624719695E9,
+ "0"
+ ],
+ [
+ 1.634624724695E9,
+ "0"
+ ],
+ [
+ 1.634624729695E9,
+ "0"
+ ],
+ [
+ 1.634624734695E9,
+ "0"
+ ],
+ [
+ 1.634624739695E9,
+ "0"
+ ],
+ [
+ 1.634624744695E9,
+ "1"
+ ],
+ [
+ 1.634624749695E9,
+ "3"
+ ],
+ [
+ 1.634624754695E9,
+ "4"
+ ],
+ [
+ 1.634624759695E9,
+ "4"
+ ],
+ [
+ 1.634624764695E9,
+ "4"
+ ],
+ [
+ 1.634624769695E9,
+ "4"
+ ],
+ [
+ 1.634624774695E9,
+ "4"
+ ],
+ [
+ 1.634624779695E9,
+ "4"
+ ],
+ [
+ 1.634624784695E9,
+ "4"
+ ],
+ [
+ 1.634624789695E9,
+ "4"
+ ],
+ [
+ 1.634624794695E9,
+ "4"
+ ],
+ [
+ 1.634624799695E9,
+ "4"
+ ],
+ [
+ 1.634624804695E9,
+ "176"
+ ],
+ [
+ 1.634624809695E9,
+ "176"
+ ],
+ [
+ 1.634624814695E9,
+ "176"
+ ],
+ [
+ 1.634624819695E9,
+ "176"
+ ],
+ [
+ 1.634624824695E9,
+ "176"
+ ],
+ [
+ 1.634624829695E9,
+ "159524"
+ ],
+ [
+ 1.634624834695E9,
+ "209870"
+ ],
+ [
+ 1.634624839695E9,
+ "278597"
+ ],
+ [
+ 1.634624844695E9,
+ "460761"
+ ],
+ [
+ 1.634624849695E9,
+ "460761"
+ ],
+ [
+ 1.634624854695E9,
+ "460761"
+ ],
+ [
+ 1.634624859695E9,
+ "460761"
+ ],
+ [
+ 1.634624864695E9,
+ "460761"
+ ],
+ [
+ 1.634624869695E9,
+ "606893"
+ ],
+ [
+ 1.634624874695E9,
+ "653534"
+ ],
+ [
+ 1.634624879695E9,
+ "755796"
+ ],
+ [
+ 1.634624884695E9,
+ "919317"
+ ],
+ [
+ 1.634624889695E9,
+ "919317"
+ ],
+ [
+ 1.634624894695E9,
+ "955926"
+ ],
+ [
+ 1.634624899695E9,
+ "955926"
+ ],
+ [
+ 1.634624904695E9,
+ "955926"
+ ],
+ [
+ 1.634624909695E9,
+ "955926"
+ ],
+ [
+ 1.634624914695E9,
+ "955926"
+ ],
+ [
+ 1.634624919695E9,
+ "1036530"
+ ],
+ [
+ 1.634624924695E9,
+ "1078477"
+ ],
+ [
+ 1.634624929695E9,
+ "1194775"
+ ],
+ [
+ 1.634624934695E9,
+ "1347755"
+ ],
+ [
+ 1.634624939695E9,
+ "1352151"
+ ],
+ [
+ 1.634624944695E9,
+ "1360428"
+ ],
+ [
+ 1.634624949695E9,
+ "1360428"
+ ],
+ [
+ 1.634624954695E9,
+ "1360428"
+ ],
+ [
+ 1.634624959695E9,
+ "1360428"
+ ],
+ [
+ 1.634624964695E9,
+ "1360428"
+ ],
+ [
+ 1.634624969695E9,
+ "1525685"
+ ],
+ [
+ 1.634624974695E9,
+ "1689296"
+ ],
+ [
+ 1.634624979695E9,
+ "1771358"
+ ],
+ [
+ 1.634624984695E9,
+ "1854284"
+ ],
+ [
+ 1.634624989695E9,
+ "1854284"
+ ]
+ ]
+ }
+ ]
+ ],
+ "metadata": {
+ "threshold": 2000000,
+ "warmup": 60
+ }
+}
\ No newline at end of file
diff --git a/slo-checker/record-lag/Dockerfile b/slo-checker/record-lag/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..032b8153a6989ca04631ba553289dacb3620a38d
--- /dev/null
+++ b/slo-checker/record-lag/Dockerfile
@@ -0,0 +1,6 @@
+FROM tiangolo/uvicorn-gunicorn-fastapi:python3.7
+
+COPY requirements.txt requirements.txt
+RUN pip install -r requirements.txt
+
+COPY ./app /app
\ No newline at end of file
diff --git a/slo-checker/record-lag/README.md b/slo-checker/record-lag/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..b4882eeaf54aadfb8cbf33a957e6052a7b74123b
--- /dev/null
+++ b/slo-checker/record-lag/README.md
@@ -0,0 +1,80 @@
+# Lag Trend SLO Evaluator
+
+## Execution
+
+For development:
+
+```sh
+uvicorn main:app --reload # run this command inside the app/ folder
+```
+
+## Build the docker image:
+
+```sh
+docker build . -t theodolite-evaluator
+```
+
+Run the Docker image:
+
+```sh
+docker run -p 80:80 theodolite-evaluator
+```
+
+## Configuration
+
+You can set the `HOST` and the `PORT` (and a lot of more parameters) via environment variables. Default is `0.0.0.0:80`.
+For more information see the [Gunicorn/FastAPI Docker docs](https://github.com/tiangolo/uvicorn-gunicorn-fastapi-docker#advanced-usage).
+
+# API Documentation
+
+The running webserver provides a REST API with the following route:
+
+* /dropped-records
+ * Method: POST
+ * Body:
+ * results
+ * metric-metadata
+ * values
+ * metadata
+ * threshold
+ * warmup
+
+The body of the request must be a JSON string that satisfies the following conditions:
+
+* **total_lag**: This property is based on the [Range Vector type](https://www.prometheus.io/docs/prometheus/latest/querying/api/#range-vectors) from Prometheus and must have the following JSON *structure*:
+
+ ```json
+ {
+ "results": [
+ [
+ {
+ "metric": {
+ "<label-name>": "<label-value>"
+ },
+ "values": [
+ [
+ <unix_timestamp>, // 1.634624989695E9
+ "<sample_value>" // integer
+ ]
+ ]
+ }
+ ]
+ ],
+ "metadata": {
+ "threshold": 2000000,
+ "warmup": 60
+ }
+ }
+ ```
+
+### description
+
+* results:
+ * metric-metadata:
+ * Labels of this metric. The `dropped-records` slo checker does not use labels in the calculation of the service level objective.
+ * results
+ * The `<unix_timestamp>` provided as the first element of each element in the "values" array must be the timestamp of the measurement value in seconds (with optional decimal precision)
+ * The `<sample_value>` must be the measurement value as string.
+* metadata: For the calculation of the service level objective require metadata.
+ * **threshold**: Must be an unsigned integer that specifies the threshold for the SLO evaluation. The SLO is considered fulfilled, if the result value is below the threshold. If the result value is equal or above the threshold, the SLO is considered not fulfilled.
+ * **warmup**: Specifieds the warmup time in seconds that are ignored for evaluating the SLO.
diff --git a/slope-evaluator/app/main.py b/slo-checker/record-lag/app/main.py
similarity index 90%
rename from slope-evaluator/app/main.py
rename to slo-checker/record-lag/app/main.py
index 6f6788f0ca84b7710be5b509ca4f0641047e963d..621fa0cfc9c27e809fd92752de93f2795fa32c05 100644
--- a/slope-evaluator/app/main.py
+++ b/slo-checker/record-lag/app/main.py
@@ -38,7 +38,7 @@ def calculate_slope_trend(results, warmup):
err_msg = 'Computing trend slope failed.'
logger.exception(err_msg)
logger.error('Mark this subexperiment as not successful and continue benchmark.')
- return False
+ return float('inf')
logger.info("Computed lag trend slope is '%s'", trend_slope)
return trend_slope
@@ -49,7 +49,7 @@ def check_service_level_objective(results, threshold):
@app.post("/evaluate-slope",response_model=bool)
async def evaluate_slope(request: Request):
data = json.loads(await request.body())
- results = [calculate_slope_trend(total_lag, data['warmup']) for total_lag in data['total_lags']]
- return check_service_level_objective(results=results, threshold=data["threshold"])
+ results = [calculate_slope_trend(total_lag, data['metadata']['warmup']) for total_lag in data['results']]
+ return check_service_level_objective(results=results, threshold=data['metadata']["threshold"])
logger.info("SLO evaluator is online")
\ No newline at end of file
diff --git a/slope-evaluator/app/test.py b/slo-checker/record-lag/app/test.py
similarity index 99%
rename from slope-evaluator/app/test.py
rename to slo-checker/record-lag/app/test.py
index 9b165ea479bb9a552edaba7692df4fd4ef3f4ab4..c8d81f86b16255dcdce5337d8f00e922b98b4f82 100644
--- a/slope-evaluator/app/test.py
+++ b/slo-checker/record-lag/app/test.py
@@ -17,7 +17,7 @@ class TestSloEvaluation(unittest.TestCase):
data = json.load(json_file)
response = self.client.post("/evaluate-slope", json=data)
self.assertEquals(response.json(), True)
-
+
def test_check_service_level_objective(self):
list = [1,2,3,4]
self.assertEquals(check_service_level_objective(list, 2), False)
diff --git a/slope-evaluator/app/trend_slope_computer.py b/slo-checker/record-lag/app/trend_slope_computer.py
similarity index 100%
rename from slope-evaluator/app/trend_slope_computer.py
rename to slo-checker/record-lag/app/trend_slope_computer.py
diff --git a/slo-checker/record-lag/requirements.txt b/slo-checker/record-lag/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..8b6c3863226c2bd5e8bcd7982b2674dee593f192
--- /dev/null
+++ b/slo-checker/record-lag/requirements.txt
@@ -0,0 +1,5 @@
+fastapi==0.65.2
+scikit-learn==0.20.3
+pandas==1.0.3
+uvicorn
+requests
diff --git a/slope-evaluator/resources/test-1-rep-success.json b/slo-checker/record-lag/resources/test-1-rep-success.json
similarity index 97%
rename from slope-evaluator/resources/test-1-rep-success.json
rename to slo-checker/record-lag/resources/test-1-rep-success.json
index 9e315c707be7b2a874c58fcb1093aa86f7676560..dfe11282720ebfcdd60582b7717da892bc85a923 100644
--- a/slope-evaluator/resources/test-1-rep-success.json
+++ b/slo-checker/record-lag/resources/test-1-rep-success.json
@@ -1,5 +1,5 @@
{
- "total_lags": [
+ "results": [
[
{
"metric": {
@@ -134,6 +134,8 @@
}
]
],
- "threshold": 2000,
- "warmup": 0
+ "metadata": {
+ "threshold": 2000,
+ "warmup": 0
+ }
}
\ No newline at end of file
diff --git a/slope-evaluator/resources/test-3-rep-success.json b/slo-checker/record-lag/resources/test-3-rep-success.json
similarity index 98%
rename from slope-evaluator/resources/test-3-rep-success.json
rename to slo-checker/record-lag/resources/test-3-rep-success.json
index 485966cba40f01e4a646e626914510ba49b707bc..cf483f42f3783aecd1f428ac7bbbe2090c4cade0 100644
--- a/slope-evaluator/resources/test-3-rep-success.json
+++ b/slo-checker/record-lag/resources/test-3-rep-success.json
@@ -1,5 +1,5 @@
{
- "total_lags": [
+ "results": [
[
{
"metric": {
@@ -284,6 +284,8 @@
}
]
],
- "threshold": 2000,
- "warmup": 0
+ "metadata": {
+ "threshold": 2000,
+ "warmup": 0
+ }
}
\ No newline at end of file
diff --git a/slope-evaluator/README.md b/slope-evaluator/README.md
deleted file mode 100644
index cd9e6820ed46452ce44d57d0c7e5cd5ae05e5a3b..0000000000000000000000000000000000000000
--- a/slope-evaluator/README.md
+++ /dev/null
@@ -1,61 +0,0 @@
-# Lag Trend SLO Evaluator
-
-## Execution
-
-For development:
-
-```sh
-uvicorn main:app --reload # run this command inside the app/ folder
-```
-
-## Build the docker image:
-
-```sh
-docker build . -t theodolite-evaluator
-```
-
-Run the Docker image:
-
-```sh
-docker run -p 80:80 theodolite-evaluator
-```
-
-## Configuration
-
-You can set the `HOST` and the `PORT` (and a lot of more parameters) via environment variables. Default is `0.0.0.0:80`.
-For more information see the [Gunicorn/FastAPI Docker docs](https://github.com/tiangolo/uvicorn-gunicorn-fastapi-docker#advanced-usage).
-
-## API Documentation
-
-The running webserver provides a REST API with the following route:
-
-* /evaluate-slope
- * Method: POST
- * Body:
- * total_lags
- * threshold
- * warmup
-
-The body of the request must be a JSON string that satisfies the following conditions:
-
-* **total_lag**: This property is based on the [Range Vector type](https://www.prometheus.io/docs/prometheus/latest/querying/api/#range-vectors) from Prometheus and must have the following JSON structure:
- ```
- {
- [
- "metric": {
- "group": "<label_value>"
- },
- "values": [
- [
- <unix_timestamp>,
- "<sample_value>"
- ]
- ]
- ]
- }
- ```
- * The `<label_value>` provided in "metric.group" must be equal to the id of the Kafka consumer group.
- * The `<unix_timestamp>` provided as the first element of each element in the "values" array must be the timestamp of the measurement value in seconds (with optional decimal precision)
- * The `<sample_value>` must be the measurement value as string.
-* **threshold**: Must be an unsigned integer that specifies the threshold for the SLO evaluation. The SLO is considered fulfilled, if the result value is below the threshold. If the result value is equal or above the threshold, the SLO is considered not fulfilled.
-* **warmup**: Specifieds the warmup time in seconds that are ignored for evaluating the SLO.
\ No newline at end of file
diff --git a/theodolite/README.md b/theodolite/README.md
index 26efe96b4756316971378be810f69f6138613958..e0d7b9b46473a877d982e912cb85e2455e09be79 100644
--- a/theodolite/README.md
+++ b/theodolite/README.md
@@ -2,13 +2,13 @@
This project uses Quarkus, the Supersonic Subatomic Java Framework.
-If you want to learn more about Quarkus, please visit its website: https://quarkus.io/ .
+If you want to learn more about Quarkus, please visit its website: <https://quarkus.io/> .
## Running the application in dev mode
You can run your application in dev mode using:
-```shell script
+```sh
./gradlew quarkusDev
```
@@ -16,7 +16,7 @@ You can run your application in dev mode using:
The application can be packaged using:
-```shell script
+```sh
./gradlew build
```
@@ -25,7 +25,7 @@ an _über-jar_ as the dependencies are copied into the `build/lib` directory.
If you want to build an _über-jar_, execute the following command:
-```shell script
+```sh
./gradlew build -Dquarkus.package.type=uber-jar
```
@@ -33,92 +33,108 @@ The application is now runnable using `java -jar build/theodolite-1.0.0-SNAPSHOT
## Creating a native executable
+It is recommended to use the native GraalVM images to create executable jars from Theodolite. For more information please visit the [Native Image guide](https://www.graalvm.org/reference-manual/native-image/).
+
You can create a native executable using:
-```shell script
+```sh
./gradlew build -Dquarkus.package.type=native
```
Or, if you don't have GraalVM installed, you can run the native executable build in a container using:
-```shell script
+```sh
./gradlew build -Dquarkus.package.type=native -Dquarkus.native.container-build=true
```
You can then execute your native executable with:
```./build/theodolite-1.0.0-SNAPSHOT-runner```
-If you want to learn more about building native executables, please consult https://quarkus.io/guides/gradle-tooling.
+If you want to learn more about building native executables, please consult <https://quarkus.io/guides/gradle-tooling>.
## Build docker images
For the jvm version use:
-```shell script
+```sh
./gradlew build
docker build -f src/main/docker/Dockerfile.jvm -t theodolite-jvm .
```
For the native image version use:
-```shell script
+```sh
./gradlew build -Dquarkus.package.type=native
docker build -f src/main/docker/Dockerfile.native -t theodolite-native .
```
-## Execute docker images:
+## Execute docker images
Remember to set the environment variables first.
Jvm version:
-```shell script
+```sh
docker run -i --rm theodolite-jvm
```
Native image version:
-```shell script
+```sh
docker run -i --rm theodolite-native
```
## Environment variables
-**Production:** (Docker-Container)
+**Execution in Docker**:
| Variables name | Default value |Usage |
| -----------------------------|:----------------------------------:| ------------:|
| `NAMESPACE` | `default` |Determines the namespace of the Theodolite will be executed in. Used in the KubernetesBenchmark|
-| `THEODOLITE_EXECUTION` | `./config/BenchmarkExecution.yaml`|The complete path to the benchmarkExecution file. Used in the TheodoliteYamlExecutor. |
-| `THEODOLITE_BENCHMARK_TYPE` | `./config/BenchmarkType.yaml` |The complete path to the benchmarkType file. Used in the TheodoliteYamlExecutor.|
-| `THEODOLITE_APP_RESOURCES` | `./config` |The path under which the yamls for the resources for the subexperiments are found. Used in the KubernetesBenchmark|
-| `MODE` | `yaml-executor` | Defines the mode of operation: either `yaml-executor` or `operator`|
+| `THEODOLITE_EXECUTION` | `execution/execution.yaml` |The complete path to the benchmarkExecution file. Used in the TheodoliteYamlExecutor. |
+| `THEODOLITE_BENCHMARK_TYPE` | `benchmark/benchmark.yaml` |The complete path to the benchmarkType file. Used in the TheodoliteYamlExecutor.|
+| `THEODOLITE_APP_RESOURCES` | `./benchmark-resources` |The path under which the yamls for the resources for the subexperiments are found. Used in the KubernetesBenchmark|
+| `MODE` | `standalone` |Defines the mode of operation: either `standalone` or `operator`
-**Development:** (local via Intellij)
+**Execution in IntelliJ**:
When running Theodolite from within IntelliJ via
-[Run Configurations](https://www.jetbrains.com/help/idea/work-with-gradle-tasks.html#gradle_run_config), set the *
-Environment variables* field to:
+[Run Configurations](https://www.jetbrains.com/help/idea/work-with-gradle-tasks.html#gradle_run_config), set the *Environment variables* field to:
+
+Set the following environment variables to run the example in the `standalone` mode within the IDE:
+
+```sh
+THEODOLITE_BENCHMARK=./../../../../examples/standalone/example-benchmark.yaml;THEODOLITE_EXECUTION=./../../../../examples/standalone/example-execution.yaml;THEODOLITE_APP_RESOURCES=./../../../../examples/resources;
+```
+
+Alternative:
+``` sh
+export THEODOLITE_BENCHMARK=./../../../../examples/standalone/example-benchmark.yaml
+export THEODOLITE_APP_RESOURCES=./../../../../examples/resources;
+export THEODOLITE_EXECUTION=./../../../../examples/standalone/example-execution.yaml
+./gradlew quarkusDev
```
-NAMESPACE=default;THEODOLITE_BENCHMARK=./../../../../config/BenchmarkType.yaml;THEODOLITE_APP_RESOURCES=./../../../../config;THEODOLITE_EXECUTION=./../../../../config/BenchmarkExecution.yaml;MODE=operator
+
+Set the following environment variables to run the example in the `operator` mode within the IDE:
+
+```sh
+THEODOLITE_APP_RESOURCES=./../../../../examples/resources;MODE=operator
```
Alternative:
``` sh
-export NAMESPACE=default
-export THEODOLITE_BENCHMARK=./../../../../config/BenchmarkType.yaml
-export THEODOLITE_APP_RESOURCES=./../../../../config
-export THEODOLITE_EXECUTION=./../../../../config/BenchmarkExecution.yaml
+export THEODOLITE_APP_RESOURCES=./../../../../examples/resources;
export MODE=operator
./gradlew quarkusDev
-
```
-#### Install Detekt Code analysis Plugin
+Additionally, the benchmark and execution resources must be installed.
+
+### Install Detekt Code analysis Plugin
-Install https://plugins.jetbrains.com/plugin/10761-detekt
+Install <https://plugins.jetbrains.com/plugin/10761-detekt>
- Install the plugin
- Navigate to Settings/Preferences -> Tools -> Detekt
@@ -127,7 +143,7 @@ Install https://plugins.jetbrains.com/plugin/10761-detekt
-> detekt issues will be annotated on-the-fly while coding
-**ingore Failures in build:** add
+**ingore Failures in build**: add
```ignoreFailures = true```
diff --git a/theodolite/examples/operator/example-benchmark.yaml b/theodolite/examples/operator/example-benchmark.yaml
index 2fd7fe26a7016d2f5df89c15fc37d487195f116d..fb2c5749dc469e30baabdbad910c302bef0b26bc 100644
--- a/theodolite/examples/operator/example-benchmark.yaml
+++ b/theodolite/examples/operator/example-benchmark.yaml
@@ -40,9 +40,6 @@ spec:
name: "example-configmap"
files:
- "uc1-kstreams-deployment.yaml"
- - "aggregation-service.yaml"
- - "jmx-configmap.yaml"
- - "uc1-service-monitor.yaml"
loadGenResourceSets:
# - fileSystem:
# path: ./../../../../../../config
diff --git a/theodolite/examples/resources/uc1-kstreams-deployment.yaml b/theodolite/examples/resources/uc1-kstreams-deployment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..fdd1ff867ac83beb10856baec53569c88169232e
--- /dev/null
+++ b/theodolite/examples/resources/uc1-kstreams-deployment.yaml
@@ -0,0 +1,34 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: titan-ccp-aggregation
+spec:
+ selector:
+ matchLabels:
+ app: titan-ccp-aggregation
+ replicas: 1
+ template:
+ metadata:
+ labels:
+ app: titan-ccp-aggregation
+ spec:
+ terminationGracePeriodSeconds: 0
+ containers:
+ - name: uc-application
+ image: ghcr.io/cau-se/theodolite-uc1-kstreams-app:latest
+ ports:
+ - containerPort: 5555
+ name: jmx
+ env:
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-cp-kafka:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
+ - name: JAVA_OPTS
+ value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555"
+ - name: COMMIT_INTERVAL_MS # Set as default for the applications
+ value: "100"
+ resources:
+ limits:
+ memory: 4Gi
+ cpu: 1000m
\ No newline at end of file
diff --git a/theodolite/examples/resources/uc1-load-generator-deployment.yaml b/theodolite/examples/resources/uc1-load-generator-deployment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..9f9ccc6ae39407bb1f027e1e23cb152944b869e0
--- /dev/null
+++ b/theodolite/examples/resources/uc1-load-generator-deployment.yaml
@@ -0,0 +1,32 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: titan-ccp-load-generator
+spec:
+ selector:
+ matchLabels:
+ app: titan-ccp-load-generator
+ replicas: 1
+ template:
+ metadata:
+ labels:
+ app: titan-ccp-load-generator
+ spec:
+ terminationGracePeriodSeconds: 0
+ containers:
+ - name: workload-generator
+ image: ghcr.io/cau-se/theodolite-uc1-workload-generator:latest
+ ports:
+ - containerPort: 5701
+ name: coordination
+ env:
+ - name: KUBERNETES_NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ - name: KUBERNETES_DNS_NAME
+ value: "titan-ccp-load-generator.$(KUBERNETES_NAMESPACE).svc.cluster.local"
+ - name: KAFKA_BOOTSTRAP_SERVERS
+ value: "theodolite-cp-kafka:9092"
+ - name: SCHEMA_REGISTRY_URL
+ value: "http://theodolite-cp-schema-registry:8081"
diff --git a/theodolite/examples/resources/uc1-load-generator-service.yaml b/theodolite/examples/resources/uc1-load-generator-service.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..f8b26b3f6dece427f9c1ad4db94e351b042749b3
--- /dev/null
+++ b/theodolite/examples/resources/uc1-load-generator-service.yaml
@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: titan-ccp-load-generator
+ labels:
+ app: titan-ccp-load-generator
+spec:
+ type: ClusterIP
+ clusterIP: None
+ selector:
+ app: titan-ccp-load-generator
+ ports:
+ - name: coordination
+ port: 5701
+ targetPort: 5701
+ protocol: TCP
diff --git a/theodolite/examples/standalone/example-benchmark.yaml b/theodolite/examples/standalone/example-benchmark.yaml
index e9397def45ac460ccffd5e33b6851b6221f54e04..be3c7d86fca788ccbdbf7285f4100341836d4ae6 100644
--- a/theodolite/examples/standalone/example-benchmark.yaml
+++ b/theodolite/examples/standalone/example-benchmark.yaml
@@ -17,7 +17,7 @@ loadTypes:
properties:
loadGenMaxRecords: "15000"
kafkaConfig:
- bootstrapServer: "localhost:31290"
+ bootstrapServer: "theodolite-cp-kafka:9092"
topics:
- name: "input"
numPartitions: 40
@@ -36,9 +36,6 @@ appResourceSets:
name: "example-configmap"
files:
- "uc1-kstreams-deployment.yaml"
- - "aggregation-service.yaml"
- - "jmx-configmap.yaml"
- - "uc1-service-monitor.yaml"
loadGenResourceSets:
# - fileSystem:
# path: ./../../../../../../config